| blob | 5b607748c5761563879a030df66943411cfb1a5b |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE
3 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX
4 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2
5 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FP
6 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FCP
7 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512
8 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512-FCP
9 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ
10 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ-FCP
11 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW
12 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW-FCP
13 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512DQ-BW
14 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ-BW-FCP
16 ; These patterns are produced by LoopVectorizer for interleaved loads.
18 define void @load_i8_stride8_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4, ptr %out.vec5, ptr %out.vec6, ptr %out.vec7) nounwind {
19 ; SSE-LABEL: load_i8_stride8_vf2:
20 ; SSE: # %bb.0:
21 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
22 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r10
23 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r11
24 ; SSE-NEXT: movdqa (%rdi), %xmm1
25 ; SSE-NEXT: movdqa {{.*#+}} xmm0 = [255,0,0,0,255,0,0,0]
26 ; SSE-NEXT: pand %xmm1, %xmm0
27 ; SSE-NEXT: packuswb %xmm0, %xmm0
28 ; SSE-NEXT: packuswb %xmm0, %xmm0
29 ; SSE-NEXT: packuswb %xmm0, %xmm0
30 ; SSE-NEXT: pxor %xmm2, %xmm2
31 ; SSE-NEXT: movdqa %xmm1, %xmm3
32 ; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
33 ; SSE-NEXT: movdqa %xmm1, %xmm4
34 ; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
35 ; SSE-NEXT: movdqa %xmm4, %xmm2
36 ; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
37 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
38 ; SSE-NEXT: packuswb %xmm5, %xmm5
39 ; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
40 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm1[0,2,2,3]
41 ; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[1,3,2,3,4,5,6,7]
42 ; SSE-NEXT: packuswb %xmm6, %xmm6
43 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
44 ; SSE-NEXT: packuswb %xmm2, %xmm2
45 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
46 ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm1[2,0,2,3,4,5,6,7]
47 ; SSE-NEXT: packuswb %xmm7, %xmm7
48 ; SSE-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
49 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,1,1]
50 ; SSE-NEXT: packuswb %xmm3, %xmm3
51 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]
52 ; SSE-NEXT: packuswb %xmm1, %xmm1
53 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
54 ; SSE-NEXT: packuswb %xmm4, %xmm4
55 ; SSE-NEXT: movd %xmm0, %edi
56 ; SSE-NEXT: movw %di, (%rsi)
57 ; SSE-NEXT: movd %xmm5, %esi
58 ; SSE-NEXT: movw %si, (%rdx)
59 ; SSE-NEXT: movd %xmm6, %edx
60 ; SSE-NEXT: movw %dx, (%rcx)
61 ; SSE-NEXT: movd %xmm2, %ecx
62 ; SSE-NEXT: movw %cx, (%r8)
63 ; SSE-NEXT: movd %xmm7, %ecx
64 ; SSE-NEXT: movw %cx, (%r9)
65 ; SSE-NEXT: movd %xmm3, %ecx
66 ; SSE-NEXT: movw %cx, (%r11)
67 ; SSE-NEXT: movd %xmm1, %ecx
68 ; SSE-NEXT: movw %cx, (%r10)
69 ; SSE-NEXT: movd %xmm4, %ecx
70 ; SSE-NEXT: movw %cx, (%rax)
71 ; SSE-NEXT: retq
72 ;
73 ; AVX-LABEL: load_i8_stride8_vf2:
74 ; AVX: # %bb.0:
75 ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax
76 ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %r10
77 ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %r11
78 ; AVX-NEXT: vmovdqa (%rdi), %xmm0
79 ; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
80 ; AVX-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
81 ; AVX-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[2,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
82 ; AVX-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[3,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
83 ; AVX-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[4,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
84 ; AVX-NEXT: vpshufb {{.*#+}} xmm6 = xmm0[5,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
85 ; AVX-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[6,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
86 ; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[7,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
87 ; AVX-NEXT: vpextrw $0, %xmm1, (%rsi)
88 ; AVX-NEXT: vpextrw $0, %xmm2, (%rdx)
89 ; AVX-NEXT: vpextrw $0, %xmm3, (%rcx)
90 ; AVX-NEXT: vpextrw $0, %xmm4, (%r8)
91 ; AVX-NEXT: vpextrw $0, %xmm5, (%r9)
92 ; AVX-NEXT: vpextrw $0, %xmm6, (%r11)
93 ; AVX-NEXT: vpextrw $0, %xmm7, (%r10)
94 ; AVX-NEXT: vpextrw $0, %xmm0, (%rax)
95 ; AVX-NEXT: retq
96 ;
97 ; AVX2-LABEL: load_i8_stride8_vf2:
98 ; AVX2: # %bb.0:
99 ; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
100 ; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
101 ; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
102 ; AVX2-NEXT: vmovdqa (%rdi), %xmm0
103 ; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
104 ; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
105 ; AVX2-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[2,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
106 ; AVX2-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[3,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
107 ; AVX2-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[4,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
108 ; AVX2-NEXT: vpshufb {{.*#+}} xmm6 = xmm0[5,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
109 ; AVX2-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[6,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
110 ; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[7,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
111 ; AVX2-NEXT: vpextrw $0, %xmm1, (%rsi)
112 ; AVX2-NEXT: vpextrw $0, %xmm2, (%rdx)
113 ; AVX2-NEXT: vpextrw $0, %xmm3, (%rcx)
114 ; AVX2-NEXT: vpextrw $0, %xmm4, (%r8)
115 ; AVX2-NEXT: vpextrw $0, %xmm5, (%r9)
116 ; AVX2-NEXT: vpextrw $0, %xmm6, (%r11)
117 ; AVX2-NEXT: vpextrw $0, %xmm7, (%r10)
118 ; AVX2-NEXT: vpextrw $0, %xmm0, (%rax)
119 ; AVX2-NEXT: retq
120 ;
121 ; AVX2-FP-LABEL: load_i8_stride8_vf2:
122 ; AVX2-FP: # %bb.0:
123 ; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %rax
124 ; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %r10
125 ; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %r11
126 ; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm0
127 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
128 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
129 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[2,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
130 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[3,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
131 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[4,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
132 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm6 = xmm0[5,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
133 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[6,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
134 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[7,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
135 ; AVX2-FP-NEXT: vpextrw $0, %xmm1, (%rsi)
136 ; AVX2-FP-NEXT: vpextrw $0, %xmm2, (%rdx)
137 ; AVX2-FP-NEXT: vpextrw $0, %xmm3, (%rcx)
138 ; AVX2-FP-NEXT: vpextrw $0, %xmm4, (%r8)
139 ; AVX2-FP-NEXT: vpextrw $0, %xmm5, (%r9)
140 ; AVX2-FP-NEXT: vpextrw $0, %xmm6, (%r11)
141 ; AVX2-FP-NEXT: vpextrw $0, %xmm7, (%r10)
142 ; AVX2-FP-NEXT: vpextrw $0, %xmm0, (%rax)
143 ; AVX2-FP-NEXT: retq
144 ;
145 ; AVX2-FCP-LABEL: load_i8_stride8_vf2:
146 ; AVX2-FCP: # %bb.0:
147 ; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
148 ; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
149 ; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11
150 ; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm0
151 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
152 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
153 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[2,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
154 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[3,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
155 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[4,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
156 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm0[5,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
157 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[6,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
158 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[7,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
159 ; AVX2-FCP-NEXT: vpextrw $0, %xmm1, (%rsi)
160 ; AVX2-FCP-NEXT: vpextrw $0, %xmm2, (%rdx)
161 ; AVX2-FCP-NEXT: vpextrw $0, %xmm3, (%rcx)
162 ; AVX2-FCP-NEXT: vpextrw $0, %xmm4, (%r8)
163 ; AVX2-FCP-NEXT: vpextrw $0, %xmm5, (%r9)
164 ; AVX2-FCP-NEXT: vpextrw $0, %xmm6, (%r11)
165 ; AVX2-FCP-NEXT: vpextrw $0, %xmm7, (%r10)
166 ; AVX2-FCP-NEXT: vpextrw $0, %xmm0, (%rax)
167 ; AVX2-FCP-NEXT: retq
168 ;
169 ; AVX512-LABEL: load_i8_stride8_vf2:
170 ; AVX512: # %bb.0:
171 ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
172 ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r10
173 ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
174 ; AVX512-NEXT: vmovdqa (%rdi), %xmm0
175 ; AVX512-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[1,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
176 ; AVX512-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[2,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
177 ; AVX512-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[3,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
178 ; AVX512-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[4,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
179 ; AVX512-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[5,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
180 ; AVX512-NEXT: vpshufb {{.*#+}} xmm6 = xmm0[6,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
181 ; AVX512-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[7,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
182 ; AVX512-NEXT: vpmovqb %xmm0, (%rsi)
183 ; AVX512-NEXT: vpextrw $0, %xmm1, (%rdx)
184 ; AVX512-NEXT: vpextrw $0, %xmm2, (%rcx)
185 ; AVX512-NEXT: vpextrw $0, %xmm3, (%r8)
186 ; AVX512-NEXT: vpextrw $0, %xmm4, (%r9)
187 ; AVX512-NEXT: vpextrw $0, %xmm5, (%r11)
188 ; AVX512-NEXT: vpextrw $0, %xmm6, (%r10)
189 ; AVX512-NEXT: vpextrw $0, %xmm7, (%rax)
190 ; AVX512-NEXT: retq
191 ;
192 ; AVX512-FCP-LABEL: load_i8_stride8_vf2:
193 ; AVX512-FCP: # %bb.0:
194 ; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
195 ; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
196 ; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11
197 ; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm0
198 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[1,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
199 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[2,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
200 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[3,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
201 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[4,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
202 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[5,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
203 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm0[6,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
204 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[7,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
205 ; AVX512-FCP-NEXT: vpmovqb %xmm0, (%rsi)
206 ; AVX512-FCP-NEXT: vpextrw $0, %xmm1, (%rdx)
207 ; AVX512-FCP-NEXT: vpextrw $0, %xmm2, (%rcx)
208 ; AVX512-FCP-NEXT: vpextrw $0, %xmm3, (%r8)
209 ; AVX512-FCP-NEXT: vpextrw $0, %xmm4, (%r9)
210 ; AVX512-FCP-NEXT: vpextrw $0, %xmm5, (%r11)
211 ; AVX512-FCP-NEXT: vpextrw $0, %xmm6, (%r10)
212 ; AVX512-FCP-NEXT: vpextrw $0, %xmm7, (%rax)
213 ; AVX512-FCP-NEXT: retq
214 ;
215 ; AVX512DQ-LABEL: load_i8_stride8_vf2:
216 ; AVX512DQ: # %bb.0:
217 ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax
218 ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %r10
219 ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %r11
220 ; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0
221 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[1,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
222 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[2,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
223 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[3,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
224 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[4,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
225 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[5,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
226 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm6 = xmm0[6,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
227 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[7,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
228 ; AVX512DQ-NEXT: vpmovqb %xmm0, (%rsi)
229 ; AVX512DQ-NEXT: vpextrw $0, %xmm1, (%rdx)
230 ; AVX512DQ-NEXT: vpextrw $0, %xmm2, (%rcx)
231 ; AVX512DQ-NEXT: vpextrw $0, %xmm3, (%r8)
232 ; AVX512DQ-NEXT: vpextrw $0, %xmm4, (%r9)
233 ; AVX512DQ-NEXT: vpextrw $0, %xmm5, (%r11)
234 ; AVX512DQ-NEXT: vpextrw $0, %xmm6, (%r10)
235 ; AVX512DQ-NEXT: vpextrw $0, %xmm7, (%rax)
236 ; AVX512DQ-NEXT: retq
237 ;
238 ; AVX512DQ-FCP-LABEL: load_i8_stride8_vf2:
239 ; AVX512DQ-FCP: # %bb.0:
240 ; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
241 ; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
242 ; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11
243 ; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm0
244 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[1,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
245 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[2,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
246 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[3,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
247 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[4,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
248 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[5,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
249 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm0[6,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
250 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[7,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
251 ; AVX512DQ-FCP-NEXT: vpmovqb %xmm0, (%rsi)
252 ; AVX512DQ-FCP-NEXT: vpextrw $0, %xmm1, (%rdx)
253 ; AVX512DQ-FCP-NEXT: vpextrw $0, %xmm2, (%rcx)
254 ; AVX512DQ-FCP-NEXT: vpextrw $0, %xmm3, (%r8)
255 ; AVX512DQ-FCP-NEXT: vpextrw $0, %xmm4, (%r9)
256 ; AVX512DQ-FCP-NEXT: vpextrw $0, %xmm5, (%r11)
257 ; AVX512DQ-FCP-NEXT: vpextrw $0, %xmm6, (%r10)
258 ; AVX512DQ-FCP-NEXT: vpextrw $0, %xmm7, (%rax)
259 ; AVX512DQ-FCP-NEXT: retq
260 ;
261 ; AVX512BW-LABEL: load_i8_stride8_vf2:
262 ; AVX512BW: # %bb.0:
263 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
264 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %r10
265 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %r11
266 ; AVX512BW-NEXT: vmovdqa (%rdi), %xmm0
267 ; AVX512BW-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[1,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
268 ; AVX512BW-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[2,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
269 ; AVX512BW-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[3,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
270 ; AVX512BW-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[4,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
271 ; AVX512BW-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[5,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
272 ; AVX512BW-NEXT: vpshufb {{.*#+}} xmm6 = xmm0[6,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
273 ; AVX512BW-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[7,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
274 ; AVX512BW-NEXT: vpmovqb %xmm0, (%rsi)
275 ; AVX512BW-NEXT: vpextrw $0, %xmm1, (%rdx)
276 ; AVX512BW-NEXT: vpextrw $0, %xmm2, (%rcx)
277 ; AVX512BW-NEXT: vpextrw $0, %xmm3, (%r8)
278 ; AVX512BW-NEXT: vpextrw $0, %xmm4, (%r9)
279 ; AVX512BW-NEXT: vpextrw $0, %xmm5, (%r11)
280 ; AVX512BW-NEXT: vpextrw $0, %xmm6, (%r10)
281 ; AVX512BW-NEXT: vpextrw $0, %xmm7, (%rax)
282 ; AVX512BW-NEXT: retq
283 ;
284 ; AVX512BW-FCP-LABEL: load_i8_stride8_vf2:
285 ; AVX512BW-FCP: # %bb.0:
286 ; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
287 ; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
288 ; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11
289 ; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %xmm0
290 ; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[1,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
291 ; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[2,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
292 ; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[3,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
293 ; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[4,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
294 ; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[5,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
295 ; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm0[6,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
296 ; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[7,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
297 ; AVX512BW-FCP-NEXT: vpmovqb %xmm0, (%rsi)
298 ; AVX512BW-FCP-NEXT: vpextrw $0, %xmm1, (%rdx)
299 ; AVX512BW-FCP-NEXT: vpextrw $0, %xmm2, (%rcx)
300 ; AVX512BW-FCP-NEXT: vpextrw $0, %xmm3, (%r8)
301 ; AVX512BW-FCP-NEXT: vpextrw $0, %xmm4, (%r9)
302 ; AVX512BW-FCP-NEXT: vpextrw $0, %xmm5, (%r11)
303 ; AVX512BW-FCP-NEXT: vpextrw $0, %xmm6, (%r10)
304 ; AVX512BW-FCP-NEXT: vpextrw $0, %xmm7, (%rax)
305 ; AVX512BW-FCP-NEXT: retq
306 ;
307 ; AVX512DQ-BW-LABEL: load_i8_stride8_vf2:
308 ; AVX512DQ-BW: # %bb.0:
309 ; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
310 ; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %r10
311 ; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %r11
312 ; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %xmm0
313 ; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[1,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
314 ; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[2,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
315 ; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[3,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
316 ; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[4,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
317 ; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[5,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
318 ; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm6 = xmm0[6,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
319 ; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[7,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
320 ; AVX512DQ-BW-NEXT: vpmovqb %xmm0, (%rsi)
321 ; AVX512DQ-BW-NEXT: vpextrw $0, %xmm1, (%rdx)
322 ; AVX512DQ-BW-NEXT: vpextrw $0, %xmm2, (%rcx)
323 ; AVX512DQ-BW-NEXT: vpextrw $0, %xmm3, (%r8)
324 ; AVX512DQ-BW-NEXT: vpextrw $0, %xmm4, (%r9)
325 ; AVX512DQ-BW-NEXT: vpextrw $0, %xmm5, (%r11)
326 ; AVX512DQ-BW-NEXT: vpextrw $0, %xmm6, (%r10)
327 ; AVX512DQ-BW-NEXT: vpextrw $0, %xmm7, (%rax)
328 ; AVX512DQ-BW-NEXT: retq
329 ;
330 ; AVX512DQ-BW-FCP-LABEL: load_i8_stride8_vf2:
331 ; AVX512DQ-BW-FCP: # %bb.0:
332 ; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
333 ; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
334 ; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11
335 ; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %xmm0
336 ; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[1,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
337 ; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[2,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
338 ; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[3,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
339 ; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[4,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
340 ; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[5,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
341 ; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm0[6,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
342 ; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[7,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
343 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %xmm0, (%rsi)
344 ; AVX512DQ-BW-FCP-NEXT: vpextrw $0, %xmm1, (%rdx)
345 ; AVX512DQ-BW-FCP-NEXT: vpextrw $0, %xmm2, (%rcx)
346 ; AVX512DQ-BW-FCP-NEXT: vpextrw $0, %xmm3, (%r8)
347 ; AVX512DQ-BW-FCP-NEXT: vpextrw $0, %xmm4, (%r9)
348 ; AVX512DQ-BW-FCP-NEXT: vpextrw $0, %xmm5, (%r11)
349 ; AVX512DQ-BW-FCP-NEXT: vpextrw $0, %xmm6, (%r10)
350 ; AVX512DQ-BW-FCP-NEXT: vpextrw $0, %xmm7, (%rax)
351 ; AVX512DQ-BW-FCP-NEXT: retq
352 %wide.vec = load <16 x i8>, ptr %in.vec, align 64
353 %strided.vec0 = shufflevector <16 x i8> %wide.vec, <16 x i8> poison, <2 x i32> <i32 0, i32 8>
354 %strided.vec1 = shufflevector <16 x i8> %wide.vec, <16 x i8> poison, <2 x i32> <i32 1, i32 9>
355 %strided.vec2 = shufflevector <16 x i8> %wide.vec, <16 x i8> poison, <2 x i32> <i32 2, i32 10>
356 %strided.vec3 = shufflevector <16 x i8> %wide.vec, <16 x i8> poison, <2 x i32> <i32 3, i32 11>
357 %strided.vec4 = shufflevector <16 x i8> %wide.vec, <16 x i8> poison, <2 x i32> <i32 4, i32 12>
358 %strided.vec5 = shufflevector <16 x i8> %wide.vec, <16 x i8> poison, <2 x i32> <i32 5, i32 13>
359 %strided.vec6 = shufflevector <16 x i8> %wide.vec, <16 x i8> poison, <2 x i32> <i32 6, i32 14>
360 %strided.vec7 = shufflevector <16 x i8> %wide.vec, <16 x i8> poison, <2 x i32> <i32 7, i32 15>
361 store <2 x i8> %strided.vec0, ptr %out.vec0, align 64
362 store <2 x i8> %strided.vec1, ptr %out.vec1, align 64
363 store <2 x i8> %strided.vec2, ptr %out.vec2, align 64
364 store <2 x i8> %strided.vec3, ptr %out.vec3, align 64
365 store <2 x i8> %strided.vec4, ptr %out.vec4, align 64
366 store <2 x i8> %strided.vec5, ptr %out.vec5, align 64
367 store <2 x i8> %strided.vec6, ptr %out.vec6, align 64
368 store <2 x i8> %strided.vec7, ptr %out.vec7, align 64
369 ret void
370 }
372 define void @load_i8_stride8_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4, ptr %out.vec5, ptr %out.vec6, ptr %out.vec7) nounwind {
373 ; SSE-LABEL: load_i8_stride8_vf4:
374 ; SSE: # %bb.0:
375 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
376 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r10
377 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r11
378 ; SSE-NEXT: movdqa (%rdi), %xmm5
379 ; SSE-NEXT: movdqa 16(%rdi), %xmm8
380 ; SSE-NEXT: movdqa {{.*#+}} xmm0 = [255,0,0,0,255,0,0,0]
381 ; SSE-NEXT: movdqa %xmm8, %xmm1
382 ; SSE-NEXT: pand %xmm0, %xmm1
383 ; SSE-NEXT: pand %xmm5, %xmm0
384 ; SSE-NEXT: packuswb %xmm1, %xmm0
385 ; SSE-NEXT: packuswb %xmm0, %xmm0
386 ; SSE-NEXT: packuswb %xmm0, %xmm0
387 ; SSE-NEXT: pxor %xmm3, %xmm3
388 ; SSE-NEXT: movdqa %xmm8, %xmm7
389 ; SSE-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm3[8],xmm7[9],xmm3[9],xmm7[10],xmm3[10],xmm7[11],xmm3[11],xmm7[12],xmm3[12],xmm7[13],xmm3[13],xmm7[14],xmm3[14],xmm7[15],xmm3[15]
390 ; SSE-NEXT: movdqa %xmm8, %xmm2
391 ; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
392 ; SSE-NEXT: movdqa %xmm2, %xmm10
393 ; SSE-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm7[0],xmm10[1],xmm7[1],xmm10[2],xmm7[2],xmm10[3],xmm7[3]
394 ; SSE-NEXT: packuswb %xmm10, %xmm10
395 ; SSE-NEXT: movdqa {{.*#+}} xmm1 = [65535,0,65535,65535,65535,65535,65535,65535]
396 ; SSE-NEXT: movdqa %xmm1, %xmm6
397 ; SSE-NEXT: pandn %xmm10, %xmm6
398 ; SSE-NEXT: movdqa %xmm5, %xmm9
399 ; SSE-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm3[8],xmm9[9],xmm3[9],xmm9[10],xmm3[10],xmm9[11],xmm3[11],xmm9[12],xmm3[12],xmm9[13],xmm3[13],xmm9[14],xmm3[14],xmm9[15],xmm3[15]
400 ; SSE-NEXT: movdqa %xmm5, %xmm4
401 ; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
402 ; SSE-NEXT: movdqa %xmm4, %xmm11
403 ; SSE-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
404 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm11[1,1,1,1]
405 ; SSE-NEXT: packuswb %xmm3, %xmm3
406 ; SSE-NEXT: pand %xmm1, %xmm3
407 ; SSE-NEXT: por %xmm6, %xmm3
408 ; SSE-NEXT: movdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]
409 ; SSE-NEXT: pand %xmm6, %xmm8
410 ; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm8[0,2,2,3]
411 ; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm12[0,1,1,3,4,5,6,7]
412 ; SSE-NEXT: packuswb %xmm12, %xmm12
413 ; SSE-NEXT: movdqa %xmm1, %xmm13
414 ; SSE-NEXT: pandn %xmm12, %xmm13
415 ; SSE-NEXT: pand %xmm6, %xmm5
416 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,2,2,3]
417 ; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[1,3,2,3,4,5,6,7]
418 ; SSE-NEXT: packuswb %xmm6, %xmm6
419 ; SSE-NEXT: pand %xmm1, %xmm6
420 ; SSE-NEXT: por %xmm13, %xmm6
421 ; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,2,3]
422 ; SSE-NEXT: movdqa %xmm1, %xmm12
423 ; SSE-NEXT: pandn %xmm10, %xmm12
424 ; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm11[3,3,3,3]
425 ; SSE-NEXT: packuswb %xmm10, %xmm10
426 ; SSE-NEXT: pand %xmm1, %xmm10
427 ; SSE-NEXT: por %xmm12, %xmm10
428 ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[3,1,2,3]
429 ; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm8[0,1,2,0,4,5,6,7]
430 ; SSE-NEXT: packuswb %xmm11, %xmm11
431 ; SSE-NEXT: movdqa %xmm1, %xmm12
432 ; SSE-NEXT: pandn %xmm11, %xmm12
433 ; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm5[3,1,2,3]
434 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm11[2,0,2,3,4,5,6,7]
435 ; SSE-NEXT: packuswb %xmm5, %xmm5
436 ; SSE-NEXT: pand %xmm1, %xmm5
437 ; SSE-NEXT: por %xmm12, %xmm5
438 ; SSE-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm7[4],xmm2[5],xmm7[5],xmm2[6],xmm7[6],xmm2[7],xmm7[7]
439 ; SSE-NEXT: packuswb %xmm2, %xmm2
440 ; SSE-NEXT: movdqa %xmm1, %xmm12
441 ; SSE-NEXT: pandn %xmm2, %xmm12
442 ; SSE-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm9[4],xmm4[5],xmm9[5],xmm4[6],xmm9[6],xmm4[7],xmm9[7]
443 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm4[1,1,1,1]
444 ; SSE-NEXT: packuswb %xmm7, %xmm7
445 ; SSE-NEXT: pand %xmm1, %xmm7
446 ; SSE-NEXT: por %xmm12, %xmm7
447 ; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[0,1,3,1,4,5,6,7]
448 ; SSE-NEXT: packuswb %xmm8, %xmm8
449 ; SSE-NEXT: movdqa %xmm1, %xmm9
450 ; SSE-NEXT: pandn %xmm8, %xmm9
451 ; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm11[3,1,2,3,4,5,6,7]
452 ; SSE-NEXT: packuswb %xmm8, %xmm8
453 ; SSE-NEXT: pand %xmm1, %xmm8
454 ; SSE-NEXT: por %xmm9, %xmm8
455 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
456 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
457 ; SSE-NEXT: packuswb %xmm4, %xmm4
458 ; SSE-NEXT: pand %xmm1, %xmm4
459 ; SSE-NEXT: pandn %xmm2, %xmm1
460 ; SSE-NEXT: por %xmm4, %xmm1
461 ; SSE-NEXT: movd %xmm0, (%rsi)
462 ; SSE-NEXT: movd %xmm3, (%rdx)
463 ; SSE-NEXT: movd %xmm6, (%rcx)
464 ; SSE-NEXT: movd %xmm10, (%r8)
465 ; SSE-NEXT: movd %xmm5, (%r9)
466 ; SSE-NEXT: movd %xmm7, (%r11)
467 ; SSE-NEXT: movd %xmm8, (%r10)
468 ; SSE-NEXT: movd %xmm1, (%rax)
469 ; SSE-NEXT: retq
470 ;
471 ; AVX-LABEL: load_i8_stride8_vf4:
472 ; AVX: # %bb.0:
473 ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax
474 ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %r10
475 ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %r11
476 ; AVX-NEXT: vmovd {{.*#+}} xmm0 = [0,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
477 ; AVX-NEXT: vmovdqa (%rdi), %xmm1
478 ; AVX-NEXT: vmovdqa 16(%rdi), %xmm2
479 ; AVX-NEXT: vpshufb %xmm0, %xmm2, %xmm3
480 ; AVX-NEXT: vpshufb %xmm0, %xmm1, %xmm0
481 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
482 ; AVX-NEXT: vmovd {{.*#+}} xmm3 = [1,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
483 ; AVX-NEXT: vpshufb %xmm3, %xmm2, %xmm4
484 ; AVX-NEXT: vpshufb %xmm3, %xmm1, %xmm3
485 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
486 ; AVX-NEXT: vmovd {{.*#+}} xmm4 = [2,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
487 ; AVX-NEXT: vpshufb %xmm4, %xmm2, %xmm5
488 ; AVX-NEXT: vpshufb %xmm4, %xmm1, %xmm4
489 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
490 ; AVX-NEXT: vmovd {{.*#+}} xmm5 = [3,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
491 ; AVX-NEXT: vpshufb %xmm5, %xmm2, %xmm6
492 ; AVX-NEXT: vpshufb %xmm5, %xmm1, %xmm5
493 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
494 ; AVX-NEXT: vmovd {{.*#+}} xmm6 = [4,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
495 ; AVX-NEXT: vpshufb %xmm6, %xmm2, %xmm7
496 ; AVX-NEXT: vpshufb %xmm6, %xmm1, %xmm6
497 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
498 ; AVX-NEXT: vmovd {{.*#+}} xmm7 = [5,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
499 ; AVX-NEXT: vpshufb %xmm7, %xmm2, %xmm8
500 ; AVX-NEXT: vpshufb %xmm7, %xmm1, %xmm7
501 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
502 ; AVX-NEXT: vmovd {{.*#+}} xmm8 = [6,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
503 ; AVX-NEXT: vpshufb %xmm8, %xmm2, %xmm9
504 ; AVX-NEXT: vpshufb %xmm8, %xmm1, %xmm8
505 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
506 ; AVX-NEXT: vmovd {{.*#+}} xmm9 = [7,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
507 ; AVX-NEXT: vpshufb %xmm9, %xmm2, %xmm2
508 ; AVX-NEXT: vpshufb %xmm9, %xmm1, %xmm1
509 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
510 ; AVX-NEXT: vmovd %xmm0, (%rsi)
511 ; AVX-NEXT: vmovd %xmm3, (%rdx)
512 ; AVX-NEXT: vmovd %xmm4, (%rcx)
513 ; AVX-NEXT: vmovd %xmm5, (%r8)
514 ; AVX-NEXT: vmovd %xmm6, (%r9)
515 ; AVX-NEXT: vmovd %xmm7, (%r11)
516 ; AVX-NEXT: vmovd %xmm8, (%r10)
517 ; AVX-NEXT: vmovd %xmm1, (%rax)
518 ; AVX-NEXT: retq
519 ;
520 ; AVX2-LABEL: load_i8_stride8_vf4:
521 ; AVX2: # %bb.0:
522 ; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
523 ; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
524 ; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
525 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm0 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
526 ; AVX2-NEXT: vmovdqa (%rdi), %xmm1
527 ; AVX2-NEXT: vmovdqa 16(%rdi), %xmm2
528 ; AVX2-NEXT: vpshufb %xmm0, %xmm2, %xmm3
529 ; AVX2-NEXT: vpshufb %xmm0, %xmm1, %xmm0
530 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
531 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm3 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
532 ; AVX2-NEXT: vpshufb %xmm3, %xmm2, %xmm4
533 ; AVX2-NEXT: vpshufb %xmm3, %xmm1, %xmm3
534 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
535 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm4 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
536 ; AVX2-NEXT: vpshufb %xmm4, %xmm2, %xmm5
537 ; AVX2-NEXT: vpshufb %xmm4, %xmm1, %xmm4
538 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
539 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm5 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
540 ; AVX2-NEXT: vpshufb %xmm5, %xmm2, %xmm6
541 ; AVX2-NEXT: vpshufb %xmm5, %xmm1, %xmm5
542 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
543 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm6 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
544 ; AVX2-NEXT: vpshufb %xmm6, %xmm2, %xmm7
545 ; AVX2-NEXT: vpshufb %xmm6, %xmm1, %xmm6
546 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
547 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm7 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
548 ; AVX2-NEXT: vpshufb %xmm7, %xmm2, %xmm8
549 ; AVX2-NEXT: vpshufb %xmm7, %xmm1, %xmm7
550 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
551 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm8 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
552 ; AVX2-NEXT: vpshufb %xmm8, %xmm2, %xmm9
553 ; AVX2-NEXT: vpshufb %xmm8, %xmm1, %xmm8
554 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
555 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm9 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
556 ; AVX2-NEXT: vpshufb %xmm9, %xmm2, %xmm2
557 ; AVX2-NEXT: vpshufb %xmm9, %xmm1, %xmm1
558 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
559 ; AVX2-NEXT: vmovd %xmm0, (%rsi)
560 ; AVX2-NEXT: vmovd %xmm3, (%rdx)
561 ; AVX2-NEXT: vmovd %xmm4, (%rcx)
562 ; AVX2-NEXT: vmovd %xmm5, (%r8)
563 ; AVX2-NEXT: vmovd %xmm6, (%r9)
564 ; AVX2-NEXT: vmovd %xmm7, (%r11)
565 ; AVX2-NEXT: vmovd %xmm8, (%r10)
566 ; AVX2-NEXT: vmovd %xmm1, (%rax)
567 ; AVX2-NEXT: retq
568 ;
569 ; AVX2-FP-LABEL: load_i8_stride8_vf4:
570 ; AVX2-FP: # %bb.0:
571 ; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %rax
572 ; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %r10
573 ; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %r11
574 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
575 ; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm1
576 ; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm2
577 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm2, %xmm3
578 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm1, %xmm0
579 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
580 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
581 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm2, %xmm4
582 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm1, %xmm3
583 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
584 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm4 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
585 ; AVX2-FP-NEXT: vpshufb %xmm4, %xmm2, %xmm5
586 ; AVX2-FP-NEXT: vpshufb %xmm4, %xmm1, %xmm4
587 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
588 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm5 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
589 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm2, %xmm6
590 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm1, %xmm5
591 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
592 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm6 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
593 ; AVX2-FP-NEXT: vpshufb %xmm6, %xmm2, %xmm7
594 ; AVX2-FP-NEXT: vpshufb %xmm6, %xmm1, %xmm6
595 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
596 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm7 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
597 ; AVX2-FP-NEXT: vpshufb %xmm7, %xmm2, %xmm8
598 ; AVX2-FP-NEXT: vpshufb %xmm7, %xmm1, %xmm7
599 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
600 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm8 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
601 ; AVX2-FP-NEXT: vpshufb %xmm8, %xmm2, %xmm9
602 ; AVX2-FP-NEXT: vpshufb %xmm8, %xmm1, %xmm8
603 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
604 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm9 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
605 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm2, %xmm2
606 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm1, %xmm1
607 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
608 ; AVX2-FP-NEXT: vmovd %xmm0, (%rsi)
609 ; AVX2-FP-NEXT: vmovd %xmm3, (%rdx)
610 ; AVX2-FP-NEXT: vmovd %xmm4, (%rcx)
611 ; AVX2-FP-NEXT: vmovd %xmm5, (%r8)
612 ; AVX2-FP-NEXT: vmovd %xmm6, (%r9)
613 ; AVX2-FP-NEXT: vmovd %xmm7, (%r11)
614 ; AVX2-FP-NEXT: vmovd %xmm8, (%r10)
615 ; AVX2-FP-NEXT: vmovd %xmm1, (%rax)
616 ; AVX2-FP-NEXT: retq
617 ;
618 ; AVX2-FCP-LABEL: load_i8_stride8_vf4:
619 ; AVX2-FCP: # %bb.0:
620 ; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
621 ; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
622 ; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11
623 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
624 ; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm1
625 ; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
626 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm2, %xmm3
627 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm1, %xmm0
628 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
629 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
630 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm4
631 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm3
632 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
633 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm4 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
634 ; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm2, %xmm5
635 ; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm1, %xmm4
636 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
637 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm5 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
638 ; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm2, %xmm6
639 ; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm1, %xmm5
640 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
641 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm6 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
642 ; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm2, %xmm7
643 ; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm1, %xmm6
644 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
645 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm7 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
646 ; AVX2-FCP-NEXT: vpshufb %xmm7, %xmm2, %xmm8
647 ; AVX2-FCP-NEXT: vpshufb %xmm7, %xmm1, %xmm7
648 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
649 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm8 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
650 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm2, %xmm9
651 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm1, %xmm8
652 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
653 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm9 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
654 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm2, %xmm2
655 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm1, %xmm1
656 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
657 ; AVX2-FCP-NEXT: vmovd %xmm0, (%rsi)
658 ; AVX2-FCP-NEXT: vmovd %xmm3, (%rdx)
659 ; AVX2-FCP-NEXT: vmovd %xmm4, (%rcx)
660 ; AVX2-FCP-NEXT: vmovd %xmm5, (%r8)
661 ; AVX2-FCP-NEXT: vmovd %xmm6, (%r9)
662 ; AVX2-FCP-NEXT: vmovd %xmm7, (%r11)
663 ; AVX2-FCP-NEXT: vmovd %xmm8, (%r10)
664 ; AVX2-FCP-NEXT: vmovd %xmm1, (%rax)
665 ; AVX2-FCP-NEXT: retq
666 ;
667 ; AVX512-LABEL: load_i8_stride8_vf4:
668 ; AVX512: # %bb.0:
669 ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
670 ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r10
671 ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
672 ; AVX512-NEXT: vmovdqa (%rdi), %ymm0
673 ; AVX512-NEXT: vpsrlq $8, %ymm0, %ymm1
674 ; AVX512-NEXT: vpsrlq $16, %ymm0, %ymm2
675 ; AVX512-NEXT: vpsrlq $24, %ymm0, %ymm3
676 ; AVX512-NEXT: vpsrlq $32, %ymm0, %ymm4
677 ; AVX512-NEXT: vpsrlq $40, %ymm0, %ymm5
678 ; AVX512-NEXT: vpsrlq $48, %ymm0, %ymm6
679 ; AVX512-NEXT: vpsrlq $56, %ymm0, %ymm7
680 ; AVX512-NEXT: vpmovqb %ymm0, (%rsi)
681 ; AVX512-NEXT: vpmovqb %ymm1, (%rdx)
682 ; AVX512-NEXT: vpmovqb %ymm2, (%rcx)
683 ; AVX512-NEXT: vpmovqb %ymm3, (%r8)
684 ; AVX512-NEXT: vpmovqb %ymm4, (%r9)
685 ; AVX512-NEXT: vpmovqb %ymm5, (%r11)
686 ; AVX512-NEXT: vpmovqb %ymm6, (%r10)
687 ; AVX512-NEXT: vpmovqb %ymm7, (%rax)
688 ; AVX512-NEXT: vzeroupper
689 ; AVX512-NEXT: retq
690 ;
691 ; AVX512-FCP-LABEL: load_i8_stride8_vf4:
692 ; AVX512-FCP: # %bb.0:
693 ; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
694 ; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
695 ; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11
696 ; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm0
697 ; AVX512-FCP-NEXT: vpsrlq $8, %ymm0, %ymm1
698 ; AVX512-FCP-NEXT: vpsrlq $16, %ymm0, %ymm2
699 ; AVX512-FCP-NEXT: vpsrlq $24, %ymm0, %ymm3
700 ; AVX512-FCP-NEXT: vpsrlq $32, %ymm0, %ymm4
701 ; AVX512-FCP-NEXT: vpsrlq $40, %ymm0, %ymm5
702 ; AVX512-FCP-NEXT: vpsrlq $48, %ymm0, %ymm6
703 ; AVX512-FCP-NEXT: vpsrlq $56, %ymm0, %ymm7
704 ; AVX512-FCP-NEXT: vpmovqb %ymm0, (%rsi)
705 ; AVX512-FCP-NEXT: vpmovqb %ymm1, (%rdx)
706 ; AVX512-FCP-NEXT: vpmovqb %ymm2, (%rcx)
707 ; AVX512-FCP-NEXT: vpmovqb %ymm3, (%r8)
708 ; AVX512-FCP-NEXT: vpmovqb %ymm4, (%r9)
709 ; AVX512-FCP-NEXT: vpmovqb %ymm5, (%r11)
710 ; AVX512-FCP-NEXT: vpmovqb %ymm6, (%r10)
711 ; AVX512-FCP-NEXT: vpmovqb %ymm7, (%rax)
712 ; AVX512-FCP-NEXT: vzeroupper
713 ; AVX512-FCP-NEXT: retq
714 ;
715 ; AVX512DQ-LABEL: load_i8_stride8_vf4:
716 ; AVX512DQ: # %bb.0:
717 ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax
718 ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %r10
719 ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %r11
720 ; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm0
721 ; AVX512DQ-NEXT: vpsrlq $8, %ymm0, %ymm1
722 ; AVX512DQ-NEXT: vpsrlq $16, %ymm0, %ymm2
723 ; AVX512DQ-NEXT: vpsrlq $24, %ymm0, %ymm3
724 ; AVX512DQ-NEXT: vpsrlq $32, %ymm0, %ymm4
725 ; AVX512DQ-NEXT: vpsrlq $40, %ymm0, %ymm5
726 ; AVX512DQ-NEXT: vpsrlq $48, %ymm0, %ymm6
727 ; AVX512DQ-NEXT: vpsrlq $56, %ymm0, %ymm7
728 ; AVX512DQ-NEXT: vpmovqb %ymm0, (%rsi)
729 ; AVX512DQ-NEXT: vpmovqb %ymm1, (%rdx)
730 ; AVX512DQ-NEXT: vpmovqb %ymm2, (%rcx)
731 ; AVX512DQ-NEXT: vpmovqb %ymm3, (%r8)
732 ; AVX512DQ-NEXT: vpmovqb %ymm4, (%r9)
733 ; AVX512DQ-NEXT: vpmovqb %ymm5, (%r11)
734 ; AVX512DQ-NEXT: vpmovqb %ymm6, (%r10)
735 ; AVX512DQ-NEXT: vpmovqb %ymm7, (%rax)
736 ; AVX512DQ-NEXT: vzeroupper
737 ; AVX512DQ-NEXT: retq
738 ;
739 ; AVX512DQ-FCP-LABEL: load_i8_stride8_vf4:
740 ; AVX512DQ-FCP: # %bb.0:
741 ; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
742 ; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
743 ; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11
744 ; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm0
745 ; AVX512DQ-FCP-NEXT: vpsrlq $8, %ymm0, %ymm1
746 ; AVX512DQ-FCP-NEXT: vpsrlq $16, %ymm0, %ymm2
747 ; AVX512DQ-FCP-NEXT: vpsrlq $24, %ymm0, %ymm3
748 ; AVX512DQ-FCP-NEXT: vpsrlq $32, %ymm0, %ymm4
749 ; AVX512DQ-FCP-NEXT: vpsrlq $40, %ymm0, %ymm5
750 ; AVX512DQ-FCP-NEXT: vpsrlq $48, %ymm0, %ymm6
751 ; AVX512DQ-FCP-NEXT: vpsrlq $56, %ymm0, %ymm7
752 ; AVX512DQ-FCP-NEXT: vpmovqb %ymm0, (%rsi)
753 ; AVX512DQ-FCP-NEXT: vpmovqb %ymm1, (%rdx)
754 ; AVX512DQ-FCP-NEXT: vpmovqb %ymm2, (%rcx)
755 ; AVX512DQ-FCP-NEXT: vpmovqb %ymm3, (%r8)
756 ; AVX512DQ-FCP-NEXT: vpmovqb %ymm4, (%r9)
757 ; AVX512DQ-FCP-NEXT: vpmovqb %ymm5, (%r11)
758 ; AVX512DQ-FCP-NEXT: vpmovqb %ymm6, (%r10)
759 ; AVX512DQ-FCP-NEXT: vpmovqb %ymm7, (%rax)
760 ; AVX512DQ-FCP-NEXT: vzeroupper
761 ; AVX512DQ-FCP-NEXT: retq
762 ;
763 ; AVX512BW-LABEL: load_i8_stride8_vf4:
764 ; AVX512BW: # %bb.0:
765 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
766 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %r10
767 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %r11
768 ; AVX512BW-NEXT: vmovdqa (%rdi), %ymm0
769 ; AVX512BW-NEXT: vpsrlq $8, %ymm0, %ymm1
770 ; AVX512BW-NEXT: vpsrlq $16, %ymm0, %ymm2
771 ; AVX512BW-NEXT: vpsrlq $24, %ymm0, %ymm3
772 ; AVX512BW-NEXT: vpsrlq $32, %ymm0, %ymm4
773 ; AVX512BW-NEXT: vpsrlq $40, %ymm0, %ymm5
774 ; AVX512BW-NEXT: vpsrlq $48, %ymm0, %ymm6
775 ; AVX512BW-NEXT: vpsrlq $56, %ymm0, %ymm7
776 ; AVX512BW-NEXT: vpmovqb %ymm0, (%rsi)
777 ; AVX512BW-NEXT: vpmovqb %ymm1, (%rdx)
778 ; AVX512BW-NEXT: vpmovqb %ymm2, (%rcx)
779 ; AVX512BW-NEXT: vpmovqb %ymm3, (%r8)
780 ; AVX512BW-NEXT: vpmovqb %ymm4, (%r9)
781 ; AVX512BW-NEXT: vpmovqb %ymm5, (%r11)
782 ; AVX512BW-NEXT: vpmovqb %ymm6, (%r10)
783 ; AVX512BW-NEXT: vpmovqb %ymm7, (%rax)
784 ; AVX512BW-NEXT: vzeroupper
785 ; AVX512BW-NEXT: retq
786 ;
787 ; AVX512BW-FCP-LABEL: load_i8_stride8_vf4:
788 ; AVX512BW-FCP: # %bb.0:
789 ; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
790 ; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
791 ; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11
792 ; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %ymm0
793 ; AVX512BW-FCP-NEXT: vpsrlq $8, %ymm0, %ymm1
794 ; AVX512BW-FCP-NEXT: vpsrlq $16, %ymm0, %ymm2
795 ; AVX512BW-FCP-NEXT: vpsrlq $24, %ymm0, %ymm3
796 ; AVX512BW-FCP-NEXT: vpsrlq $32, %ymm0, %ymm4
797 ; AVX512BW-FCP-NEXT: vpsrlq $40, %ymm0, %ymm5
798 ; AVX512BW-FCP-NEXT: vpsrlq $48, %ymm0, %ymm6
799 ; AVX512BW-FCP-NEXT: vpsrlq $56, %ymm0, %ymm7
800 ; AVX512BW-FCP-NEXT: vpmovqb %ymm0, (%rsi)
801 ; AVX512BW-FCP-NEXT: vpmovqb %ymm1, (%rdx)
802 ; AVX512BW-FCP-NEXT: vpmovqb %ymm2, (%rcx)
803 ; AVX512BW-FCP-NEXT: vpmovqb %ymm3, (%r8)
804 ; AVX512BW-FCP-NEXT: vpmovqb %ymm4, (%r9)
805 ; AVX512BW-FCP-NEXT: vpmovqb %ymm5, (%r11)
806 ; AVX512BW-FCP-NEXT: vpmovqb %ymm6, (%r10)
807 ; AVX512BW-FCP-NEXT: vpmovqb %ymm7, (%rax)
808 ; AVX512BW-FCP-NEXT: vzeroupper
809 ; AVX512BW-FCP-NEXT: retq
810 ;
811 ; AVX512DQ-BW-LABEL: load_i8_stride8_vf4:
812 ; AVX512DQ-BW: # %bb.0:
813 ; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
814 ; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %r10
815 ; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %r11
816 ; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %ymm0
817 ; AVX512DQ-BW-NEXT: vpsrlq $8, %ymm0, %ymm1
818 ; AVX512DQ-BW-NEXT: vpsrlq $16, %ymm0, %ymm2
819 ; AVX512DQ-BW-NEXT: vpsrlq $24, %ymm0, %ymm3
820 ; AVX512DQ-BW-NEXT: vpsrlq $32, %ymm0, %ymm4
821 ; AVX512DQ-BW-NEXT: vpsrlq $40, %ymm0, %ymm5
822 ; AVX512DQ-BW-NEXT: vpsrlq $48, %ymm0, %ymm6
823 ; AVX512DQ-BW-NEXT: vpsrlq $56, %ymm0, %ymm7
824 ; AVX512DQ-BW-NEXT: vpmovqb %ymm0, (%rsi)
825 ; AVX512DQ-BW-NEXT: vpmovqb %ymm1, (%rdx)
826 ; AVX512DQ-BW-NEXT: vpmovqb %ymm2, (%rcx)
827 ; AVX512DQ-BW-NEXT: vpmovqb %ymm3, (%r8)
828 ; AVX512DQ-BW-NEXT: vpmovqb %ymm4, (%r9)
829 ; AVX512DQ-BW-NEXT: vpmovqb %ymm5, (%r11)
830 ; AVX512DQ-BW-NEXT: vpmovqb %ymm6, (%r10)
831 ; AVX512DQ-BW-NEXT: vpmovqb %ymm7, (%rax)
832 ; AVX512DQ-BW-NEXT: vzeroupper
833 ; AVX512DQ-BW-NEXT: retq
834 ;
835 ; AVX512DQ-BW-FCP-LABEL: load_i8_stride8_vf4:
836 ; AVX512DQ-BW-FCP: # %bb.0:
837 ; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
838 ; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
839 ; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11
840 ; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %ymm0
841 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $8, %ymm0, %ymm1
842 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $16, %ymm0, %ymm2
843 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $24, %ymm0, %ymm3
844 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $32, %ymm0, %ymm4
845 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $40, %ymm0, %ymm5
846 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $48, %ymm0, %ymm6
847 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $56, %ymm0, %ymm7
848 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %ymm0, (%rsi)
849 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %ymm1, (%rdx)
850 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %ymm2, (%rcx)
851 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %ymm3, (%r8)
852 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %ymm4, (%r9)
853 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %ymm5, (%r11)
854 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %ymm6, (%r10)
855 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %ymm7, (%rax)
856 ; AVX512DQ-BW-FCP-NEXT: vzeroupper
857 ; AVX512DQ-BW-FCP-NEXT: retq
858 %wide.vec = load <32 x i8>, ptr %in.vec, align 64
859 %strided.vec0 = shufflevector <32 x i8> %wide.vec, <32 x i8> poison, <4 x i32> <i32 0, i32 8, i32 16, i32 24>
860 %strided.vec1 = shufflevector <32 x i8> %wide.vec, <32 x i8> poison, <4 x i32> <i32 1, i32 9, i32 17, i32 25>
861 %strided.vec2 = shufflevector <32 x i8> %wide.vec, <32 x i8> poison, <4 x i32> <i32 2, i32 10, i32 18, i32 26>
862 %strided.vec3 = shufflevector <32 x i8> %wide.vec, <32 x i8> poison, <4 x i32> <i32 3, i32 11, i32 19, i32 27>
863 %strided.vec4 = shufflevector <32 x i8> %wide.vec, <32 x i8> poison, <4 x i32> <i32 4, i32 12, i32 20, i32 28>
864 %strided.vec5 = shufflevector <32 x i8> %wide.vec, <32 x i8> poison, <4 x i32> <i32 5, i32 13, i32 21, i32 29>
865 %strided.vec6 = shufflevector <32 x i8> %wide.vec, <32 x i8> poison, <4 x i32> <i32 6, i32 14, i32 22, i32 30>
866 %strided.vec7 = shufflevector <32 x i8> %wide.vec, <32 x i8> poison, <4 x i32> <i32 7, i32 15, i32 23, i32 31>
867 store <4 x i8> %strided.vec0, ptr %out.vec0, align 64
868 store <4 x i8> %strided.vec1, ptr %out.vec1, align 64
869 store <4 x i8> %strided.vec2, ptr %out.vec2, align 64
870 store <4 x i8> %strided.vec3, ptr %out.vec3, align 64
871 store <4 x i8> %strided.vec4, ptr %out.vec4, align 64
872 store <4 x i8> %strided.vec5, ptr %out.vec5, align 64
873 store <4 x i8> %strided.vec6, ptr %out.vec6, align 64
874 store <4 x i8> %strided.vec7, ptr %out.vec7, align 64
875 ret void
876 }
878 define void @load_i8_stride8_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4, ptr %out.vec5, ptr %out.vec6, ptr %out.vec7) nounwind {
879 ; SSE-LABEL: load_i8_stride8_vf8:
880 ; SSE: # %bb.0:
881 ; SSE-NEXT: pushq %rax
882 ; SSE-NEXT: movdqa (%rdi), %xmm12
883 ; SSE-NEXT: movdqa 16(%rdi), %xmm11
884 ; SSE-NEXT: movdqa 32(%rdi), %xmm9
885 ; SSE-NEXT: movdqa 48(%rdi), %xmm10
886 ; SSE-NEXT: movdqa {{.*#+}} xmm0 = [255,0,0,0,255,0,0,0]
887 ; SSE-NEXT: movdqa %xmm11, %xmm1
888 ; SSE-NEXT: pand %xmm0, %xmm1
889 ; SSE-NEXT: movdqa %xmm12, %xmm2
890 ; SSE-NEXT: pand %xmm0, %xmm2
891 ; SSE-NEXT: packuswb %xmm1, %xmm2
892 ; SSE-NEXT: packuswb %xmm2, %xmm2
893 ; SSE-NEXT: movdqa %xmm10, %xmm1
894 ; SSE-NEXT: pand %xmm0, %xmm1
895 ; SSE-NEXT: pand %xmm9, %xmm0
896 ; SSE-NEXT: packuswb %xmm1, %xmm0
897 ; SSE-NEXT: packuswb %xmm0, %xmm0
898 ; SSE-NEXT: packuswb %xmm0, %xmm2
899 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
900 ; SSE-NEXT: pxor %xmm7, %xmm7
901 ; SSE-NEXT: movdqa %xmm11, %xmm0
902 ; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm7[8],xmm0[9],xmm7[9],xmm0[10],xmm7[10],xmm0[11],xmm7[11],xmm0[12],xmm7[12],xmm0[13],xmm7[13],xmm0[14],xmm7[14],xmm0[15],xmm7[15]
903 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
904 ; SSE-NEXT: movdqa %xmm11, %xmm14
905 ; SSE-NEXT: punpcklbw {{.*#+}} xmm14 = xmm14[0],xmm7[0],xmm14[1],xmm7[1],xmm14[2],xmm7[2],xmm14[3],xmm7[3],xmm14[4],xmm7[4],xmm14[5],xmm7[5],xmm14[6],xmm7[6],xmm14[7],xmm7[7]
906 ; SSE-NEXT: movdqa %xmm14, %xmm15
907 ; SSE-NEXT: punpcklwd {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
908 ; SSE-NEXT: packuswb %xmm15, %xmm15
909 ; SSE-NEXT: movdqa {{.*#+}} xmm3 = [65535,0,65535,65535,65535,65535,65535,65535]
910 ; SSE-NEXT: movdqa %xmm3, %xmm0
911 ; SSE-NEXT: pandn %xmm15, %xmm0
912 ; SSE-NEXT: movdqa %xmm12, %xmm1
913 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm7[8],xmm1[9],xmm7[9],xmm1[10],xmm7[10],xmm1[11],xmm7[11],xmm1[12],xmm7[12],xmm1[13],xmm7[13],xmm1[14],xmm7[14],xmm1[15],xmm7[15]
914 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
915 ; SSE-NEXT: movdqa %xmm12, %xmm5
916 ; SSE-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3],xmm5[4],xmm7[4],xmm5[5],xmm7[5],xmm5[6],xmm7[6],xmm5[7],xmm7[7]
917 ; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
918 ; SSE-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
919 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm5[1,1,1,1]
920 ; SSE-NEXT: packuswb %xmm1, %xmm1
921 ; SSE-NEXT: pand %xmm3, %xmm1
922 ; SSE-NEXT: por %xmm0, %xmm1
923 ; SSE-NEXT: movdqa %xmm10, %xmm0
924 ; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm7[8],xmm0[9],xmm7[9],xmm0[10],xmm7[10],xmm0[11],xmm7[11],xmm0[12],xmm7[12],xmm0[13],xmm7[13],xmm0[14],xmm7[14],xmm0[15],xmm7[15]
925 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
926 ; SSE-NEXT: movdqa %xmm10, %xmm13
927 ; SSE-NEXT: punpcklbw {{.*#+}} xmm13 = xmm13[0],xmm7[0],xmm13[1],xmm7[1],xmm13[2],xmm7[2],xmm13[3],xmm7[3],xmm13[4],xmm7[4],xmm13[5],xmm7[5],xmm13[6],xmm7[6],xmm13[7],xmm7[7]
928 ; SSE-NEXT: movdqa %xmm13, %xmm4
929 ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
930 ; SSE-NEXT: packuswb %xmm4, %xmm4
931 ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm4[0,0,2,3]
932 ; SSE-NEXT: movdqa {{.*#+}} xmm2 = [65535,65535,65535,0,65535,65535,65535,65535]
933 ; SSE-NEXT: movdqa %xmm2, %xmm6
934 ; SSE-NEXT: pandn %xmm8, %xmm6
935 ; SSE-NEXT: movdqa %xmm9, %xmm8
936 ; SSE-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm7[8],xmm8[9],xmm7[9],xmm8[10],xmm7[10],xmm8[11],xmm7[11],xmm8[12],xmm7[12],xmm8[13],xmm7[13],xmm8[14],xmm7[14],xmm8[15],xmm7[15]
937 ; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
938 ; SSE-NEXT: movdqa %xmm9, %xmm0
939 ; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1],xmm0[2],xmm7[2],xmm0[3],xmm7[3],xmm0[4],xmm7[4],xmm0[5],xmm7[5],xmm0[6],xmm7[6],xmm0[7],xmm7[7]
940 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
941 ; SSE-NEXT: movdqa %xmm0, %xmm7
942 ; SSE-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
943 ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,1,1,3]
944 ; SSE-NEXT: packuswb %xmm8, %xmm8
945 ; SSE-NEXT: pand %xmm2, %xmm8
946 ; SSE-NEXT: por %xmm6, %xmm8
947 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm8[1,1,1,1]
948 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1]
949 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
950 ; SSE-NEXT: movdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]
951 ; SSE-NEXT: pand %xmm6, %xmm11
952 ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm11[0,2,2,3]
953 ; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[0,1,1,3,4,5,6,7]
954 ; SSE-NEXT: packuswb %xmm8, %xmm8
955 ; SSE-NEXT: movdqa %xmm3, %xmm1
956 ; SSE-NEXT: pandn %xmm8, %xmm1
957 ; SSE-NEXT: pand %xmm6, %xmm12
958 ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm12[0,2,2,3]
959 ; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[1,3,2,3,4,5,6,7]
960 ; SSE-NEXT: packuswb %xmm8, %xmm8
961 ; SSE-NEXT: pand %xmm3, %xmm8
962 ; SSE-NEXT: por %xmm1, %xmm8
963 ; SSE-NEXT: pand %xmm6, %xmm10
964 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm10[0,1,2,0]
965 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,7,5]
966 ; SSE-NEXT: packuswb %xmm1, %xmm1
967 ; SSE-NEXT: movdqa %xmm2, %xmm0
968 ; SSE-NEXT: pandn %xmm1, %xmm0
969 ; SSE-NEXT: pand %xmm6, %xmm9
970 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm9[0,1,2,0]
971 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,7]
972 ; SSE-NEXT: packuswb %xmm1, %xmm1
973 ; SSE-NEXT: pand %xmm2, %xmm1
974 ; SSE-NEXT: por %xmm0, %xmm1
975 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
976 ; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1]
977 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm15[1,1,2,3]
978 ; SSE-NEXT: movdqa %xmm3, %xmm1
979 ; SSE-NEXT: pandn %xmm0, %xmm1
980 ; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm5[3,3,3,3]
981 ; SSE-NEXT: packuswb %xmm15, %xmm15
982 ; SSE-NEXT: pand %xmm3, %xmm15
983 ; SSE-NEXT: por %xmm1, %xmm15
984 ; SSE-NEXT: movdqa %xmm2, %xmm0
985 ; SSE-NEXT: pandn %xmm4, %xmm0
986 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[2,2,3,3]
987 ; SSE-NEXT: packuswb %xmm1, %xmm1
988 ; SSE-NEXT: pand %xmm2, %xmm1
989 ; SSE-NEXT: por %xmm0, %xmm1
990 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
991 ; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]
992 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm11[3,1,2,3]
993 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm4[0,1,2,0,4,5,6,7]
994 ; SSE-NEXT: packuswb %xmm0, %xmm0
995 ; SSE-NEXT: movdqa %xmm3, %xmm1
996 ; SSE-NEXT: pandn %xmm0, %xmm1
997 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm12[3,1,2,3]
998 ; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm5[2,0,2,3,4,5,6,7]
999 ; SSE-NEXT: packuswb %xmm11, %xmm11
1000 ; SSE-NEXT: pand %xmm3, %xmm11
1001 ; SSE-NEXT: por %xmm1, %xmm11
1002 ; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,1,1,3]
1003 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm10[0,1,2,3,4,5,4,6]
1004 ; SSE-NEXT: packuswb %xmm0, %xmm0
1005 ; SSE-NEXT: movdqa %xmm2, %xmm1
1006 ; SSE-NEXT: pandn %xmm0, %xmm1
1007 ; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm9[0,1,1,3]
1008 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm12[0,1,2,3,4,6,6,7]
1009 ; SSE-NEXT: packuswb %xmm0, %xmm0
1010 ; SSE-NEXT: pand %xmm2, %xmm0
1011 ; SSE-NEXT: por %xmm1, %xmm0
1012 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
1013 ; SSE-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm0[0],xmm11[1],xmm0[1]
1014 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload
1015 ; SSE-NEXT: # xmm14 = xmm14[4],mem[4],xmm14[5],mem[5],xmm14[6],mem[6],xmm14[7],mem[7]
1016 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
1017 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
1018 ; SSE-NEXT: # xmm6 = xmm6[4],mem[4],xmm6[5],mem[5],xmm6[6],mem[6],xmm6[7],mem[7]
1019 ; SSE-NEXT: packuswb %xmm14, %xmm14
1020 ; SSE-NEXT: movdqa %xmm3, %xmm0
1021 ; SSE-NEXT: pandn %xmm14, %xmm0
1022 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm6[1,1,1,1]
1023 ; SSE-NEXT: packuswb %xmm9, %xmm9
1024 ; SSE-NEXT: pand %xmm3, %xmm9
1025 ; SSE-NEXT: por %xmm0, %xmm9
1026 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload
1027 ; SSE-NEXT: # xmm13 = xmm13[4],mem[4],xmm13[5],mem[5],xmm13[6],mem[6],xmm13[7],mem[7]
1028 ; SSE-NEXT: packuswb %xmm13, %xmm13
1029 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm13[0,0,2,3]
1030 ; SSE-NEXT: movdqa %xmm2, %xmm1
1031 ; SSE-NEXT: pandn %xmm0, %xmm1
1032 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
1033 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
1034 ; SSE-NEXT: # xmm7 = xmm7[4],mem[4],xmm7[5],mem[5],xmm7[6],mem[6],xmm7[7],mem[7]
1035 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,1,1,3]
1036 ; SSE-NEXT: packuswb %xmm0, %xmm0
1037 ; SSE-NEXT: pand %xmm2, %xmm0
1038 ; SSE-NEXT: por %xmm1, %xmm0
1039 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
1040 ; SSE-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm0[0],xmm9[1],xmm0[1]
1041 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm4[0,1,3,1,4,5,6,7]
1042 ; SSE-NEXT: packuswb %xmm0, %xmm0
1043 ; SSE-NEXT: movdqa %xmm3, %xmm1
1044 ; SSE-NEXT: pandn %xmm0, %xmm1
1045 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm5[3,1,2,3,4,5,6,7]
1046 ; SSE-NEXT: packuswb %xmm4, %xmm4
1047 ; SSE-NEXT: pand %xmm3, %xmm4
1048 ; SSE-NEXT: por %xmm1, %xmm4
1049 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm10[0,1,2,3,4,5,5,7]
1050 ; SSE-NEXT: packuswb %xmm0, %xmm0
1051 ; SSE-NEXT: movdqa %xmm2, %xmm1
1052 ; SSE-NEXT: pandn %xmm0, %xmm1
1053 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm12[0,1,2,3,5,7,6,7]
1054 ; SSE-NEXT: packuswb %xmm0, %xmm0
1055 ; SSE-NEXT: pand %xmm2, %xmm0
1056 ; SSE-NEXT: por %xmm1, %xmm0
1057 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
1058 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
1059 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm14[1,1,2,3]
1060 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm6[3,3,3,3]
1061 ; SSE-NEXT: packuswb %xmm1, %xmm1
1062 ; SSE-NEXT: pand %xmm3, %xmm1
1063 ; SSE-NEXT: pandn %xmm0, %xmm3
1064 ; SSE-NEXT: por %xmm1, %xmm3
1065 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm7[2,2,3,3]
1066 ; SSE-NEXT: packuswb %xmm0, %xmm0
1067 ; SSE-NEXT: pand %xmm2, %xmm0
1068 ; SSE-NEXT: pandn %xmm13, %xmm2
1069 ; SSE-NEXT: por %xmm0, %xmm2
1070 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
1071 ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
1072 ; SSE-NEXT: pshufd $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1073 ; SSE-NEXT: # xmm0 = mem[0,3,2,3]
1074 ; SSE-NEXT: movq %xmm0, (%rsi)
1075 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1076 ; SSE-NEXT: movlps %xmm0, (%rdx)
1077 ; SSE-NEXT: movq %xmm8, (%rcx)
1078 ; SSE-NEXT: movq %xmm15, (%r8)
1079 ; SSE-NEXT: movq %xmm11, (%r9)
1080 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
1081 ; SSE-NEXT: movq %xmm9, (%rax)
1082 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
1083 ; SSE-NEXT: movq %xmm4, (%rax)
1084 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
1085 ; SSE-NEXT: movq %xmm3, (%rax)
1086 ; SSE-NEXT: popq %rax
1087 ; SSE-NEXT: retq
1088 ;
1089 ; AVX-LABEL: load_i8_stride8_vf8:
1090 ; AVX: # %bb.0:
1091 ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax
1092 ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %r10
1093 ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %r11
1094 ; AVX-NEXT: vmovd {{.*#+}} xmm4 = [0,0,0,8,0,0,0,0,0,0,0,0,0,0,0,0]
1095 ; AVX-NEXT: vmovdqa (%rdi), %xmm0
1096 ; AVX-NEXT: vmovdqa 16(%rdi), %xmm1
1097 ; AVX-NEXT: vmovdqa 32(%rdi), %xmm2
1098 ; AVX-NEXT: vmovdqa 48(%rdi), %xmm3
1099 ; AVX-NEXT: vpshufb %xmm4, %xmm3, %xmm5
1100 ; AVX-NEXT: vpshufb %xmm4, %xmm2, %xmm4
1101 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
1102 ; AVX-NEXT: vmovd {{.*#+}} xmm5 = [0,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
1103 ; AVX-NEXT: vpshufb %xmm5, %xmm1, %xmm6
1104 ; AVX-NEXT: vpshufb %xmm5, %xmm0, %xmm5
1105 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
1106 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3],xmm5[4,5,6,7]
1107 ; AVX-NEXT: vmovd {{.*#+}} xmm5 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
1108 ; AVX-NEXT: vpshufb %xmm5, %xmm3, %xmm6
1109 ; AVX-NEXT: vpshufb %xmm5, %xmm2, %xmm5
1110 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
1111 ; AVX-NEXT: vmovd {{.*#+}} xmm6 = [1,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
1112 ; AVX-NEXT: vpshufb %xmm6, %xmm1, %xmm7
1113 ; AVX-NEXT: vpshufb %xmm6, %xmm0, %xmm6
1114 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
1115 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5,6,7]
1116 ; AVX-NEXT: vmovd {{.*#+}} xmm6 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
1117 ; AVX-NEXT: vpshufb %xmm6, %xmm3, %xmm7
1118 ; AVX-NEXT: vpshufb %xmm6, %xmm2, %xmm6
1119 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
1120 ; AVX-NEXT: vmovd {{.*#+}} xmm7 = [2,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
1121 ; AVX-NEXT: vpshufb %xmm7, %xmm1, %xmm8
1122 ; AVX-NEXT: vpshufb %xmm7, %xmm0, %xmm7
1123 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
1124 ; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3],xmm7[4,5,6,7]
1125 ; AVX-NEXT: vmovd {{.*#+}} xmm7 = [0,0,3,11,0,0,0,0,0,0,0,0,0,0,0,0]
1126 ; AVX-NEXT: vpshufb %xmm7, %xmm3, %xmm8
1127 ; AVX-NEXT: vpshufb %xmm7, %xmm2, %xmm7
1128 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
1129 ; AVX-NEXT: vmovd {{.*#+}} xmm8 = [3,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
1130 ; AVX-NEXT: vpshufb %xmm8, %xmm1, %xmm9
1131 ; AVX-NEXT: vpshufb %xmm8, %xmm0, %xmm8
1132 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
1133 ; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3],xmm8[4,5,6,7]
1134 ; AVX-NEXT: vmovd {{.*#+}} xmm8 = [0,0,4,12,0,0,0,0,0,0,0,0,0,0,0,0]
1135 ; AVX-NEXT: vpshufb %xmm8, %xmm3, %xmm9
1136 ; AVX-NEXT: vpshufb %xmm8, %xmm2, %xmm8
1137 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
1138 ; AVX-NEXT: vmovd {{.*#+}} xmm9 = [4,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
1139 ; AVX-NEXT: vpshufb %xmm9, %xmm1, %xmm10
1140 ; AVX-NEXT: vpshufb %xmm9, %xmm0, %xmm9
1141 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
1142 ; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3],xmm9[4,5,6,7]
1143 ; AVX-NEXT: vmovd {{.*#+}} xmm9 = [0,0,5,13,0,0,0,0,0,0,0,0,0,0,0,0]
1144 ; AVX-NEXT: vpshufb %xmm9, %xmm3, %xmm10
1145 ; AVX-NEXT: vpshufb %xmm9, %xmm2, %xmm9
1146 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
1147 ; AVX-NEXT: vmovd {{.*#+}} xmm10 = [5,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
1148 ; AVX-NEXT: vpshufb %xmm10, %xmm1, %xmm11
1149 ; AVX-NEXT: vpshufb %xmm10, %xmm0, %xmm10
1150 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
1151 ; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3],xmm10[4,5,6,7]
1152 ; AVX-NEXT: vmovd {{.*#+}} xmm10 = [0,0,6,14,0,0,0,0,0,0,0,0,0,0,0,0]
1153 ; AVX-NEXT: vpshufb %xmm10, %xmm3, %xmm11
1154 ; AVX-NEXT: vpshufb %xmm10, %xmm2, %xmm10
1155 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
1156 ; AVX-NEXT: vmovd {{.*#+}} xmm11 = [6,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
1157 ; AVX-NEXT: vpshufb %xmm11, %xmm1, %xmm12
1158 ; AVX-NEXT: vpshufb %xmm11, %xmm0, %xmm11
1159 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
1160 ; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3],xmm11[4,5,6,7]
1161 ; AVX-NEXT: vmovd {{.*#+}} xmm11 = [0,0,7,15,0,0,0,0,0,0,0,0,0,0,0,0]
1162 ; AVX-NEXT: vpshufb %xmm11, %xmm3, %xmm3
1163 ; AVX-NEXT: vpshufb %xmm11, %xmm2, %xmm2
1164 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
1165 ; AVX-NEXT: vmovd {{.*#+}} xmm3 = [7,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
1166 ; AVX-NEXT: vpshufb %xmm3, %xmm1, %xmm1
1167 ; AVX-NEXT: vpshufb %xmm3, %xmm0, %xmm0
1168 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1169 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5,6,7]
1170 ; AVX-NEXT: vmovq %xmm4, (%rsi)
1171 ; AVX-NEXT: vmovq %xmm5, (%rdx)
1172 ; AVX-NEXT: vmovq %xmm6, (%rcx)
1173 ; AVX-NEXT: vmovq %xmm7, (%r8)
1174 ; AVX-NEXT: vmovq %xmm8, (%r9)
1175 ; AVX-NEXT: vmovq %xmm9, (%r11)
1176 ; AVX-NEXT: vmovq %xmm10, (%r10)
1177 ; AVX-NEXT: vmovq %xmm0, (%rax)
1178 ; AVX-NEXT: retq
1179 ;
1180 ; AVX2-LABEL: load_i8_stride8_vf8:
1181 ; AVX2: # %bb.0:
1182 ; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
1183 ; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
1184 ; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
1185 ; AVX2-NEXT: vmovdqa (%rdi), %xmm0
1186 ; AVX2-NEXT: vmovdqa 16(%rdi), %xmm1
1187 ; AVX2-NEXT: vmovdqa 32(%rdi), %xmm2
1188 ; AVX2-NEXT: vmovdqa 48(%rdi), %xmm3
1189 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm4 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
1190 ; AVX2-NEXT: vpshufb %xmm4, %xmm3, %xmm5
1191 ; AVX2-NEXT: vpshufb %xmm4, %xmm2, %xmm4
1192 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
1193 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm5 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
1194 ; AVX2-NEXT: vpshufb %xmm5, %xmm1, %xmm6
1195 ; AVX2-NEXT: vpshufb %xmm5, %xmm0, %xmm5
1196 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
1197 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0],xmm4[1],xmm5[2,3]
1198 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm5 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
1199 ; AVX2-NEXT: vpshufb %xmm5, %xmm3, %xmm6
1200 ; AVX2-NEXT: vpshufb %xmm5, %xmm2, %xmm5
1201 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
1202 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm6 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
1203 ; AVX2-NEXT: vpshufb %xmm6, %xmm1, %xmm7
1204 ; AVX2-NEXT: vpshufb %xmm6, %xmm0, %xmm6
1205 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
1206 ; AVX2-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
1207 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm6 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
1208 ; AVX2-NEXT: vpshufb %xmm6, %xmm3, %xmm7
1209 ; AVX2-NEXT: vpshufb %xmm6, %xmm2, %xmm6
1210 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
1211 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm7 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
1212 ; AVX2-NEXT: vpshufb %xmm7, %xmm1, %xmm8
1213 ; AVX2-NEXT: vpshufb %xmm7, %xmm0, %xmm7
1214 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
1215 ; AVX2-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0],xmm6[1],xmm7[2,3]
1216 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm7 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
1217 ; AVX2-NEXT: vpshufb %xmm7, %xmm3, %xmm8
1218 ; AVX2-NEXT: vpshufb %xmm7, %xmm2, %xmm7
1219 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
1220 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm8 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
1221 ; AVX2-NEXT: vpshufb %xmm8, %xmm1, %xmm9
1222 ; AVX2-NEXT: vpshufb %xmm8, %xmm0, %xmm8
1223 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
1224 ; AVX2-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0],xmm7[1],xmm8[2,3]
1225 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm8 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
1226 ; AVX2-NEXT: vpshufb %xmm8, %xmm3, %xmm9
1227 ; AVX2-NEXT: vpshufb %xmm8, %xmm2, %xmm8
1228 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
1229 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm9 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
1230 ; AVX2-NEXT: vpshufb %xmm9, %xmm1, %xmm10
1231 ; AVX2-NEXT: vpshufb %xmm9, %xmm0, %xmm9
1232 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
1233 ; AVX2-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0],xmm8[1],xmm9[2,3]
1234 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm9 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
1235 ; AVX2-NEXT: vpshufb %xmm9, %xmm3, %xmm10
1236 ; AVX2-NEXT: vpshufb %xmm9, %xmm2, %xmm9
1237 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
1238 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm10 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
1239 ; AVX2-NEXT: vpshufb %xmm10, %xmm1, %xmm11
1240 ; AVX2-NEXT: vpshufb %xmm10, %xmm0, %xmm10
1241 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
1242 ; AVX2-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0],xmm9[1],xmm10[2,3]
1243 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm10 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
1244 ; AVX2-NEXT: vpshufb %xmm10, %xmm3, %xmm11
1245 ; AVX2-NEXT: vpshufb %xmm10, %xmm2, %xmm10
1246 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
1247 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm11 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
1248 ; AVX2-NEXT: vpshufb %xmm11, %xmm1, %xmm12
1249 ; AVX2-NEXT: vpshufb %xmm11, %xmm0, %xmm11
1250 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
1251 ; AVX2-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0],xmm10[1],xmm11[2,3]
1252 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm11 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
1253 ; AVX2-NEXT: vpshufb %xmm11, %xmm3, %xmm3
1254 ; AVX2-NEXT: vpshufb %xmm11, %xmm2, %xmm2
1255 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
1256 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm3 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
1257 ; AVX2-NEXT: vpshufb %xmm3, %xmm1, %xmm1
1258 ; AVX2-NEXT: vpshufb %xmm3, %xmm0, %xmm0
1259 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1260 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
1261 ; AVX2-NEXT: vmovq %xmm4, (%rsi)
1262 ; AVX2-NEXT: vmovq %xmm5, (%rdx)
1263 ; AVX2-NEXT: vmovq %xmm6, (%rcx)
1264 ; AVX2-NEXT: vmovq %xmm7, (%r8)
1265 ; AVX2-NEXT: vmovq %xmm8, (%r9)
1266 ; AVX2-NEXT: vmovq %xmm9, (%r11)
1267 ; AVX2-NEXT: vmovq %xmm10, (%r10)
1268 ; AVX2-NEXT: vmovq %xmm0, (%rax)
1269 ; AVX2-NEXT: retq
1270 ;
1271 ; AVX2-FP-LABEL: load_i8_stride8_vf8:
1272 ; AVX2-FP: # %bb.0:
1273 ; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %rax
1274 ; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %r10
1275 ; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %r11
1276 ; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm0
1277 ; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm1
1278 ; AVX2-FP-NEXT: vmovdqa 32(%rdi), %xmm2
1279 ; AVX2-FP-NEXT: vmovdqa 48(%rdi), %xmm3
1280 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm4 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
1281 ; AVX2-FP-NEXT: vpshufb %xmm4, %xmm3, %xmm5
1282 ; AVX2-FP-NEXT: vpshufb %xmm4, %xmm2, %xmm4
1283 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
1284 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm5 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
1285 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm1, %xmm6
1286 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm0, %xmm5
1287 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
1288 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0],xmm4[1],xmm5[2,3]
1289 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm5 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
1290 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm3, %xmm6
1291 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm2, %xmm5
1292 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
1293 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm6 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
1294 ; AVX2-FP-NEXT: vpshufb %xmm6, %xmm1, %xmm7
1295 ; AVX2-FP-NEXT: vpshufb %xmm6, %xmm0, %xmm6
1296 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
1297 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
1298 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm6 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
1299 ; AVX2-FP-NEXT: vpshufb %xmm6, %xmm3, %xmm7
1300 ; AVX2-FP-NEXT: vpshufb %xmm6, %xmm2, %xmm6
1301 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
1302 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm7 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
1303 ; AVX2-FP-NEXT: vpshufb %xmm7, %xmm1, %xmm8
1304 ; AVX2-FP-NEXT: vpshufb %xmm7, %xmm0, %xmm7
1305 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
1306 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0],xmm6[1],xmm7[2,3]
1307 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm7 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
1308 ; AVX2-FP-NEXT: vpshufb %xmm7, %xmm3, %xmm8
1309 ; AVX2-FP-NEXT: vpshufb %xmm7, %xmm2, %xmm7
1310 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
1311 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm8 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
1312 ; AVX2-FP-NEXT: vpshufb %xmm8, %xmm1, %xmm9
1313 ; AVX2-FP-NEXT: vpshufb %xmm8, %xmm0, %xmm8
1314 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
1315 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0],xmm7[1],xmm8[2,3]
1316 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm8 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
1317 ; AVX2-FP-NEXT: vpshufb %xmm8, %xmm3, %xmm9
1318 ; AVX2-FP-NEXT: vpshufb %xmm8, %xmm2, %xmm8
1319 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
1320 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm9 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
1321 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm1, %xmm10
1322 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm0, %xmm9
1323 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
1324 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0],xmm8[1],xmm9[2,3]
1325 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm9 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
1326 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm3, %xmm10
1327 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm2, %xmm9
1328 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
1329 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
1330 ; AVX2-FP-NEXT: vpshufb %xmm10, %xmm1, %xmm11
1331 ; AVX2-FP-NEXT: vpshufb %xmm10, %xmm0, %xmm10
1332 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
1333 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0],xmm9[1],xmm10[2,3]
1334 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
1335 ; AVX2-FP-NEXT: vpshufb %xmm10, %xmm3, %xmm11
1336 ; AVX2-FP-NEXT: vpshufb %xmm10, %xmm2, %xmm10
1337 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
1338 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
1339 ; AVX2-FP-NEXT: vpshufb %xmm11, %xmm1, %xmm12
1340 ; AVX2-FP-NEXT: vpshufb %xmm11, %xmm0, %xmm11
1341 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
1342 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0],xmm10[1],xmm11[2,3]
1343 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
1344 ; AVX2-FP-NEXT: vpshufb %xmm11, %xmm3, %xmm3
1345 ; AVX2-FP-NEXT: vpshufb %xmm11, %xmm2, %xmm2
1346 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
1347 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
1348 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
1349 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm0, %xmm0
1350 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1351 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
1352 ; AVX2-FP-NEXT: vmovq %xmm4, (%rsi)
1353 ; AVX2-FP-NEXT: vmovq %xmm5, (%rdx)
1354 ; AVX2-FP-NEXT: vmovq %xmm6, (%rcx)
1355 ; AVX2-FP-NEXT: vmovq %xmm7, (%r8)
1356 ; AVX2-FP-NEXT: vmovq %xmm8, (%r9)
1357 ; AVX2-FP-NEXT: vmovq %xmm9, (%r11)
1358 ; AVX2-FP-NEXT: vmovq %xmm10, (%r10)
1359 ; AVX2-FP-NEXT: vmovq %xmm0, (%rax)
1360 ; AVX2-FP-NEXT: retq
1361 ;
1362 ; AVX2-FCP-LABEL: load_i8_stride8_vf8:
1363 ; AVX2-FCP: # %bb.0:
1364 ; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
1365 ; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
1366 ; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11
1367 ; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm0
1368 ; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm1
1369 ; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %xmm2
1370 ; AVX2-FCP-NEXT: vmovdqa 48(%rdi), %xmm3
1371 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm4 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
1372 ; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm3, %xmm5
1373 ; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm2, %xmm4
1374 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
1375 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm5 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
1376 ; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm1, %xmm6
1377 ; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm0, %xmm5
1378 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
1379 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0],xmm4[1],xmm5[2,3]
1380 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm5 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
1381 ; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm3, %xmm6
1382 ; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm2, %xmm5
1383 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
1384 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm6 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
1385 ; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm1, %xmm7
1386 ; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm0, %xmm6
1387 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
1388 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
1389 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm6 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
1390 ; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm3, %xmm7
1391 ; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm2, %xmm6
1392 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
1393 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm7 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
1394 ; AVX2-FCP-NEXT: vpshufb %xmm7, %xmm1, %xmm8
1395 ; AVX2-FCP-NEXT: vpshufb %xmm7, %xmm0, %xmm7
1396 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
1397 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0],xmm6[1],xmm7[2,3]
1398 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm7 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
1399 ; AVX2-FCP-NEXT: vpshufb %xmm7, %xmm3, %xmm8
1400 ; AVX2-FCP-NEXT: vpshufb %xmm7, %xmm2, %xmm7
1401 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
1402 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm8 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
1403 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm1, %xmm9
1404 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm0, %xmm8
1405 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
1406 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0],xmm7[1],xmm8[2,3]
1407 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm8 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
1408 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm3, %xmm9
1409 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm2, %xmm8
1410 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
1411 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm9 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
1412 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm1, %xmm10
1413 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm0, %xmm9
1414 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
1415 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0],xmm8[1],xmm9[2,3]
1416 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm9 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
1417 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm3, %xmm10
1418 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm2, %xmm9
1419 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
1420 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
1421 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm1, %xmm11
1422 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm0, %xmm10
1423 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
1424 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0],xmm9[1],xmm10[2,3]
1425 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
1426 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm3, %xmm11
1427 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm2, %xmm10
1428 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
1429 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
1430 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm1, %xmm12
1431 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm0, %xmm11
1432 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
1433 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0],xmm10[1],xmm11[2,3]
1434 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
1435 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm3, %xmm3
1436 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm2, %xmm2
1437 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
1438 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
1439 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
1440 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm0, %xmm0
1441 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1442 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
1443 ; AVX2-FCP-NEXT: vmovq %xmm4, (%rsi)
1444 ; AVX2-FCP-NEXT: vmovq %xmm5, (%rdx)
1445 ; AVX2-FCP-NEXT: vmovq %xmm6, (%rcx)
1446 ; AVX2-FCP-NEXT: vmovq %xmm7, (%r8)
1447 ; AVX2-FCP-NEXT: vmovq %xmm8, (%r9)
1448 ; AVX2-FCP-NEXT: vmovq %xmm9, (%r11)
1449 ; AVX2-FCP-NEXT: vmovq %xmm10, (%r10)
1450 ; AVX2-FCP-NEXT: vmovq %xmm0, (%rax)
1451 ; AVX2-FCP-NEXT: retq
1452 ;
1453 ; AVX512-LABEL: load_i8_stride8_vf8:
1454 ; AVX512: # %bb.0:
1455 ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
1456 ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r10
1457 ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
1458 ; AVX512-NEXT: vmovdqa64 (%rdi), %zmm0
1459 ; AVX512-NEXT: vpsrlq $8, %zmm0, %zmm1
1460 ; AVX512-NEXT: vpsrlq $16, %zmm0, %zmm2
1461 ; AVX512-NEXT: vpsrlq $24, %zmm0, %zmm3
1462 ; AVX512-NEXT: vpsrlq $32, %zmm0, %zmm4
1463 ; AVX512-NEXT: vpsrlq $40, %zmm0, %zmm5
1464 ; AVX512-NEXT: vpsrlq $48, %zmm0, %zmm6
1465 ; AVX512-NEXT: vpsrlq $56, %zmm0, %zmm7
1466 ; AVX512-NEXT: vpmovqb %zmm0, (%rsi)
1467 ; AVX512-NEXT: vpmovqb %zmm1, (%rdx)
1468 ; AVX512-NEXT: vpmovqb %zmm2, (%rcx)
1469 ; AVX512-NEXT: vpmovqb %zmm3, (%r8)
1470 ; AVX512-NEXT: vpmovqb %zmm4, (%r9)
1471 ; AVX512-NEXT: vpmovqb %zmm5, (%r11)
1472 ; AVX512-NEXT: vpmovqb %zmm6, (%r10)
1473 ; AVX512-NEXT: vpmovqb %zmm7, (%rax)
1474 ; AVX512-NEXT: vzeroupper
1475 ; AVX512-NEXT: retq
1476 ;
1477 ; AVX512-FCP-LABEL: load_i8_stride8_vf8:
1478 ; AVX512-FCP: # %bb.0:
1479 ; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
1480 ; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
1481 ; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11
1482 ; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
1483 ; AVX512-FCP-NEXT: vpsrlq $8, %zmm0, %zmm1
1484 ; AVX512-FCP-NEXT: vpsrlq $16, %zmm0, %zmm2
1485 ; AVX512-FCP-NEXT: vpsrlq $24, %zmm0, %zmm3
1486 ; AVX512-FCP-NEXT: vpsrlq $32, %zmm0, %zmm4
1487 ; AVX512-FCP-NEXT: vpsrlq $40, %zmm0, %zmm5
1488 ; AVX512-FCP-NEXT: vpsrlq $48, %zmm0, %zmm6
1489 ; AVX512-FCP-NEXT: vpsrlq $56, %zmm0, %zmm7
1490 ; AVX512-FCP-NEXT: vpmovqb %zmm0, (%rsi)
1491 ; AVX512-FCP-NEXT: vpmovqb %zmm1, (%rdx)
1492 ; AVX512-FCP-NEXT: vpmovqb %zmm2, (%rcx)
1493 ; AVX512-FCP-NEXT: vpmovqb %zmm3, (%r8)
1494 ; AVX512-FCP-NEXT: vpmovqb %zmm4, (%r9)
1495 ; AVX512-FCP-NEXT: vpmovqb %zmm5, (%r11)
1496 ; AVX512-FCP-NEXT: vpmovqb %zmm6, (%r10)
1497 ; AVX512-FCP-NEXT: vpmovqb %zmm7, (%rax)
1498 ; AVX512-FCP-NEXT: vzeroupper
1499 ; AVX512-FCP-NEXT: retq
1500 ;
1501 ; AVX512DQ-LABEL: load_i8_stride8_vf8:
1502 ; AVX512DQ: # %bb.0:
1503 ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax
1504 ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %r10
1505 ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %r11
1506 ; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm0
1507 ; AVX512DQ-NEXT: vpsrlq $8, %zmm0, %zmm1
1508 ; AVX512DQ-NEXT: vpsrlq $16, %zmm0, %zmm2
1509 ; AVX512DQ-NEXT: vpsrlq $24, %zmm0, %zmm3
1510 ; AVX512DQ-NEXT: vpsrlq $32, %zmm0, %zmm4
1511 ; AVX512DQ-NEXT: vpsrlq $40, %zmm0, %zmm5
1512 ; AVX512DQ-NEXT: vpsrlq $48, %zmm0, %zmm6
1513 ; AVX512DQ-NEXT: vpsrlq $56, %zmm0, %zmm7
1514 ; AVX512DQ-NEXT: vpmovqb %zmm0, (%rsi)
1515 ; AVX512DQ-NEXT: vpmovqb %zmm1, (%rdx)
1516 ; AVX512DQ-NEXT: vpmovqb %zmm2, (%rcx)
1517 ; AVX512DQ-NEXT: vpmovqb %zmm3, (%r8)
1518 ; AVX512DQ-NEXT: vpmovqb %zmm4, (%r9)
1519 ; AVX512DQ-NEXT: vpmovqb %zmm5, (%r11)
1520 ; AVX512DQ-NEXT: vpmovqb %zmm6, (%r10)
1521 ; AVX512DQ-NEXT: vpmovqb %zmm7, (%rax)
1522 ; AVX512DQ-NEXT: vzeroupper
1523 ; AVX512DQ-NEXT: retq
1524 ;
1525 ; AVX512DQ-FCP-LABEL: load_i8_stride8_vf8:
1526 ; AVX512DQ-FCP: # %bb.0:
1527 ; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
1528 ; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
1529 ; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11
1530 ; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
1531 ; AVX512DQ-FCP-NEXT: vpsrlq $8, %zmm0, %zmm1
1532 ; AVX512DQ-FCP-NEXT: vpsrlq $16, %zmm0, %zmm2
1533 ; AVX512DQ-FCP-NEXT: vpsrlq $24, %zmm0, %zmm3
1534 ; AVX512DQ-FCP-NEXT: vpsrlq $32, %zmm0, %zmm4
1535 ; AVX512DQ-FCP-NEXT: vpsrlq $40, %zmm0, %zmm5
1536 ; AVX512DQ-FCP-NEXT: vpsrlq $48, %zmm0, %zmm6
1537 ; AVX512DQ-FCP-NEXT: vpsrlq $56, %zmm0, %zmm7
1538 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm0, (%rsi)
1539 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm1, (%rdx)
1540 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm2, (%rcx)
1541 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm3, (%r8)
1542 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm4, (%r9)
1543 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm5, (%r11)
1544 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm6, (%r10)
1545 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm7, (%rax)
1546 ; AVX512DQ-FCP-NEXT: vzeroupper
1547 ; AVX512DQ-FCP-NEXT: retq
1548 ;
1549 ; AVX512BW-LABEL: load_i8_stride8_vf8:
1550 ; AVX512BW: # %bb.0:
1551 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
1552 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %r10
1553 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %r11
1554 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
1555 ; AVX512BW-NEXT: vpsrlq $8, %zmm0, %zmm1
1556 ; AVX512BW-NEXT: vpsrlq $16, %zmm0, %zmm2
1557 ; AVX512BW-NEXT: vpsrlq $24, %zmm0, %zmm3
1558 ; AVX512BW-NEXT: vpsrlq $32, %zmm0, %zmm4
1559 ; AVX512BW-NEXT: vpsrlq $40, %zmm0, %zmm5
1560 ; AVX512BW-NEXT: vpsrlq $48, %zmm0, %zmm6
1561 ; AVX512BW-NEXT: vpsrlq $56, %zmm0, %zmm7
1562 ; AVX512BW-NEXT: vpmovqb %zmm0, (%rsi)
1563 ; AVX512BW-NEXT: vpmovqb %zmm1, (%rdx)
1564 ; AVX512BW-NEXT: vpmovqb %zmm2, (%rcx)
1565 ; AVX512BW-NEXT: vpmovqb %zmm3, (%r8)
1566 ; AVX512BW-NEXT: vpmovqb %zmm4, (%r9)
1567 ; AVX512BW-NEXT: vpmovqb %zmm5, (%r11)
1568 ; AVX512BW-NEXT: vpmovqb %zmm6, (%r10)
1569 ; AVX512BW-NEXT: vpmovqb %zmm7, (%rax)
1570 ; AVX512BW-NEXT: vzeroupper
1571 ; AVX512BW-NEXT: retq
1572 ;
1573 ; AVX512BW-FCP-LABEL: load_i8_stride8_vf8:
1574 ; AVX512BW-FCP: # %bb.0:
1575 ; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
1576 ; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
1577 ; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11
1578 ; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
1579 ; AVX512BW-FCP-NEXT: vpsrlq $8, %zmm0, %zmm1
1580 ; AVX512BW-FCP-NEXT: vpsrlq $16, %zmm0, %zmm2
1581 ; AVX512BW-FCP-NEXT: vpsrlq $24, %zmm0, %zmm3
1582 ; AVX512BW-FCP-NEXT: vpsrlq $32, %zmm0, %zmm4
1583 ; AVX512BW-FCP-NEXT: vpsrlq $40, %zmm0, %zmm5
1584 ; AVX512BW-FCP-NEXT: vpsrlq $48, %zmm0, %zmm6
1585 ; AVX512BW-FCP-NEXT: vpsrlq $56, %zmm0, %zmm7
1586 ; AVX512BW-FCP-NEXT: vpmovqb %zmm0, (%rsi)
1587 ; AVX512BW-FCP-NEXT: vpmovqb %zmm1, (%rdx)
1588 ; AVX512BW-FCP-NEXT: vpmovqb %zmm2, (%rcx)
1589 ; AVX512BW-FCP-NEXT: vpmovqb %zmm3, (%r8)
1590 ; AVX512BW-FCP-NEXT: vpmovqb %zmm4, (%r9)
1591 ; AVX512BW-FCP-NEXT: vpmovqb %zmm5, (%r11)
1592 ; AVX512BW-FCP-NEXT: vpmovqb %zmm6, (%r10)
1593 ; AVX512BW-FCP-NEXT: vpmovqb %zmm7, (%rax)
1594 ; AVX512BW-FCP-NEXT: vzeroupper
1595 ; AVX512BW-FCP-NEXT: retq
1596 ;
1597 ; AVX512DQ-BW-LABEL: load_i8_stride8_vf8:
1598 ; AVX512DQ-BW: # %bb.0:
1599 ; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
1600 ; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %r10
1601 ; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %r11
1602 ; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm0
1603 ; AVX512DQ-BW-NEXT: vpsrlq $8, %zmm0, %zmm1
1604 ; AVX512DQ-BW-NEXT: vpsrlq $16, %zmm0, %zmm2
1605 ; AVX512DQ-BW-NEXT: vpsrlq $24, %zmm0, %zmm3
1606 ; AVX512DQ-BW-NEXT: vpsrlq $32, %zmm0, %zmm4
1607 ; AVX512DQ-BW-NEXT: vpsrlq $40, %zmm0, %zmm5
1608 ; AVX512DQ-BW-NEXT: vpsrlq $48, %zmm0, %zmm6
1609 ; AVX512DQ-BW-NEXT: vpsrlq $56, %zmm0, %zmm7
1610 ; AVX512DQ-BW-NEXT: vpmovqb %zmm0, (%rsi)
1611 ; AVX512DQ-BW-NEXT: vpmovqb %zmm1, (%rdx)
1612 ; AVX512DQ-BW-NEXT: vpmovqb %zmm2, (%rcx)
1613 ; AVX512DQ-BW-NEXT: vpmovqb %zmm3, (%r8)
1614 ; AVX512DQ-BW-NEXT: vpmovqb %zmm4, (%r9)
1615 ; AVX512DQ-BW-NEXT: vpmovqb %zmm5, (%r11)
1616 ; AVX512DQ-BW-NEXT: vpmovqb %zmm6, (%r10)
1617 ; AVX512DQ-BW-NEXT: vpmovqb %zmm7, (%rax)
1618 ; AVX512DQ-BW-NEXT: vzeroupper
1619 ; AVX512DQ-BW-NEXT: retq
1620 ;
1621 ; AVX512DQ-BW-FCP-LABEL: load_i8_stride8_vf8:
1622 ; AVX512DQ-BW-FCP: # %bb.0:
1623 ; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
1624 ; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
1625 ; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11
1626 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
1627 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $8, %zmm0, %zmm1
1628 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $16, %zmm0, %zmm2
1629 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $24, %zmm0, %zmm3
1630 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $32, %zmm0, %zmm4
1631 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $40, %zmm0, %zmm5
1632 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $48, %zmm0, %zmm6
1633 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $56, %zmm0, %zmm7
1634 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm0, (%rsi)
1635 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm1, (%rdx)
1636 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm2, (%rcx)
1637 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm3, (%r8)
1638 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm4, (%r9)
1639 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm5, (%r11)
1640 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm6, (%r10)
1641 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm7, (%rax)
1642 ; AVX512DQ-BW-FCP-NEXT: vzeroupper
1643 ; AVX512DQ-BW-FCP-NEXT: retq
1644 %wide.vec = load <64 x i8>, ptr %in.vec, align 64
1645 %strided.vec0 = shufflevector <64 x i8> %wide.vec, <64 x i8> poison, <8 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 40, i32 48, i32 56>
1646 %strided.vec1 = shufflevector <64 x i8> %wide.vec, <64 x i8> poison, <8 x i32> <i32 1, i32 9, i32 17, i32 25, i32 33, i32 41, i32 49, i32 57>
1647 %strided.vec2 = shufflevector <64 x i8> %wide.vec, <64 x i8> poison, <8 x i32> <i32 2, i32 10, i32 18, i32 26, i32 34, i32 42, i32 50, i32 58>
1648 %strided.vec3 = shufflevector <64 x i8> %wide.vec, <64 x i8> poison, <8 x i32> <i32 3, i32 11, i32 19, i32 27, i32 35, i32 43, i32 51, i32 59>
1649 %strided.vec4 = shufflevector <64 x i8> %wide.vec, <64 x i8> poison, <8 x i32> <i32 4, i32 12, i32 20, i32 28, i32 36, i32 44, i32 52, i32 60>
1650 %strided.vec5 = shufflevector <64 x i8> %wide.vec, <64 x i8> poison, <8 x i32> <i32 5, i32 13, i32 21, i32 29, i32 37, i32 45, i32 53, i32 61>
1651 %strided.vec6 = shufflevector <64 x i8> %wide.vec, <64 x i8> poison, <8 x i32> <i32 6, i32 14, i32 22, i32 30, i32 38, i32 46, i32 54, i32 62>
1652 %strided.vec7 = shufflevector <64 x i8> %wide.vec, <64 x i8> poison, <8 x i32> <i32 7, i32 15, i32 23, i32 31, i32 39, i32 47, i32 55, i32 63>
1653 store <8 x i8> %strided.vec0, ptr %out.vec0, align 64
1654 store <8 x i8> %strided.vec1, ptr %out.vec1, align 64
1655 store <8 x i8> %strided.vec2, ptr %out.vec2, align 64
1656 store <8 x i8> %strided.vec3, ptr %out.vec3, align 64
1657 store <8 x i8> %strided.vec4, ptr %out.vec4, align 64
1658 store <8 x i8> %strided.vec5, ptr %out.vec5, align 64
1659 store <8 x i8> %strided.vec6, ptr %out.vec6, align 64
1660 store <8 x i8> %strided.vec7, ptr %out.vec7, align 64
1661 ret void
1662 }
1664 define void @load_i8_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4, ptr %out.vec5, ptr %out.vec6, ptr %out.vec7) nounwind {
1665 ; SSE-LABEL: load_i8_stride8_vf16:
1666 ; SSE: # %bb.0:
1667 ; SSE-NEXT: subq $328, %rsp # imm = 0x148
1668 ; SSE-NEXT: movdqa (%rdi), %xmm6
1669 ; SSE-NEXT: movdqa 16(%rdi), %xmm14
1670 ; SSE-NEXT: movdqa 32(%rdi), %xmm13
1671 ; SSE-NEXT: movdqa 48(%rdi), %xmm8
1672 ; SSE-NEXT: movdqa 64(%rdi), %xmm15
1673 ; SSE-NEXT: movdqa 80(%rdi), %xmm4
1674 ; SSE-NEXT: movdqa 96(%rdi), %xmm12
1675 ; SSE-NEXT: movdqa 112(%rdi), %xmm9
1676 ; SSE-NEXT: movdqa {{.*#+}} xmm3 = [255,0,0,0,255,0,0,0]
1677 ; SSE-NEXT: movdqa %xmm9, %xmm0
1678 ; SSE-NEXT: pand %xmm3, %xmm0
1679 ; SSE-NEXT: movdqa %xmm12, %xmm1
1680 ; SSE-NEXT: pand %xmm3, %xmm1
1681 ; SSE-NEXT: packuswb %xmm0, %xmm1
1682 ; SSE-NEXT: packuswb %xmm1, %xmm0
1683 ; SSE-NEXT: movdqa %xmm4, %xmm1
1684 ; SSE-NEXT: pand %xmm3, %xmm1
1685 ; SSE-NEXT: movdqa %xmm15, %xmm2
1686 ; SSE-NEXT: pand %xmm3, %xmm2
1687 ; SSE-NEXT: packuswb %xmm1, %xmm2
1688 ; SSE-NEXT: packuswb %xmm2, %xmm2
1689 ; SSE-NEXT: packuswb %xmm0, %xmm2
1690 ; SSE-NEXT: movdqa %xmm8, %xmm0
1691 ; SSE-NEXT: pand %xmm3, %xmm0
1692 ; SSE-NEXT: movdqa %xmm13, %xmm1
1693 ; SSE-NEXT: pand %xmm3, %xmm1
1694 ; SSE-NEXT: packuswb %xmm0, %xmm1
1695 ; SSE-NEXT: packuswb %xmm1, %xmm0
1696 ; SSE-NEXT: movdqa %xmm14, %xmm1
1697 ; SSE-NEXT: pand %xmm3, %xmm1
1698 ; SSE-NEXT: pand %xmm6, %xmm3
1699 ; SSE-NEXT: packuswb %xmm1, %xmm3
1700 ; SSE-NEXT: packuswb %xmm3, %xmm3
1701 ; SSE-NEXT: packuswb %xmm0, %xmm3
1702 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,3],xmm2[0,3]
1703 ; SSE-NEXT: movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1704 ; SSE-NEXT: pxor %xmm7, %xmm7
1705 ; SSE-NEXT: movdqa %xmm14, %xmm0
1706 ; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm7[8],xmm0[9],xmm7[9],xmm0[10],xmm7[10],xmm0[11],xmm7[11],xmm0[12],xmm7[12],xmm0[13],xmm7[13],xmm0[14],xmm7[14],xmm0[15],xmm7[15]
1707 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1708 ; SSE-NEXT: movdqa %xmm14, %xmm11
1709 ; SSE-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm7[0],xmm11[1],xmm7[1],xmm11[2],xmm7[2],xmm11[3],xmm7[3],xmm11[4],xmm7[4],xmm11[5],xmm7[5],xmm11[6],xmm7[6],xmm11[7],xmm7[7]
1710 ; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1711 ; SSE-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm0[0],xmm11[1],xmm0[1],xmm11[2],xmm0[2],xmm11[3],xmm0[3]
1712 ; SSE-NEXT: packuswb %xmm11, %xmm11
1713 ; SSE-NEXT: movdqa {{.*#+}} xmm10 = [65535,0,65535,65535,65535,65535,65535,65535]
1714 ; SSE-NEXT: movdqa %xmm10, %xmm1
1715 ; SSE-NEXT: pandn %xmm11, %xmm1
1716 ; SSE-NEXT: movdqa %xmm6, %xmm0
1717 ; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1718 ; SSE-NEXT: movdqa %xmm6, %xmm2
1719 ; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm7[8],xmm2[9],xmm7[9],xmm2[10],xmm7[10],xmm2[11],xmm7[11],xmm2[12],xmm7[12],xmm2[13],xmm7[13],xmm2[14],xmm7[14],xmm2[15],xmm7[15]
1720 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1721 ; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1],xmm0[2],xmm7[2],xmm0[3],xmm7[3],xmm0[4],xmm7[4],xmm0[5],xmm7[5],xmm0[6],xmm7[6],xmm0[7],xmm7[7]
1722 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1723 ; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
1724 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1725 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
1726 ; SSE-NEXT: packuswb %xmm0, %xmm0
1727 ; SSE-NEXT: pand %xmm10, %xmm0
1728 ; SSE-NEXT: por %xmm1, %xmm0
1729 ; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1730 ; SSE-NEXT: movdqa %xmm8, %xmm1
1731 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm7[8],xmm1[9],xmm7[9],xmm1[10],xmm7[10],xmm1[11],xmm7[11],xmm1[12],xmm7[12],xmm1[13],xmm7[13],xmm1[14],xmm7[14],xmm1[15],xmm7[15]
1732 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1733 ; SSE-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3],xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]
1734 ; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1735 ; SSE-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm1[0],xmm8[1],xmm1[1],xmm8[2],xmm1[2],xmm8[3],xmm1[3]
1736 ; SSE-NEXT: packuswb %xmm8, %xmm8
1737 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm8[0,0,2,3]
1738 ; SSE-NEXT: movdqa {{.*#+}} xmm5 = [65535,65535,65535,0,65535,65535,65535,65535]
1739 ; SSE-NEXT: movdqa %xmm5, %xmm6
1740 ; SSE-NEXT: pandn %xmm1, %xmm6
1741 ; SSE-NEXT: movdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1742 ; SSE-NEXT: movdqa %xmm13, %xmm1
1743 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm7[8],xmm1[9],xmm7[9],xmm1[10],xmm7[10],xmm1[11],xmm7[11],xmm1[12],xmm7[12],xmm1[13],xmm7[13],xmm1[14],xmm7[14],xmm1[15],xmm7[15]
1744 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1745 ; SSE-NEXT: punpcklbw {{.*#+}} xmm13 = xmm13[0],xmm7[0],xmm13[1],xmm7[1],xmm13[2],xmm7[2],xmm13[3],xmm7[3],xmm13[4],xmm7[4],xmm13[5],xmm7[5],xmm13[6],xmm7[6],xmm13[7],xmm7[7]
1746 ; SSE-NEXT: movdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1747 ; SSE-NEXT: punpcklwd {{.*#+}} xmm13 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3]
1748 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm13[0,1,1,3]
1749 ; SSE-NEXT: packuswb %xmm1, %xmm1
1750 ; SSE-NEXT: pand %xmm5, %xmm1
1751 ; SSE-NEXT: por %xmm6, %xmm1
1752 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
1753 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1754 ; SSE-NEXT: movdqa %xmm9, %xmm1
1755 ; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1756 ; SSE-NEXT: movdqa %xmm9, %xmm2
1757 ; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm7[8],xmm2[9],xmm7[9],xmm2[10],xmm7[10],xmm2[11],xmm7[11],xmm2[12],xmm7[12],xmm2[13],xmm7[13],xmm2[14],xmm7[14],xmm2[15],xmm7[15]
1758 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1759 ; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm7[0],xmm1[1],xmm7[1],xmm1[2],xmm7[2],xmm1[3],xmm7[3],xmm1[4],xmm7[4],xmm1[5],xmm7[5],xmm1[6],xmm7[6],xmm1[7],xmm7[7]
1760 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1761 ; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
1762 ; SSE-NEXT: packuswb %xmm1, %xmm1
1763 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1764 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,2]
1765 ; SSE-NEXT: movdqa {{.*#+}} xmm6 = [65535,65535,65535,65535,65535,65535,65535,0]
1766 ; SSE-NEXT: movdqa %xmm6, %xmm9
1767 ; SSE-NEXT: pandn %xmm1, %xmm9
1768 ; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1769 ; SSE-NEXT: movdqa %xmm12, %xmm1
1770 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm7[8],xmm1[9],xmm7[9],xmm1[10],xmm7[10],xmm1[11],xmm7[11],xmm1[12],xmm7[12],xmm1[13],xmm7[13],xmm1[14],xmm7[14],xmm1[15],xmm7[15]
1771 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1772 ; SSE-NEXT: punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm7[0],xmm12[1],xmm7[1],xmm12[2],xmm7[2],xmm12[3],xmm7[3],xmm12[4],xmm7[4],xmm12[5],xmm7[5],xmm12[6],xmm7[6],xmm12[7],xmm7[7]
1773 ; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1774 ; SSE-NEXT: punpcklwd {{.*#+}} xmm12 = xmm12[0],xmm1[0],xmm12[1],xmm1[1],xmm12[2],xmm1[2],xmm12[3],xmm1[3]
1775 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm12[0,1,1,3]
1776 ; SSE-NEXT: packuswb %xmm1, %xmm1
1777 ; SSE-NEXT: pand %xmm6, %xmm1
1778 ; SSE-NEXT: por %xmm9, %xmm1
1779 ; SSE-NEXT: movdqa %xmm4, %xmm3
1780 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1781 ; SSE-NEXT: movdqa %xmm4, %xmm2
1782 ; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm7[8],xmm2[9],xmm7[9],xmm2[10],xmm7[10],xmm2[11],xmm7[11],xmm2[12],xmm7[12],xmm2[13],xmm7[13],xmm2[14],xmm7[14],xmm2[15],xmm7[15]
1783 ; SSE-NEXT: movdqa %xmm2, %xmm4
1784 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1785 ; SSE-NEXT: movdqa %xmm3, %xmm2
1786 ; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm7[0],xmm2[1],xmm7[1],xmm2[2],xmm7[2],xmm2[3],xmm7[3],xmm2[4],xmm7[4],xmm2[5],xmm7[5],xmm2[6],xmm7[6],xmm2[7],xmm7[7]
1787 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1788 ; SSE-NEXT: movdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1789 ; SSE-NEXT: movdqa %xmm15, %xmm3
1790 ; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm7[8],xmm3[9],xmm7[9],xmm3[10],xmm7[10],xmm3[11],xmm7[11],xmm3[12],xmm7[12],xmm3[13],xmm7[13],xmm3[14],xmm7[14],xmm3[15],xmm7[15]
1791 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1792 ; SSE-NEXT: punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm7[0],xmm15[1],xmm7[1],xmm15[2],xmm7[2],xmm15[3],xmm7[3],xmm15[4],xmm7[4],xmm15[5],xmm7[5],xmm15[6],xmm7[6],xmm15[7],xmm7[7]
1793 ; SSE-NEXT: movdqa %xmm15, (%rsp) # 16-byte Spill
1794 ; SSE-NEXT: movdqa %xmm2, %xmm7
1795 ; SSE-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3]
1796 ; SSE-NEXT: packuswb %xmm7, %xmm7
1797 ; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1798 ; SSE-NEXT: movdqa {{.*#+}} xmm4 = [65535,65535,65535,65535,65535,0,65535,65535]
1799 ; SSE-NEXT: movdqa %xmm4, %xmm9
1800 ; SSE-NEXT: pandn %xmm7, %xmm9
1801 ; SSE-NEXT: movdqa %xmm15, %xmm7
1802 ; SSE-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3]
1803 ; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm7[1,1,1,1]
1804 ; SSE-NEXT: packuswb %xmm15, %xmm15
1805 ; SSE-NEXT: pand %xmm4, %xmm15
1806 ; SSE-NEXT: por %xmm9, %xmm15
1807 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm15[2,2,2,2]
1808 ; SSE-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
1809 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
1810 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1811 ; SSE-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
1812 ; SSE-NEXT: pand %xmm1, %xmm14
1813 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm14[0,2,2,3]
1814 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]
1815 ; SSE-NEXT: packuswb %xmm0, %xmm0
1816 ; SSE-NEXT: movdqa %xmm10, %xmm9
1817 ; SSE-NEXT: pandn %xmm0, %xmm9
1818 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1819 ; SSE-NEXT: pand %xmm1, %xmm0
1820 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1821 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
1822 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
1823 ; SSE-NEXT: packuswb %xmm0, %xmm0
1824 ; SSE-NEXT: pand %xmm10, %xmm0
1825 ; SSE-NEXT: por %xmm9, %xmm0
1826 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
1827 ; SSE-NEXT: pand %xmm1, %xmm2
1828 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1829 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm2[0,1,2,0]
1830 ; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,5,7,5]
1831 ; SSE-NEXT: packuswb %xmm9, %xmm9
1832 ; SSE-NEXT: movdqa %xmm5, %xmm15
1833 ; SSE-NEXT: pandn %xmm9, %xmm15
1834 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
1835 ; SSE-NEXT: pand %xmm1, %xmm2
1836 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1837 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm2[0,1,2,0]
1838 ; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,7,5,6,7]
1839 ; SSE-NEXT: packuswb %xmm9, %xmm9
1840 ; SSE-NEXT: pand %xmm5, %xmm9
1841 ; SSE-NEXT: por %xmm15, %xmm9
1842 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[1,1,1,1]
1843 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm9[0],xmm0[1],xmm9[1]
1844 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
1845 ; SSE-NEXT: pand %xmm1, %xmm2
1846 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1847 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm2[0,1,2,0]
1848 ; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,5,7,5]
1849 ; SSE-NEXT: packuswb %xmm9, %xmm9
1850 ; SSE-NEXT: movdqa %xmm6, %xmm15
1851 ; SSE-NEXT: pandn %xmm9, %xmm15
1852 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
1853 ; SSE-NEXT: pand %xmm1, %xmm2
1854 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1855 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm2[0,1,2,0]
1856 ; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,7,5,6,7]
1857 ; SSE-NEXT: packuswb %xmm9, %xmm9
1858 ; SSE-NEXT: pand %xmm6, %xmm9
1859 ; SSE-NEXT: por %xmm15, %xmm9
1860 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
1861 ; SSE-NEXT: pand %xmm1, %xmm2
1862 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1863 ; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm2[0,2,2,3]
1864 ; SSE-NEXT: pshuflw {{.*#+}} xmm15 = xmm15[0,1,1,3,4,5,6,7]
1865 ; SSE-NEXT: packuswb %xmm15, %xmm15
1866 ; SSE-NEXT: movdqa %xmm4, %xmm2
1867 ; SSE-NEXT: pandn %xmm15, %xmm2
1868 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
1869 ; SSE-NEXT: pand %xmm1, %xmm15
1870 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm15[0,2,2,3]
1871 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]
1872 ; SSE-NEXT: packuswb %xmm1, %xmm1
1873 ; SSE-NEXT: pand %xmm4, %xmm1
1874 ; SSE-NEXT: por %xmm2, %xmm1
1875 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]
1876 ; SSE-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm9[2],xmm1[3],xmm9[3]
1877 ; SSE-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
1878 ; SSE-NEXT: movapd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1879 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm11[1,1,2,3]
1880 ; SSE-NEXT: movdqa %xmm10, %xmm1
1881 ; SSE-NEXT: pandn %xmm0, %xmm1
1882 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1883 ; SSE-NEXT: # xmm0 = mem[3,3,3,3]
1884 ; SSE-NEXT: packuswb %xmm0, %xmm0
1885 ; SSE-NEXT: pand %xmm10, %xmm0
1886 ; SSE-NEXT: por %xmm1, %xmm0
1887 ; SSE-NEXT: movdqa %xmm5, %xmm1
1888 ; SSE-NEXT: pandn %xmm8, %xmm1
1889 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm13[2,2,3,3]
1890 ; SSE-NEXT: packuswb %xmm2, %xmm2
1891 ; SSE-NEXT: pand %xmm5, %xmm2
1892 ; SSE-NEXT: por %xmm1, %xmm2
1893 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
1894 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1895 ; SSE-NEXT: movdqa %xmm6, %xmm1
1896 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
1897 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm12[2,2,3,3]
1898 ; SSE-NEXT: packuswb %xmm2, %xmm2
1899 ; SSE-NEXT: pand %xmm6, %xmm2
1900 ; SSE-NEXT: por %xmm1, %xmm2
1901 ; SSE-NEXT: pshufd $244, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
1902 ; SSE-NEXT: # xmm1 = mem[0,1,3,3]
1903 ; SSE-NEXT: movdqa %xmm4, %xmm8
1904 ; SSE-NEXT: pandn %xmm1, %xmm8
1905 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[3,3,3,3]
1906 ; SSE-NEXT: packuswb %xmm1, %xmm1
1907 ; SSE-NEXT: pand %xmm4, %xmm1
1908 ; SSE-NEXT: por %xmm8, %xmm1
1909 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]
1910 ; SSE-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
1911 ; SSE-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
1912 ; SSE-NEXT: movapd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1913 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm14[3,1,2,3]
1914 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm3[0,1,2,0,4,5,6,7]
1915 ; SSE-NEXT: packuswb %xmm0, %xmm0
1916 ; SSE-NEXT: movdqa %xmm10, %xmm1
1917 ; SSE-NEXT: pandn %xmm0, %xmm1
1918 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1919 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
1920 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1921 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]
1922 ; SSE-NEXT: packuswb %xmm0, %xmm0
1923 ; SSE-NEXT: pand %xmm10, %xmm0
1924 ; SSE-NEXT: por %xmm1, %xmm0
1925 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
1926 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
1927 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1928 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,6]
1929 ; SSE-NEXT: packuswb %xmm1, %xmm1
1930 ; SSE-NEXT: movdqa %xmm5, %xmm8
1931 ; SSE-NEXT: pandn %xmm1, %xmm8
1932 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
1933 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
1934 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1935 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
1936 ; SSE-NEXT: packuswb %xmm1, %xmm1
1937 ; SSE-NEXT: pand %xmm5, %xmm1
1938 ; SSE-NEXT: por %xmm8, %xmm1
1939 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
1940 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1941 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
1942 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
1943 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1944 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,6]
1945 ; SSE-NEXT: packuswb %xmm1, %xmm1
1946 ; SSE-NEXT: movdqa %xmm6, %xmm9
1947 ; SSE-NEXT: pandn %xmm1, %xmm9
1948 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
1949 ; SSE-NEXT: # xmm8 = mem[0,1,1,3]
1950 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm8[0,1,2,3,4,6,6,7]
1951 ; SSE-NEXT: packuswb %xmm1, %xmm14
1952 ; SSE-NEXT: pand %xmm6, %xmm14
1953 ; SSE-NEXT: por %xmm9, %xmm14
1954 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
1955 ; SSE-NEXT: # xmm12 = mem[3,1,2,3]
1956 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm12[0,1,2,0,4,5,6,7]
1957 ; SSE-NEXT: packuswb %xmm1, %xmm1
1958 ; SSE-NEXT: movdqa %xmm4, %xmm9
1959 ; SSE-NEXT: pandn %xmm1, %xmm9
1960 ; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm15[3,1,2,3]
1961 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm13[2,0,2,3,4,5,6,7]
1962 ; SSE-NEXT: packuswb %xmm1, %xmm1
1963 ; SSE-NEXT: pand %xmm4, %xmm1
1964 ; SSE-NEXT: por %xmm9, %xmm1
1965 ; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm1[2,2,2,2]
1966 ; SSE-NEXT: punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm14[2],xmm11[3],xmm14[3]
1967 ; SSE-NEXT: movsd {{.*#+}} xmm11 = xmm0[0],xmm11[1]
1968 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1969 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1970 ; SSE-NEXT: # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
1971 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
1972 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
1973 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
1974 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1975 ; SSE-NEXT: packuswb %xmm0, %xmm0
1976 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1977 ; SSE-NEXT: movdqa %xmm10, %xmm9
1978 ; SSE-NEXT: pandn %xmm0, %xmm9
1979 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
1980 ; SSE-NEXT: packuswb %xmm2, %xmm2
1981 ; SSE-NEXT: pand %xmm10, %xmm2
1982 ; SSE-NEXT: por %xmm9, %xmm2
1983 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1984 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1985 ; SSE-NEXT: # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
1986 ; SSE-NEXT: packuswb %xmm0, %xmm0
1987 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1988 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm0[0,0,2,3]
1989 ; SSE-NEXT: movdqa %xmm5, %xmm14
1990 ; SSE-NEXT: pandn %xmm9, %xmm14
1991 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1992 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1993 ; SSE-NEXT: # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
1994 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1995 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm0[0,1,1,3]
1996 ; SSE-NEXT: packuswb %xmm9, %xmm9
1997 ; SSE-NEXT: pand %xmm5, %xmm9
1998 ; SSE-NEXT: por %xmm14, %xmm9
1999 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[1,1,1,1]
2000 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1]
2001 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
2002 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2003 ; SSE-NEXT: # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
2004 ; SSE-NEXT: packuswb %xmm0, %xmm14
2005 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm14[0,1,2,2]
2006 ; SSE-NEXT: movdqa %xmm6, %xmm15
2007 ; SSE-NEXT: pandn %xmm9, %xmm15
2008 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
2009 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2010 ; SSE-NEXT: # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
2011 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2012 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm0[0,1,1,3]
2013 ; SSE-NEXT: packuswb %xmm9, %xmm1
2014 ; SSE-NEXT: pand %xmm6, %xmm1
2015 ; SSE-NEXT: por %xmm15, %xmm1
2016 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
2017 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
2018 ; SSE-NEXT: # xmm7 = xmm7[4],mem[4],xmm7[5],mem[5],xmm7[6],mem[6],xmm7[7],mem[7]
2019 ; SSE-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
2020 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2021 ; SSE-NEXT: # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
2022 ; SSE-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
2023 ; SSE-NEXT: packuswb %xmm7, %xmm9
2024 ; SSE-NEXT: movdqa %xmm4, %xmm15
2025 ; SSE-NEXT: pandn %xmm9, %xmm15
2026 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm0[1,1,1,1]
2027 ; SSE-NEXT: packuswb %xmm7, %xmm7
2028 ; SSE-NEXT: pand %xmm4, %xmm7
2029 ; SSE-NEXT: por %xmm15, %xmm7
2030 ; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm7[2,2,2,2]
2031 ; SSE-NEXT: punpckhdq {{.*#+}} xmm15 = xmm15[2],xmm1[2],xmm15[3],xmm1[3]
2032 ; SSE-NEXT: movsd {{.*#+}} xmm15 = xmm2[0],xmm15[1]
2033 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm3[0,1,3,1,4,5,6,7]
2034 ; SSE-NEXT: packuswb %xmm0, %xmm0
2035 ; SSE-NEXT: movdqa %xmm10, %xmm1
2036 ; SSE-NEXT: pandn %xmm0, %xmm1
2037 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2038 ; SSE-NEXT: # xmm0 = mem[3,1,2,3,4,5,6,7]
2039 ; SSE-NEXT: packuswb %xmm0, %xmm0
2040 ; SSE-NEXT: pand %xmm10, %xmm0
2041 ; SSE-NEXT: por %xmm1, %xmm0
2042 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2043 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,4,5,5,7]
2044 ; SSE-NEXT: packuswb %xmm1, %xmm1
2045 ; SSE-NEXT: movdqa %xmm5, %xmm7
2046 ; SSE-NEXT: pandn %xmm1, %xmm7
2047 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2048 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,5,7,6,7]
2049 ; SSE-NEXT: packuswb %xmm1, %xmm1
2050 ; SSE-NEXT: pand %xmm5, %xmm1
2051 ; SSE-NEXT: por %xmm7, %xmm1
2052 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
2053 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
2054 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2055 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,4,5,5,7]
2056 ; SSE-NEXT: packuswb %xmm1, %xmm1
2057 ; SSE-NEXT: movdqa %xmm6, %xmm7
2058 ; SSE-NEXT: pandn %xmm1, %xmm7
2059 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm8[0,1,2,3,5,7,6,7]
2060 ; SSE-NEXT: packuswb %xmm1, %xmm1
2061 ; SSE-NEXT: pand %xmm6, %xmm1
2062 ; SSE-NEXT: por %xmm7, %xmm1
2063 ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm12[0,1,3,1,4,5,6,7]
2064 ; SSE-NEXT: packuswb %xmm7, %xmm7
2065 ; SSE-NEXT: movdqa %xmm4, %xmm8
2066 ; SSE-NEXT: pandn %xmm7, %xmm8
2067 ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm13[3,1,2,3,4,5,6,7]
2068 ; SSE-NEXT: packuswb %xmm7, %xmm7
2069 ; SSE-NEXT: pand %xmm4, %xmm7
2070 ; SSE-NEXT: por %xmm8, %xmm7
2071 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[2,2,2,2]
2072 ; SSE-NEXT: punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm1[2],xmm7[3],xmm1[3]
2073 ; SSE-NEXT: movsd {{.*#+}} xmm7 = xmm0[0],xmm7[1]
2074 ; SSE-NEXT: pshufd $229, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2075 ; SSE-NEXT: # xmm0 = mem[1,1,2,3]
2076 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2077 ; SSE-NEXT: # xmm1 = mem[3,3,3,3]
2078 ; SSE-NEXT: packuswb %xmm1, %xmm1
2079 ; SSE-NEXT: pand %xmm10, %xmm1
2080 ; SSE-NEXT: pandn %xmm0, %xmm10
2081 ; SSE-NEXT: por %xmm1, %xmm10
2082 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2083 ; SSE-NEXT: # xmm0 = mem[2,2,3,3]
2084 ; SSE-NEXT: packuswb %xmm0, %xmm0
2085 ; SSE-NEXT: pand %xmm5, %xmm0
2086 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
2087 ; SSE-NEXT: por %xmm0, %xmm5
2088 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm5[1,1,1,1]
2089 ; SSE-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm0[0],xmm10[1],xmm0[1]
2090 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2091 ; SSE-NEXT: # xmm0 = mem[2,2,3,3]
2092 ; SSE-NEXT: packuswb %xmm0, %xmm0
2093 ; SSE-NEXT: pand %xmm6, %xmm0
2094 ; SSE-NEXT: pandn %xmm14, %xmm6
2095 ; SSE-NEXT: por %xmm0, %xmm6
2096 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,1,3,3]
2097 ; SSE-NEXT: pshufd $255, (%rsp), %xmm1 # 16-byte Folded Reload
2098 ; SSE-NEXT: # xmm1 = mem[3,3,3,3]
2099 ; SSE-NEXT: packuswb %xmm1, %xmm1
2100 ; SSE-NEXT: pand %xmm4, %xmm1
2101 ; SSE-NEXT: pandn %xmm0, %xmm4
2102 ; SSE-NEXT: por %xmm1, %xmm4
2103 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm4[2,2,2,2]
2104 ; SSE-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm6[2],xmm0[3],xmm6[3]
2105 ; SSE-NEXT: movsd {{.*#+}} xmm0 = xmm10[0],xmm0[1]
2106 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
2107 ; SSE-NEXT: movaps %xmm1, (%rsi)
2108 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
2109 ; SSE-NEXT: movaps %xmm1, (%rdx)
2110 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
2111 ; SSE-NEXT: movaps %xmm1, (%rcx)
2112 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
2113 ; SSE-NEXT: movaps %xmm1, (%r8)
2114 ; SSE-NEXT: movapd %xmm11, (%r9)
2115 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
2116 ; SSE-NEXT: movapd %xmm15, (%rax)
2117 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
2118 ; SSE-NEXT: movapd %xmm7, (%rax)
2119 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
2120 ; SSE-NEXT: movapd %xmm0, (%rax)
2121 ; SSE-NEXT: addq $328, %rsp # imm = 0x148
2122 ; SSE-NEXT: retq
2123 ;
2124 ; AVX-LABEL: load_i8_stride8_vf16:
2125 ; AVX: # %bb.0:
2126 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm0 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
2127 ; AVX-NEXT: vmovdqa 112(%rdi), %xmm8
2128 ; AVX-NEXT: vpshufb %xmm0, %xmm8, %xmm3
2129 ; AVX-NEXT: vmovdqa 96(%rdi), %xmm2
2130 ; AVX-NEXT: vpshufb %xmm0, %xmm2, %xmm0
2131 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
2132 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm3 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
2133 ; AVX-NEXT: vmovdqa 80(%rdi), %xmm4
2134 ; AVX-NEXT: vpshufb %xmm3, %xmm4, %xmm6
2135 ; AVX-NEXT: vmovdqa 64(%rdi), %xmm5
2136 ; AVX-NEXT: vpshufb %xmm3, %xmm5, %xmm3
2137 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3]
2138 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3,4,5],xmm0[6,7]
2139 ; AVX-NEXT: vmovd {{.*#+}} xmm9 = [0,0,0,8,0,0,0,0,0,0,0,0,0,0,0,0]
2140 ; AVX-NEXT: vmovdqa (%rdi), %xmm1
2141 ; AVX-NEXT: vmovdqa 16(%rdi), %xmm3
2142 ; AVX-NEXT: vmovdqa 32(%rdi), %xmm6
2143 ; AVX-NEXT: vmovdqa 48(%rdi), %xmm7
2144 ; AVX-NEXT: vpshufb %xmm9, %xmm7, %xmm10
2145 ; AVX-NEXT: vpshufb %xmm9, %xmm6, %xmm9
2146 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
2147 ; AVX-NEXT: vmovd {{.*#+}} xmm10 = [0,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
2148 ; AVX-NEXT: vpshufb %xmm10, %xmm3, %xmm11
2149 ; AVX-NEXT: vpshufb %xmm10, %xmm1, %xmm10
2150 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
2151 ; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3],xmm10[4,5,6,7]
2152 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm9[0,1,2,3],xmm0[4,5,6,7]
2153 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2154 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm9 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
2155 ; AVX-NEXT: vpshufb %xmm9, %xmm8, %xmm10
2156 ; AVX-NEXT: vpshufb %xmm9, %xmm2, %xmm9
2157 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
2158 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm10 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
2159 ; AVX-NEXT: vpshufb %xmm10, %xmm4, %xmm11
2160 ; AVX-NEXT: vpshufb %xmm10, %xmm5, %xmm10
2161 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
2162 ; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm10[0,1,2,3,4,5],xmm9[6,7]
2163 ; AVX-NEXT: vmovd {{.*#+}} xmm10 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
2164 ; AVX-NEXT: vpshufb %xmm10, %xmm7, %xmm11
2165 ; AVX-NEXT: vpshufb %xmm10, %xmm6, %xmm10
2166 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
2167 ; AVX-NEXT: vmovd {{.*#+}} xmm11 = [1,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
2168 ; AVX-NEXT: vpshufb %xmm11, %xmm3, %xmm12
2169 ; AVX-NEXT: vpshufb %xmm11, %xmm1, %xmm11
2170 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
2171 ; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3],xmm11[4,5,6,7]
2172 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm10[0,1,2,3],xmm9[4,5,6,7]
2173 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2174 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm10 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
2175 ; AVX-NEXT: vpshufb %xmm10, %xmm8, %xmm11
2176 ; AVX-NEXT: vpshufb %xmm10, %xmm2, %xmm10
2177 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
2178 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm11 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
2179 ; AVX-NEXT: vpshufb %xmm11, %xmm4, %xmm12
2180 ; AVX-NEXT: vpshufb %xmm11, %xmm5, %xmm11
2181 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
2182 ; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0,1,2,3,4,5],xmm10[6,7]
2183 ; AVX-NEXT: vmovd {{.*#+}} xmm11 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
2184 ; AVX-NEXT: vpshufb %xmm11, %xmm7, %xmm12
2185 ; AVX-NEXT: vpshufb %xmm11, %xmm6, %xmm11
2186 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
2187 ; AVX-NEXT: vmovd {{.*#+}} xmm12 = [2,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
2188 ; AVX-NEXT: vpshufb %xmm12, %xmm3, %xmm13
2189 ; AVX-NEXT: vpshufb %xmm12, %xmm1, %xmm12
2190 ; AVX-NEXT: vmovdqa %xmm1, %xmm9
2191 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
2192 ; AVX-NEXT: vpblendw {{.*#+}} xmm11 = xmm12[0,1],xmm11[2,3],xmm12[4,5,6,7]
2193 ; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0,1,2,3],xmm10[4,5,6,7]
2194 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm11 = [0,0,3,11,0,0,3,11,0,0,3,11,0,0,3,11]
2195 ; AVX-NEXT: vpshufb %xmm11, %xmm8, %xmm12
2196 ; AVX-NEXT: vpshufb %xmm11, %xmm2, %xmm11
2197 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
2198 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm12 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
2199 ; AVX-NEXT: vpshufb %xmm12, %xmm4, %xmm13
2200 ; AVX-NEXT: vpshufb %xmm12, %xmm5, %xmm12
2201 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
2202 ; AVX-NEXT: vpblendw {{.*#+}} xmm11 = xmm12[0,1,2,3,4,5],xmm11[6,7]
2203 ; AVX-NEXT: vmovd {{.*#+}} xmm12 = [0,0,3,11,0,0,0,0,0,0,0,0,0,0,0,0]
2204 ; AVX-NEXT: vpshufb %xmm12, %xmm7, %xmm13
2205 ; AVX-NEXT: vpshufb %xmm12, %xmm6, %xmm12
2206 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
2207 ; AVX-NEXT: vmovd {{.*#+}} xmm13 = [3,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
2208 ; AVX-NEXT: vpshufb %xmm13, %xmm3, %xmm14
2209 ; AVX-NEXT: vpshufb %xmm13, %xmm1, %xmm13
2210 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
2211 ; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm13[0,1],xmm12[2,3],xmm13[4,5,6,7]
2212 ; AVX-NEXT: vpblendw {{.*#+}} xmm11 = xmm12[0,1,2,3],xmm11[4,5,6,7]
2213 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm12 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
2214 ; AVX-NEXT: vpshufb %xmm12, %xmm8, %xmm13
2215 ; AVX-NEXT: vpshufb %xmm12, %xmm2, %xmm12
2216 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
2217 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm13 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
2218 ; AVX-NEXT: vpshufb %xmm13, %xmm4, %xmm14
2219 ; AVX-NEXT: vpshufb %xmm13, %xmm5, %xmm13
2220 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
2221 ; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm13[0,1,2,3,4,5],xmm12[6,7]
2222 ; AVX-NEXT: vmovd {{.*#+}} xmm13 = [0,0,4,12,0,0,0,0,0,0,0,0,0,0,0,0]
2223 ; AVX-NEXT: vpshufb %xmm13, %xmm7, %xmm14
2224 ; AVX-NEXT: vpshufb %xmm13, %xmm6, %xmm13
2225 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
2226 ; AVX-NEXT: vmovd {{.*#+}} xmm14 = [4,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
2227 ; AVX-NEXT: vpshufb %xmm14, %xmm3, %xmm15
2228 ; AVX-NEXT: vpshufb %xmm14, %xmm1, %xmm14
2229 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
2230 ; AVX-NEXT: vpblendw {{.*#+}} xmm13 = xmm14[0,1],xmm13[2,3],xmm14[4,5,6,7]
2231 ; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm13[0,1,2,3],xmm12[4,5,6,7]
2232 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm13 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
2233 ; AVX-NEXT: vpshufb %xmm13, %xmm8, %xmm14
2234 ; AVX-NEXT: vpshufb %xmm13, %xmm2, %xmm13
2235 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
2236 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm14 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
2237 ; AVX-NEXT: vpshufb %xmm14, %xmm4, %xmm15
2238 ; AVX-NEXT: vpshufb %xmm14, %xmm5, %xmm14
2239 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
2240 ; AVX-NEXT: vpblendw {{.*#+}} xmm13 = xmm14[0,1,2,3,4,5],xmm13[6,7]
2241 ; AVX-NEXT: vmovd {{.*#+}} xmm14 = [0,0,5,13,0,0,0,0,0,0,0,0,0,0,0,0]
2242 ; AVX-NEXT: vpshufb %xmm14, %xmm7, %xmm15
2243 ; AVX-NEXT: vpshufb %xmm14, %xmm6, %xmm14
2244 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
2245 ; AVX-NEXT: vmovd {{.*#+}} xmm15 = [5,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
2246 ; AVX-NEXT: vpshufb %xmm15, %xmm3, %xmm0
2247 ; AVX-NEXT: vpshufb %xmm15, %xmm1, %xmm15
2248 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
2249 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm14[2,3],xmm0[4,5,6,7]
2250 ; AVX-NEXT: vpblendw {{.*#+}} xmm13 = xmm0[0,1,2,3],xmm13[4,5,6,7]
2251 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm0 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
2252 ; AVX-NEXT: vpshufb %xmm0, %xmm8, %xmm14
2253 ; AVX-NEXT: vpshufb %xmm0, %xmm2, %xmm0
2254 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
2255 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm14 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
2256 ; AVX-NEXT: vpshufb %xmm14, %xmm4, %xmm15
2257 ; AVX-NEXT: vpshufb %xmm14, %xmm5, %xmm14
2258 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
2259 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm14[0,1,2,3,4,5],xmm0[6,7]
2260 ; AVX-NEXT: vmovd {{.*#+}} xmm14 = [0,0,6,14,0,0,0,0,0,0,0,0,0,0,0,0]
2261 ; AVX-NEXT: vpshufb %xmm14, %xmm7, %xmm15
2262 ; AVX-NEXT: vpshufb %xmm14, %xmm6, %xmm14
2263 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
2264 ; AVX-NEXT: vmovd {{.*#+}} xmm15 = [6,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
2265 ; AVX-NEXT: vpshufb %xmm15, %xmm3, %xmm0
2266 ; AVX-NEXT: vpshufb %xmm15, %xmm9, %xmm15
2267 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
2268 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm14[2,3],xmm0[4,5,6,7]
2269 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
2270 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm1 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
2271 ; AVX-NEXT: vpshufb %xmm1, %xmm8, %xmm8
2272 ; AVX-NEXT: vpshufb %xmm1, %xmm2, %xmm1
2273 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3]
2274 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm2 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
2275 ; AVX-NEXT: vpshufb %xmm2, %xmm4, %xmm4
2276 ; AVX-NEXT: vpshufb %xmm2, %xmm5, %xmm2
2277 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
2278 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3,4,5],xmm1[6,7]
2279 ; AVX-NEXT: vmovd {{.*#+}} xmm2 = [0,0,7,15,0,0,0,0,0,0,0,0,0,0,0,0]
2280 ; AVX-NEXT: vpshufb %xmm2, %xmm7, %xmm4
2281 ; AVX-NEXT: vpshufb %xmm2, %xmm6, %xmm2
2282 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
2283 ; AVX-NEXT: vmovd {{.*#+}} xmm4 = [7,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
2284 ; AVX-NEXT: vpshufb %xmm4, %xmm3, %xmm3
2285 ; AVX-NEXT: vpshufb %xmm4, %xmm9, %xmm4
2286 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
2287 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5,6,7]
2288 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
2289 ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
2290 ; AVX-NEXT: vmovaps %xmm2, (%rsi)
2291 ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
2292 ; AVX-NEXT: vmovaps %xmm2, (%rdx)
2293 ; AVX-NEXT: vmovdqa %xmm10, (%rcx)
2294 ; AVX-NEXT: vmovdqa %xmm11, (%r8)
2295 ; AVX-NEXT: vmovdqa %xmm12, (%r9)
2296 ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax
2297 ; AVX-NEXT: vmovdqa %xmm13, (%rax)
2298 ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax
2299 ; AVX-NEXT: vmovdqa %xmm0, (%rax)
2300 ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax
2301 ; AVX-NEXT: vmovdqa %xmm1, (%rax)
2302 ; AVX-NEXT: retq
2303 ;
2304 ; AVX2-LABEL: load_i8_stride8_vf16:
2305 ; AVX2: # %bb.0:
2306 ; AVX2-NEXT: vmovdqa 112(%rdi), %xmm8
2307 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm0 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
2308 ; AVX2-NEXT: vpshufb %xmm0, %xmm8, %xmm2
2309 ; AVX2-NEXT: vmovdqa 96(%rdi), %xmm3
2310 ; AVX2-NEXT: vpshufb %xmm0, %xmm3, %xmm0
2311 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
2312 ; AVX2-NEXT: vmovdqa 80(%rdi), %xmm4
2313 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
2314 ; AVX2-NEXT: vpshufb %xmm2, %xmm4, %xmm6
2315 ; AVX2-NEXT: vmovdqa 64(%rdi), %xmm5
2316 ; AVX2-NEXT: vpshufb %xmm2, %xmm5, %xmm2
2317 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3]
2318 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
2319 ; AVX2-NEXT: vmovdqa (%rdi), %xmm1
2320 ; AVX2-NEXT: vmovdqa 16(%rdi), %xmm2
2321 ; AVX2-NEXT: vmovdqa 32(%rdi), %xmm6
2322 ; AVX2-NEXT: vmovdqa 48(%rdi), %xmm7
2323 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm9 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
2324 ; AVX2-NEXT: vpshufb %xmm9, %xmm7, %xmm10
2325 ; AVX2-NEXT: vpshufb %xmm9, %xmm6, %xmm9
2326 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
2327 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm10 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
2328 ; AVX2-NEXT: vpshufb %xmm10, %xmm2, %xmm11
2329 ; AVX2-NEXT: vpshufb %xmm10, %xmm1, %xmm10
2330 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
2331 ; AVX2-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0],xmm9[1],xmm10[2,3]
2332 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm9[0,1],xmm0[2,3]
2333 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2334 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm9 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
2335 ; AVX2-NEXT: vpshufb %xmm9, %xmm8, %xmm10
2336 ; AVX2-NEXT: vpshufb %xmm9, %xmm3, %xmm9
2337 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
2338 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm10 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
2339 ; AVX2-NEXT: vpshufb %xmm10, %xmm4, %xmm11
2340 ; AVX2-NEXT: vpshufb %xmm10, %xmm5, %xmm10
2341 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
2342 ; AVX2-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
2343 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm10 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
2344 ; AVX2-NEXT: vpshufb %xmm10, %xmm7, %xmm11
2345 ; AVX2-NEXT: vpshufb %xmm10, %xmm6, %xmm10
2346 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
2347 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm11 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
2348 ; AVX2-NEXT: vpshufb %xmm11, %xmm2, %xmm12
2349 ; AVX2-NEXT: vpshufb %xmm11, %xmm1, %xmm11
2350 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
2351 ; AVX2-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0],xmm10[1],xmm11[2,3]
2352 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm9[2,3]
2353 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2354 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm10 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
2355 ; AVX2-NEXT: vpshufb %xmm10, %xmm8, %xmm11
2356 ; AVX2-NEXT: vpshufb %xmm10, %xmm3, %xmm10
2357 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
2358 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm11 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
2359 ; AVX2-NEXT: vpshufb %xmm11, %xmm4, %xmm12
2360 ; AVX2-NEXT: vpshufb %xmm11, %xmm5, %xmm11
2361 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
2362 ; AVX2-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1,2],xmm10[3]
2363 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm11 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
2364 ; AVX2-NEXT: vpshufb %xmm11, %xmm7, %xmm12
2365 ; AVX2-NEXT: vpshufb %xmm11, %xmm6, %xmm11
2366 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
2367 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm12 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
2368 ; AVX2-NEXT: vpshufb %xmm12, %xmm2, %xmm13
2369 ; AVX2-NEXT: vpshufb %xmm12, %xmm1, %xmm12
2370 ; AVX2-NEXT: vmovdqa %xmm1, %xmm9
2371 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
2372 ; AVX2-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0],xmm11[1],xmm12[2,3]
2373 ; AVX2-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3]
2374 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm11 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
2375 ; AVX2-NEXT: vpshufb %xmm11, %xmm8, %xmm12
2376 ; AVX2-NEXT: vpshufb %xmm11, %xmm3, %xmm11
2377 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
2378 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm12 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
2379 ; AVX2-NEXT: vpshufb %xmm12, %xmm4, %xmm13
2380 ; AVX2-NEXT: vpshufb %xmm12, %xmm5, %xmm12
2381 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
2382 ; AVX2-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1,2],xmm11[3]
2383 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm12 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
2384 ; AVX2-NEXT: vpshufb %xmm12, %xmm7, %xmm13
2385 ; AVX2-NEXT: vpshufb %xmm12, %xmm6, %xmm12
2386 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
2387 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm13 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
2388 ; AVX2-NEXT: vpshufb %xmm13, %xmm2, %xmm14
2389 ; AVX2-NEXT: vpshufb %xmm13, %xmm1, %xmm13
2390 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
2391 ; AVX2-NEXT: vpblendd {{.*#+}} xmm12 = xmm13[0],xmm12[1],xmm13[2,3]
2392 ; AVX2-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1],xmm11[2,3]
2393 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm12 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
2394 ; AVX2-NEXT: vpshufb %xmm12, %xmm8, %xmm13
2395 ; AVX2-NEXT: vpshufb %xmm12, %xmm3, %xmm12
2396 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
2397 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm13 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
2398 ; AVX2-NEXT: vpshufb %xmm13, %xmm4, %xmm14
2399 ; AVX2-NEXT: vpshufb %xmm13, %xmm5, %xmm13
2400 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
2401 ; AVX2-NEXT: vpblendd {{.*#+}} xmm12 = xmm13[0,1,2],xmm12[3]
2402 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm13 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
2403 ; AVX2-NEXT: vpshufb %xmm13, %xmm7, %xmm14
2404 ; AVX2-NEXT: vpshufb %xmm13, %xmm6, %xmm13
2405 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
2406 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm14 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
2407 ; AVX2-NEXT: vpshufb %xmm14, %xmm2, %xmm15
2408 ; AVX2-NEXT: vpshufb %xmm14, %xmm1, %xmm14
2409 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
2410 ; AVX2-NEXT: vpblendd {{.*#+}} xmm13 = xmm14[0],xmm13[1],xmm14[2,3]
2411 ; AVX2-NEXT: vpblendd {{.*#+}} xmm12 = xmm13[0,1],xmm12[2,3]
2412 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm13 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
2413 ; AVX2-NEXT: vpshufb %xmm13, %xmm8, %xmm14
2414 ; AVX2-NEXT: vpshufb %xmm13, %xmm3, %xmm13
2415 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
2416 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm14 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
2417 ; AVX2-NEXT: vpshufb %xmm14, %xmm4, %xmm15
2418 ; AVX2-NEXT: vpshufb %xmm14, %xmm5, %xmm14
2419 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
2420 ; AVX2-NEXT: vpblendd {{.*#+}} xmm13 = xmm14[0,1,2],xmm13[3]
2421 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm14 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
2422 ; AVX2-NEXT: vpshufb %xmm14, %xmm7, %xmm15
2423 ; AVX2-NEXT: vpshufb %xmm14, %xmm6, %xmm14
2424 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
2425 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm15 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
2426 ; AVX2-NEXT: vpshufb %xmm15, %xmm2, %xmm0
2427 ; AVX2-NEXT: vpshufb %xmm15, %xmm1, %xmm15
2428 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
2429 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm14[1],xmm0[2,3]
2430 ; AVX2-NEXT: vpblendd {{.*#+}} xmm13 = xmm0[0,1],xmm13[2,3]
2431 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm0 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
2432 ; AVX2-NEXT: vpshufb %xmm0, %xmm8, %xmm14
2433 ; AVX2-NEXT: vpshufb %xmm0, %xmm3, %xmm0
2434 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
2435 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm14 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
2436 ; AVX2-NEXT: vpshufb %xmm14, %xmm4, %xmm15
2437 ; AVX2-NEXT: vpshufb %xmm14, %xmm5, %xmm14
2438 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
2439 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm14[0,1,2],xmm0[3]
2440 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm14 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
2441 ; AVX2-NEXT: vpshufb %xmm14, %xmm7, %xmm15
2442 ; AVX2-NEXT: vpshufb %xmm14, %xmm6, %xmm14
2443 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
2444 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm15 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
2445 ; AVX2-NEXT: vpshufb %xmm15, %xmm2, %xmm0
2446 ; AVX2-NEXT: vpshufb %xmm15, %xmm9, %xmm15
2447 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
2448 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm14[1],xmm0[2,3]
2449 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
2450 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm1 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
2451 ; AVX2-NEXT: vpshufb %xmm1, %xmm8, %xmm8
2452 ; AVX2-NEXT: vpshufb %xmm1, %xmm3, %xmm1
2453 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3]
2454 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm3 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
2455 ; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm4
2456 ; AVX2-NEXT: vpshufb %xmm3, %xmm5, %xmm3
2457 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
2458 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
2459 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm3 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
2460 ; AVX2-NEXT: vpshufb %xmm3, %xmm7, %xmm4
2461 ; AVX2-NEXT: vpshufb %xmm3, %xmm6, %xmm3
2462 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
2463 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm4 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
2464 ; AVX2-NEXT: vpshufb %xmm4, %xmm2, %xmm2
2465 ; AVX2-NEXT: vpshufb %xmm4, %xmm9, %xmm4
2466 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
2467 ; AVX2-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3]
2468 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
2469 ; AVX2-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
2470 ; AVX2-NEXT: vmovaps %xmm2, (%rsi)
2471 ; AVX2-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
2472 ; AVX2-NEXT: vmovaps %xmm2, (%rdx)
2473 ; AVX2-NEXT: vmovdqa %xmm10, (%rcx)
2474 ; AVX2-NEXT: vmovdqa %xmm11, (%r8)
2475 ; AVX2-NEXT: vmovdqa %xmm12, (%r9)
2476 ; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
2477 ; AVX2-NEXT: vmovdqa %xmm13, (%rax)
2478 ; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
2479 ; AVX2-NEXT: vmovdqa %xmm0, (%rax)
2480 ; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
2481 ; AVX2-NEXT: vmovdqa %xmm1, (%rax)
2482 ; AVX2-NEXT: retq
2483 ;
2484 ; AVX2-FP-LABEL: load_i8_stride8_vf16:
2485 ; AVX2-FP: # %bb.0:
2486 ; AVX2-FP-NEXT: vmovdqa 112(%rdi), %xmm8
2487 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
2488 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm8, %xmm2
2489 ; AVX2-FP-NEXT: vmovdqa 96(%rdi), %xmm3
2490 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm3, %xmm0
2491 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
2492 ; AVX2-FP-NEXT: vmovdqa 80(%rdi), %xmm4
2493 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
2494 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm4, %xmm6
2495 ; AVX2-FP-NEXT: vmovdqa 64(%rdi), %xmm5
2496 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm5, %xmm2
2497 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3]
2498 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
2499 ; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm1
2500 ; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm2
2501 ; AVX2-FP-NEXT: vmovdqa 32(%rdi), %xmm6
2502 ; AVX2-FP-NEXT: vmovdqa 48(%rdi), %xmm7
2503 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm9 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
2504 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm7, %xmm10
2505 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm6, %xmm9
2506 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
2507 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
2508 ; AVX2-FP-NEXT: vpshufb %xmm10, %xmm2, %xmm11
2509 ; AVX2-FP-NEXT: vpshufb %xmm10, %xmm1, %xmm10
2510 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
2511 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0],xmm9[1],xmm10[2,3]
2512 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm9[0,1],xmm0[2,3]
2513 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2514 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm9 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
2515 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm8, %xmm10
2516 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm3, %xmm9
2517 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
2518 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
2519 ; AVX2-FP-NEXT: vpshufb %xmm10, %xmm4, %xmm11
2520 ; AVX2-FP-NEXT: vpshufb %xmm10, %xmm5, %xmm10
2521 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
2522 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
2523 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
2524 ; AVX2-FP-NEXT: vpshufb %xmm10, %xmm7, %xmm11
2525 ; AVX2-FP-NEXT: vpshufb %xmm10, %xmm6, %xmm10
2526 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
2527 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
2528 ; AVX2-FP-NEXT: vpshufb %xmm11, %xmm2, %xmm12
2529 ; AVX2-FP-NEXT: vpshufb %xmm11, %xmm1, %xmm11
2530 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
2531 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0],xmm10[1],xmm11[2,3]
2532 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm9[2,3]
2533 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2534 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
2535 ; AVX2-FP-NEXT: vpshufb %xmm10, %xmm8, %xmm11
2536 ; AVX2-FP-NEXT: vpshufb %xmm10, %xmm3, %xmm10
2537 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
2538 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
2539 ; AVX2-FP-NEXT: vpshufb %xmm11, %xmm4, %xmm12
2540 ; AVX2-FP-NEXT: vpshufb %xmm11, %xmm5, %xmm11
2541 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
2542 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1,2],xmm10[3]
2543 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
2544 ; AVX2-FP-NEXT: vpshufb %xmm11, %xmm7, %xmm12
2545 ; AVX2-FP-NEXT: vpshufb %xmm11, %xmm6, %xmm11
2546 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
2547 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm12 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
2548 ; AVX2-FP-NEXT: vpshufb %xmm12, %xmm2, %xmm13
2549 ; AVX2-FP-NEXT: vpshufb %xmm12, %xmm1, %xmm12
2550 ; AVX2-FP-NEXT: vmovdqa %xmm1, %xmm9
2551 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
2552 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0],xmm11[1],xmm12[2,3]
2553 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3]
2554 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
2555 ; AVX2-FP-NEXT: vpshufb %xmm11, %xmm8, %xmm12
2556 ; AVX2-FP-NEXT: vpshufb %xmm11, %xmm3, %xmm11
2557 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
2558 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm12 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
2559 ; AVX2-FP-NEXT: vpshufb %xmm12, %xmm4, %xmm13
2560 ; AVX2-FP-NEXT: vpshufb %xmm12, %xmm5, %xmm12
2561 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
2562 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1,2],xmm11[3]
2563 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm12 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
2564 ; AVX2-FP-NEXT: vpshufb %xmm12, %xmm7, %xmm13
2565 ; AVX2-FP-NEXT: vpshufb %xmm12, %xmm6, %xmm12
2566 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
2567 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm13 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
2568 ; AVX2-FP-NEXT: vpshufb %xmm13, %xmm2, %xmm14
2569 ; AVX2-FP-NEXT: vpshufb %xmm13, %xmm1, %xmm13
2570 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
2571 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm12 = xmm13[0],xmm12[1],xmm13[2,3]
2572 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1],xmm11[2,3]
2573 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm12 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
2574 ; AVX2-FP-NEXT: vpshufb %xmm12, %xmm8, %xmm13
2575 ; AVX2-FP-NEXT: vpshufb %xmm12, %xmm3, %xmm12
2576 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
2577 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm13 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
2578 ; AVX2-FP-NEXT: vpshufb %xmm13, %xmm4, %xmm14
2579 ; AVX2-FP-NEXT: vpshufb %xmm13, %xmm5, %xmm13
2580 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
2581 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm12 = xmm13[0,1,2],xmm12[3]
2582 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm13 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
2583 ; AVX2-FP-NEXT: vpshufb %xmm13, %xmm7, %xmm14
2584 ; AVX2-FP-NEXT: vpshufb %xmm13, %xmm6, %xmm13
2585 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
2586 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm14 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
2587 ; AVX2-FP-NEXT: vpshufb %xmm14, %xmm2, %xmm15
2588 ; AVX2-FP-NEXT: vpshufb %xmm14, %xmm1, %xmm14
2589 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
2590 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm13 = xmm14[0],xmm13[1],xmm14[2,3]
2591 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm12 = xmm13[0,1],xmm12[2,3]
2592 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm13 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
2593 ; AVX2-FP-NEXT: vpshufb %xmm13, %xmm8, %xmm14
2594 ; AVX2-FP-NEXT: vpshufb %xmm13, %xmm3, %xmm13
2595 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
2596 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm14 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
2597 ; AVX2-FP-NEXT: vpshufb %xmm14, %xmm4, %xmm15
2598 ; AVX2-FP-NEXT: vpshufb %xmm14, %xmm5, %xmm14
2599 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
2600 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm13 = xmm14[0,1,2],xmm13[3]
2601 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm14 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
2602 ; AVX2-FP-NEXT: vpshufb %xmm14, %xmm7, %xmm15
2603 ; AVX2-FP-NEXT: vpshufb %xmm14, %xmm6, %xmm14
2604 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
2605 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm15 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
2606 ; AVX2-FP-NEXT: vpshufb %xmm15, %xmm2, %xmm0
2607 ; AVX2-FP-NEXT: vpshufb %xmm15, %xmm1, %xmm15
2608 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
2609 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm14[1],xmm0[2,3]
2610 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm13 = xmm0[0,1],xmm13[2,3]
2611 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
2612 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm8, %xmm14
2613 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm3, %xmm0
2614 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
2615 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm14 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
2616 ; AVX2-FP-NEXT: vpshufb %xmm14, %xmm4, %xmm15
2617 ; AVX2-FP-NEXT: vpshufb %xmm14, %xmm5, %xmm14
2618 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
2619 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm14[0,1,2],xmm0[3]
2620 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm14 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
2621 ; AVX2-FP-NEXT: vpshufb %xmm14, %xmm7, %xmm15
2622 ; AVX2-FP-NEXT: vpshufb %xmm14, %xmm6, %xmm14
2623 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
2624 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm15 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
2625 ; AVX2-FP-NEXT: vpshufb %xmm15, %xmm2, %xmm0
2626 ; AVX2-FP-NEXT: vpshufb %xmm15, %xmm9, %xmm15
2627 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
2628 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm14[1],xmm0[2,3]
2629 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
2630 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
2631 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm8, %xmm8
2632 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm3, %xmm1
2633 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3]
2634 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
2635 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm4, %xmm4
2636 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm5, %xmm3
2637 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
2638 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
2639 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
2640 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm7, %xmm4
2641 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm6, %xmm3
2642 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
2643 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm4 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
2644 ; AVX2-FP-NEXT: vpshufb %xmm4, %xmm2, %xmm2
2645 ; AVX2-FP-NEXT: vpshufb %xmm4, %xmm9, %xmm4
2646 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
2647 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3]
2648 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
2649 ; AVX2-FP-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
2650 ; AVX2-FP-NEXT: vmovaps %xmm2, (%rsi)
2651 ; AVX2-FP-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
2652 ; AVX2-FP-NEXT: vmovaps %xmm2, (%rdx)
2653 ; AVX2-FP-NEXT: vmovdqa %xmm10, (%rcx)
2654 ; AVX2-FP-NEXT: vmovdqa %xmm11, (%r8)
2655 ; AVX2-FP-NEXT: vmovdqa %xmm12, (%r9)
2656 ; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %rax
2657 ; AVX2-FP-NEXT: vmovdqa %xmm13, (%rax)
2658 ; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %rax
2659 ; AVX2-FP-NEXT: vmovdqa %xmm0, (%rax)
2660 ; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %rax
2661 ; AVX2-FP-NEXT: vmovdqa %xmm1, (%rax)
2662 ; AVX2-FP-NEXT: retq
2663 ;
2664 ; AVX2-FCP-LABEL: load_i8_stride8_vf16:
2665 ; AVX2-FCP: # %bb.0:
2666 ; AVX2-FCP-NEXT: vmovdqa 112(%rdi), %xmm8
2667 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
2668 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm8, %xmm2
2669 ; AVX2-FCP-NEXT: vmovdqa 96(%rdi), %xmm3
2670 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm0
2671 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
2672 ; AVX2-FCP-NEXT: vmovdqa 80(%rdi), %xmm4
2673 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
2674 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm4, %xmm6
2675 ; AVX2-FCP-NEXT: vmovdqa 64(%rdi), %xmm5
2676 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm5, %xmm2
2677 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3]
2678 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
2679 ; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm1
2680 ; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
2681 ; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %xmm6
2682 ; AVX2-FCP-NEXT: vmovdqa 48(%rdi), %xmm7
2683 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm9 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
2684 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm7, %xmm10
2685 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm6, %xmm9
2686 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
2687 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
2688 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm2, %xmm11
2689 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm1, %xmm10
2690 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
2691 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0],xmm9[1],xmm10[2,3]
2692 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm9[0,1],xmm0[2,3]
2693 ; AVX2-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2694 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm9 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
2695 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm8, %xmm10
2696 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm3, %xmm9
2697 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
2698 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
2699 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm4, %xmm11
2700 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm5, %xmm10
2701 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
2702 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
2703 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
2704 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm7, %xmm11
2705 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm6, %xmm10
2706 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
2707 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
2708 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm2, %xmm12
2709 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm1, %xmm11
2710 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
2711 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0],xmm10[1],xmm11[2,3]
2712 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm9[2,3]
2713 ; AVX2-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2714 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
2715 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm8, %xmm11
2716 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm3, %xmm10
2717 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
2718 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
2719 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm4, %xmm12
2720 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm5, %xmm11
2721 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
2722 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1,2],xmm10[3]
2723 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
2724 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm7, %xmm12
2725 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm6, %xmm11
2726 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
2727 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm12 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
2728 ; AVX2-FCP-NEXT: vpshufb %xmm12, %xmm2, %xmm13
2729 ; AVX2-FCP-NEXT: vpshufb %xmm12, %xmm1, %xmm12
2730 ; AVX2-FCP-NEXT: vmovdqa %xmm1, %xmm9
2731 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
2732 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0],xmm11[1],xmm12[2,3]
2733 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3]
2734 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
2735 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm8, %xmm12
2736 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm3, %xmm11
2737 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
2738 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm12 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
2739 ; AVX2-FCP-NEXT: vpshufb %xmm12, %xmm4, %xmm13
2740 ; AVX2-FCP-NEXT: vpshufb %xmm12, %xmm5, %xmm12
2741 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
2742 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1,2],xmm11[3]
2743 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm12 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
2744 ; AVX2-FCP-NEXT: vpshufb %xmm12, %xmm7, %xmm13
2745 ; AVX2-FCP-NEXT: vpshufb %xmm12, %xmm6, %xmm12
2746 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
2747 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm13 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
2748 ; AVX2-FCP-NEXT: vpshufb %xmm13, %xmm2, %xmm14
2749 ; AVX2-FCP-NEXT: vpshufb %xmm13, %xmm1, %xmm13
2750 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
2751 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm13[0],xmm12[1],xmm13[2,3]
2752 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1],xmm11[2,3]
2753 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm12 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
2754 ; AVX2-FCP-NEXT: vpshufb %xmm12, %xmm8, %xmm13
2755 ; AVX2-FCP-NEXT: vpshufb %xmm12, %xmm3, %xmm12
2756 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
2757 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm13 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
2758 ; AVX2-FCP-NEXT: vpshufb %xmm13, %xmm4, %xmm14
2759 ; AVX2-FCP-NEXT: vpshufb %xmm13, %xmm5, %xmm13
2760 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
2761 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm13[0,1,2],xmm12[3]
2762 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm13 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
2763 ; AVX2-FCP-NEXT: vpshufb %xmm13, %xmm7, %xmm14
2764 ; AVX2-FCP-NEXT: vpshufb %xmm13, %xmm6, %xmm13
2765 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
2766 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm14 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
2767 ; AVX2-FCP-NEXT: vpshufb %xmm14, %xmm2, %xmm15
2768 ; AVX2-FCP-NEXT: vpshufb %xmm14, %xmm1, %xmm14
2769 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
2770 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm13 = xmm14[0],xmm13[1],xmm14[2,3]
2771 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm13[0,1],xmm12[2,3]
2772 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm13 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
2773 ; AVX2-FCP-NEXT: vpshufb %xmm13, %xmm8, %xmm14
2774 ; AVX2-FCP-NEXT: vpshufb %xmm13, %xmm3, %xmm13
2775 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
2776 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm14 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
2777 ; AVX2-FCP-NEXT: vpshufb %xmm14, %xmm4, %xmm15
2778 ; AVX2-FCP-NEXT: vpshufb %xmm14, %xmm5, %xmm14
2779 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
2780 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm13 = xmm14[0,1,2],xmm13[3]
2781 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm14 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
2782 ; AVX2-FCP-NEXT: vpshufb %xmm14, %xmm7, %xmm15
2783 ; AVX2-FCP-NEXT: vpshufb %xmm14, %xmm6, %xmm14
2784 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
2785 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm15 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
2786 ; AVX2-FCP-NEXT: vpshufb %xmm15, %xmm2, %xmm0
2787 ; AVX2-FCP-NEXT: vpshufb %xmm15, %xmm1, %xmm15
2788 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
2789 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm14[1],xmm0[2,3]
2790 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm13 = xmm0[0,1],xmm13[2,3]
2791 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
2792 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm8, %xmm14
2793 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm0
2794 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
2795 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm14 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
2796 ; AVX2-FCP-NEXT: vpshufb %xmm14, %xmm4, %xmm15
2797 ; AVX2-FCP-NEXT: vpshufb %xmm14, %xmm5, %xmm14
2798 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
2799 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm14[0,1,2],xmm0[3]
2800 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm14 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
2801 ; AVX2-FCP-NEXT: vpshufb %xmm14, %xmm7, %xmm15
2802 ; AVX2-FCP-NEXT: vpshufb %xmm14, %xmm6, %xmm14
2803 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
2804 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm15 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
2805 ; AVX2-FCP-NEXT: vpshufb %xmm15, %xmm2, %xmm0
2806 ; AVX2-FCP-NEXT: vpshufb %xmm15, %xmm9, %xmm15
2807 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
2808 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm14[1],xmm0[2,3]
2809 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
2810 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
2811 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm8, %xmm8
2812 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm3, %xmm1
2813 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3]
2814 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
2815 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm4, %xmm4
2816 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm5, %xmm3
2817 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
2818 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
2819 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
2820 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm7, %xmm4
2821 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm6, %xmm3
2822 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
2823 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm4 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
2824 ; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm2, %xmm2
2825 ; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm9, %xmm4
2826 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
2827 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3]
2828 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
2829 ; AVX2-FCP-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
2830 ; AVX2-FCP-NEXT: vmovaps %xmm2, (%rsi)
2831 ; AVX2-FCP-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
2832 ; AVX2-FCP-NEXT: vmovaps %xmm2, (%rdx)
2833 ; AVX2-FCP-NEXT: vmovdqa %xmm10, (%rcx)
2834 ; AVX2-FCP-NEXT: vmovdqa %xmm11, (%r8)
2835 ; AVX2-FCP-NEXT: vmovdqa %xmm12, (%r9)
2836 ; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
2837 ; AVX2-FCP-NEXT: vmovdqa %xmm13, (%rax)
2838 ; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
2839 ; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rax)
2840 ; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
2841 ; AVX2-FCP-NEXT: vmovdqa %xmm1, (%rax)
2842 ; AVX2-FCP-NEXT: retq
2843 ;
2844 ; AVX512-LABEL: load_i8_stride8_vf16:
2845 ; AVX512: # %bb.0:
2846 ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
2847 ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r10
2848 ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
2849 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
2850 ; AVX512-NEXT: vmovdqa 112(%rdi), %xmm0
2851 ; AVX512-NEXT: vpshufb %xmm2, %xmm0, %xmm3
2852 ; AVX512-NEXT: vmovdqa 96(%rdi), %xmm1
2853 ; AVX512-NEXT: vpshufb %xmm2, %xmm1, %xmm2
2854 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
2855 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm5 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
2856 ; AVX512-NEXT: vmovdqa 80(%rdi), %xmm2
2857 ; AVX512-NEXT: vpshufb %xmm5, %xmm2, %xmm6
2858 ; AVX512-NEXT: vmovdqa 64(%rdi), %xmm3
2859 ; AVX512-NEXT: vpshufb %xmm5, %xmm3, %xmm5
2860 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
2861 ; AVX512-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
2862 ; AVX512-NEXT: vmovdqa64 (%rdi), %zmm5
2863 ; AVX512-NEXT: vpmovqb %zmm5, %xmm6
2864 ; AVX512-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3]
2865 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm6 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
2866 ; AVX512-NEXT: vpshufb %xmm6, %xmm0, %xmm7
2867 ; AVX512-NEXT: vpshufb %xmm6, %xmm1, %xmm6
2868 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
2869 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm7 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
2870 ; AVX512-NEXT: vpshufb %xmm7, %xmm2, %xmm8
2871 ; AVX512-NEXT: vpshufb %xmm7, %xmm3, %xmm7
2872 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
2873 ; AVX512-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1,2],xmm6[3]
2874 ; AVX512-NEXT: vpsrlq $8, %zmm5, %zmm7
2875 ; AVX512-NEXT: vpmovqb %zmm7, %xmm7
2876 ; AVX512-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3]
2877 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm7 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
2878 ; AVX512-NEXT: vpshufb %xmm7, %xmm0, %xmm8
2879 ; AVX512-NEXT: vpshufb %xmm7, %xmm1, %xmm7
2880 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
2881 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm8 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
2882 ; AVX512-NEXT: vpshufb %xmm8, %xmm2, %xmm9
2883 ; AVX512-NEXT: vpshufb %xmm8, %xmm3, %xmm8
2884 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
2885 ; AVX512-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3]
2886 ; AVX512-NEXT: vpsrlq $16, %zmm5, %zmm8
2887 ; AVX512-NEXT: vpmovqb %zmm8, %xmm8
2888 ; AVX512-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3]
2889 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm8 = [0,0,3,11,0,0,3,11,0,0,3,11,0,0,3,11]
2890 ; AVX512-NEXT: vpshufb %xmm8, %xmm0, %xmm9
2891 ; AVX512-NEXT: vpshufb %xmm8, %xmm1, %xmm8
2892 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
2893 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm9 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
2894 ; AVX512-NEXT: vpshufb %xmm9, %xmm2, %xmm10
2895 ; AVX512-NEXT: vpshufb %xmm9, %xmm3, %xmm9
2896 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
2897 ; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
2898 ; AVX512-NEXT: vpsrlq $24, %zmm5, %zmm9
2899 ; AVX512-NEXT: vpmovqb %zmm9, %xmm9
2900 ; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
2901 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm9 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
2902 ; AVX512-NEXT: vpshufb %xmm9, %xmm0, %xmm10
2903 ; AVX512-NEXT: vpshufb %xmm9, %xmm1, %xmm9
2904 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
2905 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm10 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
2906 ; AVX512-NEXT: vpshufb %xmm10, %xmm2, %xmm11
2907 ; AVX512-NEXT: vpshufb %xmm10, %xmm3, %xmm10
2908 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
2909 ; AVX512-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
2910 ; AVX512-NEXT: vpsrlq $32, %zmm5, %zmm10
2911 ; AVX512-NEXT: vpmovqb %zmm10, %xmm10
2912 ; AVX512-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
2913 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm10 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
2914 ; AVX512-NEXT: vpshufb %xmm10, %xmm0, %xmm11
2915 ; AVX512-NEXT: vpshufb %xmm10, %xmm1, %xmm10
2916 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
2917 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm11 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
2918 ; AVX512-NEXT: vpshufb %xmm11, %xmm2, %xmm12
2919 ; AVX512-NEXT: vpshufb %xmm11, %xmm3, %xmm11
2920 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
2921 ; AVX512-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1,2],xmm10[3]
2922 ; AVX512-NEXT: vpsrlq $40, %zmm5, %zmm11
2923 ; AVX512-NEXT: vpmovqb %zmm11, %xmm11
2924 ; AVX512-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3]
2925 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm11 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
2926 ; AVX512-NEXT: vpshufb %xmm11, %xmm0, %xmm12
2927 ; AVX512-NEXT: vpshufb %xmm11, %xmm1, %xmm11
2928 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
2929 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm12 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
2930 ; AVX512-NEXT: vpshufb %xmm12, %xmm2, %xmm13
2931 ; AVX512-NEXT: vpshufb %xmm12, %xmm3, %xmm12
2932 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
2933 ; AVX512-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1,2],xmm11[3]
2934 ; AVX512-NEXT: vpsrlq $48, %zmm5, %zmm12
2935 ; AVX512-NEXT: vpmovqb %zmm12, %xmm12
2936 ; AVX512-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1],xmm11[2,3]
2937 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm12 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
2938 ; AVX512-NEXT: vpshufb %xmm12, %xmm0, %xmm0
2939 ; AVX512-NEXT: vpshufb %xmm12, %xmm1, %xmm1
2940 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2941 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
2942 ; AVX512-NEXT: vpshufb %xmm1, %xmm2, %xmm2
2943 ; AVX512-NEXT: vpshufb %xmm1, %xmm3, %xmm1
2944 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
2945 ; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
2946 ; AVX512-NEXT: vpsrlq $56, %zmm5, %zmm1
2947 ; AVX512-NEXT: vpmovqb %zmm1, %xmm1
2948 ; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
2949 ; AVX512-NEXT: vmovdqa %xmm4, (%rsi)
2950 ; AVX512-NEXT: vmovdqa %xmm6, (%rdx)
2951 ; AVX512-NEXT: vmovdqa %xmm7, (%rcx)
2952 ; AVX512-NEXT: vmovdqa %xmm8, (%r8)
2953 ; AVX512-NEXT: vmovdqa %xmm9, (%r9)
2954 ; AVX512-NEXT: vmovdqa %xmm10, (%r11)
2955 ; AVX512-NEXT: vmovdqa %xmm11, (%r10)
2956 ; AVX512-NEXT: vmovdqa %xmm0, (%rax)
2957 ; AVX512-NEXT: vzeroupper
2958 ; AVX512-NEXT: retq
2959 ;
2960 ; AVX512-FCP-LABEL: load_i8_stride8_vf16:
2961 ; AVX512-FCP: # %bb.0:
2962 ; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
2963 ; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
2964 ; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11
2965 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm2 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
2966 ; AVX512-FCP-NEXT: vmovdqa 112(%rdi), %xmm0
2967 ; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm0, %xmm3
2968 ; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %xmm1
2969 ; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm1, %xmm2
2970 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
2971 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm5 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
2972 ; AVX512-FCP-NEXT: vmovdqa 80(%rdi), %xmm2
2973 ; AVX512-FCP-NEXT: vpshufb %xmm5, %xmm2, %xmm6
2974 ; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %xmm3
2975 ; AVX512-FCP-NEXT: vpshufb %xmm5, %xmm3, %xmm5
2976 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
2977 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
2978 ; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm5
2979 ; AVX512-FCP-NEXT: vpmovqb %zmm5, %xmm6
2980 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3]
2981 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm6 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
2982 ; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm0, %xmm7
2983 ; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm1, %xmm6
2984 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
2985 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm7 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
2986 ; AVX512-FCP-NEXT: vpshufb %xmm7, %xmm2, %xmm8
2987 ; AVX512-FCP-NEXT: vpshufb %xmm7, %xmm3, %xmm7
2988 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
2989 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1,2],xmm6[3]
2990 ; AVX512-FCP-NEXT: vpsrlq $8, %zmm5, %zmm7
2991 ; AVX512-FCP-NEXT: vpmovqb %zmm7, %xmm7
2992 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3]
2993 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm7 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
2994 ; AVX512-FCP-NEXT: vpshufb %xmm7, %xmm0, %xmm8
2995 ; AVX512-FCP-NEXT: vpshufb %xmm7, %xmm1, %xmm7
2996 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
2997 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm8 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
2998 ; AVX512-FCP-NEXT: vpshufb %xmm8, %xmm2, %xmm9
2999 ; AVX512-FCP-NEXT: vpshufb %xmm8, %xmm3, %xmm8
3000 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
3001 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3]
3002 ; AVX512-FCP-NEXT: vpsrlq $16, %zmm5, %zmm8
3003 ; AVX512-FCP-NEXT: vpmovqb %zmm8, %xmm8
3004 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3]
3005 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm8 = [0,0,3,11,0,0,3,11,0,0,3,11,0,0,3,11]
3006 ; AVX512-FCP-NEXT: vpshufb %xmm8, %xmm0, %xmm9
3007 ; AVX512-FCP-NEXT: vpshufb %xmm8, %xmm1, %xmm8
3008 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
3009 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm9 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
3010 ; AVX512-FCP-NEXT: vpshufb %xmm9, %xmm2, %xmm10
3011 ; AVX512-FCP-NEXT: vpshufb %xmm9, %xmm3, %xmm9
3012 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
3013 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
3014 ; AVX512-FCP-NEXT: vpsrlq $24, %zmm5, %zmm9
3015 ; AVX512-FCP-NEXT: vpmovqb %zmm9, %xmm9
3016 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
3017 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm9 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
3018 ; AVX512-FCP-NEXT: vpshufb %xmm9, %xmm0, %xmm10
3019 ; AVX512-FCP-NEXT: vpshufb %xmm9, %xmm1, %xmm9
3020 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
3021 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm10 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
3022 ; AVX512-FCP-NEXT: vpshufb %xmm10, %xmm2, %xmm11
3023 ; AVX512-FCP-NEXT: vpshufb %xmm10, %xmm3, %xmm10
3024 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
3025 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
3026 ; AVX512-FCP-NEXT: vpsrlq $32, %zmm5, %zmm10
3027 ; AVX512-FCP-NEXT: vpmovqb %zmm10, %xmm10
3028 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
3029 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm10 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
3030 ; AVX512-FCP-NEXT: vpshufb %xmm10, %xmm0, %xmm11
3031 ; AVX512-FCP-NEXT: vpshufb %xmm10, %xmm1, %xmm10
3032 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
3033 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm11 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
3034 ; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm2, %xmm12
3035 ; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm3, %xmm11
3036 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
3037 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1,2],xmm10[3]
3038 ; AVX512-FCP-NEXT: vpsrlq $40, %zmm5, %zmm11
3039 ; AVX512-FCP-NEXT: vpmovqb %zmm11, %xmm11
3040 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3]
3041 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm11 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
3042 ; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm0, %xmm12
3043 ; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm1, %xmm11
3044 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
3045 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm12 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
3046 ; AVX512-FCP-NEXT: vpshufb %xmm12, %xmm2, %xmm13
3047 ; AVX512-FCP-NEXT: vpshufb %xmm12, %xmm3, %xmm12
3048 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
3049 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1,2],xmm11[3]
3050 ; AVX512-FCP-NEXT: vpsrlq $48, %zmm5, %zmm12
3051 ; AVX512-FCP-NEXT: vpmovqb %zmm12, %xmm12
3052 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1],xmm11[2,3]
3053 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm12 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
3054 ; AVX512-FCP-NEXT: vpshufb %xmm12, %xmm0, %xmm0
3055 ; AVX512-FCP-NEXT: vpshufb %xmm12, %xmm1, %xmm1
3056 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
3057 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm1 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
3058 ; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm2, %xmm2
3059 ; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm3, %xmm1
3060 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
3061 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
3062 ; AVX512-FCP-NEXT: vpsrlq $56, %zmm5, %zmm1
3063 ; AVX512-FCP-NEXT: vpmovqb %zmm1, %xmm1
3064 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
3065 ; AVX512-FCP-NEXT: vmovdqa %xmm4, (%rsi)
3066 ; AVX512-FCP-NEXT: vmovdqa %xmm6, (%rdx)
3067 ; AVX512-FCP-NEXT: vmovdqa %xmm7, (%rcx)
3068 ; AVX512-FCP-NEXT: vmovdqa %xmm8, (%r8)
3069 ; AVX512-FCP-NEXT: vmovdqa %xmm9, (%r9)
3070 ; AVX512-FCP-NEXT: vmovdqa %xmm10, (%r11)
3071 ; AVX512-FCP-NEXT: vmovdqa %xmm11, (%r10)
3072 ; AVX512-FCP-NEXT: vmovdqa %xmm0, (%rax)
3073 ; AVX512-FCP-NEXT: vzeroupper
3074 ; AVX512-FCP-NEXT: retq
3075 ;
3076 ; AVX512DQ-LABEL: load_i8_stride8_vf16:
3077 ; AVX512DQ: # %bb.0:
3078 ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax
3079 ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %r10
3080 ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %r11
3081 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm2 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
3082 ; AVX512DQ-NEXT: vmovdqa 112(%rdi), %xmm0
3083 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm0, %xmm3
3084 ; AVX512DQ-NEXT: vmovdqa 96(%rdi), %xmm1
3085 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm1, %xmm2
3086 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
3087 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm5 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
3088 ; AVX512DQ-NEXT: vmovdqa 80(%rdi), %xmm2
3089 ; AVX512DQ-NEXT: vpshufb %xmm5, %xmm2, %xmm6
3090 ; AVX512DQ-NEXT: vmovdqa 64(%rdi), %xmm3
3091 ; AVX512DQ-NEXT: vpshufb %xmm5, %xmm3, %xmm5
3092 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
3093 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
3094 ; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm5
3095 ; AVX512DQ-NEXT: vpmovqb %zmm5, %xmm6
3096 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3]
3097 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm6 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
3098 ; AVX512DQ-NEXT: vpshufb %xmm6, %xmm0, %xmm7
3099 ; AVX512DQ-NEXT: vpshufb %xmm6, %xmm1, %xmm6
3100 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
3101 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm7 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
3102 ; AVX512DQ-NEXT: vpshufb %xmm7, %xmm2, %xmm8
3103 ; AVX512DQ-NEXT: vpshufb %xmm7, %xmm3, %xmm7
3104 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
3105 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1,2],xmm6[3]
3106 ; AVX512DQ-NEXT: vpsrlq $8, %zmm5, %zmm7
3107 ; AVX512DQ-NEXT: vpmovqb %zmm7, %xmm7
3108 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3]
3109 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm7 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
3110 ; AVX512DQ-NEXT: vpshufb %xmm7, %xmm0, %xmm8
3111 ; AVX512DQ-NEXT: vpshufb %xmm7, %xmm1, %xmm7
3112 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
3113 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm8 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
3114 ; AVX512DQ-NEXT: vpshufb %xmm8, %xmm2, %xmm9
3115 ; AVX512DQ-NEXT: vpshufb %xmm8, %xmm3, %xmm8
3116 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
3117 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3]
3118 ; AVX512DQ-NEXT: vpsrlq $16, %zmm5, %zmm8
3119 ; AVX512DQ-NEXT: vpmovqb %zmm8, %xmm8
3120 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3]
3121 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm8 = [0,0,3,11,0,0,3,11,0,0,3,11,0,0,3,11]
3122 ; AVX512DQ-NEXT: vpshufb %xmm8, %xmm0, %xmm9
3123 ; AVX512DQ-NEXT: vpshufb %xmm8, %xmm1, %xmm8
3124 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
3125 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm9 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
3126 ; AVX512DQ-NEXT: vpshufb %xmm9, %xmm2, %xmm10
3127 ; AVX512DQ-NEXT: vpshufb %xmm9, %xmm3, %xmm9
3128 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
3129 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
3130 ; AVX512DQ-NEXT: vpsrlq $24, %zmm5, %zmm9
3131 ; AVX512DQ-NEXT: vpmovqb %zmm9, %xmm9
3132 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
3133 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm9 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
3134 ; AVX512DQ-NEXT: vpshufb %xmm9, %xmm0, %xmm10
3135 ; AVX512DQ-NEXT: vpshufb %xmm9, %xmm1, %xmm9
3136 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
3137 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm10 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
3138 ; AVX512DQ-NEXT: vpshufb %xmm10, %xmm2, %xmm11
3139 ; AVX512DQ-NEXT: vpshufb %xmm10, %xmm3, %xmm10
3140 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
3141 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
3142 ; AVX512DQ-NEXT: vpsrlq $32, %zmm5, %zmm10
3143 ; AVX512DQ-NEXT: vpmovqb %zmm10, %xmm10
3144 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
3145 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm10 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
3146 ; AVX512DQ-NEXT: vpshufb %xmm10, %xmm0, %xmm11
3147 ; AVX512DQ-NEXT: vpshufb %xmm10, %xmm1, %xmm10
3148 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
3149 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm11 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
3150 ; AVX512DQ-NEXT: vpshufb %xmm11, %xmm2, %xmm12
3151 ; AVX512DQ-NEXT: vpshufb %xmm11, %xmm3, %xmm11
3152 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
3153 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1,2],xmm10[3]
3154 ; AVX512DQ-NEXT: vpsrlq $40, %zmm5, %zmm11
3155 ; AVX512DQ-NEXT: vpmovqb %zmm11, %xmm11
3156 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3]
3157 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm11 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
3158 ; AVX512DQ-NEXT: vpshufb %xmm11, %xmm0, %xmm12
3159 ; AVX512DQ-NEXT: vpshufb %xmm11, %xmm1, %xmm11
3160 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
3161 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm12 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
3162 ; AVX512DQ-NEXT: vpshufb %xmm12, %xmm2, %xmm13
3163 ; AVX512DQ-NEXT: vpshufb %xmm12, %xmm3, %xmm12
3164 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
3165 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1,2],xmm11[3]
3166 ; AVX512DQ-NEXT: vpsrlq $48, %zmm5, %zmm12
3167 ; AVX512DQ-NEXT: vpmovqb %zmm12, %xmm12
3168 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1],xmm11[2,3]
3169 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm12 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
3170 ; AVX512DQ-NEXT: vpshufb %xmm12, %xmm0, %xmm0
3171 ; AVX512DQ-NEXT: vpshufb %xmm12, %xmm1, %xmm1
3172 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
3173 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm1 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
3174 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm2, %xmm2
3175 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm1
3176 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
3177 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
3178 ; AVX512DQ-NEXT: vpsrlq $56, %zmm5, %zmm1
3179 ; AVX512DQ-NEXT: vpmovqb %zmm1, %xmm1
3180 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
3181 ; AVX512DQ-NEXT: vmovdqa %xmm4, (%rsi)
3182 ; AVX512DQ-NEXT: vmovdqa %xmm6, (%rdx)
3183 ; AVX512DQ-NEXT: vmovdqa %xmm7, (%rcx)
3184 ; AVX512DQ-NEXT: vmovdqa %xmm8, (%r8)
3185 ; AVX512DQ-NEXT: vmovdqa %xmm9, (%r9)
3186 ; AVX512DQ-NEXT: vmovdqa %xmm10, (%r11)
3187 ; AVX512DQ-NEXT: vmovdqa %xmm11, (%r10)
3188 ; AVX512DQ-NEXT: vmovdqa %xmm0, (%rax)
3189 ; AVX512DQ-NEXT: vzeroupper
3190 ; AVX512DQ-NEXT: retq
3191 ;
3192 ; AVX512DQ-FCP-LABEL: load_i8_stride8_vf16:
3193 ; AVX512DQ-FCP: # %bb.0:
3194 ; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
3195 ; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
3196 ; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11
3197 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm2 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
3198 ; AVX512DQ-FCP-NEXT: vmovdqa 112(%rdi), %xmm0
3199 ; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm0, %xmm3
3200 ; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %xmm1
3201 ; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm1, %xmm2
3202 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
3203 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm5 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
3204 ; AVX512DQ-FCP-NEXT: vmovdqa 80(%rdi), %xmm2
3205 ; AVX512DQ-FCP-NEXT: vpshufb %xmm5, %xmm2, %xmm6
3206 ; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %xmm3
3207 ; AVX512DQ-FCP-NEXT: vpshufb %xmm5, %xmm3, %xmm5
3208 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
3209 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
3210 ; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm5
3211 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm5, %xmm6
3212 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3]
3213 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm6 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
3214 ; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm0, %xmm7
3215 ; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm1, %xmm6
3216 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
3217 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm7 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
3218 ; AVX512DQ-FCP-NEXT: vpshufb %xmm7, %xmm2, %xmm8
3219 ; AVX512DQ-FCP-NEXT: vpshufb %xmm7, %xmm3, %xmm7
3220 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
3221 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1,2],xmm6[3]
3222 ; AVX512DQ-FCP-NEXT: vpsrlq $8, %zmm5, %zmm7
3223 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm7, %xmm7
3224 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3]
3225 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm7 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
3226 ; AVX512DQ-FCP-NEXT: vpshufb %xmm7, %xmm0, %xmm8
3227 ; AVX512DQ-FCP-NEXT: vpshufb %xmm7, %xmm1, %xmm7
3228 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
3229 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm8 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
3230 ; AVX512DQ-FCP-NEXT: vpshufb %xmm8, %xmm2, %xmm9
3231 ; AVX512DQ-FCP-NEXT: vpshufb %xmm8, %xmm3, %xmm8
3232 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
3233 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3]
3234 ; AVX512DQ-FCP-NEXT: vpsrlq $16, %zmm5, %zmm8
3235 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm8, %xmm8
3236 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3]
3237 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm8 = [0,0,3,11,0,0,3,11,0,0,3,11,0,0,3,11]
3238 ; AVX512DQ-FCP-NEXT: vpshufb %xmm8, %xmm0, %xmm9
3239 ; AVX512DQ-FCP-NEXT: vpshufb %xmm8, %xmm1, %xmm8
3240 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
3241 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm9 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
3242 ; AVX512DQ-FCP-NEXT: vpshufb %xmm9, %xmm2, %xmm10
3243 ; AVX512DQ-FCP-NEXT: vpshufb %xmm9, %xmm3, %xmm9
3244 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
3245 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
3246 ; AVX512DQ-FCP-NEXT: vpsrlq $24, %zmm5, %zmm9
3247 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm9, %xmm9
3248 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
3249 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm9 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
3250 ; AVX512DQ-FCP-NEXT: vpshufb %xmm9, %xmm0, %xmm10
3251 ; AVX512DQ-FCP-NEXT: vpshufb %xmm9, %xmm1, %xmm9
3252 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
3253 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm10 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
3254 ; AVX512DQ-FCP-NEXT: vpshufb %xmm10, %xmm2, %xmm11
3255 ; AVX512DQ-FCP-NEXT: vpshufb %xmm10, %xmm3, %xmm10
3256 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
3257 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
3258 ; AVX512DQ-FCP-NEXT: vpsrlq $32, %zmm5, %zmm10
3259 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm10, %xmm10
3260 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
3261 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm10 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
3262 ; AVX512DQ-FCP-NEXT: vpshufb %xmm10, %xmm0, %xmm11
3263 ; AVX512DQ-FCP-NEXT: vpshufb %xmm10, %xmm1, %xmm10
3264 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
3265 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm11 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
3266 ; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm2, %xmm12
3267 ; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm3, %xmm11
3268 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
3269 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1,2],xmm10[3]
3270 ; AVX512DQ-FCP-NEXT: vpsrlq $40, %zmm5, %zmm11
3271 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm11, %xmm11
3272 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3]
3273 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm11 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
3274 ; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm0, %xmm12
3275 ; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm1, %xmm11
3276 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
3277 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm12 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
3278 ; AVX512DQ-FCP-NEXT: vpshufb %xmm12, %xmm2, %xmm13
3279 ; AVX512DQ-FCP-NEXT: vpshufb %xmm12, %xmm3, %xmm12
3280 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
3281 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1,2],xmm11[3]
3282 ; AVX512DQ-FCP-NEXT: vpsrlq $48, %zmm5, %zmm12
3283 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm12, %xmm12
3284 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1],xmm11[2,3]
3285 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm12 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
3286 ; AVX512DQ-FCP-NEXT: vpshufb %xmm12, %xmm0, %xmm0
3287 ; AVX512DQ-FCP-NEXT: vpshufb %xmm12, %xmm1, %xmm1
3288 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
3289 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm1 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
3290 ; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm2, %xmm2
3291 ; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm3, %xmm1
3292 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
3293 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
3294 ; AVX512DQ-FCP-NEXT: vpsrlq $56, %zmm5, %zmm1
3295 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm1, %xmm1
3296 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
3297 ; AVX512DQ-FCP-NEXT: vmovdqa %xmm4, (%rsi)
3298 ; AVX512DQ-FCP-NEXT: vmovdqa %xmm6, (%rdx)
3299 ; AVX512DQ-FCP-NEXT: vmovdqa %xmm7, (%rcx)
3300 ; AVX512DQ-FCP-NEXT: vmovdqa %xmm8, (%r8)
3301 ; AVX512DQ-FCP-NEXT: vmovdqa %xmm9, (%r9)
3302 ; AVX512DQ-FCP-NEXT: vmovdqa %xmm10, (%r11)
3303 ; AVX512DQ-FCP-NEXT: vmovdqa %xmm11, (%r10)
3304 ; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, (%rax)
3305 ; AVX512DQ-FCP-NEXT: vzeroupper
3306 ; AVX512DQ-FCP-NEXT: retq
3307 ;
3308 ; AVX512BW-LABEL: load_i8_stride8_vf16:
3309 ; AVX512BW: # %bb.0:
3310 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
3311 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %r10
3312 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %r11
3313 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm2 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
3314 ; AVX512BW-NEXT: vmovdqa 112(%rdi), %xmm0
3315 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm0, %xmm3
3316 ; AVX512BW-NEXT: vmovdqa 96(%rdi), %xmm1
3317 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm1, %xmm2
3318 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
3319 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm5 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
3320 ; AVX512BW-NEXT: vmovdqa 80(%rdi), %xmm2
3321 ; AVX512BW-NEXT: vpshufb %xmm5, %xmm2, %xmm6
3322 ; AVX512BW-NEXT: vmovdqa 64(%rdi), %xmm3
3323 ; AVX512BW-NEXT: vpshufb %xmm5, %xmm3, %xmm5
3324 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
3325 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
3326 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm5
3327 ; AVX512BW-NEXT: vpmovqb %zmm5, %xmm6
3328 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3]
3329 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm6 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
3330 ; AVX512BW-NEXT: vpshufb %xmm6, %xmm0, %xmm7
3331 ; AVX512BW-NEXT: vpshufb %xmm6, %xmm1, %xmm6
3332 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
3333 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm7 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
3334 ; AVX512BW-NEXT: vpshufb %xmm7, %xmm2, %xmm8
3335 ; AVX512BW-NEXT: vpshufb %xmm7, %xmm3, %xmm7
3336 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
3337 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1,2],xmm6[3]
3338 ; AVX512BW-NEXT: vpsrlq $8, %zmm5, %zmm7
3339 ; AVX512BW-NEXT: vpmovqb %zmm7, %xmm7
3340 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3]
3341 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm7 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
3342 ; AVX512BW-NEXT: vpshufb %xmm7, %xmm0, %xmm8
3343 ; AVX512BW-NEXT: vpshufb %xmm7, %xmm1, %xmm7
3344 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
3345 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm8 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
3346 ; AVX512BW-NEXT: vpshufb %xmm8, %xmm2, %xmm9
3347 ; AVX512BW-NEXT: vpshufb %xmm8, %xmm3, %xmm8
3348 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
3349 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3]
3350 ; AVX512BW-NEXT: vpsrlq $16, %zmm5, %zmm8
3351 ; AVX512BW-NEXT: vpmovqb %zmm8, %xmm8
3352 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3]
3353 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm8 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
3354 ; AVX512BW-NEXT: vpshufb %xmm8, %xmm0, %xmm9
3355 ; AVX512BW-NEXT: vpshufb %xmm8, %xmm1, %xmm8
3356 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
3357 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm9 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
3358 ; AVX512BW-NEXT: vpshufb %xmm9, %xmm2, %xmm10
3359 ; AVX512BW-NEXT: vpshufb %xmm9, %xmm3, %xmm9
3360 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
3361 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
3362 ; AVX512BW-NEXT: vpsrlq $24, %zmm5, %zmm9
3363 ; AVX512BW-NEXT: vpmovqb %zmm9, %xmm9
3364 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
3365 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm9 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
3366 ; AVX512BW-NEXT: vpshufb %xmm9, %xmm0, %xmm10
3367 ; AVX512BW-NEXT: vpshufb %xmm9, %xmm1, %xmm9
3368 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
3369 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm10 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
3370 ; AVX512BW-NEXT: vpshufb %xmm10, %xmm2, %xmm11
3371 ; AVX512BW-NEXT: vpshufb %xmm10, %xmm3, %xmm10
3372 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
3373 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
3374 ; AVX512BW-NEXT: vpsrlq $32, %zmm5, %zmm10
3375 ; AVX512BW-NEXT: vpmovqb %zmm10, %xmm10
3376 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
3377 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm10 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
3378 ; AVX512BW-NEXT: vpshufb %xmm10, %xmm0, %xmm11
3379 ; AVX512BW-NEXT: vpshufb %xmm10, %xmm1, %xmm10
3380 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
3381 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm11 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
3382 ; AVX512BW-NEXT: vpshufb %xmm11, %xmm2, %xmm12
3383 ; AVX512BW-NEXT: vpshufb %xmm11, %xmm3, %xmm11
3384 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
3385 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1,2],xmm10[3]
3386 ; AVX512BW-NEXT: vpsrlq $40, %zmm5, %zmm11
3387 ; AVX512BW-NEXT: vpmovqb %zmm11, %xmm11
3388 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3]
3389 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm11 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
3390 ; AVX512BW-NEXT: vpshufb %xmm11, %xmm0, %xmm12
3391 ; AVX512BW-NEXT: vpshufb %xmm11, %xmm1, %xmm11
3392 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
3393 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm12 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
3394 ; AVX512BW-NEXT: vpshufb %xmm12, %xmm2, %xmm13
3395 ; AVX512BW-NEXT: vpshufb %xmm12, %xmm3, %xmm12
3396 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
3397 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1,2],xmm11[3]
3398 ; AVX512BW-NEXT: vpsrlq $48, %zmm5, %zmm12
3399 ; AVX512BW-NEXT: vpmovqb %zmm12, %xmm12
3400 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1],xmm11[2,3]
3401 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm12 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
3402 ; AVX512BW-NEXT: vpshufb %xmm12, %xmm0, %xmm0
3403 ; AVX512BW-NEXT: vpshufb %xmm12, %xmm1, %xmm1
3404 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
3405 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm1 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
3406 ; AVX512BW-NEXT: vpshufb %xmm1, %xmm2, %xmm2
3407 ; AVX512BW-NEXT: vpshufb %xmm1, %xmm3, %xmm1
3408 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
3409 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
3410 ; AVX512BW-NEXT: vpsrlq $56, %zmm5, %zmm1
3411 ; AVX512BW-NEXT: vpmovqb %zmm1, %xmm1
3412 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
3413 ; AVX512BW-NEXT: vmovdqa %xmm4, (%rsi)
3414 ; AVX512BW-NEXT: vmovdqa %xmm6, (%rdx)
3415 ; AVX512BW-NEXT: vmovdqa %xmm7, (%rcx)
3416 ; AVX512BW-NEXT: vmovdqa %xmm8, (%r8)
3417 ; AVX512BW-NEXT: vmovdqa %xmm9, (%r9)
3418 ; AVX512BW-NEXT: vmovdqa %xmm10, (%r11)
3419 ; AVX512BW-NEXT: vmovdqa %xmm11, (%r10)
3420 ; AVX512BW-NEXT: vmovdqa %xmm0, (%rax)
3421 ; AVX512BW-NEXT: vzeroupper
3422 ; AVX512BW-NEXT: retq
3423 ;
3424 ; AVX512BW-FCP-LABEL: load_i8_stride8_vf16:
3425 ; AVX512BW-FCP: # %bb.0:
3426 ; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
3427 ; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
3428 ; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11
3429 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
3430 ; AVX512BW-FCP-NEXT: vmovdqa 112(%rdi), %xmm0
3431 ; AVX512BW-FCP-NEXT: vpshufb %xmm2, %xmm0, %xmm3
3432 ; AVX512BW-FCP-NEXT: vmovdqa 96(%rdi), %xmm1
3433 ; AVX512BW-FCP-NEXT: vpshufb %xmm2, %xmm1, %xmm2
3434 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
3435 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm5 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
3436 ; AVX512BW-FCP-NEXT: vmovdqa 80(%rdi), %xmm2
3437 ; AVX512BW-FCP-NEXT: vpshufb %xmm5, %xmm2, %xmm6
3438 ; AVX512BW-FCP-NEXT: vmovdqa 64(%rdi), %xmm3
3439 ; AVX512BW-FCP-NEXT: vpshufb %xmm5, %xmm3, %xmm5
3440 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
3441 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
3442 ; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm5
3443 ; AVX512BW-FCP-NEXT: vpmovqb %zmm5, %xmm6
3444 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3]
3445 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm6 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
3446 ; AVX512BW-FCP-NEXT: vpshufb %xmm6, %xmm0, %xmm7
3447 ; AVX512BW-FCP-NEXT: vpshufb %xmm6, %xmm1, %xmm6
3448 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
3449 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm7 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
3450 ; AVX512BW-FCP-NEXT: vpshufb %xmm7, %xmm2, %xmm8
3451 ; AVX512BW-FCP-NEXT: vpshufb %xmm7, %xmm3, %xmm7
3452 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
3453 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1,2],xmm6[3]
3454 ; AVX512BW-FCP-NEXT: vpsrlq $8, %zmm5, %zmm7
3455 ; AVX512BW-FCP-NEXT: vpmovqb %zmm7, %xmm7
3456 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3]
3457 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm7 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
3458 ; AVX512BW-FCP-NEXT: vpshufb %xmm7, %xmm0, %xmm8
3459 ; AVX512BW-FCP-NEXT: vpshufb %xmm7, %xmm1, %xmm7
3460 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
3461 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm8 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
3462 ; AVX512BW-FCP-NEXT: vpshufb %xmm8, %xmm2, %xmm9
3463 ; AVX512BW-FCP-NEXT: vpshufb %xmm8, %xmm3, %xmm8
3464 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
3465 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3]
3466 ; AVX512BW-FCP-NEXT: vpsrlq $16, %zmm5, %zmm8
3467 ; AVX512BW-FCP-NEXT: vpmovqb %zmm8, %xmm8
3468 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3]
3469 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm8 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
3470 ; AVX512BW-FCP-NEXT: vpshufb %xmm8, %xmm0, %xmm9
3471 ; AVX512BW-FCP-NEXT: vpshufb %xmm8, %xmm1, %xmm8
3472 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
3473 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm9 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
3474 ; AVX512BW-FCP-NEXT: vpshufb %xmm9, %xmm2, %xmm10
3475 ; AVX512BW-FCP-NEXT: vpshufb %xmm9, %xmm3, %xmm9
3476 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
3477 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
3478 ; AVX512BW-FCP-NEXT: vpsrlq $24, %zmm5, %zmm9
3479 ; AVX512BW-FCP-NEXT: vpmovqb %zmm9, %xmm9
3480 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
3481 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm9 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
3482 ; AVX512BW-FCP-NEXT: vpshufb %xmm9, %xmm0, %xmm10
3483 ; AVX512BW-FCP-NEXT: vpshufb %xmm9, %xmm1, %xmm9
3484 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
3485 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
3486 ; AVX512BW-FCP-NEXT: vpshufb %xmm10, %xmm2, %xmm11
3487 ; AVX512BW-FCP-NEXT: vpshufb %xmm10, %xmm3, %xmm10
3488 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
3489 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
3490 ; AVX512BW-FCP-NEXT: vpsrlq $32, %zmm5, %zmm10
3491 ; AVX512BW-FCP-NEXT: vpmovqb %zmm10, %xmm10
3492 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
3493 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
3494 ; AVX512BW-FCP-NEXT: vpshufb %xmm10, %xmm0, %xmm11
3495 ; AVX512BW-FCP-NEXT: vpshufb %xmm10, %xmm1, %xmm10
3496 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
3497 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
3498 ; AVX512BW-FCP-NEXT: vpshufb %xmm11, %xmm2, %xmm12
3499 ; AVX512BW-FCP-NEXT: vpshufb %xmm11, %xmm3, %xmm11
3500 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
3501 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1,2],xmm10[3]
3502 ; AVX512BW-FCP-NEXT: vpsrlq $40, %zmm5, %zmm11
3503 ; AVX512BW-FCP-NEXT: vpmovqb %zmm11, %xmm11
3504 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3]
3505 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
3506 ; AVX512BW-FCP-NEXT: vpshufb %xmm11, %xmm0, %xmm12
3507 ; AVX512BW-FCP-NEXT: vpshufb %xmm11, %xmm1, %xmm11
3508 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
3509 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm12 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
3510 ; AVX512BW-FCP-NEXT: vpshufb %xmm12, %xmm2, %xmm13
3511 ; AVX512BW-FCP-NEXT: vpshufb %xmm12, %xmm3, %xmm12
3512 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
3513 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1,2],xmm11[3]
3514 ; AVX512BW-FCP-NEXT: vpsrlq $48, %zmm5, %zmm12
3515 ; AVX512BW-FCP-NEXT: vpmovqb %zmm12, %xmm12
3516 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1],xmm11[2,3]
3517 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm12 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
3518 ; AVX512BW-FCP-NEXT: vpshufb %xmm12, %xmm0, %xmm0
3519 ; AVX512BW-FCP-NEXT: vpshufb %xmm12, %xmm1, %xmm1
3520 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
3521 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
3522 ; AVX512BW-FCP-NEXT: vpshufb %xmm1, %xmm2, %xmm2
3523 ; AVX512BW-FCP-NEXT: vpshufb %xmm1, %xmm3, %xmm1
3524 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
3525 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
3526 ; AVX512BW-FCP-NEXT: vpsrlq $56, %zmm5, %zmm1
3527 ; AVX512BW-FCP-NEXT: vpmovqb %zmm1, %xmm1
3528 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
3529 ; AVX512BW-FCP-NEXT: vmovdqa %xmm4, (%rsi)
3530 ; AVX512BW-FCP-NEXT: vmovdqa %xmm6, (%rdx)
3531 ; AVX512BW-FCP-NEXT: vmovdqa %xmm7, (%rcx)
3532 ; AVX512BW-FCP-NEXT: vmovdqa %xmm8, (%r8)
3533 ; AVX512BW-FCP-NEXT: vmovdqa %xmm9, (%r9)
3534 ; AVX512BW-FCP-NEXT: vmovdqa %xmm10, (%r11)
3535 ; AVX512BW-FCP-NEXT: vmovdqa %xmm11, (%r10)
3536 ; AVX512BW-FCP-NEXT: vmovdqa %xmm0, (%rax)
3537 ; AVX512BW-FCP-NEXT: vzeroupper
3538 ; AVX512BW-FCP-NEXT: retq
3539 ;
3540 ; AVX512DQ-BW-LABEL: load_i8_stride8_vf16:
3541 ; AVX512DQ-BW: # %bb.0:
3542 ; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
3543 ; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %r10
3544 ; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %r11
3545 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm2 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
3546 ; AVX512DQ-BW-NEXT: vmovdqa 112(%rdi), %xmm0
3547 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm0, %xmm3
3548 ; AVX512DQ-BW-NEXT: vmovdqa 96(%rdi), %xmm1
3549 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm1, %xmm2
3550 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
3551 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm5 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
3552 ; AVX512DQ-BW-NEXT: vmovdqa 80(%rdi), %xmm2
3553 ; AVX512DQ-BW-NEXT: vpshufb %xmm5, %xmm2, %xmm6
3554 ; AVX512DQ-BW-NEXT: vmovdqa 64(%rdi), %xmm3
3555 ; AVX512DQ-BW-NEXT: vpshufb %xmm5, %xmm3, %xmm5
3556 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
3557 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
3558 ; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm5
3559 ; AVX512DQ-BW-NEXT: vpmovqb %zmm5, %xmm6
3560 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3]
3561 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm6 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
3562 ; AVX512DQ-BW-NEXT: vpshufb %xmm6, %xmm0, %xmm7
3563 ; AVX512DQ-BW-NEXT: vpshufb %xmm6, %xmm1, %xmm6
3564 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
3565 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm7 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
3566 ; AVX512DQ-BW-NEXT: vpshufb %xmm7, %xmm2, %xmm8
3567 ; AVX512DQ-BW-NEXT: vpshufb %xmm7, %xmm3, %xmm7
3568 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
3569 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1,2],xmm6[3]
3570 ; AVX512DQ-BW-NEXT: vpsrlq $8, %zmm5, %zmm7
3571 ; AVX512DQ-BW-NEXT: vpmovqb %zmm7, %xmm7
3572 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3]
3573 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm7 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
3574 ; AVX512DQ-BW-NEXT: vpshufb %xmm7, %xmm0, %xmm8
3575 ; AVX512DQ-BW-NEXT: vpshufb %xmm7, %xmm1, %xmm7
3576 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
3577 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm8 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
3578 ; AVX512DQ-BW-NEXT: vpshufb %xmm8, %xmm2, %xmm9
3579 ; AVX512DQ-BW-NEXT: vpshufb %xmm8, %xmm3, %xmm8
3580 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
3581 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3]
3582 ; AVX512DQ-BW-NEXT: vpsrlq $16, %zmm5, %zmm8
3583 ; AVX512DQ-BW-NEXT: vpmovqb %zmm8, %xmm8
3584 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3]
3585 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm8 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
3586 ; AVX512DQ-BW-NEXT: vpshufb %xmm8, %xmm0, %xmm9
3587 ; AVX512DQ-BW-NEXT: vpshufb %xmm8, %xmm1, %xmm8
3588 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
3589 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm9 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
3590 ; AVX512DQ-BW-NEXT: vpshufb %xmm9, %xmm2, %xmm10
3591 ; AVX512DQ-BW-NEXT: vpshufb %xmm9, %xmm3, %xmm9
3592 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
3593 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
3594 ; AVX512DQ-BW-NEXT: vpsrlq $24, %zmm5, %zmm9
3595 ; AVX512DQ-BW-NEXT: vpmovqb %zmm9, %xmm9
3596 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
3597 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm9 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
3598 ; AVX512DQ-BW-NEXT: vpshufb %xmm9, %xmm0, %xmm10
3599 ; AVX512DQ-BW-NEXT: vpshufb %xmm9, %xmm1, %xmm9
3600 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
3601 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm10 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
3602 ; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm2, %xmm11
3603 ; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm3, %xmm10
3604 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
3605 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
3606 ; AVX512DQ-BW-NEXT: vpsrlq $32, %zmm5, %zmm10
3607 ; AVX512DQ-BW-NEXT: vpmovqb %zmm10, %xmm10
3608 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
3609 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm10 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
3610 ; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm0, %xmm11
3611 ; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm1, %xmm10
3612 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
3613 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm11 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
3614 ; AVX512DQ-BW-NEXT: vpshufb %xmm11, %xmm2, %xmm12
3615 ; AVX512DQ-BW-NEXT: vpshufb %xmm11, %xmm3, %xmm11
3616 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
3617 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1,2],xmm10[3]
3618 ; AVX512DQ-BW-NEXT: vpsrlq $40, %zmm5, %zmm11
3619 ; AVX512DQ-BW-NEXT: vpmovqb %zmm11, %xmm11
3620 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3]
3621 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm11 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
3622 ; AVX512DQ-BW-NEXT: vpshufb %xmm11, %xmm0, %xmm12
3623 ; AVX512DQ-BW-NEXT: vpshufb %xmm11, %xmm1, %xmm11
3624 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
3625 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm12 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
3626 ; AVX512DQ-BW-NEXT: vpshufb %xmm12, %xmm2, %xmm13
3627 ; AVX512DQ-BW-NEXT: vpshufb %xmm12, %xmm3, %xmm12
3628 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
3629 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1,2],xmm11[3]
3630 ; AVX512DQ-BW-NEXT: vpsrlq $48, %zmm5, %zmm12
3631 ; AVX512DQ-BW-NEXT: vpmovqb %zmm12, %xmm12
3632 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1],xmm11[2,3]
3633 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm12 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
3634 ; AVX512DQ-BW-NEXT: vpshufb %xmm12, %xmm0, %xmm0
3635 ; AVX512DQ-BW-NEXT: vpshufb %xmm12, %xmm1, %xmm1
3636 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
3637 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm1 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
3638 ; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm2, %xmm2
3639 ; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm3, %xmm1
3640 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
3641 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
3642 ; AVX512DQ-BW-NEXT: vpsrlq $56, %zmm5, %zmm1
3643 ; AVX512DQ-BW-NEXT: vpmovqb %zmm1, %xmm1
3644 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
3645 ; AVX512DQ-BW-NEXT: vmovdqa %xmm4, (%rsi)
3646 ; AVX512DQ-BW-NEXT: vmovdqa %xmm6, (%rdx)
3647 ; AVX512DQ-BW-NEXT: vmovdqa %xmm7, (%rcx)
3648 ; AVX512DQ-BW-NEXT: vmovdqa %xmm8, (%r8)
3649 ; AVX512DQ-BW-NEXT: vmovdqa %xmm9, (%r9)
3650 ; AVX512DQ-BW-NEXT: vmovdqa %xmm10, (%r11)
3651 ; AVX512DQ-BW-NEXT: vmovdqa %xmm11, (%r10)
3652 ; AVX512DQ-BW-NEXT: vmovdqa %xmm0, (%rax)
3653 ; AVX512DQ-BW-NEXT: vzeroupper
3654 ; AVX512DQ-BW-NEXT: retq
3655 ;
3656 ; AVX512DQ-BW-FCP-LABEL: load_i8_stride8_vf16:
3657 ; AVX512DQ-BW-FCP: # %bb.0:
3658 ; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
3659 ; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
3660 ; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11
3661 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
3662 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 112(%rdi), %xmm0
3663 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm2, %xmm0, %xmm3
3664 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 96(%rdi), %xmm1
3665 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm2, %xmm1, %xmm2
3666 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
3667 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm5 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
3668 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 80(%rdi), %xmm2
3669 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm5, %xmm2, %xmm6
3670 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 64(%rdi), %xmm3
3671 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm5, %xmm3, %xmm5
3672 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
3673 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
3674 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm5
3675 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm5, %xmm6
3676 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3]
3677 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm6 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
3678 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm6, %xmm0, %xmm7
3679 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm6, %xmm1, %xmm6
3680 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
3681 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm7 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
3682 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm7, %xmm2, %xmm8
3683 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm7, %xmm3, %xmm7
3684 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
3685 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1,2],xmm6[3]
3686 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $8, %zmm5, %zmm7
3687 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm7, %xmm7
3688 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3]
3689 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm7 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
3690 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm7, %xmm0, %xmm8
3691 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm7, %xmm1, %xmm7
3692 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
3693 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm8 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
3694 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm8, %xmm2, %xmm9
3695 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm8, %xmm3, %xmm8
3696 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
3697 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3]
3698 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $16, %zmm5, %zmm8
3699 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm8, %xmm8
3700 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3]
3701 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm8 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
3702 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm8, %xmm0, %xmm9
3703 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm8, %xmm1, %xmm8
3704 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
3705 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm9 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
3706 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm9, %xmm2, %xmm10
3707 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm9, %xmm3, %xmm9
3708 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
3709 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
3710 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $24, %zmm5, %zmm9
3711 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm9, %xmm9
3712 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
3713 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm9 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
3714 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm9, %xmm0, %xmm10
3715 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm9, %xmm1, %xmm9
3716 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
3717 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
3718 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm10, %xmm2, %xmm11
3719 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm10, %xmm3, %xmm10
3720 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
3721 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
3722 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $32, %zmm5, %zmm10
3723 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm10, %xmm10
3724 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
3725 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
3726 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm10, %xmm0, %xmm11
3727 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm10, %xmm1, %xmm10
3728 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
3729 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
3730 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm11, %xmm2, %xmm12
3731 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm11, %xmm3, %xmm11
3732 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
3733 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1,2],xmm10[3]
3734 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $40, %zmm5, %zmm11
3735 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm11, %xmm11
3736 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3]
3737 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
3738 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm11, %xmm0, %xmm12
3739 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm11, %xmm1, %xmm11
3740 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
3741 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm12 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
3742 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm12, %xmm2, %xmm13
3743 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm12, %xmm3, %xmm12
3744 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
3745 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1,2],xmm11[3]
3746 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $48, %zmm5, %zmm12
3747 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm12, %xmm12
3748 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1],xmm11[2,3]
3749 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm12 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
3750 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm12, %xmm0, %xmm0
3751 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm12, %xmm1, %xmm1
3752 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
3753 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
3754 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm1, %xmm2, %xmm2
3755 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm1, %xmm3, %xmm1
3756 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
3757 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
3758 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $56, %zmm5, %zmm1
3759 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm1, %xmm1
3760 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
3761 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm4, (%rsi)
3762 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm6, (%rdx)
3763 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm7, (%rcx)
3764 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm8, (%r8)
3765 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm9, (%r9)
3766 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm10, (%r11)
3767 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm11, (%r10)
3768 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm0, (%rax)
3769 ; AVX512DQ-BW-FCP-NEXT: vzeroupper
3770 ; AVX512DQ-BW-FCP-NEXT: retq
3771 %wide.vec = load <128 x i8>, ptr %in.vec, align 64
3772 %strided.vec0 = shufflevector <128 x i8> %wide.vec, <128 x i8> poison, <16 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 40, i32 48, i32 56, i32 64, i32 72, i32 80, i32 88, i32 96, i32 104, i32 112, i32 120>
3773 %strided.vec1 = shufflevector <128 x i8> %wide.vec, <128 x i8> poison, <16 x i32> <i32 1, i32 9, i32 17, i32 25, i32 33, i32 41, i32 49, i32 57, i32 65, i32 73, i32 81, i32 89, i32 97, i32 105, i32 113, i32 121>
3774 %strided.vec2 = shufflevector <128 x i8> %wide.vec, <128 x i8> poison, <16 x i32> <i32 2, i32 10, i32 18, i32 26, i32 34, i32 42, i32 50, i32 58, i32 66, i32 74, i32 82, i32 90, i32 98, i32 106, i32 114, i32 122>
3775 %strided.vec3 = shufflevector <128 x i8> %wide.vec, <128 x i8> poison, <16 x i32> <i32 3, i32 11, i32 19, i32 27, i32 35, i32 43, i32 51, i32 59, i32 67, i32 75, i32 83, i32 91, i32 99, i32 107, i32 115, i32 123>
3776 %strided.vec4 = shufflevector <128 x i8> %wide.vec, <128 x i8> poison, <16 x i32> <i32 4, i32 12, i32 20, i32 28, i32 36, i32 44, i32 52, i32 60, i32 68, i32 76, i32 84, i32 92, i32 100, i32 108, i32 116, i32 124>
3777 %strided.vec5 = shufflevector <128 x i8> %wide.vec, <128 x i8> poison, <16 x i32> <i32 5, i32 13, i32 21, i32 29, i32 37, i32 45, i32 53, i32 61, i32 69, i32 77, i32 85, i32 93, i32 101, i32 109, i32 117, i32 125>
3778 %strided.vec6 = shufflevector <128 x i8> %wide.vec, <128 x i8> poison, <16 x i32> <i32 6, i32 14, i32 22, i32 30, i32 38, i32 46, i32 54, i32 62, i32 70, i32 78, i32 86, i32 94, i32 102, i32 110, i32 118, i32 126>
3779 %strided.vec7 = shufflevector <128 x i8> %wide.vec, <128 x i8> poison, <16 x i32> <i32 7, i32 15, i32 23, i32 31, i32 39, i32 47, i32 55, i32 63, i32 71, i32 79, i32 87, i32 95, i32 103, i32 111, i32 119, i32 127>
3780 store <16 x i8> %strided.vec0, ptr %out.vec0, align 64
3781 store <16 x i8> %strided.vec1, ptr %out.vec1, align 64
3782 store <16 x i8> %strided.vec2, ptr %out.vec2, align 64
3783 store <16 x i8> %strided.vec3, ptr %out.vec3, align 64
3784 store <16 x i8> %strided.vec4, ptr %out.vec4, align 64
3785 store <16 x i8> %strided.vec5, ptr %out.vec5, align 64
3786 store <16 x i8> %strided.vec6, ptr %out.vec6, align 64
3787 store <16 x i8> %strided.vec7, ptr %out.vec7, align 64
3788 ret void
3789 }
3791 define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4, ptr %out.vec5, ptr %out.vec6, ptr %out.vec7) nounwind {
3792 ; SSE-LABEL: load_i8_stride8_vf32:
3793 ; SSE: # %bb.0:
3794 ; SSE-NEXT: subq $904, %rsp # imm = 0x388
3795 ; SSE-NEXT: movdqa 64(%rdi), %xmm5
3796 ; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3797 ; SSE-NEXT: movdqa 80(%rdi), %xmm10
3798 ; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3799 ; SSE-NEXT: movdqa 96(%rdi), %xmm12
3800 ; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3801 ; SSE-NEXT: movdqa 128(%rdi), %xmm6
3802 ; SSE-NEXT: movdqa 144(%rdi), %xmm13
3803 ; SSE-NEXT: movdqa 160(%rdi), %xmm11
3804 ; SSE-NEXT: movdqa 176(%rdi), %xmm14
3805 ; SSE-NEXT: movdqa 192(%rdi), %xmm2
3806 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3807 ; SSE-NEXT: movdqa 208(%rdi), %xmm7
3808 ; SSE-NEXT: movdqa 224(%rdi), %xmm8
3809 ; SSE-NEXT: movdqa 240(%rdi), %xmm9
3810 ; SSE-NEXT: movdqa {{.*#+}} xmm4 = [255,0,0,0,255,0,0,0]
3811 ; SSE-NEXT: movdqa %xmm9, %xmm0
3812 ; SSE-NEXT: pand %xmm4, %xmm0
3813 ; SSE-NEXT: movdqa %xmm8, %xmm1
3814 ; SSE-NEXT: pand %xmm4, %xmm1
3815 ; SSE-NEXT: packuswb %xmm0, %xmm1
3816 ; SSE-NEXT: packuswb %xmm1, %xmm0
3817 ; SSE-NEXT: movdqa %xmm7, %xmm1
3818 ; SSE-NEXT: pand %xmm4, %xmm1
3819 ; SSE-NEXT: pand %xmm4, %xmm2
3820 ; SSE-NEXT: packuswb %xmm1, %xmm2
3821 ; SSE-NEXT: packuswb %xmm2, %xmm2
3822 ; SSE-NEXT: packuswb %xmm0, %xmm2
3823 ; SSE-NEXT: movdqa %xmm14, %xmm0
3824 ; SSE-NEXT: pand %xmm4, %xmm0
3825 ; SSE-NEXT: movdqa %xmm11, %xmm1
3826 ; SSE-NEXT: pand %xmm4, %xmm1
3827 ; SSE-NEXT: packuswb %xmm0, %xmm1
3828 ; SSE-NEXT: movdqa %xmm13, %xmm0
3829 ; SSE-NEXT: pand %xmm4, %xmm0
3830 ; SSE-NEXT: movdqa %xmm6, %xmm3
3831 ; SSE-NEXT: pand %xmm4, %xmm3
3832 ; SSE-NEXT: packuswb %xmm0, %xmm3
3833 ; SSE-NEXT: movdqa 112(%rdi), %xmm15
3834 ; SSE-NEXT: movdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3835 ; SSE-NEXT: packuswb %xmm1, %xmm0
3836 ; SSE-NEXT: packuswb %xmm3, %xmm3
3837 ; SSE-NEXT: packuswb %xmm0, %xmm3
3838 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,3],xmm2[0,3]
3839 ; SSE-NEXT: movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3840 ; SSE-NEXT: movdqa %xmm15, %xmm0
3841 ; SSE-NEXT: pand %xmm4, %xmm0
3842 ; SSE-NEXT: movdqa %xmm12, %xmm1
3843 ; SSE-NEXT: pand %xmm4, %xmm1
3844 ; SSE-NEXT: packuswb %xmm0, %xmm1
3845 ; SSE-NEXT: movdqa %xmm10, %xmm0
3846 ; SSE-NEXT: pand %xmm4, %xmm0
3847 ; SSE-NEXT: movdqa %xmm5, %xmm2
3848 ; SSE-NEXT: pand %xmm4, %xmm2
3849 ; SSE-NEXT: packuswb %xmm0, %xmm2
3850 ; SSE-NEXT: packuswb %xmm1, %xmm0
3851 ; SSE-NEXT: packuswb %xmm2, %xmm2
3852 ; SSE-NEXT: packuswb %xmm0, %xmm2
3853 ; SSE-NEXT: movdqa 48(%rdi), %xmm15
3854 ; SSE-NEXT: movdqa %xmm15, %xmm0
3855 ; SSE-NEXT: pand %xmm4, %xmm0
3856 ; SSE-NEXT: movdqa 32(%rdi), %xmm1
3857 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3858 ; SSE-NEXT: pand %xmm4, %xmm1
3859 ; SSE-NEXT: packuswb %xmm0, %xmm1
3860 ; SSE-NEXT: movdqa 16(%rdi), %xmm10
3861 ; SSE-NEXT: movdqa %xmm10, %xmm0
3862 ; SSE-NEXT: pand %xmm4, %xmm0
3863 ; SSE-NEXT: movdqa (%rdi), %xmm12
3864 ; SSE-NEXT: pand %xmm12, %xmm4
3865 ; SSE-NEXT: packuswb %xmm0, %xmm4
3866 ; SSE-NEXT: packuswb %xmm1, %xmm0
3867 ; SSE-NEXT: packuswb %xmm4, %xmm4
3868 ; SSE-NEXT: packuswb %xmm0, %xmm4
3869 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,3],xmm2[0,3]
3870 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3871 ; SSE-NEXT: pxor %xmm5, %xmm5
3872 ; SSE-NEXT: movdqa %xmm13, %xmm0
3873 ; SSE-NEXT: movdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3874 ; SSE-NEXT: movdqa %xmm13, %xmm1
3875 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm5[8],xmm1[9],xmm5[9],xmm1[10],xmm5[10],xmm1[11],xmm5[11],xmm1[12],xmm5[12],xmm1[13],xmm5[13],xmm1[14],xmm5[14],xmm1[15],xmm5[15]
3876 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3877 ; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3],xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7]
3878 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3879 ; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
3880 ; SSE-NEXT: packuswb %xmm0, %xmm0
3881 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3882 ; SSE-NEXT: movdqa {{.*#+}} xmm13 = [65535,0,65535,65535,65535,65535,65535,65535]
3883 ; SSE-NEXT: movdqa %xmm13, %xmm1
3884 ; SSE-NEXT: pandn %xmm0, %xmm1
3885 ; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3886 ; SSE-NEXT: movdqa %xmm6, %xmm2
3887 ; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15]
3888 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3889 ; SSE-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
3890 ; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3891 ; SSE-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
3892 ; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3893 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,1,1]
3894 ; SSE-NEXT: packuswb %xmm0, %xmm0
3895 ; SSE-NEXT: pand %xmm13, %xmm0
3896 ; SSE-NEXT: por %xmm1, %xmm0
3897 ; SSE-NEXT: movdqa %xmm14, %xmm3
3898 ; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3899 ; SSE-NEXT: movdqa %xmm14, %xmm1
3900 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm5[8],xmm1[9],xmm5[9],xmm1[10],xmm5[10],xmm1[11],xmm5[11],xmm1[12],xmm5[12],xmm1[13],xmm5[13],xmm1[14],xmm5[14],xmm1[15],xmm5[15]
3901 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3902 ; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
3903 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3904 ; SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
3905 ; SSE-NEXT: packuswb %xmm3, %xmm3
3906 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3907 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,0,2,3]
3908 ; SSE-NEXT: movdqa {{.*#+}} xmm14 = [65535,65535,65535,0,65535,65535,65535,65535]
3909 ; SSE-NEXT: movdqa %xmm14, %xmm4
3910 ; SSE-NEXT: pandn %xmm1, %xmm4
3911 ; SSE-NEXT: movdqa %xmm11, %xmm2
3912 ; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3913 ; SSE-NEXT: movdqa %xmm11, %xmm1
3914 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm5[8],xmm1[9],xmm5[9],xmm1[10],xmm5[10],xmm1[11],xmm5[11],xmm1[12],xmm5[12],xmm1[13],xmm5[13],xmm1[14],xmm5[14],xmm1[15],xmm5[15]
3915 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3916 ; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7]
3917 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3918 ; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
3919 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3920 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,1,1,3]
3921 ; SSE-NEXT: packuswb %xmm1, %xmm1
3922 ; SSE-NEXT: pand %xmm14, %xmm1
3923 ; SSE-NEXT: por %xmm4, %xmm1
3924 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
3925 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
3926 ; SSE-NEXT: movdqa %xmm9, %xmm1
3927 ; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3928 ; SSE-NEXT: movdqa %xmm9, %xmm2
3929 ; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15]
3930 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3931 ; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3],xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7]
3932 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3933 ; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
3934 ; SSE-NEXT: packuswb %xmm1, %xmm1
3935 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3936 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,2]
3937 ; SSE-NEXT: movdqa {{.*#+}} xmm11 = [65535,65535,65535,65535,65535,65535,65535,0]
3938 ; SSE-NEXT: movdqa %xmm11, %xmm4
3939 ; SSE-NEXT: pandn %xmm1, %xmm4
3940 ; SSE-NEXT: movdqa %xmm8, %xmm2
3941 ; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3942 ; SSE-NEXT: movdqa %xmm8, %xmm1
3943 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm5[8],xmm1[9],xmm5[9],xmm1[10],xmm5[10],xmm1[11],xmm5[11],xmm1[12],xmm5[12],xmm1[13],xmm5[13],xmm1[14],xmm5[14],xmm1[15],xmm5[15]
3944 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3945 ; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7]
3946 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3947 ; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
3948 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3949 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,1,1,3]
3950 ; SSE-NEXT: packuswb %xmm1, %xmm1
3951 ; SSE-NEXT: pand %xmm11, %xmm1
3952 ; SSE-NEXT: por %xmm4, %xmm1
3953 ; SSE-NEXT: movdqa %xmm7, %xmm4
3954 ; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3955 ; SSE-NEXT: movdqa %xmm7, %xmm2
3956 ; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15]
3957 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3958 ; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3],xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]
3959 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3960 ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
3961 ; SSE-NEXT: packuswb %xmm4, %xmm2
3962 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3963 ; SSE-NEXT: movdqa {{.*#+}} xmm9 = [65535,65535,65535,65535,65535,0,65535,65535]
3964 ; SSE-NEXT: movdqa %xmm9, %xmm4
3965 ; SSE-NEXT: pandn %xmm2, %xmm4
3966 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
3967 ; SSE-NEXT: movdqa %xmm7, %xmm2
3968 ; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15]
3969 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3970 ; SSE-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3],xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]
3971 ; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3972 ; SSE-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3]
3973 ; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3974 ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm7[1,1,1,1]
3975 ; SSE-NEXT: packuswb %xmm8, %xmm8
3976 ; SSE-NEXT: pand %xmm9, %xmm8
3977 ; SSE-NEXT: por %xmm4, %xmm8
3978 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm8[2,2,2,2]
3979 ; SSE-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm1[2],xmm4[3],xmm1[3]
3980 ; SSE-NEXT: movsd {{.*#+}} xmm4 = xmm0[0],xmm4[1]
3981 ; SSE-NEXT: movapd %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3982 ; SSE-NEXT: movdqa %xmm10, (%rsp) # 16-byte Spill
3983 ; SSE-NEXT: movdqa %xmm10, %xmm0
3984 ; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm5[8],xmm0[9],xmm5[9],xmm0[10],xmm5[10],xmm0[11],xmm5[11],xmm0[12],xmm5[12],xmm0[13],xmm5[13],xmm0[14],xmm5[14],xmm0[15],xmm5[15]
3985 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3986 ; SSE-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3],xmm10[4],xmm5[4],xmm10[5],xmm5[5],xmm10[6],xmm5[6],xmm10[7],xmm5[7]
3987 ; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3988 ; SSE-NEXT: movdqa %xmm10, %xmm8
3989 ; SSE-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3]
3990 ; SSE-NEXT: packuswb %xmm8, %xmm8
3991 ; SSE-NEXT: movdqa %xmm13, %xmm1
3992 ; SSE-NEXT: pandn %xmm8, %xmm1
3993 ; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3994 ; SSE-NEXT: movdqa %xmm12, %xmm2
3995 ; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15]
3996 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3997 ; SSE-NEXT: punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm5[0],xmm12[1],xmm5[1],xmm12[2],xmm5[2],xmm12[3],xmm5[3],xmm12[4],xmm5[4],xmm12[5],xmm5[5],xmm12[6],xmm5[6],xmm12[7],xmm5[7]
3998 ; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3999 ; SSE-NEXT: punpcklwd {{.*#+}} xmm12 = xmm12[0],xmm2[0],xmm12[1],xmm2[1],xmm12[2],xmm2[2],xmm12[3],xmm2[3]
4000 ; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4001 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm12[1,1,1,1]
4002 ; SSE-NEXT: packuswb %xmm6, %xmm6
4003 ; SSE-NEXT: pand %xmm13, %xmm6
4004 ; SSE-NEXT: por %xmm1, %xmm6
4005 ; SSE-NEXT: movdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4006 ; SSE-NEXT: movdqa %xmm15, %xmm1
4007 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm5[8],xmm1[9],xmm5[9],xmm1[10],xmm5[10],xmm1[11],xmm5[11],xmm1[12],xmm5[12],xmm1[13],xmm5[13],xmm1[14],xmm5[14],xmm1[15],xmm5[15]
4008 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4009 ; SSE-NEXT: punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm5[0],xmm15[1],xmm5[1],xmm15[2],xmm5[2],xmm15[3],xmm5[3],xmm15[4],xmm5[4],xmm15[5],xmm5[5],xmm15[6],xmm5[6],xmm15[7],xmm5[7]
4010 ; SSE-NEXT: movdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4011 ; SSE-NEXT: movdqa %xmm15, %xmm4
4012 ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
4013 ; SSE-NEXT: packuswb %xmm4, %xmm4
4014 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm4[0,0,2,3]
4015 ; SSE-NEXT: movdqa %xmm14, %xmm12
4016 ; SSE-NEXT: pandn %xmm1, %xmm12
4017 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
4018 ; SSE-NEXT: movdqa %xmm2, %xmm1
4019 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm5[8],xmm1[9],xmm5[9],xmm1[10],xmm5[10],xmm1[11],xmm5[11],xmm1[12],xmm5[12],xmm1[13],xmm5[13],xmm1[14],xmm5[14],xmm1[15],xmm5[15]
4020 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4021 ; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7]
4022 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4023 ; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
4024 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4025 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,1,1,3]
4026 ; SSE-NEXT: packuswb %xmm1, %xmm1
4027 ; SSE-NEXT: pand %xmm14, %xmm1
4028 ; SSE-NEXT: por %xmm12, %xmm1
4029 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
4030 ; SSE-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]
4031 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
4032 ; SSE-NEXT: movdqa %xmm1, %xmm2
4033 ; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15]
4034 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4035 ; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3],xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7]
4036 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4037 ; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
4038 ; SSE-NEXT: packuswb %xmm1, %xmm0
4039 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4040 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,1,2,2]
4041 ; SSE-NEXT: movdqa %xmm11, %xmm7
4042 ; SSE-NEXT: pandn %xmm1, %xmm7
4043 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
4044 ; SSE-NEXT: movdqa %xmm12, %xmm1
4045 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm5[8],xmm1[9],xmm5[9],xmm1[10],xmm5[10],xmm1[11],xmm5[11],xmm1[12],xmm5[12],xmm1[13],xmm5[13],xmm1[14],xmm5[14],xmm1[15],xmm5[15]
4046 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4047 ; SSE-NEXT: punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm5[0],xmm12[1],xmm5[1],xmm12[2],xmm5[2],xmm12[3],xmm5[3],xmm12[4],xmm5[4],xmm12[5],xmm5[5],xmm12[6],xmm5[6],xmm12[7],xmm5[7]
4048 ; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4049 ; SSE-NEXT: punpcklwd {{.*#+}} xmm12 = xmm12[0],xmm1[0],xmm12[1],xmm1[1],xmm12[2],xmm1[2],xmm12[3],xmm1[3]
4050 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm12[0,1,1,3]
4051 ; SSE-NEXT: packuswb %xmm1, %xmm3
4052 ; SSE-NEXT: pand %xmm11, %xmm3
4053 ; SSE-NEXT: por %xmm7, %xmm3
4054 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
4055 ; SSE-NEXT: movdqa %xmm0, %xmm2
4056 ; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15]
4057 ; SSE-NEXT: movdqa %xmm2, %xmm7
4058 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4059 ; SSE-NEXT: movdqa %xmm0, %xmm2
4060 ; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7]
4061 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4062 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
4063 ; SSE-NEXT: movdqa %xmm0, %xmm1
4064 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm5[8],xmm1[9],xmm5[9],xmm1[10],xmm5[10],xmm1[11],xmm5[11],xmm1[12],xmm5[12],xmm1[13],xmm5[13],xmm1[14],xmm5[14],xmm1[15],xmm5[15]
4065 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4066 ; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3],xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7]
4067 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4068 ; SSE-NEXT: movdqa %xmm2, %xmm5
4069 ; SSE-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3]
4070 ; SSE-NEXT: packuswb %xmm5, %xmm2
4071 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4072 ; SSE-NEXT: movdqa %xmm9, %xmm7
4073 ; SSE-NEXT: pandn %xmm2, %xmm7
4074 ; SSE-NEXT: movdqa %xmm0, %xmm5
4075 ; SSE-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
4076 ; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm5[1,1,1,1]
4077 ; SSE-NEXT: packuswb %xmm15, %xmm15
4078 ; SSE-NEXT: pand %xmm9, %xmm15
4079 ; SSE-NEXT: por %xmm7, %xmm15
4080 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm15[2,2,2,2]
4081 ; SSE-NEXT: punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm3[2],xmm7[3],xmm3[3]
4082 ; SSE-NEXT: movsd {{.*#+}} xmm7 = xmm6[0],xmm7[1]
4083 ; SSE-NEXT: movapd %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4084 ; SSE-NEXT: movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
4085 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
4086 ; SSE-NEXT: pand %xmm2, %xmm10
4087 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm10[0,2,2,3]
4088 ; SSE-NEXT: movdqa %xmm10, %xmm3
4089 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]
4090 ; SSE-NEXT: packuswb %xmm0, %xmm0
4091 ; SSE-NEXT: movdqa %xmm13, %xmm7
4092 ; SSE-NEXT: pandn %xmm0, %xmm7
4093 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
4094 ; SSE-NEXT: pand %xmm2, %xmm10
4095 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm10[0,2,2,3]
4096 ; SSE-NEXT: movdqa %xmm10, %xmm6
4097 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm0[1,3,2,3,4,5,6,7]
4098 ; SSE-NEXT: packuswb %xmm1, %xmm1
4099 ; SSE-NEXT: pand %xmm13, %xmm1
4100 ; SSE-NEXT: por %xmm7, %xmm1
4101 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
4102 ; SSE-NEXT: pand %xmm2, %xmm0
4103 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4104 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm0[0,1,2,0]
4105 ; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5,7,5]
4106 ; SSE-NEXT: packuswb %xmm7, %xmm7
4107 ; SSE-NEXT: movdqa %xmm14, %xmm15
4108 ; SSE-NEXT: pandn %xmm7, %xmm15
4109 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
4110 ; SSE-NEXT: pand %xmm2, %xmm0
4111 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4112 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm0[0,1,2,0]
4113 ; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,7,5,6,7]
4114 ; SSE-NEXT: packuswb %xmm7, %xmm7
4115 ; SSE-NEXT: pand %xmm14, %xmm7
4116 ; SSE-NEXT: por %xmm15, %xmm7
4117 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,1,1]
4118 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm7[0],xmm1[1],xmm7[1]
4119 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
4120 ; SSE-NEXT: pand %xmm2, %xmm0
4121 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4122 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm0[0,1,2,0]
4123 ; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5,7,5]
4124 ; SSE-NEXT: packuswb %xmm7, %xmm7
4125 ; SSE-NEXT: movdqa %xmm11, %xmm15
4126 ; SSE-NEXT: pandn %xmm7, %xmm15
4127 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
4128 ; SSE-NEXT: pand %xmm2, %xmm0
4129 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4130 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm0[0,1,2,0]
4131 ; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,7,5,6,7]
4132 ; SSE-NEXT: packuswb %xmm7, %xmm7
4133 ; SSE-NEXT: pand %xmm11, %xmm7
4134 ; SSE-NEXT: por %xmm15, %xmm7
4135 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
4136 ; SSE-NEXT: pand %xmm2, %xmm0
4137 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4138 ; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm0[0,2,2,3]
4139 ; SSE-NEXT: pshuflw {{.*#+}} xmm15 = xmm15[0,1,1,3,4,5,6,7]
4140 ; SSE-NEXT: packuswb %xmm15, %xmm15
4141 ; SSE-NEXT: movdqa %xmm9, %xmm10
4142 ; SSE-NEXT: pandn %xmm15, %xmm10
4143 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
4144 ; SSE-NEXT: pand %xmm2, %xmm0
4145 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4146 ; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm0[0,2,2,3]
4147 ; SSE-NEXT: pshuflw {{.*#+}} xmm15 = xmm15[1,3,2,3,4,5,6,7]
4148 ; SSE-NEXT: packuswb %xmm15, %xmm15
4149 ; SSE-NEXT: pand %xmm9, %xmm15
4150 ; SSE-NEXT: por %xmm10, %xmm15
4151 ; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm15[2,2,2,2]
4152 ; SSE-NEXT: punpckhdq {{.*#+}} xmm10 = xmm10[2],xmm7[2],xmm10[3],xmm7[3]
4153 ; SSE-NEXT: movsd {{.*#+}} xmm10 = xmm1[0],xmm10[1]
4154 ; SSE-NEXT: movapd %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4155 ; SSE-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
4156 ; SSE-NEXT: pand %xmm2, %xmm0
4157 ; SSE-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
4158 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
4159 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]
4160 ; SSE-NEXT: packuswb %xmm0, %xmm0
4161 ; SSE-NEXT: movdqa %xmm13, %xmm7
4162 ; SSE-NEXT: pandn %xmm0, %xmm7
4163 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
4164 ; SSE-NEXT: pand %xmm2, %xmm0
4165 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4166 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
4167 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
4168 ; SSE-NEXT: packuswb %xmm0, %xmm0
4169 ; SSE-NEXT: pand %xmm13, %xmm0
4170 ; SSE-NEXT: por %xmm7, %xmm0
4171 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
4172 ; SSE-NEXT: pand %xmm2, %xmm1
4173 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4174 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm1[0,1,2,0]
4175 ; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5,7,5]
4176 ; SSE-NEXT: packuswb %xmm7, %xmm7
4177 ; SSE-NEXT: movdqa %xmm14, %xmm10
4178 ; SSE-NEXT: pandn %xmm7, %xmm10
4179 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
4180 ; SSE-NEXT: pand %xmm2, %xmm1
4181 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4182 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm1[0,1,2,0]
4183 ; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,7,5,6,7]
4184 ; SSE-NEXT: packuswb %xmm7, %xmm7
4185 ; SSE-NEXT: pand %xmm14, %xmm7
4186 ; SSE-NEXT: por %xmm10, %xmm7
4187 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,1,1]
4188 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1]
4189 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
4190 ; SSE-NEXT: pand %xmm2, %xmm1
4191 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4192 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm1[0,1,2,0]
4193 ; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5,7,5]
4194 ; SSE-NEXT: packuswb %xmm7, %xmm7
4195 ; SSE-NEXT: movdqa %xmm11, %xmm10
4196 ; SSE-NEXT: pandn %xmm7, %xmm10
4197 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
4198 ; SSE-NEXT: pand %xmm2, %xmm1
4199 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4200 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm1[0,1,2,0]
4201 ; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,7,5,6,7]
4202 ; SSE-NEXT: packuswb %xmm7, %xmm7
4203 ; SSE-NEXT: pand %xmm11, %xmm7
4204 ; SSE-NEXT: por %xmm10, %xmm7
4205 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
4206 ; SSE-NEXT: pand %xmm2, %xmm1
4207 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4208 ; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm1[0,2,2,3]
4209 ; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7]
4210 ; SSE-NEXT: packuswb %xmm10, %xmm10
4211 ; SSE-NEXT: movdqa %xmm9, %xmm15
4212 ; SSE-NEXT: pandn %xmm10, %xmm15
4213 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
4214 ; SSE-NEXT: pand %xmm2, %xmm1
4215 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4216 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
4217 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]
4218 ; SSE-NEXT: packuswb %xmm1, %xmm1
4219 ; SSE-NEXT: pand %xmm9, %xmm1
4220 ; SSE-NEXT: por %xmm15, %xmm1
4221 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]
4222 ; SSE-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm7[2],xmm1[3],xmm7[3]
4223 ; SSE-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
4224 ; SSE-NEXT: movapd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4225 ; SSE-NEXT: pshufd $229, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
4226 ; SSE-NEXT: # xmm0 = mem[1,1,2,3]
4227 ; SSE-NEXT: movdqa %xmm13, %xmm1
4228 ; SSE-NEXT: pandn %xmm0, %xmm1
4229 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
4230 ; SSE-NEXT: # xmm0 = mem[3,3,3,3]
4231 ; SSE-NEXT: packuswb %xmm0, %xmm0
4232 ; SSE-NEXT: pand %xmm13, %xmm0
4233 ; SSE-NEXT: por %xmm1, %xmm0
4234 ; SSE-NEXT: movdqa %xmm14, %xmm1
4235 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4236 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
4237 ; SSE-NEXT: # xmm7 = mem[2,2,3,3]
4238 ; SSE-NEXT: packuswb %xmm7, %xmm7
4239 ; SSE-NEXT: pand %xmm14, %xmm7
4240 ; SSE-NEXT: por %xmm1, %xmm7
4241 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[1,1,1,1]
4242 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
4243 ; SSE-NEXT: movdqa %xmm11, %xmm1
4244 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4245 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
4246 ; SSE-NEXT: # xmm7 = mem[2,2,3,3]
4247 ; SSE-NEXT: packuswb %xmm7, %xmm7
4248 ; SSE-NEXT: pand %xmm11, %xmm7
4249 ; SSE-NEXT: por %xmm1, %xmm7
4250 ; SSE-NEXT: pshufd $244, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4251 ; SSE-NEXT: # xmm1 = mem[0,1,3,3]
4252 ; SSE-NEXT: movdqa %xmm9, %xmm10
4253 ; SSE-NEXT: pandn %xmm1, %xmm10
4254 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4255 ; SSE-NEXT: # xmm1 = mem[3,3,3,3]
4256 ; SSE-NEXT: packuswb %xmm1, %xmm1
4257 ; SSE-NEXT: pand %xmm9, %xmm1
4258 ; SSE-NEXT: por %xmm10, %xmm1
4259 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]
4260 ; SSE-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm7[2],xmm1[3],xmm7[3]
4261 ; SSE-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
4262 ; SSE-NEXT: movapd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4263 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,2,3]
4264 ; SSE-NEXT: movdqa %xmm13, %xmm1
4265 ; SSE-NEXT: pandn %xmm0, %xmm1
4266 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
4267 ; SSE-NEXT: # xmm0 = mem[3,3,3,3]
4268 ; SSE-NEXT: packuswb %xmm0, %xmm0
4269 ; SSE-NEXT: pand %xmm13, %xmm0
4270 ; SSE-NEXT: por %xmm1, %xmm0
4271 ; SSE-NEXT: movdqa %xmm14, %xmm1
4272 ; SSE-NEXT: pandn %xmm4, %xmm1
4273 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
4274 ; SSE-NEXT: # xmm4 = mem[2,2,3,3]
4275 ; SSE-NEXT: packuswb %xmm4, %xmm4
4276 ; SSE-NEXT: pand %xmm14, %xmm4
4277 ; SSE-NEXT: por %xmm1, %xmm4
4278 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm4[1,1,1,1]
4279 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
4280 ; SSE-NEXT: movdqa %xmm11, %xmm1
4281 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4282 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm12[2,2,3,3]
4283 ; SSE-NEXT: packuswb %xmm4, %xmm4
4284 ; SSE-NEXT: pand %xmm11, %xmm4
4285 ; SSE-NEXT: por %xmm1, %xmm4
4286 ; SSE-NEXT: pshufd $244, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4287 ; SSE-NEXT: # xmm1 = mem[0,1,3,3]
4288 ; SSE-NEXT: movdqa %xmm9, %xmm7
4289 ; SSE-NEXT: pandn %xmm1, %xmm7
4290 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm5[3,3,3,3]
4291 ; SSE-NEXT: packuswb %xmm1, %xmm1
4292 ; SSE-NEXT: pand %xmm9, %xmm1
4293 ; SSE-NEXT: por %xmm7, %xmm1
4294 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]
4295 ; SSE-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm4[2],xmm1[3],xmm4[3]
4296 ; SSE-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
4297 ; SSE-NEXT: movapd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4298 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm3[3,1,2,3]
4299 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4300 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]
4301 ; SSE-NEXT: packuswb %xmm0, %xmm0
4302 ; SSE-NEXT: movdqa %xmm13, %xmm1
4303 ; SSE-NEXT: pandn %xmm0, %xmm1
4304 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm6[3,1,2,3]
4305 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4306 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]
4307 ; SSE-NEXT: packuswb %xmm0, %xmm0
4308 ; SSE-NEXT: pand %xmm13, %xmm0
4309 ; SSE-NEXT: por %xmm1, %xmm0
4310 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4311 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
4312 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4313 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,6]
4314 ; SSE-NEXT: packuswb %xmm1, %xmm1
4315 ; SSE-NEXT: movdqa %xmm14, %xmm4
4316 ; SSE-NEXT: pandn %xmm1, %xmm4
4317 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4318 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
4319 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4320 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
4321 ; SSE-NEXT: packuswb %xmm1, %xmm1
4322 ; SSE-NEXT: pand %xmm14, %xmm1
4323 ; SSE-NEXT: por %xmm4, %xmm1
4324 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
4325 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
4326 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4327 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
4328 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4329 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,6]
4330 ; SSE-NEXT: packuswb %xmm1, %xmm1
4331 ; SSE-NEXT: movdqa %xmm11, %xmm4
4332 ; SSE-NEXT: pandn %xmm1, %xmm4
4333 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4334 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
4335 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4336 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
4337 ; SSE-NEXT: packuswb %xmm1, %xmm1
4338 ; SSE-NEXT: pand %xmm11, %xmm1
4339 ; SSE-NEXT: por %xmm4, %xmm1
4340 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
4341 ; SSE-NEXT: # xmm2 = mem[3,1,2,3]
4342 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4343 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm2[0,1,2,0,4,5,6,7]
4344 ; SSE-NEXT: packuswb %xmm4, %xmm4
4345 ; SSE-NEXT: movdqa %xmm9, %xmm5
4346 ; SSE-NEXT: pandn %xmm4, %xmm5
4347 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
4348 ; SSE-NEXT: # xmm2 = mem[3,1,2,3]
4349 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4350 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm2[2,0,2,3,4,5,6,7]
4351 ; SSE-NEXT: packuswb %xmm4, %xmm4
4352 ; SSE-NEXT: pand %xmm9, %xmm4
4353 ; SSE-NEXT: por %xmm5, %xmm4
4354 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm4[2,2,2,2]
4355 ; SSE-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm1[2],xmm6[3],xmm1[3]
4356 ; SSE-NEXT: movsd {{.*#+}} xmm6 = xmm0[0],xmm6[1]
4357 ; SSE-NEXT: pshufd $231, (%rsp), %xmm0 # 16-byte Folded Reload
4358 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
4359 ; SSE-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
4360 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]
4361 ; SSE-NEXT: packuswb %xmm0, %xmm0
4362 ; SSE-NEXT: movdqa %xmm13, %xmm1
4363 ; SSE-NEXT: pandn %xmm0, %xmm1
4364 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
4365 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
4366 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4367 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]
4368 ; SSE-NEXT: packuswb %xmm0, %xmm0
4369 ; SSE-NEXT: pand %xmm13, %xmm0
4370 ; SSE-NEXT: por %xmm1, %xmm0
4371 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4372 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
4373 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4374 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,6]
4375 ; SSE-NEXT: packuswb %xmm1, %xmm1
4376 ; SSE-NEXT: movdqa %xmm14, %xmm4
4377 ; SSE-NEXT: pandn %xmm1, %xmm4
4378 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4379 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
4380 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4381 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
4382 ; SSE-NEXT: packuswb %xmm1, %xmm1
4383 ; SSE-NEXT: pand %xmm14, %xmm1
4384 ; SSE-NEXT: por %xmm4, %xmm1
4385 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
4386 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
4387 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4388 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
4389 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4390 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,6]
4391 ; SSE-NEXT: packuswb %xmm1, %xmm1
4392 ; SSE-NEXT: movdqa %xmm11, %xmm4
4393 ; SSE-NEXT: pandn %xmm1, %xmm4
4394 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4395 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
4396 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4397 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
4398 ; SSE-NEXT: packuswb %xmm1, %xmm1
4399 ; SSE-NEXT: pand %xmm11, %xmm1
4400 ; SSE-NEXT: por %xmm4, %xmm1
4401 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
4402 ; SSE-NEXT: # xmm2 = mem[3,1,2,3]
4403 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4404 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm2[0,1,2,0,4,5,6,7]
4405 ; SSE-NEXT: packuswb %xmm4, %xmm4
4406 ; SSE-NEXT: movdqa %xmm9, %xmm5
4407 ; SSE-NEXT: pandn %xmm4, %xmm5
4408 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
4409 ; SSE-NEXT: # xmm12 = mem[3,1,2,3]
4410 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm12[2,0,2,3,4,5,6,7]
4411 ; SSE-NEXT: packuswb %xmm4, %xmm4
4412 ; SSE-NEXT: pand %xmm9, %xmm4
4413 ; SSE-NEXT: por %xmm5, %xmm4
4414 ; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm4[2,2,2,2]
4415 ; SSE-NEXT: punpckhdq {{.*#+}} xmm10 = xmm10[2],xmm1[2],xmm10[3],xmm1[3]
4416 ; SSE-NEXT: movsd {{.*#+}} xmm10 = xmm0[0],xmm10[1]
4417 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
4418 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
4419 ; SSE-NEXT: # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
4420 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
4421 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
4422 ; SSE-NEXT: # xmm2 = xmm2[4],mem[4],xmm2[5],mem[5],xmm2[6],mem[6],xmm2[7],mem[7]
4423 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4424 ; SSE-NEXT: packuswb %xmm0, %xmm0
4425 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4426 ; SSE-NEXT: movdqa %xmm13, %xmm1
4427 ; SSE-NEXT: pandn %xmm0, %xmm1
4428 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
4429 ; SSE-NEXT: packuswb %xmm0, %xmm0
4430 ; SSE-NEXT: pand %xmm13, %xmm0
4431 ; SSE-NEXT: por %xmm1, %xmm0
4432 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
4433 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4434 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
4435 ; SSE-NEXT: packuswb %xmm1, %xmm1
4436 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4437 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,2,3]
4438 ; SSE-NEXT: movdqa %xmm14, %xmm4
4439 ; SSE-NEXT: pandn %xmm1, %xmm4
4440 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
4441 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4442 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
4443 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4444 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
4445 ; SSE-NEXT: packuswb %xmm1, %xmm1
4446 ; SSE-NEXT: pand %xmm14, %xmm1
4447 ; SSE-NEXT: por %xmm4, %xmm1
4448 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
4449 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
4450 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
4451 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4452 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
4453 ; SSE-NEXT: packuswb %xmm1, %xmm1
4454 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4455 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,2]
4456 ; SSE-NEXT: movdqa %xmm11, %xmm4
4457 ; SSE-NEXT: pandn %xmm1, %xmm4
4458 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
4459 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4460 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
4461 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4462 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
4463 ; SSE-NEXT: packuswb %xmm1, %xmm1
4464 ; SSE-NEXT: pand %xmm11, %xmm1
4465 ; SSE-NEXT: por %xmm4, %xmm1
4466 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
4467 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
4468 ; SSE-NEXT: # xmm3 = xmm3[4],mem[4],xmm3[5],mem[5],xmm3[6],mem[6],xmm3[7],mem[7]
4469 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
4470 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
4471 ; SSE-NEXT: # xmm2 = xmm2[4],mem[4],xmm2[5],mem[5],xmm2[6],mem[6],xmm2[7],mem[7]
4472 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4473 ; SSE-NEXT: packuswb %xmm3, %xmm3
4474 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4475 ; SSE-NEXT: movdqa %xmm9, %xmm4
4476 ; SSE-NEXT: pandn %xmm3, %xmm4
4477 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
4478 ; SSE-NEXT: packuswb %xmm5, %xmm5
4479 ; SSE-NEXT: pand %xmm9, %xmm5
4480 ; SSE-NEXT: por %xmm4, %xmm5
4481 ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm5[2,2,2,2]
4482 ; SSE-NEXT: punpckhdq {{.*#+}} xmm8 = xmm8[2],xmm1[2],xmm8[3],xmm1[3]
4483 ; SSE-NEXT: movsd {{.*#+}} xmm8 = xmm0[0],xmm8[1]
4484 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
4485 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
4486 ; SSE-NEXT: # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
4487 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
4488 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
4489 ; SSE-NEXT: # xmm2 = xmm2[4],mem[4],xmm2[5],mem[5],xmm2[6],mem[6],xmm2[7],mem[7]
4490 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4491 ; SSE-NEXT: packuswb %xmm0, %xmm0
4492 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4493 ; SSE-NEXT: movdqa %xmm13, %xmm1
4494 ; SSE-NEXT: pandn %xmm0, %xmm1
4495 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
4496 ; SSE-NEXT: packuswb %xmm0, %xmm0
4497 ; SSE-NEXT: pand %xmm13, %xmm0
4498 ; SSE-NEXT: por %xmm1, %xmm0
4499 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
4500 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4501 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
4502 ; SSE-NEXT: packuswb %xmm1, %xmm1
4503 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4504 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,2,3]
4505 ; SSE-NEXT: movdqa %xmm14, %xmm4
4506 ; SSE-NEXT: pandn %xmm1, %xmm4
4507 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
4508 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4509 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
4510 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4511 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
4512 ; SSE-NEXT: packuswb %xmm1, %xmm1
4513 ; SSE-NEXT: pand %xmm14, %xmm1
4514 ; SSE-NEXT: por %xmm4, %xmm1
4515 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
4516 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
4517 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
4518 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4519 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
4520 ; SSE-NEXT: packuswb %xmm1, %xmm7
4521 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[0,1,2,2]
4522 ; SSE-NEXT: movdqa %xmm11, %xmm4
4523 ; SSE-NEXT: pandn %xmm1, %xmm4
4524 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
4525 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4526 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
4527 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4528 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
4529 ; SSE-NEXT: packuswb %xmm1, %xmm1
4530 ; SSE-NEXT: pand %xmm11, %xmm1
4531 ; SSE-NEXT: por %xmm4, %xmm1
4532 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
4533 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
4534 ; SSE-NEXT: # xmm3 = xmm3[4],mem[4],xmm3[5],mem[5],xmm3[6],mem[6],xmm3[7],mem[7]
4535 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
4536 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
4537 ; SSE-NEXT: # xmm2 = xmm2[4],mem[4],xmm2[5],mem[5],xmm2[6],mem[6],xmm2[7],mem[7]
4538 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4539 ; SSE-NEXT: packuswb %xmm3, %xmm5
4540 ; SSE-NEXT: movdqa %xmm9, %xmm4
4541 ; SSE-NEXT: pandn %xmm5, %xmm4
4542 ; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm2[1,1,1,1]
4543 ; SSE-NEXT: packuswb %xmm15, %xmm15
4544 ; SSE-NEXT: pand %xmm9, %xmm15
4545 ; SSE-NEXT: por %xmm4, %xmm15
4546 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm15[2,2,2,2]
4547 ; SSE-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm1[2],xmm4[3],xmm1[3]
4548 ; SSE-NEXT: movsd {{.*#+}} xmm4 = xmm0[0],xmm4[1]
4549 ; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
4550 ; SSE-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]
4551 ; SSE-NEXT: packuswb %xmm0, %xmm0
4552 ; SSE-NEXT: movdqa %xmm13, %xmm1
4553 ; SSE-NEXT: pandn %xmm0, %xmm1
4554 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
4555 ; SSE-NEXT: # xmm0 = mem[3,1,2,3,4,5,6,7]
4556 ; SSE-NEXT: packuswb %xmm0, %xmm0
4557 ; SSE-NEXT: pand %xmm13, %xmm0
4558 ; SSE-NEXT: por %xmm1, %xmm0
4559 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4560 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,4,5,5,7]
4561 ; SSE-NEXT: packuswb %xmm1, %xmm1
4562 ; SSE-NEXT: movdqa %xmm14, %xmm15
4563 ; SSE-NEXT: pandn %xmm1, %xmm15
4564 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4565 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,5,7,6,7]
4566 ; SSE-NEXT: packuswb %xmm1, %xmm1
4567 ; SSE-NEXT: pand %xmm14, %xmm1
4568 ; SSE-NEXT: por %xmm15, %xmm1
4569 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
4570 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
4571 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4572 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,4,5,5,7]
4573 ; SSE-NEXT: packuswb %xmm1, %xmm1
4574 ; SSE-NEXT: movdqa %xmm11, %xmm15
4575 ; SSE-NEXT: pandn %xmm1, %xmm15
4576 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4577 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,5,7,6,7]
4578 ; SSE-NEXT: packuswb %xmm1, %xmm1
4579 ; SSE-NEXT: pand %xmm11, %xmm1
4580 ; SSE-NEXT: por %xmm15, %xmm1
4581 ; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
4582 ; SSE-NEXT: # xmm15 = mem[0,1,3,1,4,5,6,7]
4583 ; SSE-NEXT: packuswb %xmm15, %xmm15
4584 ; SSE-NEXT: movdqa %xmm9, %xmm2
4585 ; SSE-NEXT: pandn %xmm15, %xmm2
4586 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
4587 ; SSE-NEXT: # xmm15 = mem[3,1,2,3,4,5,6,7]
4588 ; SSE-NEXT: packuswb %xmm15, %xmm15
4589 ; SSE-NEXT: pand %xmm9, %xmm15
4590 ; SSE-NEXT: por %xmm2, %xmm15
4591 ; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm15[2,2,2,2]
4592 ; SSE-NEXT: punpckhdq {{.*#+}} xmm15 = xmm15[2],xmm1[2],xmm15[3],xmm1[3]
4593 ; SSE-NEXT: movsd {{.*#+}} xmm15 = xmm0[0],xmm15[1]
4594 ; SSE-NEXT: pshuflw $116, (%rsp), %xmm0 # 16-byte Folded Reload
4595 ; SSE-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]
4596 ; SSE-NEXT: packuswb %xmm0, %xmm0
4597 ; SSE-NEXT: movdqa %xmm13, %xmm1
4598 ; SSE-NEXT: pandn %xmm0, %xmm1
4599 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
4600 ; SSE-NEXT: # xmm0 = mem[3,1,2,3,4,5,6,7]
4601 ; SSE-NEXT: packuswb %xmm0, %xmm0
4602 ; SSE-NEXT: pand %xmm13, %xmm0
4603 ; SSE-NEXT: por %xmm1, %xmm0
4604 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4605 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,4,5,5,7]
4606 ; SSE-NEXT: packuswb %xmm1, %xmm1
4607 ; SSE-NEXT: movdqa %xmm14, %xmm2
4608 ; SSE-NEXT: pandn %xmm1, %xmm2
4609 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4610 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,5,7,6,7]
4611 ; SSE-NEXT: packuswb %xmm1, %xmm1
4612 ; SSE-NEXT: pand %xmm14, %xmm1
4613 ; SSE-NEXT: por %xmm2, %xmm1
4614 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
4615 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
4616 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4617 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,4,5,5,7]
4618 ; SSE-NEXT: packuswb %xmm1, %xmm1
4619 ; SSE-NEXT: movdqa %xmm11, %xmm2
4620 ; SSE-NEXT: pandn %xmm1, %xmm2
4621 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4622 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,5,7,6,7]
4623 ; SSE-NEXT: packuswb %xmm1, %xmm1
4624 ; SSE-NEXT: pand %xmm11, %xmm1
4625 ; SSE-NEXT: por %xmm2, %xmm1
4626 ; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
4627 ; SSE-NEXT: # xmm2 = mem[0,1,3,1,4,5,6,7]
4628 ; SSE-NEXT: packuswb %xmm2, %xmm2
4629 ; SSE-NEXT: movdqa %xmm9, %xmm3
4630 ; SSE-NEXT: pandn %xmm2, %xmm3
4631 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm12[3,1,2,3,4,5,6,7]
4632 ; SSE-NEXT: packuswb %xmm2, %xmm2
4633 ; SSE-NEXT: pand %xmm9, %xmm2
4634 ; SSE-NEXT: por %xmm3, %xmm2
4635 ; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm2[2,2,2,2]
4636 ; SSE-NEXT: punpckhdq {{.*#+}} xmm12 = xmm12[2],xmm1[2],xmm12[3],xmm1[3]
4637 ; SSE-NEXT: movsd {{.*#+}} xmm12 = xmm0[0],xmm12[1]
4638 ; SSE-NEXT: pshufd $229, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
4639 ; SSE-NEXT: # xmm0 = mem[1,1,2,3]
4640 ; SSE-NEXT: movdqa %xmm13, %xmm1
4641 ; SSE-NEXT: pandn %xmm0, %xmm1
4642 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
4643 ; SSE-NEXT: # xmm0 = mem[3,3,3,3]
4644 ; SSE-NEXT: packuswb %xmm0, %xmm0
4645 ; SSE-NEXT: pand %xmm13, %xmm0
4646 ; SSE-NEXT: por %xmm1, %xmm0
4647 ; SSE-NEXT: movdqa %xmm14, %xmm1
4648 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4649 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
4650 ; SSE-NEXT: # xmm2 = mem[2,2,3,3]
4651 ; SSE-NEXT: packuswb %xmm2, %xmm2
4652 ; SSE-NEXT: pand %xmm14, %xmm2
4653 ; SSE-NEXT: por %xmm1, %xmm2
4654 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
4655 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
4656 ; SSE-NEXT: movdqa %xmm11, %xmm1
4657 ; SSE-NEXT: pandn %xmm7, %xmm1
4658 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
4659 ; SSE-NEXT: # xmm2 = mem[2,2,3,3]
4660 ; SSE-NEXT: packuswb %xmm2, %xmm2
4661 ; SSE-NEXT: pand %xmm11, %xmm2
4662 ; SSE-NEXT: por %xmm1, %xmm2
4663 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm5[0,1,3,3]
4664 ; SSE-NEXT: movdqa %xmm9, %xmm3
4665 ; SSE-NEXT: pandn %xmm1, %xmm3
4666 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4667 ; SSE-NEXT: # xmm1 = mem[3,3,3,3]
4668 ; SSE-NEXT: packuswb %xmm1, %xmm1
4669 ; SSE-NEXT: pand %xmm9, %xmm1
4670 ; SSE-NEXT: por %xmm3, %xmm1
4671 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[2,2,2,2]
4672 ; SSE-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm2[2],xmm5[3],xmm2[3]
4673 ; SSE-NEXT: movsd {{.*#+}} xmm5 = xmm0[0],xmm5[1]
4674 ; SSE-NEXT: pshufd $229, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
4675 ; SSE-NEXT: # xmm0 = mem[1,1,2,3]
4676 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4677 ; SSE-NEXT: # xmm1 = mem[3,3,3,3]
4678 ; SSE-NEXT: packuswb %xmm1, %xmm1
4679 ; SSE-NEXT: pand %xmm13, %xmm1
4680 ; SSE-NEXT: pandn %xmm0, %xmm13
4681 ; SSE-NEXT: por %xmm1, %xmm13
4682 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
4683 ; SSE-NEXT: # xmm0 = mem[2,2,3,3]
4684 ; SSE-NEXT: packuswb %xmm0, %xmm0
4685 ; SSE-NEXT: pand %xmm14, %xmm0
4686 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload
4687 ; SSE-NEXT: por %xmm0, %xmm14
4688 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm14[1,1,1,1]
4689 ; SSE-NEXT: punpckldq {{.*#+}} xmm13 = xmm13[0],xmm0[0],xmm13[1],xmm0[1]
4690 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
4691 ; SSE-NEXT: # xmm0 = mem[2,2,3,3]
4692 ; SSE-NEXT: packuswb %xmm0, %xmm0
4693 ; SSE-NEXT: pand %xmm11, %xmm0
4694 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
4695 ; SSE-NEXT: por %xmm0, %xmm11
4696 ; SSE-NEXT: pshufd $244, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
4697 ; SSE-NEXT: # xmm0 = mem[0,1,3,3]
4698 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4699 ; SSE-NEXT: # xmm1 = mem[3,3,3,3]
4700 ; SSE-NEXT: packuswb %xmm1, %xmm1
4701 ; SSE-NEXT: pand %xmm9, %xmm1
4702 ; SSE-NEXT: pandn %xmm0, %xmm9
4703 ; SSE-NEXT: por %xmm1, %xmm9
4704 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm9[2,2,2,2]
4705 ; SSE-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm11[2],xmm0[3],xmm11[3]
4706 ; SSE-NEXT: movsd {{.*#+}} xmm0 = xmm13[0],xmm0[1]
4707 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
4708 ; SSE-NEXT: movaps %xmm1, (%rsi)
4709 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
4710 ; SSE-NEXT: movaps %xmm1, 16(%rsi)
4711 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
4712 ; SSE-NEXT: movaps %xmm1, (%rdx)
4713 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
4714 ; SSE-NEXT: movaps %xmm1, 16(%rdx)
4715 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
4716 ; SSE-NEXT: movaps %xmm1, (%rcx)
4717 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
4718 ; SSE-NEXT: movaps %xmm1, 16(%rcx)
4719 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
4720 ; SSE-NEXT: movaps %xmm1, (%r8)
4721 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
4722 ; SSE-NEXT: movaps %xmm1, 16(%r8)
4723 ; SSE-NEXT: movapd %xmm10, (%r9)
4724 ; SSE-NEXT: movapd %xmm6, 16(%r9)
4725 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
4726 ; SSE-NEXT: movapd %xmm4, (%rax)
4727 ; SSE-NEXT: movapd %xmm8, 16(%rax)
4728 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
4729 ; SSE-NEXT: movapd %xmm12, (%rax)
4730 ; SSE-NEXT: movapd %xmm15, 16(%rax)
4731 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
4732 ; SSE-NEXT: movapd %xmm0, 16(%rax)
4733 ; SSE-NEXT: movapd %xmm5, (%rax)
4734 ; SSE-NEXT: addq $904, %rsp # imm = 0x388
4735 ; SSE-NEXT: retq
4736 ;
4737 ; AVX-LABEL: load_i8_stride8_vf32:
4738 ; AVX: # %bb.0:
4739 ; AVX-NEXT: subq $360, %rsp # imm = 0x168
4740 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm10 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
4741 ; AVX-NEXT: vmovdqa 112(%rdi), %xmm4
4742 ; AVX-NEXT: vpshufb %xmm10, %xmm4, %xmm0
4743 ; AVX-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4744 ; AVX-NEXT: vmovdqa 96(%rdi), %xmm7
4745 ; AVX-NEXT: vpshufb %xmm10, %xmm7, %xmm3
4746 ; AVX-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4747 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
4748 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm12 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
4749 ; AVX-NEXT: vmovdqa 80(%rdi), %xmm1
4750 ; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4751 ; AVX-NEXT: vpshufb %xmm12, %xmm1, %xmm3
4752 ; AVX-NEXT: vmovdqa 64(%rdi), %xmm5
4753 ; AVX-NEXT: vpshufb %xmm12, %xmm5, %xmm6
4754 ; AVX-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4755 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3]
4756 ; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm3[0,1,2,3,4,5],xmm0[6,7]
4757 ; AVX-NEXT: vmovd {{.*#+}} xmm14 = [0,0,0,8,0,0,0,0,0,0,0,0,0,0,0,0]
4758 ; AVX-NEXT: vmovdqa (%rdi), %xmm0
4759 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4760 ; AVX-NEXT: vmovdqa 16(%rdi), %xmm1
4761 ; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4762 ; AVX-NEXT: vmovdqa 32(%rdi), %xmm2
4763 ; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4764 ; AVX-NEXT: vmovdqa 48(%rdi), %xmm3
4765 ; AVX-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4766 ; AVX-NEXT: vpshufb %xmm14, %xmm3, %xmm9
4767 ; AVX-NEXT: vpshufb %xmm14, %xmm2, %xmm11
4768 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
4769 ; AVX-NEXT: vmovd {{.*#+}} xmm3 = [0,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
4770 ; AVX-NEXT: vpshufb %xmm3, %xmm1, %xmm11
4771 ; AVX-NEXT: vpshufb %xmm3, %xmm0, %xmm13
4772 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1],xmm13[2],xmm11[2],xmm13[3],xmm11[3]
4773 ; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm11[0,1],xmm9[2,3],xmm11[4,5,6,7]
4774 ; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm9[0,1,2,3],xmm8[4,5,6,7]
4775 ; AVX-NEXT: vmovdqa 240(%rdi), %xmm0
4776 ; AVX-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill
4777 ; AVX-NEXT: vpshufb %xmm10, %xmm0, %xmm11
4778 ; AVX-NEXT: vmovdqa 224(%rdi), %xmm0
4779 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4780 ; AVX-NEXT: vpshufb %xmm10, %xmm0, %xmm10
4781 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
4782 ; AVX-NEXT: vmovdqa 208(%rdi), %xmm0
4783 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4784 ; AVX-NEXT: vpshufb %xmm12, %xmm0, %xmm15
4785 ; AVX-NEXT: vmovdqa 192(%rdi), %xmm0
4786 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4787 ; AVX-NEXT: vpshufb %xmm12, %xmm0, %xmm12
4788 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm15[0],xmm12[1],xmm15[1],xmm12[2],xmm15[2],xmm12[3],xmm15[3]
4789 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm12[0,1,2,3,4,5],xmm13[6,7]
4790 ; AVX-NEXT: vmovdqa 176(%rdi), %xmm0
4791 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4792 ; AVX-NEXT: vpshufb %xmm14, %xmm0, %xmm15
4793 ; AVX-NEXT: vmovdqa 160(%rdi), %xmm0
4794 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4795 ; AVX-NEXT: vpshufb %xmm14, %xmm0, %xmm14
4796 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
4797 ; AVX-NEXT: vmovdqa 144(%rdi), %xmm15
4798 ; AVX-NEXT: vpshufb %xmm3, %xmm15, %xmm0
4799 ; AVX-NEXT: vmovdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4800 ; AVX-NEXT: vmovdqa 128(%rdi), %xmm14
4801 ; AVX-NEXT: vpshufb %xmm3, %xmm14, %xmm3
4802 ; AVX-NEXT: vmovdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4803 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
4804 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
4805 ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm1
4806 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
4807 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
4808 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm0[4,5,6,7]
4809 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4810 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm8 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
4811 ; AVX-NEXT: vpshufb %xmm8, %xmm4, %xmm1
4812 ; AVX-NEXT: vpshufb %xmm8, %xmm7, %xmm2
4813 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
4814 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm2 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
4815 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
4816 ; AVX-NEXT: vpshufb %xmm2, %xmm6, %xmm3
4817 ; AVX-NEXT: vpshufb %xmm2, %xmm5, %xmm4
4818 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
4819 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3,4,5],xmm1[6,7]
4820 ; AVX-NEXT: vmovd {{.*#+}} xmm3 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
4821 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
4822 ; AVX-NEXT: vpshufb %xmm3, %xmm13, %xmm4
4823 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
4824 ; AVX-NEXT: vpshufb %xmm3, %xmm9, %xmm5
4825 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
4826 ; AVX-NEXT: vmovd {{.*#+}} xmm5 = [1,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
4827 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
4828 ; AVX-NEXT: vpshufb %xmm5, %xmm11, %xmm0
4829 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
4830 ; AVX-NEXT: vpshufb %xmm5, %xmm7, %xmm7
4831 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3]
4832 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm4[2,3],xmm0[4,5,6,7]
4833 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
4834 ; AVX-NEXT: vmovdqa (%rsp), %xmm12 # 16-byte Reload
4835 ; AVX-NEXT: vpshufb %xmm8, %xmm12, %xmm1
4836 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
4837 ; AVX-NEXT: vpshufb %xmm8, %xmm10, %xmm4
4838 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
4839 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
4840 ; AVX-NEXT: vpshufb %xmm2, %xmm4, %xmm4
4841 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
4842 ; AVX-NEXT: vpshufb %xmm2, %xmm7, %xmm2
4843 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
4844 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3,4,5],xmm1[6,7]
4845 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
4846 ; AVX-NEXT: vpshufb %xmm3, %xmm2, %xmm2
4847 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
4848 ; AVX-NEXT: vpshufb %xmm3, %xmm4, %xmm3
4849 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
4850 ; AVX-NEXT: vpshufb %xmm5, %xmm15, %xmm3
4851 ; AVX-NEXT: vpshufb %xmm5, %xmm14, %xmm4
4852 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
4853 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5,6,7]
4854 ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
4855 ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2
4856 ; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
4857 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
4858 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4859 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm0 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
4860 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
4861 ; AVX-NEXT: vpshufb %xmm0, %xmm14, %xmm1
4862 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
4863 ; AVX-NEXT: vpshufb %xmm0, %xmm15, %xmm2
4864 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
4865 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm2 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
4866 ; AVX-NEXT: vpshufb %xmm2, %xmm6, %xmm3
4867 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
4868 ; AVX-NEXT: vpshufb %xmm2, %xmm8, %xmm4
4869 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
4870 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3,4,5],xmm1[6,7]
4871 ; AVX-NEXT: vmovd {{.*#+}} xmm3 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
4872 ; AVX-NEXT: vpshufb %xmm3, %xmm13, %xmm4
4873 ; AVX-NEXT: vpshufb %xmm3, %xmm9, %xmm5
4874 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
4875 ; AVX-NEXT: vmovd {{.*#+}} xmm5 = [2,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
4876 ; AVX-NEXT: vpshufb %xmm5, %xmm11, %xmm6
4877 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
4878 ; AVX-NEXT: vpshufb %xmm5, %xmm13, %xmm7
4879 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
4880 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3],xmm6[4,5,6,7]
4881 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm4[0,1,2,3],xmm1[4,5,6,7]
4882 ; AVX-NEXT: vpshufb %xmm0, %xmm12, %xmm4
4883 ; AVX-NEXT: vpshufb %xmm0, %xmm10, %xmm0
4884 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
4885 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
4886 ; AVX-NEXT: vpshufb %xmm2, %xmm10, %xmm4
4887 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
4888 ; AVX-NEXT: vpshufb %xmm2, %xmm11, %xmm2
4889 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
4890 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3,4,5],xmm0[6,7]
4891 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
4892 ; AVX-NEXT: vpshufb %xmm3, %xmm12, %xmm2
4893 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
4894 ; AVX-NEXT: vpshufb %xmm3, %xmm4, %xmm3
4895 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
4896 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
4897 ; AVX-NEXT: vpshufb %xmm5, %xmm3, %xmm3
4898 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
4899 ; AVX-NEXT: vpshufb %xmm5, %xmm4, %xmm4
4900 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
4901 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5,6,7]
4902 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
4903 ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2
4904 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
4905 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
4906 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4907 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm0 = [0,0,3,11,0,0,3,11,0,0,3,11,0,0,3,11]
4908 ; AVX-NEXT: vpshufb %xmm0, %xmm14, %xmm1
4909 ; AVX-NEXT: vpshufb %xmm0, %xmm15, %xmm2
4910 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
4911 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm2 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
4912 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
4913 ; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm3
4914 ; AVX-NEXT: vpshufb %xmm2, %xmm8, %xmm4
4915 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
4916 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3,4,5],xmm1[6,7]
4917 ; AVX-NEXT: vmovd {{.*#+}} xmm3 = [0,0,3,11,0,0,0,0,0,0,0,0,0,0,0,0]
4918 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
4919 ; AVX-NEXT: vpshufb %xmm3, %xmm9, %xmm4
4920 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
4921 ; AVX-NEXT: vpshufb %xmm3, %xmm5, %xmm5
4922 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
4923 ; AVX-NEXT: vmovd {{.*#+}} xmm5 = [3,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
4924 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
4925 ; AVX-NEXT: vpshufb %xmm5, %xmm6, %xmm6
4926 ; AVX-NEXT: vmovdqa %xmm13, %xmm8
4927 ; AVX-NEXT: vpshufb %xmm5, %xmm13, %xmm7
4928 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
4929 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3],xmm6[4,5,6,7]
4930 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm4[0,1,2,3],xmm1[4,5,6,7]
4931 ; AVX-NEXT: vmovdqa (%rsp), %xmm4 # 16-byte Reload
4932 ; AVX-NEXT: vpshufb %xmm0, %xmm4, %xmm4
4933 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
4934 ; AVX-NEXT: vpshufb %xmm0, %xmm6, %xmm0
4935 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
4936 ; AVX-NEXT: vpshufb %xmm2, %xmm10, %xmm4
4937 ; AVX-NEXT: vpshufb %xmm2, %xmm11, %xmm2
4938 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
4939 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3,4,5],xmm0[6,7]
4940 ; AVX-NEXT: vpshufb %xmm3, %xmm12, %xmm2
4941 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
4942 ; AVX-NEXT: vpshufb %xmm3, %xmm14, %xmm3
4943 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
4944 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
4945 ; AVX-NEXT: vpshufb %xmm5, %xmm12, %xmm3
4946 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
4947 ; AVX-NEXT: vpshufb %xmm5, %xmm15, %xmm4
4948 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
4949 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5,6,7]
4950 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
4951 ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2
4952 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
4953 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
4954 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4955 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm0 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
4956 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
4957 ; AVX-NEXT: vpshufb %xmm0, %xmm10, %xmm1
4958 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
4959 ; AVX-NEXT: vpshufb %xmm0, %xmm2, %xmm2
4960 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
4961 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm2 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
4962 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
4963 ; AVX-NEXT: vpshufb %xmm2, %xmm11, %xmm3
4964 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
4965 ; AVX-NEXT: vpshufb %xmm2, %xmm13, %xmm4
4966 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
4967 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3,4,5],xmm1[6,7]
4968 ; AVX-NEXT: vmovd {{.*#+}} xmm3 = [0,0,4,12,0,0,0,0,0,0,0,0,0,0,0,0]
4969 ; AVX-NEXT: vpshufb %xmm3, %xmm9, %xmm4
4970 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
4971 ; AVX-NEXT: vpshufb %xmm3, %xmm5, %xmm5
4972 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
4973 ; AVX-NEXT: vmovd {{.*#+}} xmm5 = [4,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
4974 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
4975 ; AVX-NEXT: vpshufb %xmm5, %xmm6, %xmm6
4976 ; AVX-NEXT: vpshufb %xmm5, %xmm8, %xmm7
4977 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
4978 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3],xmm6[4,5,6,7]
4979 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm4[0,1,2,3],xmm1[4,5,6,7]
4980 ; AVX-NEXT: vmovdqa (%rsp), %xmm9 # 16-byte Reload
4981 ; AVX-NEXT: vpshufb %xmm0, %xmm9, %xmm4
4982 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
4983 ; AVX-NEXT: vpshufb %xmm0, %xmm6, %xmm0
4984 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
4985 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
4986 ; AVX-NEXT: vpshufb %xmm2, %xmm4, %xmm4
4987 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
4988 ; AVX-NEXT: vpshufb %xmm2, %xmm6, %xmm2
4989 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
4990 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3,4,5],xmm0[6,7]
4991 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
4992 ; AVX-NEXT: vpshufb %xmm3, %xmm2, %xmm2
4993 ; AVX-NEXT: vpshufb %xmm3, %xmm14, %xmm3
4994 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
4995 ; AVX-NEXT: vpshufb %xmm5, %xmm12, %xmm3
4996 ; AVX-NEXT: vpshufb %xmm5, %xmm15, %xmm4
4997 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
4998 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5,6,7]
4999 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
5000 ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2
5001 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
5002 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
5003 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
5004 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm0 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
5005 ; AVX-NEXT: vpshufb %xmm0, %xmm10, %xmm1
5006 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
5007 ; AVX-NEXT: vpshufb %xmm0, %xmm8, %xmm2
5008 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
5009 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm2 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
5010 ; AVX-NEXT: vpshufb %xmm2, %xmm11, %xmm3
5011 ; AVX-NEXT: vpshufb %xmm2, %xmm13, %xmm4
5012 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5013 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3,4,5],xmm1[6,7]
5014 ; AVX-NEXT: vmovd {{.*#+}} xmm3 = [0,0,5,13,0,0,0,0,0,0,0,0,0,0,0,0]
5015 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5016 ; AVX-NEXT: vpshufb %xmm3, %xmm4, %xmm4
5017 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
5018 ; AVX-NEXT: vpshufb %xmm3, %xmm11, %xmm5
5019 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
5020 ; AVX-NEXT: vmovd {{.*#+}} xmm5 = [5,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
5021 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
5022 ; AVX-NEXT: vpshufb %xmm5, %xmm10, %xmm6
5023 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
5024 ; AVX-NEXT: vpshufb %xmm5, %xmm14, %xmm7
5025 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
5026 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3],xmm6[4,5,6,7]
5027 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm4[0,1,2,3],xmm1[4,5,6,7]
5028 ; AVX-NEXT: vpshufb %xmm0, %xmm9, %xmm4
5029 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
5030 ; AVX-NEXT: vpshufb %xmm0, %xmm15, %xmm0
5031 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
5032 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
5033 ; AVX-NEXT: vpshufb %xmm2, %xmm9, %xmm4
5034 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
5035 ; AVX-NEXT: vpshufb %xmm2, %xmm12, %xmm2
5036 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
5037 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3,4,5],xmm0[6,7]
5038 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
5039 ; AVX-NEXT: vpshufb %xmm3, %xmm13, %xmm2
5040 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5041 ; AVX-NEXT: vpshufb %xmm3, %xmm4, %xmm3
5042 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
5043 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
5044 ; AVX-NEXT: vpshufb %xmm5, %xmm3, %xmm3
5045 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5046 ; AVX-NEXT: vpshufb %xmm5, %xmm4, %xmm4
5047 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5048 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5,6,7]
5049 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
5050 ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2
5051 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
5052 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
5053 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
5054 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm0 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
5055 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5056 ; AVX-NEXT: vpshufb %xmm0, %xmm1, %xmm1
5057 ; AVX-NEXT: vpshufb %xmm0, %xmm8, %xmm2
5058 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
5059 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm2 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
5060 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
5061 ; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm3
5062 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5063 ; AVX-NEXT: vpshufb %xmm2, %xmm4, %xmm4
5064 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5065 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3,4,5],xmm1[6,7]
5066 ; AVX-NEXT: vmovd {{.*#+}} xmm3 = [0,0,6,14,0,0,0,0,0,0,0,0,0,0,0,0]
5067 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
5068 ; AVX-NEXT: vpshufb %xmm3, %xmm8, %xmm4
5069 ; AVX-NEXT: vpshufb %xmm3, %xmm11, %xmm5
5070 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
5071 ; AVX-NEXT: vmovd {{.*#+}} xmm5 = [6,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
5072 ; AVX-NEXT: vpshufb %xmm5, %xmm10, %xmm6
5073 ; AVX-NEXT: vpshufb %xmm5, %xmm14, %xmm7
5074 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
5075 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3],xmm6[4,5,6,7]
5076 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm4[0,1,2,3],xmm1[4,5,6,7]
5077 ; AVX-NEXT: vmovdqa (%rsp), %xmm14 # 16-byte Reload
5078 ; AVX-NEXT: vpshufb %xmm0, %xmm14, %xmm4
5079 ; AVX-NEXT: vmovdqa %xmm15, %xmm10
5080 ; AVX-NEXT: vpshufb %xmm0, %xmm15, %xmm0
5081 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
5082 ; AVX-NEXT: vpshufb %xmm2, %xmm9, %xmm4
5083 ; AVX-NEXT: vpshufb %xmm2, %xmm12, %xmm2
5084 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
5085 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3,4,5],xmm0[6,7]
5086 ; AVX-NEXT: vpshufb %xmm3, %xmm13, %xmm2
5087 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
5088 ; AVX-NEXT: vpshufb %xmm3, %xmm15, %xmm3
5089 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
5090 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
5091 ; AVX-NEXT: vpshufb %xmm5, %xmm9, %xmm3
5092 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
5093 ; AVX-NEXT: vpshufb %xmm5, %xmm11, %xmm4
5094 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5095 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5,6,7]
5096 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
5097 ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2
5098 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
5099 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
5100 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
5101 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm0 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
5102 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5103 ; AVX-NEXT: vpshufb %xmm0, %xmm1, %xmm1
5104 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
5105 ; AVX-NEXT: vpshufb %xmm0, %xmm2, %xmm2
5106 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
5107 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm2 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
5108 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
5109 ; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm3
5110 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5111 ; AVX-NEXT: vpshufb %xmm2, %xmm4, %xmm4
5112 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5113 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3,4,5],xmm1[6,7]
5114 ; AVX-NEXT: vmovd {{.*#+}} xmm3 = [0,0,7,15,0,0,0,0,0,0,0,0,0,0,0,0]
5115 ; AVX-NEXT: vpshufb %xmm3, %xmm8, %xmm4
5116 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
5117 ; AVX-NEXT: vpshufb %xmm3, %xmm5, %xmm5
5118 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
5119 ; AVX-NEXT: vmovd {{.*#+}} xmm5 = [7,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
5120 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
5121 ; AVX-NEXT: vpshufb %xmm5, %xmm6, %xmm6
5122 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
5123 ; AVX-NEXT: vpshufb %xmm5, %xmm7, %xmm7
5124 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
5125 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3],xmm6[4,5,6,7]
5126 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm4[0,1,2,3],xmm1[4,5,6,7]
5127 ; AVX-NEXT: vpshufb %xmm0, %xmm14, %xmm4
5128 ; AVX-NEXT: vpshufb %xmm0, %xmm10, %xmm0
5129 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
5130 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5131 ; AVX-NEXT: vpshufb %xmm2, %xmm4, %xmm4
5132 ; AVX-NEXT: vpshufb %xmm2, %xmm12, %xmm2
5133 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
5134 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3,4,5],xmm0[6,7]
5135 ; AVX-NEXT: vpshufb %xmm3, %xmm13, %xmm2
5136 ; AVX-NEXT: vpshufb %xmm3, %xmm15, %xmm3
5137 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
5138 ; AVX-NEXT: vpshufb %xmm5, %xmm9, %xmm3
5139 ; AVX-NEXT: vpshufb %xmm5, %xmm11, %xmm4
5140 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5141 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5,6,7]
5142 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
5143 ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2
5144 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
5145 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
5146 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
5147 ; AVX-NEXT: vmovaps %ymm1, (%rsi)
5148 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
5149 ; AVX-NEXT: vmovaps %ymm1, (%rdx)
5150 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
5151 ; AVX-NEXT: vmovaps %ymm1, (%rcx)
5152 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
5153 ; AVX-NEXT: vmovaps %ymm1, (%r8)
5154 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
5155 ; AVX-NEXT: vmovaps %ymm1, (%r9)
5156 ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax
5157 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
5158 ; AVX-NEXT: vmovaps %ymm1, (%rax)
5159 ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax
5160 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
5161 ; AVX-NEXT: vmovaps %ymm1, (%rax)
5162 ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax
5163 ; AVX-NEXT: vmovaps %ymm0, (%rax)
5164 ; AVX-NEXT: addq $360, %rsp # imm = 0x168
5165 ; AVX-NEXT: vzeroupper
5166 ; AVX-NEXT: retq
5167 ;
5168 ; AVX2-LABEL: load_i8_stride8_vf32:
5169 ; AVX2: # %bb.0:
5170 ; AVX2-NEXT: subq $360, %rsp # imm = 0x168
5171 ; AVX2-NEXT: vmovdqa 112(%rdi), %xmm0
5172 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5173 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm10 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
5174 ; AVX2-NEXT: vpshufb %xmm10, %xmm0, %xmm0
5175 ; AVX2-NEXT: vmovdqa 96(%rdi), %xmm4
5176 ; AVX2-NEXT: vpshufb %xmm10, %xmm4, %xmm2
5177 ; AVX2-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5178 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
5179 ; AVX2-NEXT: vmovdqa 80(%rdi), %xmm1
5180 ; AVX2-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5181 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm12 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
5182 ; AVX2-NEXT: vpshufb %xmm12, %xmm1, %xmm2
5183 ; AVX2-NEXT: vmovdqa 64(%rdi), %xmm5
5184 ; AVX2-NEXT: vpshufb %xmm12, %xmm5, %xmm6
5185 ; AVX2-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5186 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
5187 ; AVX2-NEXT: vpblendd {{.*#+}} xmm8 = xmm2[0,1,2],xmm0[3]
5188 ; AVX2-NEXT: vmovdqa (%rdi), %xmm0
5189 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5190 ; AVX2-NEXT: vmovdqa 16(%rdi), %xmm7
5191 ; AVX2-NEXT: vmovdqa 32(%rdi), %xmm2
5192 ; AVX2-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5193 ; AVX2-NEXT: vmovdqa 48(%rdi), %xmm3
5194 ; AVX2-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5195 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm14 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
5196 ; AVX2-NEXT: vpshufb %xmm14, %xmm3, %xmm9
5197 ; AVX2-NEXT: vpshufb %xmm14, %xmm2, %xmm11
5198 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
5199 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm3 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
5200 ; AVX2-NEXT: vpshufb %xmm3, %xmm7, %xmm11
5201 ; AVX2-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5202 ; AVX2-NEXT: vpshufb %xmm3, %xmm0, %xmm13
5203 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1],xmm13[2],xmm11[2],xmm13[3],xmm11[3]
5204 ; AVX2-NEXT: vpblendd {{.*#+}} xmm9 = xmm11[0],xmm9[1],xmm11[2,3]
5205 ; AVX2-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
5206 ; AVX2-NEXT: vmovdqa 240(%rdi), %xmm0
5207 ; AVX2-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill
5208 ; AVX2-NEXT: vpshufb %xmm10, %xmm0, %xmm11
5209 ; AVX2-NEXT: vmovdqa 224(%rdi), %xmm0
5210 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5211 ; AVX2-NEXT: vpshufb %xmm10, %xmm0, %xmm10
5212 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
5213 ; AVX2-NEXT: vmovdqa 208(%rdi), %xmm11
5214 ; AVX2-NEXT: vpshufb %xmm12, %xmm11, %xmm15
5215 ; AVX2-NEXT: vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5216 ; AVX2-NEXT: vmovdqa 192(%rdi), %xmm0
5217 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5218 ; AVX2-NEXT: vpshufb %xmm12, %xmm0, %xmm12
5219 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm15[0],xmm12[1],xmm15[1],xmm12[2],xmm15[2],xmm12[3],xmm15[3]
5220 ; AVX2-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
5221 ; AVX2-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm12
5222 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm12[0,1,2,3,4,5,6],ymm13[7]
5223 ; AVX2-NEXT: vmovdqa 176(%rdi), %xmm0
5224 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5225 ; AVX2-NEXT: vpshufb %xmm14, %xmm0, %xmm15
5226 ; AVX2-NEXT: vmovdqa 160(%rdi), %xmm0
5227 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5228 ; AVX2-NEXT: vpshufb %xmm14, %xmm0, %xmm14
5229 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
5230 ; AVX2-NEXT: vmovdqa 144(%rdi), %xmm0
5231 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5232 ; AVX2-NEXT: vpshufb %xmm3, %xmm0, %xmm0
5233 ; AVX2-NEXT: vmovdqa 128(%rdi), %xmm13
5234 ; AVX2-NEXT: vpshufb %xmm3, %xmm13, %xmm3
5235 ; AVX2-NEXT: vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5236 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
5237 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
5238 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
5239 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
5240 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
5241 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm0[4,5,6,7]
5242 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
5243 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm8 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
5244 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5245 ; AVX2-NEXT: vpshufb %xmm8, %xmm0, %xmm1
5246 ; AVX2-NEXT: vpshufb %xmm8, %xmm4, %xmm2
5247 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
5248 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
5249 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
5250 ; AVX2-NEXT: vpshufb %xmm2, %xmm6, %xmm3
5251 ; AVX2-NEXT: vpshufb %xmm2, %xmm5, %xmm4
5252 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5253 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
5254 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm3 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
5255 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
5256 ; AVX2-NEXT: vpshufb %xmm3, %xmm15, %xmm4
5257 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
5258 ; AVX2-NEXT: vpshufb %xmm3, %xmm9, %xmm5
5259 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
5260 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm5 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
5261 ; AVX2-NEXT: vpshufb %xmm5, %xmm7, %xmm0
5262 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
5263 ; AVX2-NEXT: vpshufb %xmm5, %xmm14, %xmm7
5264 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3]
5265 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm4[1],xmm0[2,3]
5266 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
5267 ; AVX2-NEXT: vmovdqa (%rsp), %xmm12 # 16-byte Reload
5268 ; AVX2-NEXT: vpshufb %xmm8, %xmm12, %xmm1
5269 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
5270 ; AVX2-NEXT: vpshufb %xmm8, %xmm10, %xmm4
5271 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
5272 ; AVX2-NEXT: vpshufb %xmm2, %xmm11, %xmm4
5273 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
5274 ; AVX2-NEXT: vpshufb %xmm2, %xmm11, %xmm2
5275 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
5276 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
5277 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5278 ; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
5279 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
5280 ; AVX2-NEXT: vpshufb %xmm3, %xmm2, %xmm2
5281 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5282 ; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm3
5283 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
5284 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
5285 ; AVX2-NEXT: vpshufb %xmm5, %xmm3, %xmm3
5286 ; AVX2-NEXT: vpshufb %xmm5, %xmm13, %xmm4
5287 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5288 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5289 ; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
5290 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
5291 ; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
5292 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
5293 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
5294 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm0 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
5295 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5296 ; AVX2-NEXT: vpshufb %xmm0, %xmm1, %xmm1
5297 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
5298 ; AVX2-NEXT: vpshufb %xmm0, %xmm13, %xmm2
5299 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
5300 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
5301 ; AVX2-NEXT: vpshufb %xmm2, %xmm6, %xmm3
5302 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
5303 ; AVX2-NEXT: vpshufb %xmm2, %xmm8, %xmm4
5304 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5305 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
5306 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm3 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
5307 ; AVX2-NEXT: vpshufb %xmm3, %xmm15, %xmm4
5308 ; AVX2-NEXT: vpshufb %xmm3, %xmm9, %xmm5
5309 ; AVX2-NEXT: vmovdqa %xmm9, %xmm15
5310 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
5311 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm5 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
5312 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
5313 ; AVX2-NEXT: vpshufb %xmm5, %xmm6, %xmm6
5314 ; AVX2-NEXT: vpshufb %xmm5, %xmm14, %xmm7
5315 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
5316 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0],xmm4[1],xmm6[2,3]
5317 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm4[0,1],xmm1[2,3]
5318 ; AVX2-NEXT: vpshufb %xmm0, %xmm12, %xmm4
5319 ; AVX2-NEXT: vpshufb %xmm0, %xmm10, %xmm0
5320 ; AVX2-NEXT: vmovdqa %xmm10, %xmm14
5321 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
5322 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
5323 ; AVX2-NEXT: vpshufb %xmm2, %xmm10, %xmm4
5324 ; AVX2-NEXT: vpshufb %xmm2, %xmm11, %xmm2
5325 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
5326 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
5327 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5328 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5,6],ymm0[7]
5329 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
5330 ; AVX2-NEXT: vpshufb %xmm3, %xmm12, %xmm2
5331 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5332 ; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm3
5333 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
5334 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
5335 ; AVX2-NEXT: vpshufb %xmm5, %xmm3, %xmm3
5336 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5337 ; AVX2-NEXT: vpshufb %xmm5, %xmm4, %xmm4
5338 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5339 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5340 ; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
5341 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
5342 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
5343 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
5344 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
5345 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm0 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
5346 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5347 ; AVX2-NEXT: vpshufb %xmm0, %xmm1, %xmm1
5348 ; AVX2-NEXT: vpshufb %xmm0, %xmm13, %xmm2
5349 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
5350 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
5351 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
5352 ; AVX2-NEXT: vpshufb %xmm2, %xmm3, %xmm3
5353 ; AVX2-NEXT: vpshufb %xmm2, %xmm8, %xmm4
5354 ; AVX2-NEXT: vmovdqa %xmm8, %xmm13
5355 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5356 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
5357 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm3 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
5358 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
5359 ; AVX2-NEXT: vpshufb %xmm3, %xmm9, %xmm4
5360 ; AVX2-NEXT: vpshufb %xmm3, %xmm15, %xmm5
5361 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
5362 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm5 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
5363 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
5364 ; AVX2-NEXT: vpshufb %xmm5, %xmm11, %xmm6
5365 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
5366 ; AVX2-NEXT: vpshufb %xmm5, %xmm15, %xmm7
5367 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
5368 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0],xmm4[1],xmm6[2,3]
5369 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm4[0,1],xmm1[2,3]
5370 ; AVX2-NEXT: vmovdqa (%rsp), %xmm4 # 16-byte Reload
5371 ; AVX2-NEXT: vpshufb %xmm0, %xmm4, %xmm4
5372 ; AVX2-NEXT: vpshufb %xmm0, %xmm14, %xmm0
5373 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
5374 ; AVX2-NEXT: vpshufb %xmm2, %xmm10, %xmm4
5375 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
5376 ; AVX2-NEXT: vpshufb %xmm2, %xmm8, %xmm2
5377 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
5378 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
5379 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5380 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5,6],ymm0[7]
5381 ; AVX2-NEXT: vpshufb %xmm3, %xmm12, %xmm2
5382 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
5383 ; AVX2-NEXT: vpshufb %xmm3, %xmm14, %xmm3
5384 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
5385 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
5386 ; AVX2-NEXT: vpshufb %xmm5, %xmm3, %xmm3
5387 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5388 ; AVX2-NEXT: vpshufb %xmm5, %xmm4, %xmm4
5389 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5390 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5391 ; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
5392 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
5393 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
5394 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
5395 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
5396 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm0 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
5397 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
5398 ; AVX2-NEXT: vpshufb %xmm0, %xmm10, %xmm1
5399 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
5400 ; AVX2-NEXT: vpshufb %xmm0, %xmm2, %xmm2
5401 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
5402 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
5403 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
5404 ; AVX2-NEXT: vpshufb %xmm2, %xmm12, %xmm3
5405 ; AVX2-NEXT: vpshufb %xmm2, %xmm13, %xmm4
5406 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5407 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
5408 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm3 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
5409 ; AVX2-NEXT: vpshufb %xmm3, %xmm9, %xmm4
5410 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
5411 ; AVX2-NEXT: vpshufb %xmm3, %xmm13, %xmm5
5412 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
5413 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm5 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
5414 ; AVX2-NEXT: vpshufb %xmm5, %xmm11, %xmm6
5415 ; AVX2-NEXT: vpshufb %xmm5, %xmm15, %xmm7
5416 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
5417 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0],xmm4[1],xmm6[2,3]
5418 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm4[0,1],xmm1[2,3]
5419 ; AVX2-NEXT: vmovdqa (%rsp), %xmm9 # 16-byte Reload
5420 ; AVX2-NEXT: vpshufb %xmm0, %xmm9, %xmm4
5421 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
5422 ; AVX2-NEXT: vpshufb %xmm0, %xmm6, %xmm0
5423 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
5424 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5425 ; AVX2-NEXT: vpshufb %xmm2, %xmm4, %xmm4
5426 ; AVX2-NEXT: vpshufb %xmm2, %xmm8, %xmm2
5427 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
5428 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
5429 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5430 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5,6],ymm0[7]
5431 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
5432 ; AVX2-NEXT: vpshufb %xmm3, %xmm15, %xmm2
5433 ; AVX2-NEXT: vpshufb %xmm3, %xmm14, %xmm3
5434 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
5435 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
5436 ; AVX2-NEXT: vpshufb %xmm5, %xmm14, %xmm3
5437 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5438 ; AVX2-NEXT: vpshufb %xmm5, %xmm4, %xmm4
5439 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5440 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5441 ; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
5442 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
5443 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
5444 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
5445 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
5446 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm0 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
5447 ; AVX2-NEXT: vpshufb %xmm0, %xmm10, %xmm1
5448 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
5449 ; AVX2-NEXT: vpshufb %xmm0, %xmm8, %xmm2
5450 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
5451 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
5452 ; AVX2-NEXT: vmovdqa %xmm12, %xmm11
5453 ; AVX2-NEXT: vpshufb %xmm2, %xmm12, %xmm3
5454 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5455 ; AVX2-NEXT: vpshufb %xmm2, %xmm4, %xmm4
5456 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5457 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
5458 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm3 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
5459 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5460 ; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm4
5461 ; AVX2-NEXT: vpshufb %xmm3, %xmm13, %xmm5
5462 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
5463 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm5 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
5464 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
5465 ; AVX2-NEXT: vpshufb %xmm5, %xmm6, %xmm6
5466 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
5467 ; AVX2-NEXT: vpshufb %xmm5, %xmm7, %xmm7
5468 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
5469 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0],xmm4[1],xmm6[2,3]
5470 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm4[0,1],xmm1[2,3]
5471 ; AVX2-NEXT: vpshufb %xmm0, %xmm9, %xmm4
5472 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
5473 ; AVX2-NEXT: vpshufb %xmm0, %xmm9, %xmm0
5474 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
5475 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
5476 ; AVX2-NEXT: vpshufb %xmm2, %xmm13, %xmm4
5477 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
5478 ; AVX2-NEXT: vpshufb %xmm2, %xmm12, %xmm2
5479 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
5480 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
5481 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5482 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5,6],ymm0[7]
5483 ; AVX2-NEXT: vpshufb %xmm3, %xmm15, %xmm2
5484 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5485 ; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm3
5486 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
5487 ; AVX2-NEXT: vpshufb %xmm5, %xmm14, %xmm3
5488 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
5489 ; AVX2-NEXT: vpshufb %xmm5, %xmm10, %xmm4
5490 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5491 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5492 ; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
5493 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
5494 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
5495 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
5496 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
5497 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm0 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
5498 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5499 ; AVX2-NEXT: vpshufb %xmm0, %xmm1, %xmm1
5500 ; AVX2-NEXT: vpshufb %xmm0, %xmm8, %xmm2
5501 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
5502 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
5503 ; AVX2-NEXT: vpshufb %xmm2, %xmm11, %xmm3
5504 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
5505 ; AVX2-NEXT: vpshufb %xmm2, %xmm14, %xmm4
5506 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5507 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
5508 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm3 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
5509 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
5510 ; AVX2-NEXT: vpshufb %xmm3, %xmm11, %xmm4
5511 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
5512 ; AVX2-NEXT: vpshufb %xmm3, %xmm5, %xmm5
5513 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
5514 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm5 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
5515 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
5516 ; AVX2-NEXT: vpshufb %xmm5, %xmm6, %xmm6
5517 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
5518 ; AVX2-NEXT: vpshufb %xmm5, %xmm8, %xmm7
5519 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
5520 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0],xmm4[1],xmm6[2,3]
5521 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm4[0,1],xmm1[2,3]
5522 ; AVX2-NEXT: vmovdqa (%rsp), %xmm15 # 16-byte Reload
5523 ; AVX2-NEXT: vpshufb %xmm0, %xmm15, %xmm4
5524 ; AVX2-NEXT: vpshufb %xmm0, %xmm9, %xmm0
5525 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
5526 ; AVX2-NEXT: vpshufb %xmm2, %xmm13, %xmm4
5527 ; AVX2-NEXT: vpshufb %xmm2, %xmm12, %xmm2
5528 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
5529 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
5530 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5531 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5,6],ymm0[7]
5532 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
5533 ; AVX2-NEXT: vpshufb %xmm3, %xmm13, %xmm2
5534 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
5535 ; AVX2-NEXT: vpshufb %xmm3, %xmm9, %xmm3
5536 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
5537 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
5538 ; AVX2-NEXT: vpshufb %xmm5, %xmm12, %xmm3
5539 ; AVX2-NEXT: vpshufb %xmm5, %xmm10, %xmm4
5540 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5541 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5542 ; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
5543 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
5544 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
5545 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
5546 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
5547 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm0 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
5548 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5549 ; AVX2-NEXT: vpshufb %xmm0, %xmm1, %xmm1
5550 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
5551 ; AVX2-NEXT: vpshufb %xmm0, %xmm2, %xmm2
5552 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
5553 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
5554 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
5555 ; AVX2-NEXT: vpshufb %xmm2, %xmm3, %xmm3
5556 ; AVX2-NEXT: vpshufb %xmm2, %xmm14, %xmm4
5557 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5558 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
5559 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm3 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
5560 ; AVX2-NEXT: vpshufb %xmm3, %xmm11, %xmm4
5561 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
5562 ; AVX2-NEXT: vpshufb %xmm3, %xmm5, %xmm5
5563 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
5564 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm5 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
5565 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
5566 ; AVX2-NEXT: vpshufb %xmm5, %xmm6, %xmm6
5567 ; AVX2-NEXT: vpshufb %xmm5, %xmm8, %xmm7
5568 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
5569 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0],xmm4[1],xmm6[2,3]
5570 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm4[0,1],xmm1[2,3]
5571 ; AVX2-NEXT: vpshufb %xmm0, %xmm15, %xmm4
5572 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
5573 ; AVX2-NEXT: vpshufb %xmm0, %xmm6, %xmm0
5574 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
5575 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5576 ; AVX2-NEXT: vpshufb %xmm2, %xmm4, %xmm4
5577 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
5578 ; AVX2-NEXT: vpshufb %xmm2, %xmm6, %xmm2
5579 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
5580 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
5581 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5582 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5,6],ymm0[7]
5583 ; AVX2-NEXT: vpshufb %xmm3, %xmm13, %xmm2
5584 ; AVX2-NEXT: vpshufb %xmm3, %xmm9, %xmm3
5585 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
5586 ; AVX2-NEXT: vpshufb %xmm5, %xmm12, %xmm3
5587 ; AVX2-NEXT: vpshufb %xmm5, %xmm10, %xmm4
5588 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5589 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5590 ; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
5591 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
5592 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
5593 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
5594 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
5595 ; AVX2-NEXT: vmovaps %ymm1, (%rsi)
5596 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
5597 ; AVX2-NEXT: vmovaps %ymm1, (%rdx)
5598 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
5599 ; AVX2-NEXT: vmovaps %ymm1, (%rcx)
5600 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
5601 ; AVX2-NEXT: vmovaps %ymm1, (%r8)
5602 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
5603 ; AVX2-NEXT: vmovaps %ymm1, (%r9)
5604 ; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
5605 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
5606 ; AVX2-NEXT: vmovaps %ymm1, (%rax)
5607 ; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
5608 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
5609 ; AVX2-NEXT: vmovaps %ymm1, (%rax)
5610 ; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
5611 ; AVX2-NEXT: vmovdqa %ymm0, (%rax)
5612 ; AVX2-NEXT: addq $360, %rsp # imm = 0x168
5613 ; AVX2-NEXT: vzeroupper
5614 ; AVX2-NEXT: retq
5615 ;
5616 ; AVX2-FP-LABEL: load_i8_stride8_vf32:
5617 ; AVX2-FP: # %bb.0:
5618 ; AVX2-FP-NEXT: subq $360, %rsp # imm = 0x168
5619 ; AVX2-FP-NEXT: vmovdqa 112(%rdi), %xmm0
5620 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5621 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
5622 ; AVX2-FP-NEXT: vpshufb %xmm10, %xmm0, %xmm0
5623 ; AVX2-FP-NEXT: vmovdqa 96(%rdi), %xmm4
5624 ; AVX2-FP-NEXT: vpshufb %xmm10, %xmm4, %xmm2
5625 ; AVX2-FP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5626 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
5627 ; AVX2-FP-NEXT: vmovdqa 80(%rdi), %xmm1
5628 ; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5629 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm12 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
5630 ; AVX2-FP-NEXT: vpshufb %xmm12, %xmm1, %xmm2
5631 ; AVX2-FP-NEXT: vmovdqa 64(%rdi), %xmm5
5632 ; AVX2-FP-NEXT: vpshufb %xmm12, %xmm5, %xmm6
5633 ; AVX2-FP-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5634 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
5635 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm8 = xmm2[0,1,2],xmm0[3]
5636 ; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm0
5637 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5638 ; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm7
5639 ; AVX2-FP-NEXT: vmovdqa 32(%rdi), %xmm2
5640 ; AVX2-FP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5641 ; AVX2-FP-NEXT: vmovdqa 48(%rdi), %xmm3
5642 ; AVX2-FP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5643 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm14 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
5644 ; AVX2-FP-NEXT: vpshufb %xmm14, %xmm3, %xmm9
5645 ; AVX2-FP-NEXT: vpshufb %xmm14, %xmm2, %xmm11
5646 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
5647 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
5648 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm7, %xmm11
5649 ; AVX2-FP-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5650 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm0, %xmm13
5651 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1],xmm13[2],xmm11[2],xmm13[3],xmm11[3]
5652 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm9 = xmm11[0],xmm9[1],xmm11[2,3]
5653 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
5654 ; AVX2-FP-NEXT: vmovdqa 240(%rdi), %xmm0
5655 ; AVX2-FP-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill
5656 ; AVX2-FP-NEXT: vpshufb %xmm10, %xmm0, %xmm11
5657 ; AVX2-FP-NEXT: vmovdqa 224(%rdi), %xmm0
5658 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5659 ; AVX2-FP-NEXT: vpshufb %xmm10, %xmm0, %xmm10
5660 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
5661 ; AVX2-FP-NEXT: vmovdqa 208(%rdi), %xmm11
5662 ; AVX2-FP-NEXT: vpshufb %xmm12, %xmm11, %xmm15
5663 ; AVX2-FP-NEXT: vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5664 ; AVX2-FP-NEXT: vmovdqa 192(%rdi), %xmm0
5665 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5666 ; AVX2-FP-NEXT: vpshufb %xmm12, %xmm0, %xmm12
5667 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm15[0],xmm12[1],xmm15[1],xmm12[2],xmm15[2],xmm12[3],xmm15[3]
5668 ; AVX2-FP-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
5669 ; AVX2-FP-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm12
5670 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm12[0,1,2,3,4,5,6],ymm13[7]
5671 ; AVX2-FP-NEXT: vmovdqa 176(%rdi), %xmm0
5672 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5673 ; AVX2-FP-NEXT: vpshufb %xmm14, %xmm0, %xmm15
5674 ; AVX2-FP-NEXT: vmovdqa 160(%rdi), %xmm0
5675 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5676 ; AVX2-FP-NEXT: vpshufb %xmm14, %xmm0, %xmm14
5677 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
5678 ; AVX2-FP-NEXT: vmovdqa 144(%rdi), %xmm0
5679 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5680 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm0, %xmm0
5681 ; AVX2-FP-NEXT: vmovdqa 128(%rdi), %xmm13
5682 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm13, %xmm3
5683 ; AVX2-FP-NEXT: vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5684 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
5685 ; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
5686 ; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
5687 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
5688 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
5689 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm0[4,5,6,7]
5690 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
5691 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm8 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
5692 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5693 ; AVX2-FP-NEXT: vpshufb %xmm8, %xmm0, %xmm1
5694 ; AVX2-FP-NEXT: vpshufb %xmm8, %xmm4, %xmm2
5695 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
5696 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
5697 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
5698 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm6, %xmm3
5699 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm5, %xmm4
5700 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5701 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
5702 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
5703 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
5704 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm15, %xmm4
5705 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
5706 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm9, %xmm5
5707 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
5708 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm5 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
5709 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm7, %xmm0
5710 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
5711 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm14, %xmm7
5712 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3]
5713 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm4[1],xmm0[2,3]
5714 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
5715 ; AVX2-FP-NEXT: vmovdqa (%rsp), %xmm12 # 16-byte Reload
5716 ; AVX2-FP-NEXT: vpshufb %xmm8, %xmm12, %xmm1
5717 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
5718 ; AVX2-FP-NEXT: vpshufb %xmm8, %xmm10, %xmm4
5719 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
5720 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm11, %xmm4
5721 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
5722 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm11, %xmm2
5723 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
5724 ; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
5725 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5726 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
5727 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
5728 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
5729 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5730 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm4, %xmm3
5731 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
5732 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
5733 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm3, %xmm3
5734 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm13, %xmm4
5735 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5736 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5737 ; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
5738 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
5739 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
5740 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
5741 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
5742 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
5743 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5744 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm1, %xmm1
5745 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
5746 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm13, %xmm2
5747 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
5748 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
5749 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm6, %xmm3
5750 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
5751 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm8, %xmm4
5752 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5753 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
5754 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
5755 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm15, %xmm4
5756 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm9, %xmm5
5757 ; AVX2-FP-NEXT: vmovdqa %xmm9, %xmm15
5758 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
5759 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm5 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
5760 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
5761 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm6, %xmm6
5762 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm14, %xmm7
5763 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
5764 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0],xmm4[1],xmm6[2,3]
5765 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm4[0,1],xmm1[2,3]
5766 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm12, %xmm4
5767 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm10, %xmm0
5768 ; AVX2-FP-NEXT: vmovdqa %xmm10, %xmm14
5769 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
5770 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
5771 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm10, %xmm4
5772 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm11, %xmm2
5773 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
5774 ; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
5775 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5776 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5,6],ymm0[7]
5777 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
5778 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm12, %xmm2
5779 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5780 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm4, %xmm3
5781 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
5782 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
5783 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm3, %xmm3
5784 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5785 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm4, %xmm4
5786 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5787 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5788 ; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
5789 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
5790 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
5791 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
5792 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
5793 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
5794 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5795 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm1, %xmm1
5796 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm13, %xmm2
5797 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
5798 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
5799 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
5800 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm3, %xmm3
5801 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm8, %xmm4
5802 ; AVX2-FP-NEXT: vmovdqa %xmm8, %xmm13
5803 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5804 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
5805 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
5806 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
5807 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm9, %xmm4
5808 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm15, %xmm5
5809 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
5810 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm5 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
5811 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
5812 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm11, %xmm6
5813 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
5814 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm15, %xmm7
5815 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
5816 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0],xmm4[1],xmm6[2,3]
5817 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm4[0,1],xmm1[2,3]
5818 ; AVX2-FP-NEXT: vmovdqa (%rsp), %xmm4 # 16-byte Reload
5819 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm4, %xmm4
5820 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm14, %xmm0
5821 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
5822 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm10, %xmm4
5823 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
5824 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm8, %xmm2
5825 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
5826 ; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
5827 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5828 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5,6],ymm0[7]
5829 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm12, %xmm2
5830 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
5831 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm14, %xmm3
5832 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
5833 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
5834 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm3, %xmm3
5835 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5836 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm4, %xmm4
5837 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5838 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5839 ; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
5840 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
5841 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
5842 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
5843 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
5844 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
5845 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
5846 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm10, %xmm1
5847 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
5848 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm2, %xmm2
5849 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
5850 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
5851 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
5852 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm12, %xmm3
5853 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm13, %xmm4
5854 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5855 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
5856 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
5857 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm9, %xmm4
5858 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
5859 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm13, %xmm5
5860 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
5861 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm5 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
5862 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm11, %xmm6
5863 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm15, %xmm7
5864 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
5865 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0],xmm4[1],xmm6[2,3]
5866 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm4[0,1],xmm1[2,3]
5867 ; AVX2-FP-NEXT: vmovdqa (%rsp), %xmm9 # 16-byte Reload
5868 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm9, %xmm4
5869 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
5870 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm6, %xmm0
5871 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
5872 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5873 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm4, %xmm4
5874 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm8, %xmm2
5875 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
5876 ; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
5877 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5878 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5,6],ymm0[7]
5879 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
5880 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm15, %xmm2
5881 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm14, %xmm3
5882 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
5883 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
5884 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm14, %xmm3
5885 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5886 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm4, %xmm4
5887 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5888 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5889 ; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
5890 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
5891 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
5892 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
5893 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
5894 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
5895 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm10, %xmm1
5896 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
5897 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm8, %xmm2
5898 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
5899 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
5900 ; AVX2-FP-NEXT: vmovdqa %xmm12, %xmm11
5901 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm12, %xmm3
5902 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5903 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm4, %xmm4
5904 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5905 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
5906 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
5907 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5908 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm4, %xmm4
5909 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm13, %xmm5
5910 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
5911 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm5 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
5912 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
5913 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm6, %xmm6
5914 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
5915 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm7, %xmm7
5916 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
5917 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0],xmm4[1],xmm6[2,3]
5918 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm4[0,1],xmm1[2,3]
5919 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm9, %xmm4
5920 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
5921 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm9, %xmm0
5922 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
5923 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
5924 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm13, %xmm4
5925 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
5926 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm12, %xmm2
5927 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
5928 ; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
5929 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5930 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5,6],ymm0[7]
5931 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm15, %xmm2
5932 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5933 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm4, %xmm3
5934 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
5935 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm14, %xmm3
5936 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
5937 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm10, %xmm4
5938 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5939 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5940 ; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
5941 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
5942 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
5943 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
5944 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
5945 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
5946 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5947 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm1, %xmm1
5948 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm8, %xmm2
5949 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
5950 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
5951 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm11, %xmm3
5952 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
5953 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm14, %xmm4
5954 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5955 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
5956 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
5957 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
5958 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm11, %xmm4
5959 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
5960 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm5, %xmm5
5961 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
5962 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm5 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
5963 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
5964 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm6, %xmm6
5965 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
5966 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm8, %xmm7
5967 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
5968 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0],xmm4[1],xmm6[2,3]
5969 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm4[0,1],xmm1[2,3]
5970 ; AVX2-FP-NEXT: vmovdqa (%rsp), %xmm15 # 16-byte Reload
5971 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm15, %xmm4
5972 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm9, %xmm0
5973 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
5974 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm13, %xmm4
5975 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm12, %xmm2
5976 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
5977 ; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
5978 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5979 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5,6],ymm0[7]
5980 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
5981 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm13, %xmm2
5982 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
5983 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm9, %xmm3
5984 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
5985 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
5986 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm12, %xmm3
5987 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm10, %xmm4
5988 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
5989 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5990 ; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
5991 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
5992 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
5993 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
5994 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
5995 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
5996 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5997 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm1, %xmm1
5998 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
5999 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm2, %xmm2
6000 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
6001 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
6002 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
6003 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm3, %xmm3
6004 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm14, %xmm4
6005 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
6006 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
6007 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
6008 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm11, %xmm4
6009 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
6010 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm5, %xmm5
6011 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
6012 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm5 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
6013 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
6014 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm6, %xmm6
6015 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm8, %xmm7
6016 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
6017 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0],xmm4[1],xmm6[2,3]
6018 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm4[0,1],xmm1[2,3]
6019 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm15, %xmm4
6020 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
6021 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm6, %xmm0
6022 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
6023 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
6024 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm4, %xmm4
6025 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
6026 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm6, %xmm2
6027 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
6028 ; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
6029 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
6030 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5,6],ymm0[7]
6031 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm13, %xmm2
6032 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm9, %xmm3
6033 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
6034 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm12, %xmm3
6035 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm10, %xmm4
6036 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
6037 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
6038 ; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
6039 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
6040 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
6041 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
6042 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
6043 ; AVX2-FP-NEXT: vmovaps %ymm1, (%rsi)
6044 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
6045 ; AVX2-FP-NEXT: vmovaps %ymm1, (%rdx)
6046 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
6047 ; AVX2-FP-NEXT: vmovaps %ymm1, (%rcx)
6048 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
6049 ; AVX2-FP-NEXT: vmovaps %ymm1, (%r8)
6050 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
6051 ; AVX2-FP-NEXT: vmovaps %ymm1, (%r9)
6052 ; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %rax
6053 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
6054 ; AVX2-FP-NEXT: vmovaps %ymm1, (%rax)
6055 ; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %rax
6056 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
6057 ; AVX2-FP-NEXT: vmovaps %ymm1, (%rax)
6058 ; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %rax
6059 ; AVX2-FP-NEXT: vmovdqa %ymm0, (%rax)
6060 ; AVX2-FP-NEXT: addq $360, %rsp # imm = 0x168
6061 ; AVX2-FP-NEXT: vzeroupper
6062 ; AVX2-FP-NEXT: retq
6063 ;
6064 ; AVX2-FCP-LABEL: load_i8_stride8_vf32:
6065 ; AVX2-FCP: # %bb.0:
6066 ; AVX2-FCP-NEXT: subq $248, %rsp
6067 ; AVX2-FCP-NEXT: vmovdqa 192(%rdi), %ymm4
6068 ; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6069 ; AVX2-FCP-NEXT: vmovdqa 224(%rdi), %ymm7
6070 ; AVX2-FCP-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6071 ; AVX2-FCP-NEXT: vmovdqa 112(%rdi), %xmm8
6072 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
6073 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm8, %xmm1
6074 ; AVX2-FCP-NEXT: vmovdqa 96(%rdi), %xmm13
6075 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm13, %xmm0
6076 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
6077 ; AVX2-FCP-NEXT: vmovdqa 80(%rdi), %xmm10
6078 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
6079 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm10, %xmm2
6080 ; AVX2-FCP-NEXT: vmovdqa 64(%rdi), %xmm3
6081 ; AVX2-FCP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6082 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm3, %xmm1
6083 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
6084 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
6085 ; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm5
6086 ; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm3
6087 ; AVX2-FCP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6088 ; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %xmm6
6089 ; AVX2-FCP-NEXT: vmovdqa 48(%rdi), %xmm15
6090 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
6091 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm15, %xmm9
6092 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm6, %xmm1
6093 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm9[0],xmm1[1],xmm9[1],xmm1[2],xmm9[2],xmm1[3],xmm9[3]
6094 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm9 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
6095 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm3, %xmm11
6096 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm5, %xmm9
6097 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1],xmm9[2],xmm11[2],xmm9[3],xmm11[3]
6098 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm9[0],xmm1[1],xmm9[2,3]
6099 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm3 = xmm1[0,1],xmm0[2,3]
6100 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,2,2,3,0,2,4,6]
6101 ; AVX2-FCP-NEXT: vpermd %ymm7, %ymm1, %ymm9
6102 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28]
6103 ; AVX2-FCP-NEXT: vpermd %ymm4, %ymm1, %ymm0
6104 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6105 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm11 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u]
6106 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm11[0,1,2,3,4,5,6],ymm2[7]
6107 ; AVX2-FCP-NEXT: vmovdqa 128(%rdi), %ymm0
6108 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6109 ; AVX2-FCP-NEXT: vmovdqa 160(%rdi), %ymm4
6110 ; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6111 ; AVX2-FCP-NEXT: vpermd %ymm4, %ymm1, %ymm11
6112 ; AVX2-FCP-NEXT: vpermd %ymm0, %ymm1, %ymm14
6113 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u]
6114 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u,u,u,u,u]
6115 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm12[0,1,2,3,4],ymm0[5],ymm12[6,7]
6116 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
6117 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
6118 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6119 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
6120 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm8, %xmm2
6121 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm13, %xmm0
6122 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
6123 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
6124 ; AVX2-FCP-NEXT: vmovdqa %xmm10, %xmm4
6125 ; AVX2-FCP-NEXT: vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6126 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm10, %xmm3
6127 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6128 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm1, %xmm2
6129 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
6130 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
6131 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
6132 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm15, %xmm3
6133 ; AVX2-FCP-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6134 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm6, %xmm2
6135 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
6136 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
6137 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
6138 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm7, %xmm12
6139 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm5, %xmm3
6140 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm12[0],xmm3[1],xmm12[1],xmm3[2],xmm12[2],xmm3[3],xmm12[3]
6141 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2,3]
6142 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
6143 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29]
6144 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
6145 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u]
6146 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
6147 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u,u,u,u,u]
6148 ; AVX2-FCP-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6149 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u,u,u,u,u,u,u,u,u]
6150 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm12[0,1,2,3,4],ymm3[5],ymm12[6,7]
6151 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
6152 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
6153 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6154 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
6155 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm8, %xmm2
6156 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm13, %xmm0
6157 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
6158 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
6159 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm4, %xmm3
6160 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm1, %xmm2
6161 ; AVX2-FCP-NEXT: vmovdqa %xmm1, %xmm4
6162 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
6163 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
6164 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
6165 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm15, %xmm3
6166 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm6, %xmm2
6167 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
6168 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
6169 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm7, %xmm12
6170 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm5, %xmm3
6171 ; AVX2-FCP-NEXT: vmovdqa %xmm5, %xmm1
6172 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm12[0],xmm3[1],xmm12[1],xmm3[2],xmm12[2],xmm3[3],xmm12[3]
6173 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2,3]
6174 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
6175 ; AVX2-FCP-NEXT: vmovdqa %ymm9, %ymm7
6176 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,22,26,30]
6177 ; AVX2-FCP-NEXT: vmovdqa %ymm10, %ymm9
6178 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,22,26,30,u,u,u,u]
6179 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
6180 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,22,26,30,u,u,u,u,u,u,u,u]
6181 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,22,26,30,u,u,u,u,u,u,u,u,u,u,u,u]
6182 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm12[0,1,2,3,4],ymm3[5],ymm12[6,7]
6183 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
6184 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
6185 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6186 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
6187 ; AVX2-FCP-NEXT: vmovdqa %xmm8, %xmm5
6188 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm8, %xmm2
6189 ; AVX2-FCP-NEXT: vmovdqa %xmm13, %xmm8
6190 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm13, %xmm0
6191 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
6192 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
6193 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
6194 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm6, %xmm3
6195 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm4, %xmm2
6196 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
6197 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
6198 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
6199 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm15, %xmm3
6200 ; AVX2-FCP-NEXT: vmovdqa %xmm15, %xmm10
6201 ; AVX2-FCP-NEXT: vmovdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6202 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
6203 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm11, %xmm2
6204 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
6205 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
6206 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
6207 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm13, %xmm12
6208 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm3
6209 ; AVX2-FCP-NEXT: vmovdqa %xmm1, %xmm4
6210 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm12[0],xmm3[1],xmm12[1],xmm3[2],xmm12[2],xmm3[3],xmm12[3]
6211 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2,3]
6212 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
6213 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,19,23,27,31]
6214 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,19,23,27,31,u,u,u,u]
6215 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
6216 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
6217 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,19,23,27,31,u,u,u,u,u,u,u,u]
6218 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,19,23,27,31,u,u,u,u,u,u,u,u,u,u,u,u]
6219 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5],ymm1[6,7]
6220 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
6221 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
6222 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6223 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
6224 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm5, %xmm1
6225 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm8, %xmm0
6226 ; AVX2-FCP-NEXT: vmovdqa %xmm8, %xmm9
6227 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
6228 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
6229 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm6, %xmm2
6230 ; AVX2-FCP-NEXT: vmovdqa %xmm6, %xmm8
6231 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
6232 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm15, %xmm1
6233 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
6234 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
6235 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
6236 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm10, %xmm2
6237 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm11, %xmm1
6238 ; AVX2-FCP-NEXT: vmovdqa %xmm11, %xmm14
6239 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
6240 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
6241 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm13, %xmm3
6242 ; AVX2-FCP-NEXT: vmovdqa %xmm13, %xmm6
6243 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm4, %xmm2
6244 ; AVX2-FCP-NEXT: vmovdqa %xmm4, %xmm10
6245 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
6246 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2,3]
6247 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,3]
6248 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm2 = [1,3,2,3,1,3,5,7]
6249 ; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm7 # 32-byte Folded Reload
6250 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28]
6251 ; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm12 # 32-byte Folded Reload
6252 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u]
6253 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5,6],ymm3[7]
6254 ; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm11 # 32-byte Folded Reload
6255 ; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm13 # 32-byte Folded Reload
6256 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u]
6257 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u,u,u,u,u]
6258 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4],ymm2[5],ymm4[6,7]
6259 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
6260 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm2[4,5,6,7]
6261 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6262 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
6263 ; AVX2-FCP-NEXT: vmovdqa %xmm5, %xmm0
6264 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm5, %xmm2
6265 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm9, %xmm1
6266 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
6267 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
6268 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm8, %xmm3
6269 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm15, %xmm2
6270 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
6271 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
6272 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
6273 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
6274 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm5, %xmm3
6275 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm14, %xmm2
6276 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
6277 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
6278 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm6, %xmm4
6279 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm10, %xmm3
6280 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
6281 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2,3]
6282 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
6283 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29]
6284 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u]
6285 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
6286 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u,u,u,u,u]
6287 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u,u,u,u,u,u,u,u,u]
6288 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4],ymm3[5],ymm4[6,7]
6289 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
6290 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
6291 ; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6292 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
6293 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm0, %xmm3
6294 ; AVX2-FCP-NEXT: vmovdqa %xmm0, %xmm2
6295 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm9, %xmm1
6296 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
6297 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
6298 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm8, %xmm4
6299 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm15, %xmm3
6300 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
6301 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
6302 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
6303 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm5, %xmm4
6304 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm14, %xmm3
6305 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
6306 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm4 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
6307 ; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm6, %xmm0
6308 ; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm10, %xmm4
6309 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
6310 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2,3]
6311 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
6312 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,22,26,30]
6313 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,22,26,30,u,u,u,u]
6314 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
6315 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,22,26,30,u,u,u,u,u,u,u,u]
6316 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,22,26,30,u,u,u,u,u,u,u,u,u,u,u,u]
6317 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4],ymm3[5],ymm4[6,7]
6318 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
6319 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
6320 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
6321 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm2, %xmm3
6322 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm9, %xmm0
6323 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
6324 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
6325 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm8, %xmm4
6326 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm15, %xmm3
6327 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
6328 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[3]
6329 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
6330 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm5, %xmm4
6331 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm14, %xmm3
6332 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
6333 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm4 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
6334 ; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm6, %xmm6
6335 ; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm10, %xmm4
6336 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
6337 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3]
6338 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
6339 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,19,23,27,31]
6340 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,19,23,27,31,u,u,u,u]
6341 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5,6],ymm3[7]
6342 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,19,23,27,31,u,u,u,u,u,u,u,u]
6343 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm5 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,19,23,27,31,u,u,u,u,u,u,u,u,u,u,u,u]
6344 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
6345 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
6346 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]
6347 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
6348 ; AVX2-FCP-NEXT: vmovaps %ymm2, (%rsi)
6349 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
6350 ; AVX2-FCP-NEXT: vmovaps %ymm2, (%rdx)
6351 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
6352 ; AVX2-FCP-NEXT: vmovaps %ymm2, (%rcx)
6353 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
6354 ; AVX2-FCP-NEXT: vmovaps %ymm2, (%r8)
6355 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
6356 ; AVX2-FCP-NEXT: vmovaps %ymm2, (%r9)
6357 ; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
6358 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
6359 ; AVX2-FCP-NEXT: vmovaps %ymm2, (%rax)
6360 ; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
6361 ; AVX2-FCP-NEXT: vmovdqa %ymm1, (%rax)
6362 ; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
6363 ; AVX2-FCP-NEXT: vmovdqa %ymm0, (%rax)
6364 ; AVX2-FCP-NEXT: addq $248, %rsp
6365 ; AVX2-FCP-NEXT: vzeroupper
6366 ; AVX2-FCP-NEXT: retq
6367 ;
6368 ; AVX512-LABEL: load_i8_stride8_vf32:
6369 ; AVX512: # %bb.0:
6370 ; AVX512-NEXT: vmovdqa64 (%rdi), %zmm16
6371 ; AVX512-NEXT: vmovdqa64 128(%rdi), %zmm0
6372 ; AVX512-NEXT: vpmovqb %zmm0, %xmm0
6373 ; AVX512-NEXT: vmovdqa 240(%rdi), %xmm12
6374 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm3 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
6375 ; AVX512-NEXT: vpshufb %xmm3, %xmm12, %xmm1
6376 ; AVX512-NEXT: vmovdqa 224(%rdi), %xmm7
6377 ; AVX512-NEXT: vpshufb %xmm3, %xmm7, %xmm2
6378 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
6379 ; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
6380 ; AVX512-NEXT: vmovdqa 208(%rdi), %xmm9
6381 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm4 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
6382 ; AVX512-NEXT: vpshufb %xmm4, %xmm9, %xmm2
6383 ; AVX512-NEXT: vmovdqa 192(%rdi), %xmm8
6384 ; AVX512-NEXT: vpshufb %xmm4, %xmm8, %xmm6
6385 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
6386 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
6387 ; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
6388 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
6389 ; AVX512-NEXT: vmovdqa 128(%rdi), %ymm2
6390 ; AVX512-NEXT: vpmovqb %ymm2, %xmm2
6391 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
6392 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5],ymm2[6,7]
6393 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
6394 ; AVX512-NEXT: vmovdqa 112(%rdi), %xmm5
6395 ; AVX512-NEXT: vpshufb %xmm3, %xmm5, %xmm6
6396 ; AVX512-NEXT: vmovdqa 96(%rdi), %xmm1
6397 ; AVX512-NEXT: vpshufb %xmm3, %xmm1, %xmm3
6398 ; AVX512-NEXT: vmovdqa64 %xmm1, %xmm18
6399 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3]
6400 ; AVX512-NEXT: vmovdqa 80(%rdi), %xmm1
6401 ; AVX512-NEXT: vpshufb %xmm4, %xmm1, %xmm11
6402 ; AVX512-NEXT: vmovdqa64 %xmm1, %xmm21
6403 ; AVX512-NEXT: vmovdqa 64(%rdi), %xmm3
6404 ; AVX512-NEXT: vpshufb %xmm4, %xmm3, %xmm4
6405 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm11[0],xmm4[1],xmm11[1],xmm4[2],xmm11[2],xmm4[3],xmm11[3]
6406 ; AVX512-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0,1,2],xmm10[3]
6407 ; AVX512-NEXT: vpmovqb %zmm16, %xmm10
6408 ; AVX512-NEXT: vpblendd {{.*#+}} xmm4 = xmm10[0,1],xmm4[2,3]
6409 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
6410 ; AVX512-NEXT: vmovdqa64 %ymm0, %ymm17
6411 ; AVX512-NEXT: vmovdqa 160(%rdi), %xmm10
6412 ; AVX512-NEXT: vmovdqa 176(%rdi), %xmm6
6413 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
6414 ; AVX512-NEXT: vpshufb %xmm2, %xmm12, %xmm4
6415 ; AVX512-NEXT: vmovdqa64 %xmm12, %xmm19
6416 ; AVX512-NEXT: vpshufb %xmm2, %xmm7, %xmm12
6417 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm12[0],xmm4[0],xmm12[1],xmm4[1],xmm12[2],xmm4[2],xmm12[3],xmm4[3]
6418 ; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
6419 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm14 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
6420 ; AVX512-NEXT: vpshufb %xmm14, %xmm9, %xmm12
6421 ; AVX512-NEXT: vmovdqa64 %xmm9, %xmm22
6422 ; AVX512-NEXT: vpshufb %xmm14, %xmm8, %xmm13
6423 ; AVX512-NEXT: vmovdqa64 %xmm8, %xmm20
6424 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
6425 ; AVX512-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm12
6426 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm12[0,1,2,3,4,5,6],ymm4[7]
6427 ; AVX512-NEXT: vmovd {{.*#+}} xmm12 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
6428 ; AVX512-NEXT: vpshufb %xmm12, %xmm6, %xmm13
6429 ; AVX512-NEXT: vpshufb %xmm12, %xmm10, %xmm12
6430 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
6431 ; AVX512-NEXT: vmovdqa 128(%rdi), %xmm8
6432 ; AVX512-NEXT: vmovdqa 144(%rdi), %xmm13
6433 ; AVX512-NEXT: vmovd {{.*#+}} xmm1 = [1,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
6434 ; AVX512-NEXT: vpshufb %xmm1, %xmm13, %xmm0
6435 ; AVX512-NEXT: vpshufb %xmm1, %xmm8, %xmm1
6436 ; AVX512-NEXT: vmovdqa64 %xmm8, %xmm26
6437 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
6438 ; AVX512-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm1
6439 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
6440 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
6441 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm4[6,7]
6442 ; AVX512-NEXT: vpshufb %xmm2, %xmm5, %xmm1
6443 ; AVX512-NEXT: vmovdqa64 %xmm5, %xmm24
6444 ; AVX512-NEXT: vmovdqa64 %xmm18, %xmm5
6445 ; AVX512-NEXT: vpshufb %xmm2, %xmm5, %xmm2
6446 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
6447 ; AVX512-NEXT: vmovdqa64 %xmm21, %xmm4
6448 ; AVX512-NEXT: vpshufb %xmm14, %xmm4, %xmm2
6449 ; AVX512-NEXT: vpshufb %xmm14, %xmm3, %xmm4
6450 ; AVX512-NEXT: vmovdqa %xmm3, %xmm11
6451 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
6452 ; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
6453 ; AVX512-NEXT: vpsrlq $8, %zmm16, %zmm2
6454 ; AVX512-NEXT: vpmovqb %zmm2, %xmm2
6455 ; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
6456 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
6457 ; AVX512-NEXT: vmovdqa64 %ymm0, %ymm18
6458 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
6459 ; AVX512-NEXT: vmovdqa64 %xmm19, %xmm3
6460 ; AVX512-NEXT: vpshufb %xmm0, %xmm3, %xmm1
6461 ; AVX512-NEXT: vpshufb %xmm0, %xmm7, %xmm2
6462 ; AVX512-NEXT: vmovdqa %xmm7, %xmm9
6463 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
6464 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
6465 ; AVX512-NEXT: vmovdqa64 %xmm22, %xmm8
6466 ; AVX512-NEXT: vpshufb %xmm2, %xmm8, %xmm4
6467 ; AVX512-NEXT: vmovdqa64 %xmm20, %xmm7
6468 ; AVX512-NEXT: vpshufb %xmm2, %xmm7, %xmm14
6469 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm14[0],xmm4[0],xmm14[1],xmm4[1],xmm14[2],xmm4[2],xmm14[3],xmm4[3]
6470 ; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
6471 ; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
6472 ; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
6473 ; AVX512-NEXT: vmovd {{.*#+}} xmm4 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
6474 ; AVX512-NEXT: vpshufb %xmm4, %xmm6, %xmm14
6475 ; AVX512-NEXT: vmovdqa %xmm6, %xmm12
6476 ; AVX512-NEXT: vpshufb %xmm4, %xmm10, %xmm4
6477 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3]
6478 ; AVX512-NEXT: vmovd {{.*#+}} xmm14 = [2,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
6479 ; AVX512-NEXT: vpshufb %xmm14, %xmm13, %xmm15
6480 ; AVX512-NEXT: vmovdqa64 %xmm26, %xmm6
6481 ; AVX512-NEXT: vpshufb %xmm14, %xmm6, %xmm14
6482 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
6483 ; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
6484 ; AVX512-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
6485 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm14[0,1,2,3,4],ymm4[5],ymm14[6,7]
6486 ; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
6487 ; AVX512-NEXT: vmovdqa64 %xmm24, %xmm4
6488 ; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm4
6489 ; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm0
6490 ; AVX512-NEXT: vmovdqa64 %xmm5, %xmm20
6491 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
6492 ; AVX512-NEXT: vmovdqa64 %xmm21, %xmm6
6493 ; AVX512-NEXT: vpshufb %xmm2, %xmm6, %xmm4
6494 ; AVX512-NEXT: vpshufb %xmm2, %xmm11, %xmm2
6495 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
6496 ; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
6497 ; AVX512-NEXT: vpsrlq $16, %zmm16, %zmm2
6498 ; AVX512-NEXT: vpmovqb %zmm2, %xmm2
6499 ; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
6500 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
6501 ; AVX512-NEXT: vmovdqa64 %ymm0, %ymm19
6502 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,3,11,0,0,3,11,0,0,3,11,0,0,3,11]
6503 ; AVX512-NEXT: vpshufb %xmm0, %xmm3, %xmm1
6504 ; AVX512-NEXT: vmovdqa64 %xmm3, %xmm25
6505 ; AVX512-NEXT: vpshufb %xmm0, %xmm9, %xmm2
6506 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
6507 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
6508 ; AVX512-NEXT: vpshufb %xmm2, %xmm8, %xmm4
6509 ; AVX512-NEXT: vpshufb %xmm2, %xmm7, %xmm14
6510 ; AVX512-NEXT: vmovdqa64 %xmm7, %xmm22
6511 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm14[0],xmm4[0],xmm14[1],xmm4[1],xmm14[2],xmm4[2],xmm14[3],xmm4[3]
6512 ; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
6513 ; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
6514 ; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
6515 ; AVX512-NEXT: vmovd {{.*#+}} xmm4 = [0,0,3,11,0,0,0,0,0,0,0,0,0,0,0,0]
6516 ; AVX512-NEXT: vpshufb %xmm4, %xmm12, %xmm14
6517 ; AVX512-NEXT: vmovdqa64 %xmm12, %xmm27
6518 ; AVX512-NEXT: vpshufb %xmm4, %xmm10, %xmm4
6519 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3]
6520 ; AVX512-NEXT: vmovd {{.*#+}} xmm14 = [3,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
6521 ; AVX512-NEXT: vpshufb %xmm14, %xmm13, %xmm15
6522 ; AVX512-NEXT: vmovdqa64 %xmm26, %xmm12
6523 ; AVX512-NEXT: vpshufb %xmm14, %xmm12, %xmm14
6524 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
6525 ; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
6526 ; AVX512-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
6527 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm14[0,1,2,3,4],ymm4[5],ymm14[6,7]
6528 ; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
6529 ; AVX512-NEXT: vmovdqa64 %xmm24, %xmm5
6530 ; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm4
6531 ; AVX512-NEXT: vmovdqa64 %xmm20, %xmm15
6532 ; AVX512-NEXT: vpshufb %xmm0, %xmm15, %xmm0
6533 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
6534 ; AVX512-NEXT: vpshufb %xmm2, %xmm6, %xmm4
6535 ; AVX512-NEXT: vpshufb %xmm2, %xmm11, %xmm2
6536 ; AVX512-NEXT: vmovdqa %xmm11, %xmm6
6537 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
6538 ; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
6539 ; AVX512-NEXT: vpsrlq $24, %zmm16, %zmm2
6540 ; AVX512-NEXT: vpmovqb %zmm2, %xmm2
6541 ; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
6542 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
6543 ; AVX512-NEXT: vmovdqa64 %ymm0, %ymm20
6544 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm3 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
6545 ; AVX512-NEXT: vmovdqa64 %xmm25, %xmm11
6546 ; AVX512-NEXT: vpshufb %xmm3, %xmm11, %xmm1
6547 ; AVX512-NEXT: vpshufb %xmm3, %xmm9, %xmm2
6548 ; AVX512-NEXT: vmovdqa %xmm9, %xmm7
6549 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
6550 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
6551 ; AVX512-NEXT: vpshufb %xmm2, %xmm8, %xmm4
6552 ; AVX512-NEXT: vmovdqa64 %xmm22, %xmm9
6553 ; AVX512-NEXT: vpshufb %xmm2, %xmm9, %xmm14
6554 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm14[0],xmm4[0],xmm14[1],xmm4[1],xmm14[2],xmm4[2],xmm14[3],xmm4[3]
6555 ; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
6556 ; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
6557 ; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
6558 ; AVX512-NEXT: vmovd {{.*#+}} xmm4 = [0,0,4,12,0,0,0,0,0,0,0,0,0,0,0,0]
6559 ; AVX512-NEXT: vmovdqa64 %xmm27, %xmm0
6560 ; AVX512-NEXT: vpshufb %xmm4, %xmm0, %xmm14
6561 ; AVX512-NEXT: vpshufb %xmm4, %xmm10, %xmm4
6562 ; AVX512-NEXT: vmovdqa64 %xmm10, %xmm22
6563 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3]
6564 ; AVX512-NEXT: vmovd {{.*#+}} xmm14 = [4,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
6565 ; AVX512-NEXT: vpshufb %xmm14, %xmm13, %xmm0
6566 ; AVX512-NEXT: vmovdqa64 %xmm13, %xmm25
6567 ; AVX512-NEXT: vpshufb %xmm14, %xmm12, %xmm14
6568 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1],xmm14[2],xmm0[2],xmm14[3],xmm0[3]
6569 ; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
6570 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
6571 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5],ymm0[6,7]
6572 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
6573 ; AVX512-NEXT: vpshufb %xmm3, %xmm5, %xmm1
6574 ; AVX512-NEXT: vpshufb %xmm3, %xmm15, %xmm3
6575 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
6576 ; AVX512-NEXT: vmovdqa64 %xmm21, %xmm10
6577 ; AVX512-NEXT: vpshufb %xmm2, %xmm10, %xmm3
6578 ; AVX512-NEXT: vpshufb %xmm2, %xmm6, %xmm2
6579 ; AVX512-NEXT: vmovdqa64 %xmm6, %xmm23
6580 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
6581 ; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
6582 ; AVX512-NEXT: vpsrlq $32, %zmm16, %zmm2
6583 ; AVX512-NEXT: vpmovqb %zmm2, %xmm2
6584 ; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
6585 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
6586 ; AVX512-NEXT: vmovdqa64 %ymm0, %ymm21
6587 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm5 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
6588 ; AVX512-NEXT: vpshufb %xmm5, %xmm11, %xmm1
6589 ; AVX512-NEXT: vpshufb %xmm5, %xmm7, %xmm2
6590 ; AVX512-NEXT: vmovdqa %xmm7, %xmm13
6591 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
6592 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
6593 ; AVX512-NEXT: vpshufb %xmm2, %xmm8, %xmm3
6594 ; AVX512-NEXT: vmovdqa64 %xmm8, %xmm26
6595 ; AVX512-NEXT: vpshufb %xmm2, %xmm9, %xmm4
6596 ; AVX512-NEXT: vmovdqa %xmm9, %xmm14
6597 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
6598 ; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
6599 ; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
6600 ; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
6601 ; AVX512-NEXT: vmovd {{.*#+}} xmm3 = [0,0,5,13,0,0,0,0,0,0,0,0,0,0,0,0]
6602 ; AVX512-NEXT: vmovdqa64 %xmm27, %xmm6
6603 ; AVX512-NEXT: vpshufb %xmm3, %xmm6, %xmm4
6604 ; AVX512-NEXT: vmovdqa64 %xmm22, %xmm7
6605 ; AVX512-NEXT: vpshufb %xmm3, %xmm7, %xmm3
6606 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
6607 ; AVX512-NEXT: vmovd {{.*#+}} xmm4 = [5,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
6608 ; AVX512-NEXT: vmovdqa64 %xmm25, %xmm9
6609 ; AVX512-NEXT: vpshufb %xmm4, %xmm9, %xmm0
6610 ; AVX512-NEXT: vpshufb %xmm4, %xmm12, %xmm4
6611 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
6612 ; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
6613 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
6614 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5],ymm0[6,7]
6615 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
6616 ; AVX512-NEXT: vmovdqa64 %xmm24, %xmm8
6617 ; AVX512-NEXT: vpshufb %xmm5, %xmm8, %xmm1
6618 ; AVX512-NEXT: vpshufb %xmm5, %xmm15, %xmm3
6619 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
6620 ; AVX512-NEXT: vpshufb %xmm2, %xmm10, %xmm3
6621 ; AVX512-NEXT: vmovdqa64 %xmm10, %xmm24
6622 ; AVX512-NEXT: vmovdqa64 %xmm23, %xmm4
6623 ; AVX512-NEXT: vpshufb %xmm2, %xmm4, %xmm2
6624 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
6625 ; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
6626 ; AVX512-NEXT: vpsrlq $40, %zmm16, %zmm2
6627 ; AVX512-NEXT: vpmovqb %zmm2, %xmm2
6628 ; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
6629 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
6630 ; AVX512-NEXT: vmovdqa64 %ymm0, %ymm22
6631 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm10 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
6632 ; AVX512-NEXT: vpshufb %xmm10, %xmm11, %xmm1
6633 ; AVX512-NEXT: vmovdqa64 %xmm11, %xmm23
6634 ; AVX512-NEXT: vpshufb %xmm10, %xmm13, %xmm2
6635 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
6636 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
6637 ; AVX512-NEXT: vmovdqa64 %xmm26, %xmm11
6638 ; AVX512-NEXT: vpshufb %xmm2, %xmm11, %xmm3
6639 ; AVX512-NEXT: vpshufb %xmm2, %xmm14, %xmm5
6640 ; AVX512-NEXT: vmovdqa64 %xmm14, %xmm25
6641 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
6642 ; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
6643 ; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
6644 ; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
6645 ; AVX512-NEXT: vmovd {{.*#+}} xmm3 = [0,0,6,14,0,0,0,0,0,0,0,0,0,0,0,0]
6646 ; AVX512-NEXT: vpshufb %xmm3, %xmm6, %xmm5
6647 ; AVX512-NEXT: vpshufb %xmm3, %xmm7, %xmm3
6648 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
6649 ; AVX512-NEXT: vmovd {{.*#+}} xmm5 = [6,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
6650 ; AVX512-NEXT: vpshufb %xmm5, %xmm9, %xmm0
6651 ; AVX512-NEXT: vmovdqa %xmm9, %xmm14
6652 ; AVX512-NEXT: vpshufb %xmm5, %xmm12, %xmm5
6653 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
6654 ; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
6655 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
6656 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5],ymm0[6,7]
6657 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
6658 ; AVX512-NEXT: vpshufb %xmm10, %xmm8, %xmm1
6659 ; AVX512-NEXT: vmovdqa %xmm8, %xmm9
6660 ; AVX512-NEXT: vpshufb %xmm10, %xmm15, %xmm3
6661 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
6662 ; AVX512-NEXT: vmovdqa64 %xmm24, %xmm10
6663 ; AVX512-NEXT: vpshufb %xmm2, %xmm10, %xmm3
6664 ; AVX512-NEXT: vpshufb %xmm2, %xmm4, %xmm2
6665 ; AVX512-NEXT: vmovdqa %xmm4, %xmm8
6666 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
6667 ; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
6668 ; AVX512-NEXT: vpsrlq $48, %zmm16, %zmm2
6669 ; AVX512-NEXT: vpmovqb %zmm2, %xmm2
6670 ; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
6671 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
6672 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
6673 ; AVX512-NEXT: vmovdqa64 %xmm23, %xmm2
6674 ; AVX512-NEXT: vpshufb %xmm1, %xmm2, %xmm2
6675 ; AVX512-NEXT: vpshufb %xmm1, %xmm13, %xmm3
6676 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
6677 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm3 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
6678 ; AVX512-NEXT: vpshufb %xmm3, %xmm11, %xmm4
6679 ; AVX512-NEXT: vmovdqa64 %xmm25, %xmm5
6680 ; AVX512-NEXT: vpshufb %xmm3, %xmm5, %xmm5
6681 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
6682 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
6683 ; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
6684 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
6685 ; AVX512-NEXT: vmovd {{.*#+}} xmm4 = [0,0,7,15,0,0,0,0,0,0,0,0,0,0,0,0]
6686 ; AVX512-NEXT: vpshufb %xmm4, %xmm6, %xmm5
6687 ; AVX512-NEXT: vpshufb %xmm4, %xmm7, %xmm4
6688 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
6689 ; AVX512-NEXT: vmovd {{.*#+}} xmm5 = [7,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
6690 ; AVX512-NEXT: vpshufb %xmm5, %xmm14, %xmm7
6691 ; AVX512-NEXT: vpshufb %xmm5, %xmm12, %xmm5
6692 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3]
6693 ; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
6694 ; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
6695 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
6696 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
6697 ; AVX512-NEXT: vpshufb %xmm1, %xmm9, %xmm4
6698 ; AVX512-NEXT: vpshufb %xmm1, %xmm15, %xmm1
6699 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
6700 ; AVX512-NEXT: vpshufb %xmm3, %xmm10, %xmm4
6701 ; AVX512-NEXT: vpshufb %xmm3, %xmm8, %xmm3
6702 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
6703 ; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
6704 ; AVX512-NEXT: vpsrlq $56, %zmm16, %zmm3
6705 ; AVX512-NEXT: vpmovqb %zmm3, %xmm3
6706 ; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
6707 ; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
6708 ; AVX512-NEXT: vmovdqa64 %ymm17, (%rsi)
6709 ; AVX512-NEXT: vmovdqa64 %ymm18, (%rdx)
6710 ; AVX512-NEXT: vmovdqa64 %ymm19, (%rcx)
6711 ; AVX512-NEXT: vmovdqa64 %ymm20, (%r8)
6712 ; AVX512-NEXT: vmovdqa64 %ymm21, (%r9)
6713 ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
6714 ; AVX512-NEXT: vmovdqa64 %ymm22, (%rax)
6715 ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
6716 ; AVX512-NEXT: vmovdqa %ymm0, (%rax)
6717 ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
6718 ; AVX512-NEXT: vmovdqa %ymm1, (%rax)
6719 ; AVX512-NEXT: vzeroupper
6720 ; AVX512-NEXT: retq
6721 ;
6722 ; AVX512-FCP-LABEL: load_i8_stride8_vf32:
6723 ; AVX512-FCP: # %bb.0:
6724 ; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm18
6725 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm14 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
6726 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,2,2,3,0,2,4,6]
6727 ; AVX512-FCP-NEXT: vmovdqa64 224(%rdi), %ymm16
6728 ; AVX512-FCP-NEXT: vpermd %ymm16, %ymm0, %ymm4
6729 ; AVX512-FCP-NEXT: vpshufb %ymm14, %ymm4, %ymm1
6730 ; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm17
6731 ; AVX512-FCP-NEXT: vpermd %ymm17, %ymm0, %ymm6
6732 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u]
6733 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
6734 ; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %ymm13
6735 ; AVX512-FCP-NEXT: vpermd %ymm13, %ymm0, %ymm9
6736 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u]
6737 ; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm15
6738 ; AVX512-FCP-NEXT: vpermd %ymm15, %ymm0, %ymm11
6739 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u,u,u,u,u]
6740 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5],ymm0[6,7]
6741 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
6742 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm3 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
6743 ; AVX512-FCP-NEXT: vmovdqa 112(%rdi), %xmm2
6744 ; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm5
6745 ; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %xmm1
6746 ; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm3
6747 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
6748 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm8 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
6749 ; AVX512-FCP-NEXT: vmovdqa 80(%rdi), %xmm12
6750 ; AVX512-FCP-NEXT: vpshufb %xmm8, %xmm12, %xmm10
6751 ; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %xmm5
6752 ; AVX512-FCP-NEXT: vpshufb %xmm8, %xmm5, %xmm8
6753 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3]
6754 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3]
6755 ; AVX512-FCP-NEXT: vpmovqb %zmm18, %xmm8
6756 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3]
6757 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm0[4,5,6,7]
6758 ; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm22
6759 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29]
6760 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm7 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u]
6761 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3,4,5,6],ymm0[7]
6762 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm3 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
6763 ; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm9, %ymm7
6764 ; AVX512-FCP-NEXT: vmovdqa64 %ymm3, %ymm27
6765 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm3 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
6766 ; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm11, %ymm8
6767 ; AVX512-FCP-NEXT: vmovdqa64 %ymm3, %ymm28
6768 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3,4],ymm7[5],ymm8[6,7]
6769 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3,4,5],ymm0[6,7]
6770 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm7 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
6771 ; AVX512-FCP-NEXT: vpshufb %xmm7, %xmm2, %xmm8
6772 ; AVX512-FCP-NEXT: vpshufb %xmm7, %xmm1, %xmm7
6773 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
6774 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm8 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
6775 ; AVX512-FCP-NEXT: vpshufb %xmm8, %xmm12, %xmm10
6776 ; AVX512-FCP-NEXT: vpshufb %xmm8, %xmm5, %xmm8
6777 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3]
6778 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3]
6779 ; AVX512-FCP-NEXT: vpsrlq $8, %zmm18, %zmm8
6780 ; AVX512-FCP-NEXT: vpmovqb %zmm8, %xmm8
6781 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3]
6782 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm0[4,5,6,7]
6783 ; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm20
6784 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm3 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
6785 ; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm4, %ymm0
6786 ; AVX512-FCP-NEXT: vmovdqa64 %ymm3, %ymm29
6787 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm3 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
6788 ; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm6, %ymm7
6789 ; AVX512-FCP-NEXT: vmovdqa64 %ymm3, %ymm30
6790 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3,4,5,6],ymm0[7]
6791 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm3 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
6792 ; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm9, %ymm7
6793 ; AVX512-FCP-NEXT: vmovdqa64 %ymm3, %ymm31
6794 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm3 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
6795 ; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm11, %ymm8
6796 ; AVX512-FCP-NEXT: vmovdqa64 %ymm3, %ymm26
6797 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3,4],ymm7[5],ymm8[6,7]
6798 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3,4,5],ymm0[6,7]
6799 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm7 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
6800 ; AVX512-FCP-NEXT: vpshufb %xmm7, %xmm2, %xmm8
6801 ; AVX512-FCP-NEXT: vpshufb %xmm7, %xmm1, %xmm7
6802 ; AVX512-FCP-NEXT: vmovdqa %xmm1, %xmm3
6803 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
6804 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm8 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
6805 ; AVX512-FCP-NEXT: vpshufb %xmm8, %xmm12, %xmm10
6806 ; AVX512-FCP-NEXT: vpshufb %xmm8, %xmm5, %xmm8
6807 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3]
6808 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3]
6809 ; AVX512-FCP-NEXT: vpsrlq $16, %zmm18, %zmm8
6810 ; AVX512-FCP-NEXT: vpmovqb %zmm8, %xmm8
6811 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3]
6812 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm0[4,5,6,7]
6813 ; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm21
6814 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm1 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
6815 ; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm4, %ymm0
6816 ; AVX512-FCP-NEXT: vmovdqa64 %ymm1, %ymm25
6817 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm1 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
6818 ; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm6, %ymm4
6819 ; AVX512-FCP-NEXT: vmovdqa64 %ymm1, %ymm24
6820 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5,6],ymm0[7]
6821 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm1 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
6822 ; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm9, %ymm4
6823 ; AVX512-FCP-NEXT: vmovdqa64 %ymm1, %ymm23
6824 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm8 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
6825 ; AVX512-FCP-NEXT: vpshufb %ymm8, %ymm11, %ymm6
6826 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm6[0,1,2,3,4],ymm4[5],ymm6[6,7]
6827 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
6828 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm4 = [0,0,3,11,0,0,3,11,0,0,3,11,0,0,3,11]
6829 ; AVX512-FCP-NEXT: vpshufb %xmm4, %xmm2, %xmm6
6830 ; AVX512-FCP-NEXT: vpshufb %xmm4, %xmm3, %xmm4
6831 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
6832 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm6 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
6833 ; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm12, %xmm9
6834 ; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm5, %xmm6
6835 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm9[0],xmm6[1],xmm9[1],xmm6[2],xmm9[2],xmm6[3],xmm9[3]
6836 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0,1,2],xmm4[3]
6837 ; AVX512-FCP-NEXT: vpsrlq $24, %zmm18, %zmm6
6838 ; AVX512-FCP-NEXT: vpmovqb %zmm6, %xmm6
6839 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3]
6840 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
6841 ; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm19
6842 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [1,3,2,3,1,3,5,7]
6843 ; AVX512-FCP-NEXT: vpermd %ymm16, %ymm0, %ymm11
6844 ; AVX512-FCP-NEXT: vpshufb %ymm14, %ymm11, %ymm4
6845 ; AVX512-FCP-NEXT: vpermd %ymm17, %ymm0, %ymm9
6846 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm14 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u]
6847 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm14[0,1,2,3,4,5,6],ymm4[7]
6848 ; AVX512-FCP-NEXT: vpermd %ymm13, %ymm0, %ymm13
6849 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm14 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u]
6850 ; AVX512-FCP-NEXT: vpermd %ymm15, %ymm0, %ymm1
6851 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm15 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u,u,u,u,u]
6852 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
6853 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm14[0,1,2,3,4,5],ymm4[6,7]
6854 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm14 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
6855 ; AVX512-FCP-NEXT: vmovdqa %xmm2, %xmm6
6856 ; AVX512-FCP-NEXT: vpshufb %xmm14, %xmm2, %xmm15
6857 ; AVX512-FCP-NEXT: vpshufb %xmm14, %xmm3, %xmm14
6858 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
6859 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm15 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
6860 ; AVX512-FCP-NEXT: vpshufb %xmm15, %xmm12, %xmm0
6861 ; AVX512-FCP-NEXT: vpshufb %xmm15, %xmm5, %xmm15
6862 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
6863 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm14[3]
6864 ; AVX512-FCP-NEXT: vpsrlq $32, %zmm18, %zmm14
6865 ; AVX512-FCP-NEXT: vpmovqb %zmm14, %xmm14
6866 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm14[0,1],xmm0[2,3]
6867 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm15 = ymm0[0,1,2,3],ymm4[4,5,6,7]
6868 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29]
6869 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u]
6870 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5,6],ymm0[7]
6871 ; AVX512-FCP-NEXT: vmovdqa64 %ymm27, %ymm2
6872 ; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm13, %ymm4
6873 ; AVX512-FCP-NEXT: vmovdqa64 %ymm28, %ymm2
6874 ; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm1, %ymm14
6875 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm14[0,1,2,3,4],ymm4[5],ymm14[6,7]
6876 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm0[6,7]
6877 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm4 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
6878 ; AVX512-FCP-NEXT: vpshufb %xmm4, %xmm6, %xmm14
6879 ; AVX512-FCP-NEXT: vmovdqa %xmm3, %xmm7
6880 ; AVX512-FCP-NEXT: vpshufb %xmm4, %xmm3, %xmm4
6881 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3]
6882 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm14 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
6883 ; AVX512-FCP-NEXT: vpshufb %xmm14, %xmm12, %xmm0
6884 ; AVX512-FCP-NEXT: vpshufb %xmm14, %xmm5, %xmm14
6885 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1],xmm14[2],xmm0[2],xmm14[3],xmm0[3]
6886 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm4[3]
6887 ; AVX512-FCP-NEXT: vpsrlq $40, %zmm18, %zmm4
6888 ; AVX512-FCP-NEXT: vpmovqb %zmm4, %xmm4
6889 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm4[0,1],xmm0[2,3]
6890 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm0[0,1,2,3],ymm2[4,5,6,7]
6891 ; AVX512-FCP-NEXT: vmovdqa64 %ymm29, %ymm0
6892 ; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm11, %ymm0
6893 ; AVX512-FCP-NEXT: vmovdqa64 %ymm30, %ymm2
6894 ; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm9, %ymm2
6895 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5,6],ymm0[7]
6896 ; AVX512-FCP-NEXT: vmovdqa64 %ymm31, %ymm2
6897 ; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm13, %ymm2
6898 ; AVX512-FCP-NEXT: vmovdqa64 %ymm26, %ymm4
6899 ; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm1, %ymm14
6900 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm14[0,1,2,3,4],ymm2[5],ymm14[6,7]
6901 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm2[0,1,2,3,4,5],ymm0[6,7]
6902 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm2 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
6903 ; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm6, %xmm14
6904 ; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm3, %xmm2
6905 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm14[0],xmm2[1],xmm14[1],xmm2[2],xmm14[2],xmm2[3],xmm14[3]
6906 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm14 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
6907 ; AVX512-FCP-NEXT: vpshufb %xmm14, %xmm12, %xmm0
6908 ; AVX512-FCP-NEXT: vpshufb %xmm14, %xmm5, %xmm14
6909 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1],xmm14[2],xmm0[2],xmm14[3],xmm0[3]
6910 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm2[3]
6911 ; AVX512-FCP-NEXT: vpsrlq $48, %zmm18, %zmm2
6912 ; AVX512-FCP-NEXT: vpmovqb %zmm2, %xmm2
6913 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
6914 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7]
6915 ; AVX512-FCP-NEXT: vmovdqa64 %ymm25, %ymm2
6916 ; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm11, %ymm2
6917 ; AVX512-FCP-NEXT: vmovdqa64 %ymm24, %ymm3
6918 ; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm9, %ymm3
6919 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
6920 ; AVX512-FCP-NEXT: vmovdqa64 %ymm23, %ymm3
6921 ; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm13, %ymm3
6922 ; AVX512-FCP-NEXT: vpshufb %ymm8, %ymm1, %ymm1
6923 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5],ymm1[6,7]
6924 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
6925 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm2 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
6926 ; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm6, %xmm3
6927 ; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm7, %xmm2
6928 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
6929 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm3 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
6930 ; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm12, %xmm8
6931 ; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm5, %xmm3
6932 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm8[0],xmm3[1],xmm8[1],xmm3[2],xmm8[2],xmm3[3],xmm8[3]
6933 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3]
6934 ; AVX512-FCP-NEXT: vpsrlq $56, %zmm18, %zmm3
6935 ; AVX512-FCP-NEXT: vpmovqb %zmm3, %xmm3
6936 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]
6937 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
6938 ; AVX512-FCP-NEXT: vmovdqa64 %ymm22, (%rsi)
6939 ; AVX512-FCP-NEXT: vmovdqa64 %ymm20, (%rdx)
6940 ; AVX512-FCP-NEXT: vmovdqa64 %ymm21, (%rcx)
6941 ; AVX512-FCP-NEXT: vmovdqa64 %ymm19, (%r8)
6942 ; AVX512-FCP-NEXT: vmovdqa %ymm15, (%r9)
6943 ; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
6944 ; AVX512-FCP-NEXT: vmovdqa %ymm10, (%rax)
6945 ; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
6946 ; AVX512-FCP-NEXT: vmovdqa %ymm0, (%rax)
6947 ; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
6948 ; AVX512-FCP-NEXT: vmovdqa %ymm1, (%rax)
6949 ; AVX512-FCP-NEXT: vzeroupper
6950 ; AVX512-FCP-NEXT: retq
6951 ;
6952 ; AVX512DQ-LABEL: load_i8_stride8_vf32:
6953 ; AVX512DQ: # %bb.0:
6954 ; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm16
6955 ; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %zmm0
6956 ; AVX512DQ-NEXT: vpmovqb %zmm0, %xmm0
6957 ; AVX512DQ-NEXT: vmovdqa 240(%rdi), %xmm12
6958 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm3 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
6959 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm12, %xmm1
6960 ; AVX512DQ-NEXT: vmovdqa 224(%rdi), %xmm7
6961 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm7, %xmm2
6962 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
6963 ; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
6964 ; AVX512DQ-NEXT: vmovdqa 208(%rdi), %xmm9
6965 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm4 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
6966 ; AVX512DQ-NEXT: vpshufb %xmm4, %xmm9, %xmm2
6967 ; AVX512DQ-NEXT: vmovdqa 192(%rdi), %xmm8
6968 ; AVX512DQ-NEXT: vpshufb %xmm4, %xmm8, %xmm6
6969 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
6970 ; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
6971 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
6972 ; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
6973 ; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm2
6974 ; AVX512DQ-NEXT: vpmovqb %ymm2, %xmm2
6975 ; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
6976 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5],ymm2[6,7]
6977 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
6978 ; AVX512DQ-NEXT: vmovdqa 112(%rdi), %xmm5
6979 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm5, %xmm6
6980 ; AVX512DQ-NEXT: vmovdqa 96(%rdi), %xmm1
6981 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm1, %xmm3
6982 ; AVX512DQ-NEXT: vmovdqa64 %xmm1, %xmm18
6983 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3]
6984 ; AVX512DQ-NEXT: vmovdqa 80(%rdi), %xmm1
6985 ; AVX512DQ-NEXT: vpshufb %xmm4, %xmm1, %xmm11
6986 ; AVX512DQ-NEXT: vmovdqa64 %xmm1, %xmm21
6987 ; AVX512DQ-NEXT: vmovdqa 64(%rdi), %xmm3
6988 ; AVX512DQ-NEXT: vpshufb %xmm4, %xmm3, %xmm4
6989 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm11[0],xmm4[1],xmm11[1],xmm4[2],xmm11[2],xmm4[3],xmm11[3]
6990 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0,1,2],xmm10[3]
6991 ; AVX512DQ-NEXT: vpmovqb %zmm16, %xmm10
6992 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm4 = xmm10[0,1],xmm4[2,3]
6993 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
6994 ; AVX512DQ-NEXT: vmovdqa64 %ymm0, %ymm17
6995 ; AVX512DQ-NEXT: vmovdqa 160(%rdi), %xmm10
6996 ; AVX512DQ-NEXT: vmovdqa 176(%rdi), %xmm6
6997 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm2 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
6998 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm12, %xmm4
6999 ; AVX512DQ-NEXT: vmovdqa64 %xmm12, %xmm19
7000 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm7, %xmm12
7001 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm12[0],xmm4[0],xmm12[1],xmm4[1],xmm12[2],xmm4[2],xmm12[3],xmm4[3]
7002 ; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
7003 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm14 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
7004 ; AVX512DQ-NEXT: vpshufb %xmm14, %xmm9, %xmm12
7005 ; AVX512DQ-NEXT: vmovdqa64 %xmm9, %xmm22
7006 ; AVX512DQ-NEXT: vpshufb %xmm14, %xmm8, %xmm13
7007 ; AVX512DQ-NEXT: vmovdqa64 %xmm8, %xmm20
7008 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
7009 ; AVX512DQ-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm12
7010 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm12[0,1,2,3,4,5,6],ymm4[7]
7011 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm12 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
7012 ; AVX512DQ-NEXT: vpshufb %xmm12, %xmm6, %xmm13
7013 ; AVX512DQ-NEXT: vpshufb %xmm12, %xmm10, %xmm12
7014 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
7015 ; AVX512DQ-NEXT: vmovdqa 128(%rdi), %xmm8
7016 ; AVX512DQ-NEXT: vmovdqa 144(%rdi), %xmm13
7017 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm1 = [1,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
7018 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm13, %xmm0
7019 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm8, %xmm1
7020 ; AVX512DQ-NEXT: vmovdqa64 %xmm8, %xmm26
7021 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
7022 ; AVX512DQ-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm1
7023 ; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
7024 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
7025 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm4[6,7]
7026 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm5, %xmm1
7027 ; AVX512DQ-NEXT: vmovdqa64 %xmm5, %xmm24
7028 ; AVX512DQ-NEXT: vmovdqa64 %xmm18, %xmm5
7029 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm5, %xmm2
7030 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
7031 ; AVX512DQ-NEXT: vmovdqa64 %xmm21, %xmm4
7032 ; AVX512DQ-NEXT: vpshufb %xmm14, %xmm4, %xmm2
7033 ; AVX512DQ-NEXT: vpshufb %xmm14, %xmm3, %xmm4
7034 ; AVX512DQ-NEXT: vmovdqa %xmm3, %xmm11
7035 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
7036 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
7037 ; AVX512DQ-NEXT: vpsrlq $8, %zmm16, %zmm2
7038 ; AVX512DQ-NEXT: vpmovqb %zmm2, %xmm2
7039 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
7040 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
7041 ; AVX512DQ-NEXT: vmovdqa64 %ymm0, %ymm18
7042 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
7043 ; AVX512DQ-NEXT: vmovdqa64 %xmm19, %xmm3
7044 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm3, %xmm1
7045 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm7, %xmm2
7046 ; AVX512DQ-NEXT: vmovdqa %xmm7, %xmm9
7047 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
7048 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm2 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
7049 ; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm8
7050 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm8, %xmm4
7051 ; AVX512DQ-NEXT: vmovdqa64 %xmm20, %xmm7
7052 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm7, %xmm14
7053 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm14[0],xmm4[0],xmm14[1],xmm4[1],xmm14[2],xmm4[2],xmm14[3],xmm4[3]
7054 ; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
7055 ; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
7056 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
7057 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm4 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
7058 ; AVX512DQ-NEXT: vpshufb %xmm4, %xmm6, %xmm14
7059 ; AVX512DQ-NEXT: vmovdqa %xmm6, %xmm12
7060 ; AVX512DQ-NEXT: vpshufb %xmm4, %xmm10, %xmm4
7061 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3]
7062 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm14 = [2,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
7063 ; AVX512DQ-NEXT: vpshufb %xmm14, %xmm13, %xmm15
7064 ; AVX512DQ-NEXT: vmovdqa64 %xmm26, %xmm6
7065 ; AVX512DQ-NEXT: vpshufb %xmm14, %xmm6, %xmm14
7066 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
7067 ; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
7068 ; AVX512DQ-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
7069 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm14[0,1,2,3,4],ymm4[5],ymm14[6,7]
7070 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
7071 ; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm4
7072 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm4
7073 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm0
7074 ; AVX512DQ-NEXT: vmovdqa64 %xmm5, %xmm20
7075 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
7076 ; AVX512DQ-NEXT: vmovdqa64 %xmm21, %xmm6
7077 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm6, %xmm4
7078 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm11, %xmm2
7079 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
7080 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
7081 ; AVX512DQ-NEXT: vpsrlq $16, %zmm16, %zmm2
7082 ; AVX512DQ-NEXT: vpmovqb %zmm2, %xmm2
7083 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
7084 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
7085 ; AVX512DQ-NEXT: vmovdqa64 %ymm0, %ymm19
7086 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,3,11,0,0,3,11,0,0,3,11,0,0,3,11]
7087 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm3, %xmm1
7088 ; AVX512DQ-NEXT: vmovdqa64 %xmm3, %xmm25
7089 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm9, %xmm2
7090 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
7091 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm2 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
7092 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm8, %xmm4
7093 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm7, %xmm14
7094 ; AVX512DQ-NEXT: vmovdqa64 %xmm7, %xmm22
7095 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm14[0],xmm4[0],xmm14[1],xmm4[1],xmm14[2],xmm4[2],xmm14[3],xmm4[3]
7096 ; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
7097 ; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
7098 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
7099 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm4 = [0,0,3,11,0,0,0,0,0,0,0,0,0,0,0,0]
7100 ; AVX512DQ-NEXT: vpshufb %xmm4, %xmm12, %xmm14
7101 ; AVX512DQ-NEXT: vmovdqa64 %xmm12, %xmm27
7102 ; AVX512DQ-NEXT: vpshufb %xmm4, %xmm10, %xmm4
7103 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3]
7104 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm14 = [3,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
7105 ; AVX512DQ-NEXT: vpshufb %xmm14, %xmm13, %xmm15
7106 ; AVX512DQ-NEXT: vmovdqa64 %xmm26, %xmm12
7107 ; AVX512DQ-NEXT: vpshufb %xmm14, %xmm12, %xmm14
7108 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
7109 ; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
7110 ; AVX512DQ-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
7111 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm14[0,1,2,3,4],ymm4[5],ymm14[6,7]
7112 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
7113 ; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm5
7114 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm4
7115 ; AVX512DQ-NEXT: vmovdqa64 %xmm20, %xmm15
7116 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm15, %xmm0
7117 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
7118 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm6, %xmm4
7119 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm11, %xmm2
7120 ; AVX512DQ-NEXT: vmovdqa %xmm11, %xmm6
7121 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
7122 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
7123 ; AVX512DQ-NEXT: vpsrlq $24, %zmm16, %zmm2
7124 ; AVX512DQ-NEXT: vpmovqb %zmm2, %xmm2
7125 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
7126 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
7127 ; AVX512DQ-NEXT: vmovdqa64 %ymm0, %ymm20
7128 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm3 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
7129 ; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm11
7130 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm11, %xmm1
7131 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm9, %xmm2
7132 ; AVX512DQ-NEXT: vmovdqa %xmm9, %xmm7
7133 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
7134 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm2 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
7135 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm8, %xmm4
7136 ; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm9
7137 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm9, %xmm14
7138 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm14[0],xmm4[0],xmm14[1],xmm4[1],xmm14[2],xmm4[2],xmm14[3],xmm4[3]
7139 ; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
7140 ; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
7141 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
7142 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm4 = [0,0,4,12,0,0,0,0,0,0,0,0,0,0,0,0]
7143 ; AVX512DQ-NEXT: vmovdqa64 %xmm27, %xmm0
7144 ; AVX512DQ-NEXT: vpshufb %xmm4, %xmm0, %xmm14
7145 ; AVX512DQ-NEXT: vpshufb %xmm4, %xmm10, %xmm4
7146 ; AVX512DQ-NEXT: vmovdqa64 %xmm10, %xmm22
7147 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3]
7148 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm14 = [4,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
7149 ; AVX512DQ-NEXT: vpshufb %xmm14, %xmm13, %xmm0
7150 ; AVX512DQ-NEXT: vmovdqa64 %xmm13, %xmm25
7151 ; AVX512DQ-NEXT: vpshufb %xmm14, %xmm12, %xmm14
7152 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1],xmm14[2],xmm0[2],xmm14[3],xmm0[3]
7153 ; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
7154 ; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
7155 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5],ymm0[6,7]
7156 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
7157 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm5, %xmm1
7158 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm15, %xmm3
7159 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
7160 ; AVX512DQ-NEXT: vmovdqa64 %xmm21, %xmm10
7161 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm10, %xmm3
7162 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm6, %xmm2
7163 ; AVX512DQ-NEXT: vmovdqa64 %xmm6, %xmm23
7164 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
7165 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
7166 ; AVX512DQ-NEXT: vpsrlq $32, %zmm16, %zmm2
7167 ; AVX512DQ-NEXT: vpmovqb %zmm2, %xmm2
7168 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
7169 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
7170 ; AVX512DQ-NEXT: vmovdqa64 %ymm0, %ymm21
7171 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm5 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
7172 ; AVX512DQ-NEXT: vpshufb %xmm5, %xmm11, %xmm1
7173 ; AVX512DQ-NEXT: vpshufb %xmm5, %xmm7, %xmm2
7174 ; AVX512DQ-NEXT: vmovdqa %xmm7, %xmm13
7175 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
7176 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm2 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
7177 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm8, %xmm3
7178 ; AVX512DQ-NEXT: vmovdqa64 %xmm8, %xmm26
7179 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm9, %xmm4
7180 ; AVX512DQ-NEXT: vmovdqa %xmm9, %xmm14
7181 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
7182 ; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
7183 ; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
7184 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
7185 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm3 = [0,0,5,13,0,0,0,0,0,0,0,0,0,0,0,0]
7186 ; AVX512DQ-NEXT: vmovdqa64 %xmm27, %xmm6
7187 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm6, %xmm4
7188 ; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm7
7189 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm7, %xmm3
7190 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
7191 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm4 = [5,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
7192 ; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm9
7193 ; AVX512DQ-NEXT: vpshufb %xmm4, %xmm9, %xmm0
7194 ; AVX512DQ-NEXT: vpshufb %xmm4, %xmm12, %xmm4
7195 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
7196 ; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
7197 ; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
7198 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5],ymm0[6,7]
7199 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
7200 ; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm8
7201 ; AVX512DQ-NEXT: vpshufb %xmm5, %xmm8, %xmm1
7202 ; AVX512DQ-NEXT: vpshufb %xmm5, %xmm15, %xmm3
7203 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
7204 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm10, %xmm3
7205 ; AVX512DQ-NEXT: vmovdqa64 %xmm10, %xmm24
7206 ; AVX512DQ-NEXT: vmovdqa64 %xmm23, %xmm4
7207 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm4, %xmm2
7208 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
7209 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
7210 ; AVX512DQ-NEXT: vpsrlq $40, %zmm16, %zmm2
7211 ; AVX512DQ-NEXT: vpmovqb %zmm2, %xmm2
7212 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
7213 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
7214 ; AVX512DQ-NEXT: vmovdqa64 %ymm0, %ymm22
7215 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm10 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
7216 ; AVX512DQ-NEXT: vpshufb %xmm10, %xmm11, %xmm1
7217 ; AVX512DQ-NEXT: vmovdqa64 %xmm11, %xmm23
7218 ; AVX512DQ-NEXT: vpshufb %xmm10, %xmm13, %xmm2
7219 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
7220 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm2 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
7221 ; AVX512DQ-NEXT: vmovdqa64 %xmm26, %xmm11
7222 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm11, %xmm3
7223 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm14, %xmm5
7224 ; AVX512DQ-NEXT: vmovdqa64 %xmm14, %xmm25
7225 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
7226 ; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
7227 ; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
7228 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
7229 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm3 = [0,0,6,14,0,0,0,0,0,0,0,0,0,0,0,0]
7230 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm6, %xmm5
7231 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm7, %xmm3
7232 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
7233 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm5 = [6,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
7234 ; AVX512DQ-NEXT: vpshufb %xmm5, %xmm9, %xmm0
7235 ; AVX512DQ-NEXT: vmovdqa %xmm9, %xmm14
7236 ; AVX512DQ-NEXT: vpshufb %xmm5, %xmm12, %xmm5
7237 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
7238 ; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
7239 ; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
7240 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5],ymm0[6,7]
7241 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
7242 ; AVX512DQ-NEXT: vpshufb %xmm10, %xmm8, %xmm1
7243 ; AVX512DQ-NEXT: vmovdqa %xmm8, %xmm9
7244 ; AVX512DQ-NEXT: vpshufb %xmm10, %xmm15, %xmm3
7245 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
7246 ; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm10
7247 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm10, %xmm3
7248 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm4, %xmm2
7249 ; AVX512DQ-NEXT: vmovdqa %xmm4, %xmm8
7250 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
7251 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
7252 ; AVX512DQ-NEXT: vpsrlq $48, %zmm16, %zmm2
7253 ; AVX512DQ-NEXT: vpmovqb %zmm2, %xmm2
7254 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
7255 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
7256 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm1 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
7257 ; AVX512DQ-NEXT: vmovdqa64 %xmm23, %xmm2
7258 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm2, %xmm2
7259 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm13, %xmm3
7260 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
7261 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm3 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
7262 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm11, %xmm4
7263 ; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm5
7264 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm5, %xmm5
7265 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
7266 ; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
7267 ; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
7268 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
7269 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm4 = [0,0,7,15,0,0,0,0,0,0,0,0,0,0,0,0]
7270 ; AVX512DQ-NEXT: vpshufb %xmm4, %xmm6, %xmm5
7271 ; AVX512DQ-NEXT: vpshufb %xmm4, %xmm7, %xmm4
7272 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
7273 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm5 = [7,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
7274 ; AVX512DQ-NEXT: vpshufb %xmm5, %xmm14, %xmm7
7275 ; AVX512DQ-NEXT: vpshufb %xmm5, %xmm12, %xmm5
7276 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3]
7277 ; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
7278 ; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
7279 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
7280 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
7281 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm9, %xmm4
7282 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm15, %xmm1
7283 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
7284 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm10, %xmm4
7285 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm8, %xmm3
7286 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
7287 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
7288 ; AVX512DQ-NEXT: vpsrlq $56, %zmm16, %zmm3
7289 ; AVX512DQ-NEXT: vpmovqb %zmm3, %xmm3
7290 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
7291 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
7292 ; AVX512DQ-NEXT: vmovdqa64 %ymm17, (%rsi)
7293 ; AVX512DQ-NEXT: vmovdqa64 %ymm18, (%rdx)
7294 ; AVX512DQ-NEXT: vmovdqa64 %ymm19, (%rcx)
7295 ; AVX512DQ-NEXT: vmovdqa64 %ymm20, (%r8)
7296 ; AVX512DQ-NEXT: vmovdqa64 %ymm21, (%r9)
7297 ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax
7298 ; AVX512DQ-NEXT: vmovdqa64 %ymm22, (%rax)
7299 ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax
7300 ; AVX512DQ-NEXT: vmovdqa %ymm0, (%rax)
7301 ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax
7302 ; AVX512DQ-NEXT: vmovdqa %ymm1, (%rax)
7303 ; AVX512DQ-NEXT: vzeroupper
7304 ; AVX512DQ-NEXT: retq
7305 ;
7306 ; AVX512DQ-FCP-LABEL: load_i8_stride8_vf32:
7307 ; AVX512DQ-FCP: # %bb.0:
7308 ; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm18
7309 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm14 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
7310 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,2,2,3,0,2,4,6]
7311 ; AVX512DQ-FCP-NEXT: vmovdqa64 224(%rdi), %ymm16
7312 ; AVX512DQ-FCP-NEXT: vpermd %ymm16, %ymm0, %ymm4
7313 ; AVX512DQ-FCP-NEXT: vpshufb %ymm14, %ymm4, %ymm1
7314 ; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm17
7315 ; AVX512DQ-FCP-NEXT: vpermd %ymm17, %ymm0, %ymm6
7316 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u]
7317 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
7318 ; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %ymm13
7319 ; AVX512DQ-FCP-NEXT: vpermd %ymm13, %ymm0, %ymm9
7320 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u]
7321 ; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm15
7322 ; AVX512DQ-FCP-NEXT: vpermd %ymm15, %ymm0, %ymm11
7323 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u,u,u,u,u]
7324 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5],ymm0[6,7]
7325 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
7326 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm3 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
7327 ; AVX512DQ-FCP-NEXT: vmovdqa 112(%rdi), %xmm2
7328 ; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm5
7329 ; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %xmm1
7330 ; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm3
7331 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
7332 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm8 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
7333 ; AVX512DQ-FCP-NEXT: vmovdqa 80(%rdi), %xmm12
7334 ; AVX512DQ-FCP-NEXT: vpshufb %xmm8, %xmm12, %xmm10
7335 ; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %xmm5
7336 ; AVX512DQ-FCP-NEXT: vpshufb %xmm8, %xmm5, %xmm8
7337 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3]
7338 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3]
7339 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm18, %xmm8
7340 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3]
7341 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm0[4,5,6,7]
7342 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm22
7343 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29]
7344 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm7 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u]
7345 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3,4,5,6],ymm0[7]
7346 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm3 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
7347 ; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm9, %ymm7
7348 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm3, %ymm27
7349 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm3 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
7350 ; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm11, %ymm8
7351 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm3, %ymm28
7352 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3,4],ymm7[5],ymm8[6,7]
7353 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3,4,5],ymm0[6,7]
7354 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm7 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
7355 ; AVX512DQ-FCP-NEXT: vpshufb %xmm7, %xmm2, %xmm8
7356 ; AVX512DQ-FCP-NEXT: vpshufb %xmm7, %xmm1, %xmm7
7357 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
7358 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm8 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
7359 ; AVX512DQ-FCP-NEXT: vpshufb %xmm8, %xmm12, %xmm10
7360 ; AVX512DQ-FCP-NEXT: vpshufb %xmm8, %xmm5, %xmm8
7361 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3]
7362 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3]
7363 ; AVX512DQ-FCP-NEXT: vpsrlq $8, %zmm18, %zmm8
7364 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm8, %xmm8
7365 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3]
7366 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm0[4,5,6,7]
7367 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm20
7368 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm3 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
7369 ; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm4, %ymm0
7370 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm3, %ymm29
7371 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm3 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
7372 ; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm6, %ymm7
7373 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm3, %ymm30
7374 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3,4,5,6],ymm0[7]
7375 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm3 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
7376 ; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm9, %ymm7
7377 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm3, %ymm31
7378 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm3 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
7379 ; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm11, %ymm8
7380 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm3, %ymm26
7381 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3,4],ymm7[5],ymm8[6,7]
7382 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3,4,5],ymm0[6,7]
7383 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm7 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
7384 ; AVX512DQ-FCP-NEXT: vpshufb %xmm7, %xmm2, %xmm8
7385 ; AVX512DQ-FCP-NEXT: vpshufb %xmm7, %xmm1, %xmm7
7386 ; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, %xmm3
7387 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
7388 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm8 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
7389 ; AVX512DQ-FCP-NEXT: vpshufb %xmm8, %xmm12, %xmm10
7390 ; AVX512DQ-FCP-NEXT: vpshufb %xmm8, %xmm5, %xmm8
7391 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3]
7392 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3]
7393 ; AVX512DQ-FCP-NEXT: vpsrlq $16, %zmm18, %zmm8
7394 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm8, %xmm8
7395 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3]
7396 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm0[4,5,6,7]
7397 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm21
7398 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm1 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
7399 ; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm4, %ymm0
7400 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, %ymm25
7401 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm1 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
7402 ; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm6, %ymm4
7403 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, %ymm24
7404 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5,6],ymm0[7]
7405 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm1 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
7406 ; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm9, %ymm4
7407 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, %ymm23
7408 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm8 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
7409 ; AVX512DQ-FCP-NEXT: vpshufb %ymm8, %ymm11, %ymm6
7410 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm6[0,1,2,3,4],ymm4[5],ymm6[6,7]
7411 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
7412 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm4 = [0,0,3,11,0,0,3,11,0,0,3,11,0,0,3,11]
7413 ; AVX512DQ-FCP-NEXT: vpshufb %xmm4, %xmm2, %xmm6
7414 ; AVX512DQ-FCP-NEXT: vpshufb %xmm4, %xmm3, %xmm4
7415 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
7416 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm6 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
7417 ; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm12, %xmm9
7418 ; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm5, %xmm6
7419 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm9[0],xmm6[1],xmm9[1],xmm6[2],xmm9[2],xmm6[3],xmm9[3]
7420 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0,1,2],xmm4[3]
7421 ; AVX512DQ-FCP-NEXT: vpsrlq $24, %zmm18, %zmm6
7422 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm6, %xmm6
7423 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3]
7424 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
7425 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm19
7426 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [1,3,2,3,1,3,5,7]
7427 ; AVX512DQ-FCP-NEXT: vpermd %ymm16, %ymm0, %ymm11
7428 ; AVX512DQ-FCP-NEXT: vpshufb %ymm14, %ymm11, %ymm4
7429 ; AVX512DQ-FCP-NEXT: vpermd %ymm17, %ymm0, %ymm9
7430 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm14 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u]
7431 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm14[0,1,2,3,4,5,6],ymm4[7]
7432 ; AVX512DQ-FCP-NEXT: vpermd %ymm13, %ymm0, %ymm13
7433 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm14 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u]
7434 ; AVX512DQ-FCP-NEXT: vpermd %ymm15, %ymm0, %ymm1
7435 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm15 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u,u,u,u,u]
7436 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
7437 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm14[0,1,2,3,4,5],ymm4[6,7]
7438 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm14 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
7439 ; AVX512DQ-FCP-NEXT: vmovdqa %xmm2, %xmm6
7440 ; AVX512DQ-FCP-NEXT: vpshufb %xmm14, %xmm2, %xmm15
7441 ; AVX512DQ-FCP-NEXT: vpshufb %xmm14, %xmm3, %xmm14
7442 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
7443 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm15 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
7444 ; AVX512DQ-FCP-NEXT: vpshufb %xmm15, %xmm12, %xmm0
7445 ; AVX512DQ-FCP-NEXT: vpshufb %xmm15, %xmm5, %xmm15
7446 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
7447 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm14[3]
7448 ; AVX512DQ-FCP-NEXT: vpsrlq $32, %zmm18, %zmm14
7449 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm14, %xmm14
7450 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm14[0,1],xmm0[2,3]
7451 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm15 = ymm0[0,1,2,3],ymm4[4,5,6,7]
7452 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29]
7453 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u]
7454 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5,6],ymm0[7]
7455 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm27, %ymm2
7456 ; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm13, %ymm4
7457 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm28, %ymm2
7458 ; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm1, %ymm14
7459 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm14[0,1,2,3,4],ymm4[5],ymm14[6,7]
7460 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm0[6,7]
7461 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm4 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
7462 ; AVX512DQ-FCP-NEXT: vpshufb %xmm4, %xmm6, %xmm14
7463 ; AVX512DQ-FCP-NEXT: vmovdqa %xmm3, %xmm7
7464 ; AVX512DQ-FCP-NEXT: vpshufb %xmm4, %xmm3, %xmm4
7465 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3]
7466 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm14 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
7467 ; AVX512DQ-FCP-NEXT: vpshufb %xmm14, %xmm12, %xmm0
7468 ; AVX512DQ-FCP-NEXT: vpshufb %xmm14, %xmm5, %xmm14
7469 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1],xmm14[2],xmm0[2],xmm14[3],xmm0[3]
7470 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm4[3]
7471 ; AVX512DQ-FCP-NEXT: vpsrlq $40, %zmm18, %zmm4
7472 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm4, %xmm4
7473 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm4[0,1],xmm0[2,3]
7474 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm0[0,1,2,3],ymm2[4,5,6,7]
7475 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm29, %ymm0
7476 ; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm11, %ymm0
7477 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm30, %ymm2
7478 ; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm9, %ymm2
7479 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5,6],ymm0[7]
7480 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm31, %ymm2
7481 ; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm13, %ymm2
7482 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm26, %ymm4
7483 ; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm1, %ymm14
7484 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm14[0,1,2,3,4],ymm2[5],ymm14[6,7]
7485 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm2[0,1,2,3,4,5],ymm0[6,7]
7486 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm2 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
7487 ; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm6, %xmm14
7488 ; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm3, %xmm2
7489 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm14[0],xmm2[1],xmm14[1],xmm2[2],xmm14[2],xmm2[3],xmm14[3]
7490 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm14 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
7491 ; AVX512DQ-FCP-NEXT: vpshufb %xmm14, %xmm12, %xmm0
7492 ; AVX512DQ-FCP-NEXT: vpshufb %xmm14, %xmm5, %xmm14
7493 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1],xmm14[2],xmm0[2],xmm14[3],xmm0[3]
7494 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm2[3]
7495 ; AVX512DQ-FCP-NEXT: vpsrlq $48, %zmm18, %zmm2
7496 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm2, %xmm2
7497 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
7498 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7]
7499 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm25, %ymm2
7500 ; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm11, %ymm2
7501 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm24, %ymm3
7502 ; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm9, %ymm3
7503 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
7504 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm23, %ymm3
7505 ; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm13, %ymm3
7506 ; AVX512DQ-FCP-NEXT: vpshufb %ymm8, %ymm1, %ymm1
7507 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5],ymm1[6,7]
7508 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
7509 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm2 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
7510 ; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm6, %xmm3
7511 ; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm7, %xmm2
7512 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
7513 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm3 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
7514 ; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm12, %xmm8
7515 ; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm5, %xmm3
7516 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm8[0],xmm3[1],xmm8[1],xmm3[2],xmm8[2],xmm3[3],xmm8[3]
7517 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3]
7518 ; AVX512DQ-FCP-NEXT: vpsrlq $56, %zmm18, %zmm3
7519 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm3, %xmm3
7520 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]
7521 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
7522 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm22, (%rsi)
7523 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm20, (%rdx)
7524 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm21, (%rcx)
7525 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm19, (%r8)
7526 ; AVX512DQ-FCP-NEXT: vmovdqa %ymm15, (%r9)
7527 ; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
7528 ; AVX512DQ-FCP-NEXT: vmovdqa %ymm10, (%rax)
7529 ; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
7530 ; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, (%rax)
7531 ; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
7532 ; AVX512DQ-FCP-NEXT: vmovdqa %ymm1, (%rax)
7533 ; AVX512DQ-FCP-NEXT: vzeroupper
7534 ; AVX512DQ-FCP-NEXT: retq
7535 ;
7536 ; AVX512BW-LABEL: load_i8_stride8_vf32:
7537 ; AVX512BW: # %bb.0:
7538 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
7539 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %r10
7540 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %r11
7541 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm16
7542 ; AVX512BW-NEXT: vmovdqa64 128(%rdi), %zmm0
7543 ; AVX512BW-NEXT: vpmovqb %zmm0, %xmm0
7544 ; AVX512BW-NEXT: vmovdqa 240(%rdi), %xmm2
7545 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
7546 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm2, %xmm1
7547 ; AVX512BW-NEXT: vmovdqa 224(%rdi), %xmm6
7548 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm6, %xmm3
7549 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
7550 ; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
7551 ; AVX512BW-NEXT: vmovdqa 208(%rdi), %xmm7
7552 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm5 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
7553 ; AVX512BW-NEXT: vpshufb %xmm5, %xmm7, %xmm3
7554 ; AVX512BW-NEXT: vmovdqa 192(%rdi), %xmm9
7555 ; AVX512BW-NEXT: vpshufb %xmm5, %xmm9, %xmm8
7556 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
7557 ; AVX512BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
7558 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
7559 ; AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
7560 ; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm3
7561 ; AVX512BW-NEXT: vpmovqb %ymm3, %xmm3
7562 ; AVX512BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
7563 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4],ymm0[5],ymm3[6,7]
7564 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
7565 ; AVX512BW-NEXT: vmovdqa 112(%rdi), %xmm1
7566 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm1, %xmm8
7567 ; AVX512BW-NEXT: vmovdqa 96(%rdi), %xmm3
7568 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm3, %xmm4
7569 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm4[0],xmm8[0],xmm4[1],xmm8[1],xmm4[2],xmm8[2],xmm4[3],xmm8[3]
7570 ; AVX512BW-NEXT: vmovdqa 80(%rdi), %xmm4
7571 ; AVX512BW-NEXT: vpshufb %xmm5, %xmm4, %xmm11
7572 ; AVX512BW-NEXT: vmovdqa 64(%rdi), %xmm8
7573 ; AVX512BW-NEXT: vpshufb %xmm5, %xmm8, %xmm5
7574 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm11[0],xmm5[1],xmm11[1],xmm5[2],xmm11[2],xmm5[3],xmm11[3]
7575 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1,2],xmm10[3]
7576 ; AVX512BW-NEXT: vpmovqb %zmm16, %xmm10
7577 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm5 = xmm10[0,1],xmm5[2,3]
7578 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3],ymm0[4,5,6,7]
7579 ; AVX512BW-NEXT: vmovdqa64 %ymm0, %ymm20
7580 ; AVX512BW-NEXT: vmovdqa64 128(%rdi), %xmm19
7581 ; AVX512BW-NEXT: vmovdqa 144(%rdi), %xmm11
7582 ; AVX512BW-NEXT: vmovdqa 160(%rdi), %xmm12
7583 ; AVX512BW-NEXT: vmovdqa 176(%rdi), %xmm13
7584 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
7585 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm2, %xmm5
7586 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm6, %xmm14
7587 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm14[0],xmm5[0],xmm14[1],xmm5[1],xmm14[2],xmm5[2],xmm14[3],xmm5[3]
7588 ; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
7589 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm14 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
7590 ; AVX512BW-NEXT: vpshufb %xmm14, %xmm7, %xmm15
7591 ; AVX512BW-NEXT: vpshufb %xmm14, %xmm9, %xmm17
7592 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm17[0],xmm15[0],xmm17[1],xmm15[1],xmm17[2],xmm15[2],xmm17[3],xmm15[3]
7593 ; AVX512BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
7594 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm15[0,1,2,3,4,5,6],ymm5[7]
7595 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm15 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
7596 ; AVX512BW-NEXT: vpshufb %xmm15, %xmm13, %xmm17
7597 ; AVX512BW-NEXT: vpshufb %xmm15, %xmm12, %xmm15
7598 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
7599 ; AVX512BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
7600 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm17 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
7601 ; AVX512BW-NEXT: vpshufb %xmm17, %xmm11, %xmm18
7602 ; AVX512BW-NEXT: vpshufb %xmm17, %xmm19, %xmm17
7603 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
7604 ; AVX512BW-NEXT: vinserti32x4 $1, %xmm17, %ymm0, %ymm10
7605 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm15[5],ymm10[6,7]
7606 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5],ymm5[6,7]
7607 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm1, %xmm10
7608 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
7609 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
7610 ; AVX512BW-NEXT: vpshufb %xmm14, %xmm4, %xmm10
7611 ; AVX512BW-NEXT: vpshufb %xmm14, %xmm8, %xmm14
7612 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm14[0],xmm10[0],xmm14[1],xmm10[1],xmm14[2],xmm10[2],xmm14[3],xmm10[3]
7613 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
7614 ; AVX512BW-NEXT: vpsrlq $8, %zmm16, %zmm10
7615 ; AVX512BW-NEXT: vpmovqb %zmm10, %xmm10
7616 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
7617 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
7618 ; AVX512BW-NEXT: vmovdqa64 %ymm0, %ymm21
7619 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
7620 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm2, %xmm5
7621 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm6, %xmm10
7622 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
7623 ; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
7624 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm10 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
7625 ; AVX512BW-NEXT: vpshufb %xmm10, %xmm7, %xmm14
7626 ; AVX512BW-NEXT: vpshufb %xmm10, %xmm9, %xmm15
7627 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
7628 ; AVX512BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
7629 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5,6],ymm5[7]
7630 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm14 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
7631 ; AVX512BW-NEXT: vpshufb %xmm14, %xmm13, %xmm15
7632 ; AVX512BW-NEXT: vpshufb %xmm14, %xmm12, %xmm14
7633 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
7634 ; AVX512BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
7635 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm15 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
7636 ; AVX512BW-NEXT: vpshufb %xmm15, %xmm11, %xmm17
7637 ; AVX512BW-NEXT: vpshufb %xmm15, %xmm19, %xmm15
7638 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
7639 ; AVX512BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
7640 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
7641 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
7642 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm1, %xmm14
7643 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
7644 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
7645 ; AVX512BW-NEXT: vpshufb %xmm10, %xmm4, %xmm14
7646 ; AVX512BW-NEXT: vpshufb %xmm10, %xmm8, %xmm10
7647 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
7648 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
7649 ; AVX512BW-NEXT: vpsrlq $16, %zmm16, %zmm10
7650 ; AVX512BW-NEXT: vpmovqb %zmm10, %xmm10
7651 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
7652 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
7653 ; AVX512BW-NEXT: vmovdqa64 %ymm0, %ymm22
7654 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
7655 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm2, %xmm5
7656 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm6, %xmm10
7657 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
7658 ; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
7659 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm10 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
7660 ; AVX512BW-NEXT: vpshufb %xmm10, %xmm7, %xmm14
7661 ; AVX512BW-NEXT: vpshufb %xmm10, %xmm9, %xmm15
7662 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
7663 ; AVX512BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
7664 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5,6],ymm5[7]
7665 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm14 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
7666 ; AVX512BW-NEXT: vpshufb %xmm14, %xmm13, %xmm15
7667 ; AVX512BW-NEXT: vpshufb %xmm14, %xmm12, %xmm14
7668 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
7669 ; AVX512BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
7670 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm15 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
7671 ; AVX512BW-NEXT: vpshufb %xmm15, %xmm11, %xmm17
7672 ; AVX512BW-NEXT: vpshufb %xmm15, %xmm19, %xmm15
7673 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
7674 ; AVX512BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
7675 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
7676 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
7677 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm1, %xmm14
7678 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
7679 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
7680 ; AVX512BW-NEXT: vpshufb %xmm10, %xmm4, %xmm14
7681 ; AVX512BW-NEXT: vpshufb %xmm10, %xmm8, %xmm10
7682 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
7683 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
7684 ; AVX512BW-NEXT: vpsrlq $24, %zmm16, %zmm10
7685 ; AVX512BW-NEXT: vpmovqb %zmm10, %xmm10
7686 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
7687 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
7688 ; AVX512BW-NEXT: vmovdqa64 %ymm0, %ymm23
7689 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
7690 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm2, %xmm5
7691 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm6, %xmm10
7692 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
7693 ; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
7694 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm10 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
7695 ; AVX512BW-NEXT: vpshufb %xmm10, %xmm7, %xmm14
7696 ; AVX512BW-NEXT: vpshufb %xmm10, %xmm9, %xmm17
7697 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm17[0],xmm14[0],xmm17[1],xmm14[1],xmm17[2],xmm14[2],xmm17[3],xmm14[3]
7698 ; AVX512BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
7699 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5,6],ymm5[7]
7700 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm14 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
7701 ; AVX512BW-NEXT: vpshufb %xmm14, %xmm13, %xmm17
7702 ; AVX512BW-NEXT: vpshufb %xmm14, %xmm12, %xmm14
7703 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm17[0],xmm14[1],xmm17[1],xmm14[2],xmm17[2],xmm14[3],xmm17[3]
7704 ; AVX512BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
7705 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm17 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
7706 ; AVX512BW-NEXT: vpshufb %xmm17, %xmm11, %xmm18
7707 ; AVX512BW-NEXT: vpshufb %xmm17, %xmm19, %xmm17
7708 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
7709 ; AVX512BW-NEXT: vinserti32x4 $1, %xmm17, %ymm0, %ymm15
7710 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
7711 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
7712 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm1, %xmm14
7713 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
7714 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
7715 ; AVX512BW-NEXT: vpshufb %xmm10, %xmm4, %xmm14
7716 ; AVX512BW-NEXT: vpshufb %xmm10, %xmm8, %xmm10
7717 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
7718 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
7719 ; AVX512BW-NEXT: vpsrlq $32, %zmm16, %zmm10
7720 ; AVX512BW-NEXT: vpmovqb %zmm10, %xmm10
7721 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
7722 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
7723 ; AVX512BW-NEXT: vmovdqa64 %ymm0, %ymm24
7724 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
7725 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm2, %xmm5
7726 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm6, %xmm10
7727 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
7728 ; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
7729 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm10 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
7730 ; AVX512BW-NEXT: vpshufb %xmm10, %xmm7, %xmm15
7731 ; AVX512BW-NEXT: vpshufb %xmm10, %xmm9, %xmm17
7732 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm17[0],xmm15[0],xmm17[1],xmm15[1],xmm17[2],xmm15[2],xmm17[3],xmm15[3]
7733 ; AVX512BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
7734 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm15[0,1,2,3,4,5,6],ymm5[7]
7735 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm15 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
7736 ; AVX512BW-NEXT: vpshufb %xmm15, %xmm13, %xmm17
7737 ; AVX512BW-NEXT: vpshufb %xmm15, %xmm12, %xmm15
7738 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
7739 ; AVX512BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
7740 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm17 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
7741 ; AVX512BW-NEXT: vpshufb %xmm17, %xmm11, %xmm18
7742 ; AVX512BW-NEXT: vpshufb %xmm17, %xmm19, %xmm17
7743 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
7744 ; AVX512BW-NEXT: vinserti32x4 $1, %xmm17, %ymm0, %ymm14
7745 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm14 = ymm14[0,1,2,3,4],ymm15[5],ymm14[6,7]
7746 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
7747 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm1, %xmm14
7748 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
7749 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
7750 ; AVX512BW-NEXT: vpshufb %xmm10, %xmm4, %xmm14
7751 ; AVX512BW-NEXT: vpshufb %xmm10, %xmm8, %xmm10
7752 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
7753 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
7754 ; AVX512BW-NEXT: vpsrlq $40, %zmm16, %zmm10
7755 ; AVX512BW-NEXT: vpmovqb %zmm10, %xmm10
7756 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
7757 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
7758 ; AVX512BW-NEXT: vmovdqa64 %ymm0, %ymm25
7759 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
7760 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm2, %xmm10
7761 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm6, %xmm14
7762 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm14[0],xmm10[0],xmm14[1],xmm10[1],xmm14[2],xmm10[2],xmm14[3],xmm10[3]
7763 ; AVX512BW-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
7764 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm14 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
7765 ; AVX512BW-NEXT: vpshufb %xmm14, %xmm7, %xmm15
7766 ; AVX512BW-NEXT: vpshufb %xmm14, %xmm9, %xmm17
7767 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm17[0],xmm15[0],xmm17[1],xmm15[1],xmm17[2],xmm15[2],xmm17[3],xmm15[3]
7768 ; AVX512BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
7769 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm10 = ymm15[0,1,2,3,4,5,6],ymm10[7]
7770 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm15 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
7771 ; AVX512BW-NEXT: vpshufb %xmm15, %xmm13, %xmm17
7772 ; AVX512BW-NEXT: vpshufb %xmm15, %xmm12, %xmm15
7773 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
7774 ; AVX512BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
7775 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm17 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
7776 ; AVX512BW-NEXT: vpshufb %xmm17, %xmm11, %xmm18
7777 ; AVX512BW-NEXT: vpshufb %xmm17, %xmm19, %xmm17
7778 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
7779 ; AVX512BW-NEXT: vinserti32x4 $1, %xmm17, %ymm0, %ymm5
7780 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4],ymm15[5],ymm5[6,7]
7781 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5],ymm10[6,7]
7782 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm1, %xmm10
7783 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
7784 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
7785 ; AVX512BW-NEXT: vpshufb %xmm14, %xmm4, %xmm10
7786 ; AVX512BW-NEXT: vpshufb %xmm14, %xmm8, %xmm14
7787 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm14[0],xmm10[0],xmm14[1],xmm10[1],xmm14[2],xmm10[2],xmm14[3],xmm10[3]
7788 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
7789 ; AVX512BW-NEXT: vpsrlq $48, %zmm16, %zmm10
7790 ; AVX512BW-NEXT: vpmovqb %zmm10, %xmm10
7791 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
7792 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
7793 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm5 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
7794 ; AVX512BW-NEXT: vpshufb %xmm5, %xmm2, %xmm2
7795 ; AVX512BW-NEXT: vpshufb %xmm5, %xmm6, %xmm6
7796 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
7797 ; AVX512BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
7798 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm6 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
7799 ; AVX512BW-NEXT: vpshufb %xmm6, %xmm7, %xmm7
7800 ; AVX512BW-NEXT: vpshufb %xmm6, %xmm9, %xmm9
7801 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]
7802 ; AVX512BW-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
7803 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3,4,5,6],ymm2[7]
7804 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm7 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
7805 ; AVX512BW-NEXT: vpshufb %xmm7, %xmm13, %xmm9
7806 ; AVX512BW-NEXT: vpshufb %xmm7, %xmm12, %xmm7
7807 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1],xmm7[2],xmm9[2],xmm7[3],xmm9[3]
7808 ; AVX512BW-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
7809 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm9 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
7810 ; AVX512BW-NEXT: vpshufb %xmm9, %xmm11, %xmm10
7811 ; AVX512BW-NEXT: vpshufb %xmm9, %xmm19, %xmm9
7812 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
7813 ; AVX512BW-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
7814 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm7 = ymm9[0,1,2,3,4],ymm7[5],ymm9[6,7]
7815 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3,4,5],ymm2[6,7]
7816 ; AVX512BW-NEXT: vpshufb %xmm5, %xmm1, %xmm1
7817 ; AVX512BW-NEXT: vpshufb %xmm5, %xmm3, %xmm3
7818 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
7819 ; AVX512BW-NEXT: vpshufb %xmm6, %xmm4, %xmm3
7820 ; AVX512BW-NEXT: vpshufb %xmm6, %xmm8, %xmm4
7821 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
7822 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
7823 ; AVX512BW-NEXT: vpsrlq $56, %zmm16, %zmm3
7824 ; AVX512BW-NEXT: vpmovqb %zmm3, %xmm3
7825 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
7826 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
7827 ; AVX512BW-NEXT: vmovdqa64 %ymm20, (%rsi)
7828 ; AVX512BW-NEXT: vmovdqa64 %ymm21, (%rdx)
7829 ; AVX512BW-NEXT: vmovdqa64 %ymm22, (%rcx)
7830 ; AVX512BW-NEXT: vmovdqa64 %ymm23, (%r8)
7831 ; AVX512BW-NEXT: vmovdqa64 %ymm24, (%r9)
7832 ; AVX512BW-NEXT: vmovdqa64 %ymm25, (%r11)
7833 ; AVX512BW-NEXT: vmovdqa %ymm0, (%r10)
7834 ; AVX512BW-NEXT: vmovdqa %ymm1, (%rax)
7835 ; AVX512BW-NEXT: vzeroupper
7836 ; AVX512BW-NEXT: retq
7837 ;
7838 ; AVX512BW-FCP-LABEL: load_i8_stride8_vf32:
7839 ; AVX512BW-FCP: # %bb.0:
7840 ; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
7841 ; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
7842 ; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11
7843 ; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
7844 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm12 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
7845 ; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,2,2,3,0,2,4,6]
7846 ; AVX512BW-FCP-NEXT: vmovdqa 224(%rdi), %ymm9
7847 ; AVX512BW-FCP-NEXT: vpermd %ymm9, %ymm1, %ymm4
7848 ; AVX512BW-FCP-NEXT: vpshufb %ymm12, %ymm4, %ymm2
7849 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm30 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
7850 ; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm11
7851 ; AVX512BW-FCP-NEXT: vpermd %ymm11, %ymm1, %ymm26
7852 ; AVX512BW-FCP-NEXT: vpshufb %ymm30, %ymm26, %ymm3
7853 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
7854 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm31 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
7855 ; AVX512BW-FCP-NEXT: vmovdqa 160(%rdi), %ymm13
7856 ; AVX512BW-FCP-NEXT: vpermd %ymm13, %ymm1, %ymm27
7857 ; AVX512BW-FCP-NEXT: vpshufb %ymm31, %ymm27, %ymm3
7858 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm14 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
7859 ; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm15
7860 ; AVX512BW-FCP-NEXT: vpermd %ymm15, %ymm1, %ymm28
7861 ; AVX512BW-FCP-NEXT: vpshufb %ymm14, %ymm28, %ymm1
7862 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5],ymm1[6,7]
7863 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm1[0,1,2,3,4,5],ymm2[6,7]
7864 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
7865 ; AVX512BW-FCP-NEXT: vmovdqa 112(%rdi), %xmm1
7866 ; AVX512BW-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm5
7867 ; AVX512BW-FCP-NEXT: vmovdqa 96(%rdi), %xmm2
7868 ; AVX512BW-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm3
7869 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
7870 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm16 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
7871 ; AVX512BW-FCP-NEXT: vmovdqa 80(%rdi), %xmm3
7872 ; AVX512BW-FCP-NEXT: vpshufb %xmm16, %xmm3, %xmm17
7873 ; AVX512BW-FCP-NEXT: vmovdqa 64(%rdi), %xmm5
7874 ; AVX512BW-FCP-NEXT: vpshufb %xmm16, %xmm5, %xmm16
7875 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm16[0],xmm17[0],xmm16[1],xmm17[1],xmm16[2],xmm17[2],xmm16[3],xmm17[3]
7876 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3]
7877 ; AVX512BW-FCP-NEXT: vpmovqb %zmm0, %xmm8
7878 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3]
7879 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7]
7880 ; AVX512BW-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7881 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm16 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
7882 ; AVX512BW-FCP-NEXT: vpshufb %ymm16, %ymm4, %ymm6
7883 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm17 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
7884 ; AVX512BW-FCP-NEXT: vpshufb %ymm17, %ymm26, %ymm7
7885 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3,4,5,6],ymm6[7]
7886 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm18 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
7887 ; AVX512BW-FCP-NEXT: vpshufb %ymm18, %ymm27, %ymm7
7888 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm19 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
7889 ; AVX512BW-FCP-NEXT: vpshufb %ymm19, %ymm28, %ymm8
7890 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3,4],ymm7[5],ymm8[6,7]
7891 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3,4,5],ymm6[6,7]
7892 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm7 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
7893 ; AVX512BW-FCP-NEXT: vpshufb %xmm7, %xmm1, %xmm8
7894 ; AVX512BW-FCP-NEXT: vpshufb %xmm7, %xmm2, %xmm7
7895 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
7896 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm8 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
7897 ; AVX512BW-FCP-NEXT: vpshufb %xmm8, %xmm3, %xmm20
7898 ; AVX512BW-FCP-NEXT: vpshufb %xmm8, %xmm5, %xmm8
7899 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm20[0],xmm8[1],xmm20[1],xmm8[2],xmm20[2],xmm8[3],xmm20[3]
7900 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3]
7901 ; AVX512BW-FCP-NEXT: vpsrlq $8, %zmm0, %zmm8
7902 ; AVX512BW-FCP-NEXT: vpmovqb %zmm8, %xmm8
7903 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3]
7904 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7]
7905 ; AVX512BW-FCP-NEXT: vmovdqa64 %ymm6, %ymm29
7906 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm20 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
7907 ; AVX512BW-FCP-NEXT: vpshufb %ymm20, %ymm4, %ymm7
7908 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm21 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
7909 ; AVX512BW-FCP-NEXT: vpshufb %ymm21, %ymm26, %ymm8
7910 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3,4,5,6],ymm7[7]
7911 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm22 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
7912 ; AVX512BW-FCP-NEXT: vpshufb %ymm22, %ymm27, %ymm8
7913 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm23 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
7914 ; AVX512BW-FCP-NEXT: vpshufb %ymm23, %ymm28, %ymm10
7915 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm10[0,1,2,3,4],ymm8[5],ymm10[6,7]
7916 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3,4,5],ymm7[6,7]
7917 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm8 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
7918 ; AVX512BW-FCP-NEXT: vpshufb %xmm8, %xmm1, %xmm10
7919 ; AVX512BW-FCP-NEXT: vpshufb %xmm8, %xmm2, %xmm8
7920 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3]
7921 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
7922 ; AVX512BW-FCP-NEXT: vpshufb %xmm10, %xmm3, %xmm24
7923 ; AVX512BW-FCP-NEXT: vpshufb %xmm10, %xmm5, %xmm10
7924 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm24[0],xmm10[1],xmm24[1],xmm10[2],xmm24[2],xmm10[3],xmm24[3]
7925 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm10[0,1,2],xmm8[3]
7926 ; AVX512BW-FCP-NEXT: vpsrlq $16, %zmm0, %zmm10
7927 ; AVX512BW-FCP-NEXT: vpmovqb %zmm10, %xmm10
7928 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm10[0,1],xmm8[2,3]
7929 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7]
7930 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm24 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
7931 ; AVX512BW-FCP-NEXT: vpshufb %ymm24, %ymm4, %ymm4
7932 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm25 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
7933 ; AVX512BW-FCP-NEXT: vpshufb %ymm25, %ymm26, %ymm8
7934 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5,6],ymm4[7]
7935 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm26 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
7936 ; AVX512BW-FCP-NEXT: vpshufb %ymm26, %ymm27, %ymm8
7937 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm27 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
7938 ; AVX512BW-FCP-NEXT: vpshufb %ymm27, %ymm28, %ymm10
7939 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm10[0,1,2,3,4],ymm8[5],ymm10[6,7]
7940 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
7941 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm8 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
7942 ; AVX512BW-FCP-NEXT: vpshufb %xmm8, %xmm1, %xmm10
7943 ; AVX512BW-FCP-NEXT: vpshufb %xmm8, %xmm2, %xmm8
7944 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3]
7945 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
7946 ; AVX512BW-FCP-NEXT: vpshufb %xmm10, %xmm3, %xmm28
7947 ; AVX512BW-FCP-NEXT: vpshufb %xmm10, %xmm5, %xmm10
7948 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm28[0],xmm10[1],xmm28[1],xmm10[2],xmm28[2],xmm10[3],xmm28[3]
7949 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm10[0,1,2],xmm8[3]
7950 ; AVX512BW-FCP-NEXT: vpsrlq $24, %zmm0, %zmm10
7951 ; AVX512BW-FCP-NEXT: vpmovqb %zmm10, %xmm10
7952 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm10[0,1],xmm8[2,3]
7953 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3],ymm4[4,5,6,7]
7954 ; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm8 = [1,3,2,3,1,3,5,7]
7955 ; AVX512BW-FCP-NEXT: vpermd %ymm9, %ymm8, %ymm9
7956 ; AVX512BW-FCP-NEXT: vpshufb %ymm12, %ymm9, %ymm10
7957 ; AVX512BW-FCP-NEXT: vpermd %ymm11, %ymm8, %ymm11
7958 ; AVX512BW-FCP-NEXT: vpshufb %ymm30, %ymm11, %ymm12
7959 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5,6],ymm10[7]
7960 ; AVX512BW-FCP-NEXT: vpermd %ymm13, %ymm8, %ymm10
7961 ; AVX512BW-FCP-NEXT: vpshufb %ymm31, %ymm10, %ymm13
7962 ; AVX512BW-FCP-NEXT: vpermd %ymm15, %ymm8, %ymm15
7963 ; AVX512BW-FCP-NEXT: vpshufb %ymm14, %ymm15, %ymm8
7964 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm13[5],ymm8[6,7]
7965 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm12[6,7]
7966 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm12 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
7967 ; AVX512BW-FCP-NEXT: vpshufb %xmm12, %xmm1, %xmm13
7968 ; AVX512BW-FCP-NEXT: vpshufb %xmm12, %xmm2, %xmm12
7969 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
7970 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm13 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
7971 ; AVX512BW-FCP-NEXT: vpshufb %xmm13, %xmm3, %xmm14
7972 ; AVX512BW-FCP-NEXT: vpshufb %xmm13, %xmm5, %xmm13
7973 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
7974 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm13[0,1,2],xmm12[3]
7975 ; AVX512BW-FCP-NEXT: vpsrlq $32, %zmm0, %zmm13
7976 ; AVX512BW-FCP-NEXT: vpmovqb %zmm13, %xmm13
7977 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm13[0,1],xmm12[2,3]
7978 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm12[0,1,2,3],ymm8[4,5,6,7]
7979 ; AVX512BW-FCP-NEXT: vpshufb %ymm16, %ymm9, %ymm12
7980 ; AVX512BW-FCP-NEXT: vpshufb %ymm17, %ymm11, %ymm13
7981 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3,4,5,6],ymm12[7]
7982 ; AVX512BW-FCP-NEXT: vpshufb %ymm18, %ymm10, %ymm13
7983 ; AVX512BW-FCP-NEXT: vpshufb %ymm19, %ymm15, %ymm14
7984 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm13 = ymm14[0,1,2,3,4],ymm13[5],ymm14[6,7]
7985 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3,4,5],ymm12[6,7]
7986 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm13 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
7987 ; AVX512BW-FCP-NEXT: vpshufb %xmm13, %xmm1, %xmm14
7988 ; AVX512BW-FCP-NEXT: vpshufb %xmm13, %xmm2, %xmm13
7989 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
7990 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm14 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
7991 ; AVX512BW-FCP-NEXT: vpshufb %xmm14, %xmm3, %xmm16
7992 ; AVX512BW-FCP-NEXT: vpshufb %xmm14, %xmm5, %xmm14
7993 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm16[0],xmm14[1],xmm16[1],xmm14[2],xmm16[2],xmm14[3],xmm16[3]
7994 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm13 = xmm14[0,1,2],xmm13[3]
7995 ; AVX512BW-FCP-NEXT: vpsrlq $40, %zmm0, %zmm14
7996 ; AVX512BW-FCP-NEXT: vpmovqb %zmm14, %xmm14
7997 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm13 = xmm14[0,1],xmm13[2,3]
7998 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3],ymm12[4,5,6,7]
7999 ; AVX512BW-FCP-NEXT: vpshufb %ymm20, %ymm9, %ymm13
8000 ; AVX512BW-FCP-NEXT: vpshufb %ymm21, %ymm11, %ymm14
8001 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm13 = ymm14[0,1,2,3,4,5,6],ymm13[7]
8002 ; AVX512BW-FCP-NEXT: vpshufb %ymm22, %ymm10, %ymm14
8003 ; AVX512BW-FCP-NEXT: vpshufb %ymm23, %ymm15, %ymm6
8004 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4],ymm14[5],ymm6[6,7]
8005 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],ymm13[6,7]
8006 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm13 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
8007 ; AVX512BW-FCP-NEXT: vpshufb %xmm13, %xmm1, %xmm14
8008 ; AVX512BW-FCP-NEXT: vpshufb %xmm13, %xmm2, %xmm13
8009 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
8010 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm14 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
8011 ; AVX512BW-FCP-NEXT: vpshufb %xmm14, %xmm3, %xmm16
8012 ; AVX512BW-FCP-NEXT: vpshufb %xmm14, %xmm5, %xmm14
8013 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm16[0],xmm14[1],xmm16[1],xmm14[2],xmm16[2],xmm14[3],xmm16[3]
8014 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm13 = xmm14[0,1,2],xmm13[3]
8015 ; AVX512BW-FCP-NEXT: vpsrlq $48, %zmm0, %zmm14
8016 ; AVX512BW-FCP-NEXT: vpmovqb %zmm14, %xmm14
8017 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm13 = xmm14[0,1],xmm13[2,3]
8018 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm13[0,1,2,3],ymm6[4,5,6,7]
8019 ; AVX512BW-FCP-NEXT: vpshufb %ymm24, %ymm9, %ymm9
8020 ; AVX512BW-FCP-NEXT: vpshufb %ymm25, %ymm11, %ymm11
8021 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3,4,5,6],ymm9[7]
8022 ; AVX512BW-FCP-NEXT: vpshufb %ymm26, %ymm10, %ymm10
8023 ; AVX512BW-FCP-NEXT: vpshufb %ymm27, %ymm15, %ymm11
8024 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3,4],ymm10[5],ymm11[6,7]
8025 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4,5],ymm9[6,7]
8026 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
8027 ; AVX512BW-FCP-NEXT: vpshufb %xmm10, %xmm1, %xmm1
8028 ; AVX512BW-FCP-NEXT: vpshufb %xmm10, %xmm2, %xmm2
8029 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
8030 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
8031 ; AVX512BW-FCP-NEXT: vpshufb %xmm2, %xmm3, %xmm3
8032 ; AVX512BW-FCP-NEXT: vpshufb %xmm2, %xmm5, %xmm2
8033 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
8034 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
8035 ; AVX512BW-FCP-NEXT: vpsrlq $56, %zmm0, %zmm0
8036 ; AVX512BW-FCP-NEXT: vpmovqb %zmm0, %xmm0
8037 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
8038 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm9[4,5,6,7]
8039 ; AVX512BW-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8040 ; AVX512BW-FCP-NEXT: vmovaps %ymm1, (%rsi)
8041 ; AVX512BW-FCP-NEXT: vmovdqa64 %ymm29, (%rdx)
8042 ; AVX512BW-FCP-NEXT: vmovdqa %ymm7, (%rcx)
8043 ; AVX512BW-FCP-NEXT: vmovdqa %ymm4, (%r8)
8044 ; AVX512BW-FCP-NEXT: vmovdqa %ymm8, (%r9)
8045 ; AVX512BW-FCP-NEXT: vmovdqa %ymm12, (%r11)
8046 ; AVX512BW-FCP-NEXT: vmovdqa %ymm6, (%r10)
8047 ; AVX512BW-FCP-NEXT: vmovdqa %ymm0, (%rax)
8048 ; AVX512BW-FCP-NEXT: vzeroupper
8049 ; AVX512BW-FCP-NEXT: retq
8050 ;
8051 ; AVX512DQ-BW-LABEL: load_i8_stride8_vf32:
8052 ; AVX512DQ-BW: # %bb.0:
8053 ; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
8054 ; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %r10
8055 ; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %r11
8056 ; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm16
8057 ; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %zmm0
8058 ; AVX512DQ-BW-NEXT: vpmovqb %zmm0, %xmm0
8059 ; AVX512DQ-BW-NEXT: vmovdqa 240(%rdi), %xmm2
8060 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
8061 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm2, %xmm1
8062 ; AVX512DQ-BW-NEXT: vmovdqa 224(%rdi), %xmm6
8063 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm6, %xmm3
8064 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
8065 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
8066 ; AVX512DQ-BW-NEXT: vmovdqa 208(%rdi), %xmm7
8067 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm5 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
8068 ; AVX512DQ-BW-NEXT: vpshufb %xmm5, %xmm7, %xmm3
8069 ; AVX512DQ-BW-NEXT: vmovdqa 192(%rdi), %xmm9
8070 ; AVX512DQ-BW-NEXT: vpshufb %xmm5, %xmm9, %xmm8
8071 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
8072 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
8073 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
8074 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
8075 ; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm3
8076 ; AVX512DQ-BW-NEXT: vpmovqb %ymm3, %xmm3
8077 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
8078 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4],ymm0[5],ymm3[6,7]
8079 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
8080 ; AVX512DQ-BW-NEXT: vmovdqa 112(%rdi), %xmm1
8081 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm1, %xmm8
8082 ; AVX512DQ-BW-NEXT: vmovdqa 96(%rdi), %xmm3
8083 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm3, %xmm4
8084 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm4[0],xmm8[0],xmm4[1],xmm8[1],xmm4[2],xmm8[2],xmm4[3],xmm8[3]
8085 ; AVX512DQ-BW-NEXT: vmovdqa 80(%rdi), %xmm4
8086 ; AVX512DQ-BW-NEXT: vpshufb %xmm5, %xmm4, %xmm11
8087 ; AVX512DQ-BW-NEXT: vmovdqa 64(%rdi), %xmm8
8088 ; AVX512DQ-BW-NEXT: vpshufb %xmm5, %xmm8, %xmm5
8089 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm11[0],xmm5[1],xmm11[1],xmm5[2],xmm11[2],xmm5[3],xmm11[3]
8090 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1,2],xmm10[3]
8091 ; AVX512DQ-BW-NEXT: vpmovqb %zmm16, %xmm10
8092 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm5 = xmm10[0,1],xmm5[2,3]
8093 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3],ymm0[4,5,6,7]
8094 ; AVX512DQ-BW-NEXT: vmovdqa64 %ymm0, %ymm20
8095 ; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %xmm19
8096 ; AVX512DQ-BW-NEXT: vmovdqa 144(%rdi), %xmm11
8097 ; AVX512DQ-BW-NEXT: vmovdqa 160(%rdi), %xmm12
8098 ; AVX512DQ-BW-NEXT: vmovdqa 176(%rdi), %xmm13
8099 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
8100 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm2, %xmm5
8101 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm6, %xmm14
8102 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm14[0],xmm5[0],xmm14[1],xmm5[1],xmm14[2],xmm5[2],xmm14[3],xmm5[3]
8103 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
8104 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm14 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
8105 ; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm7, %xmm15
8106 ; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm9, %xmm17
8107 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm17[0],xmm15[0],xmm17[1],xmm15[1],xmm17[2],xmm15[2],xmm17[3],xmm15[3]
8108 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
8109 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm15[0,1,2,3,4,5,6],ymm5[7]
8110 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm15 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
8111 ; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm13, %xmm17
8112 ; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm12, %xmm15
8113 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
8114 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
8115 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm17 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
8116 ; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm11, %xmm18
8117 ; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm19, %xmm17
8118 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
8119 ; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm17, %ymm0, %ymm10
8120 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm15[5],ymm10[6,7]
8121 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5],ymm5[6,7]
8122 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm1, %xmm10
8123 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
8124 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
8125 ; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm4, %xmm10
8126 ; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm8, %xmm14
8127 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm14[0],xmm10[0],xmm14[1],xmm10[1],xmm14[2],xmm10[2],xmm14[3],xmm10[3]
8128 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
8129 ; AVX512DQ-BW-NEXT: vpsrlq $8, %zmm16, %zmm10
8130 ; AVX512DQ-BW-NEXT: vpmovqb %zmm10, %xmm10
8131 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
8132 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
8133 ; AVX512DQ-BW-NEXT: vmovdqa64 %ymm0, %ymm21
8134 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
8135 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm2, %xmm5
8136 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm6, %xmm10
8137 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
8138 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
8139 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm10 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
8140 ; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm7, %xmm14
8141 ; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm9, %xmm15
8142 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
8143 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
8144 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5,6],ymm5[7]
8145 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm14 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
8146 ; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm13, %xmm15
8147 ; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm12, %xmm14
8148 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
8149 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
8150 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm15 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
8151 ; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm11, %xmm17
8152 ; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm19, %xmm15
8153 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
8154 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
8155 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
8156 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
8157 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm1, %xmm14
8158 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
8159 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
8160 ; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm4, %xmm14
8161 ; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm8, %xmm10
8162 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
8163 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
8164 ; AVX512DQ-BW-NEXT: vpsrlq $16, %zmm16, %zmm10
8165 ; AVX512DQ-BW-NEXT: vpmovqb %zmm10, %xmm10
8166 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
8167 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
8168 ; AVX512DQ-BW-NEXT: vmovdqa64 %ymm0, %ymm22
8169 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
8170 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm2, %xmm5
8171 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm6, %xmm10
8172 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
8173 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
8174 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm10 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
8175 ; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm7, %xmm14
8176 ; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm9, %xmm15
8177 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
8178 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
8179 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5,6],ymm5[7]
8180 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm14 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
8181 ; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm13, %xmm15
8182 ; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm12, %xmm14
8183 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
8184 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
8185 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm15 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
8186 ; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm11, %xmm17
8187 ; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm19, %xmm15
8188 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
8189 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
8190 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
8191 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
8192 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm1, %xmm14
8193 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
8194 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
8195 ; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm4, %xmm14
8196 ; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm8, %xmm10
8197 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
8198 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
8199 ; AVX512DQ-BW-NEXT: vpsrlq $24, %zmm16, %zmm10
8200 ; AVX512DQ-BW-NEXT: vpmovqb %zmm10, %xmm10
8201 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
8202 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
8203 ; AVX512DQ-BW-NEXT: vmovdqa64 %ymm0, %ymm23
8204 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
8205 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm2, %xmm5
8206 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm6, %xmm10
8207 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
8208 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
8209 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm10 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
8210 ; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm7, %xmm14
8211 ; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm9, %xmm17
8212 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm17[0],xmm14[0],xmm17[1],xmm14[1],xmm17[2],xmm14[2],xmm17[3],xmm14[3]
8213 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
8214 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5,6],ymm5[7]
8215 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm14 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
8216 ; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm13, %xmm17
8217 ; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm12, %xmm14
8218 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm17[0],xmm14[1],xmm17[1],xmm14[2],xmm17[2],xmm14[3],xmm17[3]
8219 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
8220 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm17 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
8221 ; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm11, %xmm18
8222 ; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm19, %xmm17
8223 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
8224 ; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm17, %ymm0, %ymm15
8225 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
8226 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
8227 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm1, %xmm14
8228 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
8229 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
8230 ; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm4, %xmm14
8231 ; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm8, %xmm10
8232 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
8233 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
8234 ; AVX512DQ-BW-NEXT: vpsrlq $32, %zmm16, %zmm10
8235 ; AVX512DQ-BW-NEXT: vpmovqb %zmm10, %xmm10
8236 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
8237 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
8238 ; AVX512DQ-BW-NEXT: vmovdqa64 %ymm0, %ymm24
8239 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
8240 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm2, %xmm5
8241 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm6, %xmm10
8242 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
8243 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
8244 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm10 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
8245 ; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm7, %xmm15
8246 ; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm9, %xmm17
8247 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm17[0],xmm15[0],xmm17[1],xmm15[1],xmm17[2],xmm15[2],xmm17[3],xmm15[3]
8248 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
8249 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm15[0,1,2,3,4,5,6],ymm5[7]
8250 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm15 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
8251 ; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm13, %xmm17
8252 ; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm12, %xmm15
8253 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
8254 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
8255 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm17 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
8256 ; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm11, %xmm18
8257 ; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm19, %xmm17
8258 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
8259 ; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm17, %ymm0, %ymm14
8260 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm14 = ymm14[0,1,2,3,4],ymm15[5],ymm14[6,7]
8261 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
8262 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm1, %xmm14
8263 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
8264 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
8265 ; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm4, %xmm14
8266 ; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm8, %xmm10
8267 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
8268 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
8269 ; AVX512DQ-BW-NEXT: vpsrlq $40, %zmm16, %zmm10
8270 ; AVX512DQ-BW-NEXT: vpmovqb %zmm10, %xmm10
8271 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
8272 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
8273 ; AVX512DQ-BW-NEXT: vmovdqa64 %ymm0, %ymm25
8274 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
8275 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm2, %xmm10
8276 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm6, %xmm14
8277 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm14[0],xmm10[0],xmm14[1],xmm10[1],xmm14[2],xmm10[2],xmm14[3],xmm10[3]
8278 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
8279 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm14 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
8280 ; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm7, %xmm15
8281 ; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm9, %xmm17
8282 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm17[0],xmm15[0],xmm17[1],xmm15[1],xmm17[2],xmm15[2],xmm17[3],xmm15[3]
8283 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
8284 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm10 = ymm15[0,1,2,3,4,5,6],ymm10[7]
8285 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm15 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
8286 ; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm13, %xmm17
8287 ; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm12, %xmm15
8288 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
8289 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
8290 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm17 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
8291 ; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm11, %xmm18
8292 ; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm19, %xmm17
8293 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
8294 ; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm17, %ymm0, %ymm5
8295 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4],ymm15[5],ymm5[6,7]
8296 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5],ymm10[6,7]
8297 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm1, %xmm10
8298 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
8299 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
8300 ; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm4, %xmm10
8301 ; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm8, %xmm14
8302 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm14[0],xmm10[0],xmm14[1],xmm10[1],xmm14[2],xmm10[2],xmm14[3],xmm10[3]
8303 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
8304 ; AVX512DQ-BW-NEXT: vpsrlq $48, %zmm16, %zmm10
8305 ; AVX512DQ-BW-NEXT: vpmovqb %zmm10, %xmm10
8306 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
8307 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
8308 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm5 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
8309 ; AVX512DQ-BW-NEXT: vpshufb %xmm5, %xmm2, %xmm2
8310 ; AVX512DQ-BW-NEXT: vpshufb %xmm5, %xmm6, %xmm6
8311 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
8312 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
8313 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm6 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
8314 ; AVX512DQ-BW-NEXT: vpshufb %xmm6, %xmm7, %xmm7
8315 ; AVX512DQ-BW-NEXT: vpshufb %xmm6, %xmm9, %xmm9
8316 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]
8317 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
8318 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3,4,5,6],ymm2[7]
8319 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm7 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
8320 ; AVX512DQ-BW-NEXT: vpshufb %xmm7, %xmm13, %xmm9
8321 ; AVX512DQ-BW-NEXT: vpshufb %xmm7, %xmm12, %xmm7
8322 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1],xmm7[2],xmm9[2],xmm7[3],xmm9[3]
8323 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
8324 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm9 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
8325 ; AVX512DQ-BW-NEXT: vpshufb %xmm9, %xmm11, %xmm10
8326 ; AVX512DQ-BW-NEXT: vpshufb %xmm9, %xmm19, %xmm9
8327 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
8328 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
8329 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm7 = ymm9[0,1,2,3,4],ymm7[5],ymm9[6,7]
8330 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3,4,5],ymm2[6,7]
8331 ; AVX512DQ-BW-NEXT: vpshufb %xmm5, %xmm1, %xmm1
8332 ; AVX512DQ-BW-NEXT: vpshufb %xmm5, %xmm3, %xmm3
8333 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
8334 ; AVX512DQ-BW-NEXT: vpshufb %xmm6, %xmm4, %xmm3
8335 ; AVX512DQ-BW-NEXT: vpshufb %xmm6, %xmm8, %xmm4
8336 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
8337 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
8338 ; AVX512DQ-BW-NEXT: vpsrlq $56, %zmm16, %zmm3
8339 ; AVX512DQ-BW-NEXT: vpmovqb %zmm3, %xmm3
8340 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
8341 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
8342 ; AVX512DQ-BW-NEXT: vmovdqa64 %ymm20, (%rsi)
8343 ; AVX512DQ-BW-NEXT: vmovdqa64 %ymm21, (%rdx)
8344 ; AVX512DQ-BW-NEXT: vmovdqa64 %ymm22, (%rcx)
8345 ; AVX512DQ-BW-NEXT: vmovdqa64 %ymm23, (%r8)
8346 ; AVX512DQ-BW-NEXT: vmovdqa64 %ymm24, (%r9)
8347 ; AVX512DQ-BW-NEXT: vmovdqa64 %ymm25, (%r11)
8348 ; AVX512DQ-BW-NEXT: vmovdqa %ymm0, (%r10)
8349 ; AVX512DQ-BW-NEXT: vmovdqa %ymm1, (%rax)
8350 ; AVX512DQ-BW-NEXT: vzeroupper
8351 ; AVX512DQ-BW-NEXT: retq
8352 ;
8353 ; AVX512DQ-BW-FCP-LABEL: load_i8_stride8_vf32:
8354 ; AVX512DQ-BW-FCP: # %bb.0:
8355 ; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
8356 ; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
8357 ; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r11
8358 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
8359 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm12 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
8360 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,2,2,3,0,2,4,6]
8361 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 224(%rdi), %ymm9
8362 ; AVX512DQ-BW-FCP-NEXT: vpermd %ymm9, %ymm1, %ymm4
8363 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm12, %ymm4, %ymm2
8364 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm30 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
8365 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm11
8366 ; AVX512DQ-BW-FCP-NEXT: vpermd %ymm11, %ymm1, %ymm26
8367 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm30, %ymm26, %ymm3
8368 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
8369 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm31 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
8370 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 160(%rdi), %ymm13
8371 ; AVX512DQ-BW-FCP-NEXT: vpermd %ymm13, %ymm1, %ymm27
8372 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm31, %ymm27, %ymm3
8373 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm14 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
8374 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm15
8375 ; AVX512DQ-BW-FCP-NEXT: vpermd %ymm15, %ymm1, %ymm28
8376 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm14, %ymm28, %ymm1
8377 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5],ymm1[6,7]
8378 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm1[0,1,2,3,4,5],ymm2[6,7]
8379 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
8380 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 112(%rdi), %xmm1
8381 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm5
8382 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 96(%rdi), %xmm2
8383 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm3
8384 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
8385 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm16 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
8386 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 80(%rdi), %xmm3
8387 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm16, %xmm3, %xmm17
8388 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 64(%rdi), %xmm5
8389 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm16, %xmm5, %xmm16
8390 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm16[0],xmm17[0],xmm16[1],xmm17[1],xmm16[2],xmm17[2],xmm16[3],xmm17[3]
8391 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3]
8392 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm0, %xmm8
8393 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3]
8394 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7]
8395 ; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8396 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm16 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
8397 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm16, %ymm4, %ymm6
8398 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm17 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
8399 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm17, %ymm26, %ymm7
8400 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3,4,5,6],ymm6[7]
8401 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm18 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
8402 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm18, %ymm27, %ymm7
8403 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm19 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
8404 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm19, %ymm28, %ymm8
8405 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3,4],ymm7[5],ymm8[6,7]
8406 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3,4,5],ymm6[6,7]
8407 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm7 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
8408 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm7, %xmm1, %xmm8
8409 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm7, %xmm2, %xmm7
8410 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
8411 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm8 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
8412 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm8, %xmm3, %xmm20
8413 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm8, %xmm5, %xmm8
8414 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm20[0],xmm8[1],xmm20[1],xmm8[2],xmm20[2],xmm8[3],xmm20[3]
8415 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3]
8416 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $8, %zmm0, %zmm8
8417 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm8, %xmm8
8418 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3]
8419 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7]
8420 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %ymm6, %ymm29
8421 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm20 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
8422 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm20, %ymm4, %ymm7
8423 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm21 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
8424 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm21, %ymm26, %ymm8
8425 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3,4,5,6],ymm7[7]
8426 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm22 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
8427 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm22, %ymm27, %ymm8
8428 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm23 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
8429 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm23, %ymm28, %ymm10
8430 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm10[0,1,2,3,4],ymm8[5],ymm10[6,7]
8431 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3,4,5],ymm7[6,7]
8432 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm8 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
8433 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm8, %xmm1, %xmm10
8434 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm8, %xmm2, %xmm8
8435 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3]
8436 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
8437 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm10, %xmm3, %xmm24
8438 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm10, %xmm5, %xmm10
8439 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm24[0],xmm10[1],xmm24[1],xmm10[2],xmm24[2],xmm10[3],xmm24[3]
8440 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm10[0,1,2],xmm8[3]
8441 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $16, %zmm0, %zmm10
8442 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm10, %xmm10
8443 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm10[0,1],xmm8[2,3]
8444 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7]
8445 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm24 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
8446 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm24, %ymm4, %ymm4
8447 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm25 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
8448 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm25, %ymm26, %ymm8
8449 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5,6],ymm4[7]
8450 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm26 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
8451 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm26, %ymm27, %ymm8
8452 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm27 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
8453 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm27, %ymm28, %ymm10
8454 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm10[0,1,2,3,4],ymm8[5],ymm10[6,7]
8455 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
8456 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm8 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
8457 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm8, %xmm1, %xmm10
8458 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm8, %xmm2, %xmm8
8459 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3]
8460 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
8461 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm10, %xmm3, %xmm28
8462 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm10, %xmm5, %xmm10
8463 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm28[0],xmm10[1],xmm28[1],xmm10[2],xmm28[2],xmm10[3],xmm28[3]
8464 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm10[0,1,2],xmm8[3]
8465 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $24, %zmm0, %zmm10
8466 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm10, %xmm10
8467 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm10[0,1],xmm8[2,3]
8468 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3],ymm4[4,5,6,7]
8469 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm8 = [1,3,2,3,1,3,5,7]
8470 ; AVX512DQ-BW-FCP-NEXT: vpermd %ymm9, %ymm8, %ymm9
8471 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm12, %ymm9, %ymm10
8472 ; AVX512DQ-BW-FCP-NEXT: vpermd %ymm11, %ymm8, %ymm11
8473 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm30, %ymm11, %ymm12
8474 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5,6],ymm10[7]
8475 ; AVX512DQ-BW-FCP-NEXT: vpermd %ymm13, %ymm8, %ymm10
8476 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm31, %ymm10, %ymm13
8477 ; AVX512DQ-BW-FCP-NEXT: vpermd %ymm15, %ymm8, %ymm15
8478 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm14, %ymm15, %ymm8
8479 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm13[5],ymm8[6,7]
8480 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm12[6,7]
8481 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm12 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
8482 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm12, %xmm1, %xmm13
8483 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm12, %xmm2, %xmm12
8484 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
8485 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm13 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
8486 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm13, %xmm3, %xmm14
8487 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm13, %xmm5, %xmm13
8488 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
8489 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm13[0,1,2],xmm12[3]
8490 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $32, %zmm0, %zmm13
8491 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm13, %xmm13
8492 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm13[0,1],xmm12[2,3]
8493 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm12[0,1,2,3],ymm8[4,5,6,7]
8494 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm16, %ymm9, %ymm12
8495 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm17, %ymm11, %ymm13
8496 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3,4,5,6],ymm12[7]
8497 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm18, %ymm10, %ymm13
8498 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm19, %ymm15, %ymm14
8499 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm13 = ymm14[0,1,2,3,4],ymm13[5],ymm14[6,7]
8500 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3,4,5],ymm12[6,7]
8501 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm13 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
8502 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm13, %xmm1, %xmm14
8503 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm13, %xmm2, %xmm13
8504 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
8505 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm14 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
8506 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm14, %xmm3, %xmm16
8507 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm14, %xmm5, %xmm14
8508 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm16[0],xmm14[1],xmm16[1],xmm14[2],xmm16[2],xmm14[3],xmm16[3]
8509 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm13 = xmm14[0,1,2],xmm13[3]
8510 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $40, %zmm0, %zmm14
8511 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm14, %xmm14
8512 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm13 = xmm14[0,1],xmm13[2,3]
8513 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3],ymm12[4,5,6,7]
8514 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm20, %ymm9, %ymm13
8515 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm21, %ymm11, %ymm14
8516 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm13 = ymm14[0,1,2,3,4,5,6],ymm13[7]
8517 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm22, %ymm10, %ymm14
8518 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm23, %ymm15, %ymm6
8519 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4],ymm14[5],ymm6[6,7]
8520 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],ymm13[6,7]
8521 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm13 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
8522 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm13, %xmm1, %xmm14
8523 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm13, %xmm2, %xmm13
8524 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
8525 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm14 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
8526 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm14, %xmm3, %xmm16
8527 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm14, %xmm5, %xmm14
8528 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm16[0],xmm14[1],xmm16[1],xmm14[2],xmm16[2],xmm14[3],xmm16[3]
8529 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm13 = xmm14[0,1,2],xmm13[3]
8530 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $48, %zmm0, %zmm14
8531 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm14, %xmm14
8532 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm13 = xmm14[0,1],xmm13[2,3]
8533 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm13[0,1,2,3],ymm6[4,5,6,7]
8534 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm24, %ymm9, %ymm9
8535 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm25, %ymm11, %ymm11
8536 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3,4,5,6],ymm9[7]
8537 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm26, %ymm10, %ymm10
8538 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm27, %ymm15, %ymm11
8539 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3,4],ymm10[5],ymm11[6,7]
8540 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4,5],ymm9[6,7]
8541 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
8542 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm10, %xmm1, %xmm1
8543 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm10, %xmm2, %xmm2
8544 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
8545 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
8546 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm2, %xmm3, %xmm3
8547 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm2, %xmm5, %xmm2
8548 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
8549 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
8550 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $56, %zmm0, %zmm0
8551 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm0, %xmm0
8552 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
8553 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm9[4,5,6,7]
8554 ; AVX512DQ-BW-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8555 ; AVX512DQ-BW-FCP-NEXT: vmovaps %ymm1, (%rsi)
8556 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %ymm29, (%rdx)
8557 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm7, (%rcx)
8558 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm4, (%r8)
8559 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm8, (%r9)
8560 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm12, (%r11)
8561 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm6, (%r10)
8562 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm0, (%rax)
8563 ; AVX512DQ-BW-FCP-NEXT: vzeroupper
8564 ; AVX512DQ-BW-FCP-NEXT: retq
8565 %wide.vec = load <256 x i8>, ptr %in.vec, align 64
8566 %strided.vec0 = shufflevector <256 x i8> %wide.vec, <256 x i8> poison, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 40, i32 48, i32 56, i32 64, i32 72, i32 80, i32 88, i32 96, i32 104, i32 112, i32 120, i32 128, i32 136, i32 144, i32 152, i32 160, i32 168, i32 176, i32 184, i32 192, i32 200, i32 208, i32 216, i32 224, i32 232, i32 240, i32 248>
8567 %strided.vec1 = shufflevector <256 x i8> %wide.vec, <256 x i8> poison, <32 x i32> <i32 1, i32 9, i32 17, i32 25, i32 33, i32 41, i32 49, i32 57, i32 65, i32 73, i32 81, i32 89, i32 97, i32 105, i32 113, i32 121, i32 129, i32 137, i32 145, i32 153, i32 161, i32 169, i32 177, i32 185, i32 193, i32 201, i32 209, i32 217, i32 225, i32 233, i32 241, i32 249>
8568 %strided.vec2 = shufflevector <256 x i8> %wide.vec, <256 x i8> poison, <32 x i32> <i32 2, i32 10, i32 18, i32 26, i32 34, i32 42, i32 50, i32 58, i32 66, i32 74, i32 82, i32 90, i32 98, i32 106, i32 114, i32 122, i32 130, i32 138, i32 146, i32 154, i32 162, i32 170, i32 178, i32 186, i32 194, i32 202, i32 210, i32 218, i32 226, i32 234, i32 242, i32 250>
8569 %strided.vec3 = shufflevector <256 x i8> %wide.vec, <256 x i8> poison, <32 x i32> <i32 3, i32 11, i32 19, i32 27, i32 35, i32 43, i32 51, i32 59, i32 67, i32 75, i32 83, i32 91, i32 99, i32 107, i32 115, i32 123, i32 131, i32 139, i32 147, i32 155, i32 163, i32 171, i32 179, i32 187, i32 195, i32 203, i32 211, i32 219, i32 227, i32 235, i32 243, i32 251>
8570 %strided.vec4 = shufflevector <256 x i8> %wide.vec, <256 x i8> poison, <32 x i32> <i32 4, i32 12, i32 20, i32 28, i32 36, i32 44, i32 52, i32 60, i32 68, i32 76, i32 84, i32 92, i32 100, i32 108, i32 116, i32 124, i32 132, i32 140, i32 148, i32 156, i32 164, i32 172, i32 180, i32 188, i32 196, i32 204, i32 212, i32 220, i32 228, i32 236, i32 244, i32 252>
8571 %strided.vec5 = shufflevector <256 x i8> %wide.vec, <256 x i8> poison, <32 x i32> <i32 5, i32 13, i32 21, i32 29, i32 37, i32 45, i32 53, i32 61, i32 69, i32 77, i32 85, i32 93, i32 101, i32 109, i32 117, i32 125, i32 133, i32 141, i32 149, i32 157, i32 165, i32 173, i32 181, i32 189, i32 197, i32 205, i32 213, i32 221, i32 229, i32 237, i32 245, i32 253>
8572 %strided.vec6 = shufflevector <256 x i8> %wide.vec, <256 x i8> poison, <32 x i32> <i32 6, i32 14, i32 22, i32 30, i32 38, i32 46, i32 54, i32 62, i32 70, i32 78, i32 86, i32 94, i32 102, i32 110, i32 118, i32 126, i32 134, i32 142, i32 150, i32 158, i32 166, i32 174, i32 182, i32 190, i32 198, i32 206, i32 214, i32 222, i32 230, i32 238, i32 246, i32 254>
8573 %strided.vec7 = shufflevector <256 x i8> %wide.vec, <256 x i8> poison, <32 x i32> <i32 7, i32 15, i32 23, i32 31, i32 39, i32 47, i32 55, i32 63, i32 71, i32 79, i32 87, i32 95, i32 103, i32 111, i32 119, i32 127, i32 135, i32 143, i32 151, i32 159, i32 167, i32 175, i32 183, i32 191, i32 199, i32 207, i32 215, i32 223, i32 231, i32 239, i32 247, i32 255>
8574 store <32 x i8> %strided.vec0, ptr %out.vec0, align 64
8575 store <32 x i8> %strided.vec1, ptr %out.vec1, align 64
8576 store <32 x i8> %strided.vec2, ptr %out.vec2, align 64
8577 store <32 x i8> %strided.vec3, ptr %out.vec3, align 64
8578 store <32 x i8> %strided.vec4, ptr %out.vec4, align 64
8579 store <32 x i8> %strided.vec5, ptr %out.vec5, align 64
8580 store <32 x i8> %strided.vec6, ptr %out.vec6, align 64
8581 store <32 x i8> %strided.vec7, ptr %out.vec7, align 64
8582 ret void
8583 }
8585 define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4, ptr %out.vec5, ptr %out.vec6, ptr %out.vec7) nounwind {
8586 ; SSE-LABEL: load_i8_stride8_vf64:
8587 ; SSE: # %bb.0:
8588 ; SSE-NEXT: subq $2040, %rsp # imm = 0x7F8
8589 ; SSE-NEXT: movdqa 64(%rdi), %xmm6
8590 ; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8591 ; SSE-NEXT: movdqa 80(%rdi), %xmm8
8592 ; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8593 ; SSE-NEXT: movdqa 96(%rdi), %xmm11
8594 ; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8595 ; SSE-NEXT: movdqa 128(%rdi), %xmm5
8596 ; SSE-NEXT: movdqa 144(%rdi), %xmm10
8597 ; SSE-NEXT: movdqa 160(%rdi), %xmm7
8598 ; SSE-NEXT: movdqa 176(%rdi), %xmm13
8599 ; SSE-NEXT: movdqa 192(%rdi), %xmm3
8600 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8601 ; SSE-NEXT: movdqa 208(%rdi), %xmm2
8602 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8603 ; SSE-NEXT: movdqa 224(%rdi), %xmm9
8604 ; SSE-NEXT: movdqa 240(%rdi), %xmm12
8605 ; SSE-NEXT: movdqa {{.*#+}} xmm4 = [255,0,0,0,255,0,0,0]
8606 ; SSE-NEXT: movdqa %xmm12, %xmm0
8607 ; SSE-NEXT: pand %xmm4, %xmm0
8608 ; SSE-NEXT: movdqa %xmm9, %xmm1
8609 ; SSE-NEXT: pand %xmm4, %xmm1
8610 ; SSE-NEXT: packuswb %xmm0, %xmm1
8611 ; SSE-NEXT: packuswb %xmm1, %xmm0
8612 ; SSE-NEXT: movdqa %xmm2, %xmm1
8613 ; SSE-NEXT: pand %xmm4, %xmm1
8614 ; SSE-NEXT: movdqa %xmm3, %xmm2
8615 ; SSE-NEXT: pand %xmm4, %xmm2
8616 ; SSE-NEXT: packuswb %xmm1, %xmm2
8617 ; SSE-NEXT: packuswb %xmm2, %xmm2
8618 ; SSE-NEXT: packuswb %xmm0, %xmm2
8619 ; SSE-NEXT: movdqa %xmm13, %xmm0
8620 ; SSE-NEXT: pand %xmm4, %xmm0
8621 ; SSE-NEXT: movdqa %xmm7, %xmm1
8622 ; SSE-NEXT: pand %xmm4, %xmm1
8623 ; SSE-NEXT: packuswb %xmm0, %xmm1
8624 ; SSE-NEXT: movdqa %xmm10, %xmm0
8625 ; SSE-NEXT: pand %xmm4, %xmm0
8626 ; SSE-NEXT: movdqa %xmm5, %xmm3
8627 ; SSE-NEXT: pand %xmm4, %xmm3
8628 ; SSE-NEXT: packuswb %xmm0, %xmm3
8629 ; SSE-NEXT: movdqa 112(%rdi), %xmm14
8630 ; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8631 ; SSE-NEXT: packuswb %xmm1, %xmm0
8632 ; SSE-NEXT: packuswb %xmm3, %xmm3
8633 ; SSE-NEXT: packuswb %xmm0, %xmm3
8634 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,3],xmm2[0,3]
8635 ; SSE-NEXT: movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8636 ; SSE-NEXT: movdqa %xmm14, %xmm0
8637 ; SSE-NEXT: pand %xmm4, %xmm0
8638 ; SSE-NEXT: movdqa %xmm11, %xmm1
8639 ; SSE-NEXT: pand %xmm4, %xmm1
8640 ; SSE-NEXT: packuswb %xmm0, %xmm1
8641 ; SSE-NEXT: movdqa %xmm8, %xmm0
8642 ; SSE-NEXT: pand %xmm4, %xmm0
8643 ; SSE-NEXT: movdqa %xmm6, %xmm2
8644 ; SSE-NEXT: pand %xmm4, %xmm2
8645 ; SSE-NEXT: packuswb %xmm0, %xmm2
8646 ; SSE-NEXT: packuswb %xmm1, %xmm0
8647 ; SSE-NEXT: packuswb %xmm2, %xmm2
8648 ; SSE-NEXT: packuswb %xmm0, %xmm2
8649 ; SSE-NEXT: movdqa 48(%rdi), %xmm14
8650 ; SSE-NEXT: movdqa %xmm14, %xmm0
8651 ; SSE-NEXT: pand %xmm4, %xmm0
8652 ; SSE-NEXT: movdqa 32(%rdi), %xmm15
8653 ; SSE-NEXT: movdqa %xmm15, %xmm1
8654 ; SSE-NEXT: pand %xmm4, %xmm1
8655 ; SSE-NEXT: packuswb %xmm0, %xmm1
8656 ; SSE-NEXT: movdqa 16(%rdi), %xmm8
8657 ; SSE-NEXT: movdqa %xmm8, %xmm0
8658 ; SSE-NEXT: pand %xmm4, %xmm0
8659 ; SSE-NEXT: movdqa (%rdi), %xmm11
8660 ; SSE-NEXT: movdqa %xmm11, %xmm3
8661 ; SSE-NEXT: pand %xmm4, %xmm3
8662 ; SSE-NEXT: packuswb %xmm0, %xmm3
8663 ; SSE-NEXT: packuswb %xmm1, %xmm0
8664 ; SSE-NEXT: packuswb %xmm3, %xmm3
8665 ; SSE-NEXT: packuswb %xmm0, %xmm3
8666 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,3],xmm2[0,3]
8667 ; SSE-NEXT: movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8668 ; SSE-NEXT: movdqa 496(%rdi), %xmm0
8669 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8670 ; SSE-NEXT: pand %xmm4, %xmm0
8671 ; SSE-NEXT: movdqa 480(%rdi), %xmm1
8672 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8673 ; SSE-NEXT: pand %xmm4, %xmm1
8674 ; SSE-NEXT: packuswb %xmm0, %xmm1
8675 ; SSE-NEXT: movdqa 464(%rdi), %xmm0
8676 ; SSE-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
8677 ; SSE-NEXT: pand %xmm4, %xmm0
8678 ; SSE-NEXT: movdqa 448(%rdi), %xmm2
8679 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8680 ; SSE-NEXT: pand %xmm4, %xmm2
8681 ; SSE-NEXT: packuswb %xmm0, %xmm2
8682 ; SSE-NEXT: packuswb %xmm1, %xmm0
8683 ; SSE-NEXT: packuswb %xmm2, %xmm2
8684 ; SSE-NEXT: packuswb %xmm0, %xmm2
8685 ; SSE-NEXT: movdqa 432(%rdi), %xmm0
8686 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8687 ; SSE-NEXT: pand %xmm4, %xmm0
8688 ; SSE-NEXT: movdqa 416(%rdi), %xmm1
8689 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8690 ; SSE-NEXT: pand %xmm4, %xmm1
8691 ; SSE-NEXT: packuswb %xmm0, %xmm1
8692 ; SSE-NEXT: movdqa 400(%rdi), %xmm0
8693 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8694 ; SSE-NEXT: pand %xmm4, %xmm0
8695 ; SSE-NEXT: movdqa 384(%rdi), %xmm3
8696 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8697 ; SSE-NEXT: pand %xmm4, %xmm3
8698 ; SSE-NEXT: packuswb %xmm0, %xmm3
8699 ; SSE-NEXT: packuswb %xmm1, %xmm0
8700 ; SSE-NEXT: packuswb %xmm3, %xmm3
8701 ; SSE-NEXT: packuswb %xmm0, %xmm3
8702 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,3],xmm2[0,3]
8703 ; SSE-NEXT: movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8704 ; SSE-NEXT: movdqa 368(%rdi), %xmm0
8705 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8706 ; SSE-NEXT: pand %xmm4, %xmm0
8707 ; SSE-NEXT: movdqa 352(%rdi), %xmm1
8708 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8709 ; SSE-NEXT: pand %xmm4, %xmm1
8710 ; SSE-NEXT: packuswb %xmm0, %xmm1
8711 ; SSE-NEXT: movdqa 336(%rdi), %xmm0
8712 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8713 ; SSE-NEXT: pand %xmm4, %xmm0
8714 ; SSE-NEXT: movdqa 320(%rdi), %xmm2
8715 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8716 ; SSE-NEXT: pand %xmm4, %xmm2
8717 ; SSE-NEXT: packuswb %xmm0, %xmm2
8718 ; SSE-NEXT: packuswb %xmm1, %xmm0
8719 ; SSE-NEXT: packuswb %xmm2, %xmm2
8720 ; SSE-NEXT: packuswb %xmm0, %xmm2
8721 ; SSE-NEXT: movdqa 304(%rdi), %xmm0
8722 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8723 ; SSE-NEXT: pand %xmm4, %xmm0
8724 ; SSE-NEXT: movdqa 288(%rdi), %xmm1
8725 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8726 ; SSE-NEXT: pand %xmm4, %xmm1
8727 ; SSE-NEXT: packuswb %xmm0, %xmm1
8728 ; SSE-NEXT: movdqa 272(%rdi), %xmm0
8729 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8730 ; SSE-NEXT: pand %xmm4, %xmm0
8731 ; SSE-NEXT: movdqa 256(%rdi), %xmm3
8732 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8733 ; SSE-NEXT: pand %xmm3, %xmm4
8734 ; SSE-NEXT: packuswb %xmm0, %xmm4
8735 ; SSE-NEXT: packuswb %xmm1, %xmm0
8736 ; SSE-NEXT: packuswb %xmm4, %xmm4
8737 ; SSE-NEXT: packuswb %xmm0, %xmm4
8738 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,3],xmm2[0,3]
8739 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8740 ; SSE-NEXT: pxor %xmm6, %xmm6
8741 ; SSE-NEXT: movdqa %xmm10, %xmm0
8742 ; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8743 ; SSE-NEXT: movdqa %xmm10, %xmm1
8744 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm6[8],xmm1[9],xmm6[9],xmm1[10],xmm6[10],xmm1[11],xmm6[11],xmm1[12],xmm6[12],xmm1[13],xmm6[13],xmm1[14],xmm6[14],xmm1[15],xmm6[15]
8745 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8746 ; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3],xmm0[4],xmm6[4],xmm0[5],xmm6[5],xmm0[6],xmm6[6],xmm0[7],xmm6[7]
8747 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8748 ; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
8749 ; SSE-NEXT: packuswb %xmm0, %xmm0
8750 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8751 ; SSE-NEXT: movdqa {{.*#+}} xmm10 = [65535,0,65535,65535,65535,65535,65535,65535]
8752 ; SSE-NEXT: movdqa %xmm10, %xmm1
8753 ; SSE-NEXT: pandn %xmm0, %xmm1
8754 ; SSE-NEXT: movdqa %xmm5, %xmm0
8755 ; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8756 ; SSE-NEXT: movdqa %xmm5, %xmm2
8757 ; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm6[8],xmm2[9],xmm6[9],xmm2[10],xmm6[10],xmm2[11],xmm6[11],xmm2[12],xmm6[12],xmm2[13],xmm6[13],xmm2[14],xmm6[14],xmm2[15],xmm6[15]
8758 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8759 ; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3],xmm0[4],xmm6[4],xmm0[5],xmm6[5],xmm0[6],xmm6[6],xmm0[7],xmm6[7]
8760 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8761 ; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
8762 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8763 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,1,1]
8764 ; SSE-NEXT: packuswb %xmm5, %xmm5
8765 ; SSE-NEXT: pand %xmm10, %xmm5
8766 ; SSE-NEXT: por %xmm1, %xmm5
8767 ; SSE-NEXT: movdqa %xmm13, %xmm2
8768 ; SSE-NEXT: movdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8769 ; SSE-NEXT: movdqa %xmm13, %xmm1
8770 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm6[8],xmm1[9],xmm6[9],xmm1[10],xmm6[10],xmm1[11],xmm6[11],xmm1[12],xmm6[12],xmm1[13],xmm6[13],xmm1[14],xmm6[14],xmm1[15],xmm6[15]
8771 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8772 ; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3],xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
8773 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8774 ; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
8775 ; SSE-NEXT: packuswb %xmm2, %xmm2
8776 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8777 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,0,2,3]
8778 ; SSE-NEXT: movdqa {{.*#+}} xmm13 = [65535,65535,65535,0,65535,65535,65535,65535]
8779 ; SSE-NEXT: movdqa %xmm13, %xmm2
8780 ; SSE-NEXT: pandn %xmm1, %xmm2
8781 ; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8782 ; SSE-NEXT: movdqa %xmm7, %xmm1
8783 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm6[8],xmm1[9],xmm6[9],xmm1[10],xmm6[10],xmm1[11],xmm6[11],xmm1[12],xmm6[12],xmm1[13],xmm6[13],xmm1[14],xmm6[14],xmm1[15],xmm6[15]
8784 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8785 ; SSE-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
8786 ; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8787 ; SSE-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1],xmm7[2],xmm1[2],xmm7[3],xmm1[3]
8788 ; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8789 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[0,1,1,3]
8790 ; SSE-NEXT: packuswb %xmm1, %xmm1
8791 ; SSE-NEXT: pand %xmm13, %xmm1
8792 ; SSE-NEXT: por %xmm2, %xmm1
8793 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
8794 ; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
8795 ; SSE-NEXT: movdqa %xmm12, %xmm1
8796 ; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8797 ; SSE-NEXT: movdqa %xmm12, %xmm2
8798 ; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm6[8],xmm2[9],xmm6[9],xmm2[10],xmm6[10],xmm2[11],xmm6[11],xmm2[12],xmm6[12],xmm2[13],xmm6[13],xmm2[14],xmm6[14],xmm2[15],xmm6[15]
8799 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8800 ; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3],xmm1[4],xmm6[4],xmm1[5],xmm6[5],xmm1[6],xmm6[6],xmm1[7],xmm6[7]
8801 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8802 ; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
8803 ; SSE-NEXT: packuswb %xmm1, %xmm1
8804 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8805 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,2]
8806 ; SSE-NEXT: movdqa {{.*#+}} xmm12 = [65535,65535,65535,65535,65535,65535,65535,0]
8807 ; SSE-NEXT: movdqa %xmm12, %xmm2
8808 ; SSE-NEXT: pandn %xmm1, %xmm2
8809 ; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8810 ; SSE-NEXT: movdqa %xmm9, %xmm1
8811 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm6[8],xmm1[9],xmm6[9],xmm1[10],xmm6[10],xmm1[11],xmm6[11],xmm1[12],xmm6[12],xmm1[13],xmm6[13],xmm1[14],xmm6[14],xmm1[15],xmm6[15]
8812 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8813 ; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm6[0],xmm9[1],xmm6[1],xmm9[2],xmm6[2],xmm9[3],xmm6[3],xmm9[4],xmm6[4],xmm9[5],xmm6[5],xmm9[6],xmm6[6],xmm9[7],xmm6[7]
8814 ; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8815 ; SSE-NEXT: punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm1[0],xmm9[1],xmm1[1],xmm9[2],xmm1[2],xmm9[3],xmm1[3]
8816 ; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8817 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm9[0,1,1,3]
8818 ; SSE-NEXT: packuswb %xmm1, %xmm1
8819 ; SSE-NEXT: pand %xmm12, %xmm1
8820 ; SSE-NEXT: por %xmm2, %xmm1
8821 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
8822 ; SSE-NEXT: movdqa %xmm2, %xmm3
8823 ; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm6[8],xmm3[9],xmm6[9],xmm3[10],xmm6[10],xmm3[11],xmm6[11],xmm3[12],xmm6[12],xmm3[13],xmm6[13],xmm3[14],xmm6[14],xmm3[15],xmm6[15]
8824 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8825 ; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3],xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
8826 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8827 ; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
8828 ; SSE-NEXT: packuswb %xmm2, %xmm3
8829 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8830 ; SSE-NEXT: movdqa {{.*#+}} xmm9 = [65535,65535,65535,65535,65535,0,65535,65535]
8831 ; SSE-NEXT: movdqa %xmm9, %xmm2
8832 ; SSE-NEXT: pandn %xmm3, %xmm2
8833 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
8834 ; SSE-NEXT: movdqa %xmm3, %xmm4
8835 ; SSE-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm6[8],xmm4[9],xmm6[9],xmm4[10],xmm6[10],xmm4[11],xmm6[11],xmm4[12],xmm6[12],xmm4[13],xmm6[13],xmm4[14],xmm6[14],xmm4[15],xmm6[15]
8836 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8837 ; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3],xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]
8838 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8839 ; SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
8840 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8841 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,1,1]
8842 ; SSE-NEXT: packuswb %xmm3, %xmm3
8843 ; SSE-NEXT: pand %xmm9, %xmm3
8844 ; SSE-NEXT: por %xmm2, %xmm3
8845 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,2,2,2]
8846 ; SSE-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
8847 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm5[0],xmm2[1]
8848 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8849 ; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8850 ; SSE-NEXT: movdqa %xmm8, %xmm1
8851 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm6[8],xmm1[9],xmm6[9],xmm1[10],xmm6[10],xmm1[11],xmm6[11],xmm1[12],xmm6[12],xmm1[13],xmm6[13],xmm1[14],xmm6[14],xmm1[15],xmm6[15]
8852 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8853 ; SSE-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3],xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]
8854 ; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8855 ; SSE-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm1[0],xmm8[1],xmm1[1],xmm8[2],xmm1[2],xmm8[3],xmm1[3]
8856 ; SSE-NEXT: packuswb %xmm8, %xmm8
8857 ; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8858 ; SSE-NEXT: movdqa %xmm10, %xmm1
8859 ; SSE-NEXT: pandn %xmm8, %xmm1
8860 ; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8861 ; SSE-NEXT: movdqa %xmm11, %xmm2
8862 ; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm6[8],xmm2[9],xmm6[9],xmm2[10],xmm6[10],xmm2[11],xmm6[11],xmm2[12],xmm6[12],xmm2[13],xmm6[13],xmm2[14],xmm6[14],xmm2[15],xmm6[15]
8863 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8864 ; SSE-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm6[0],xmm11[1],xmm6[1],xmm11[2],xmm6[2],xmm11[3],xmm6[3],xmm11[4],xmm6[4],xmm11[5],xmm6[5],xmm11[6],xmm6[6],xmm11[7],xmm6[7]
8865 ; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8866 ; SSE-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm2[0],xmm11[1],xmm2[1],xmm11[2],xmm2[2],xmm11[3],xmm2[3]
8867 ; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8868 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm11[1,1,1,1]
8869 ; SSE-NEXT: packuswb %xmm0, %xmm0
8870 ; SSE-NEXT: pand %xmm10, %xmm0
8871 ; SSE-NEXT: por %xmm1, %xmm0
8872 ; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8873 ; SSE-NEXT: movdqa %xmm14, %xmm1
8874 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm6[8],xmm1[9],xmm6[9],xmm1[10],xmm6[10],xmm1[11],xmm6[11],xmm1[12],xmm6[12],xmm1[13],xmm6[13],xmm1[14],xmm6[14],xmm1[15],xmm6[15]
8875 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8876 ; SSE-NEXT: punpcklbw {{.*#+}} xmm14 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3],xmm14[4],xmm6[4],xmm14[5],xmm6[5],xmm14[6],xmm6[6],xmm14[7],xmm6[7]
8877 ; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8878 ; SSE-NEXT: punpcklwd {{.*#+}} xmm14 = xmm14[0],xmm1[0],xmm14[1],xmm1[1],xmm14[2],xmm1[2],xmm14[3],xmm1[3]
8879 ; SSE-NEXT: packuswb %xmm14, %xmm14
8880 ; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8881 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm14[0,0,2,3]
8882 ; SSE-NEXT: movdqa %xmm13, %xmm2
8883 ; SSE-NEXT: pandn %xmm1, %xmm2
8884 ; SSE-NEXT: movdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8885 ; SSE-NEXT: movdqa %xmm15, %xmm1
8886 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm6[8],xmm1[9],xmm6[9],xmm1[10],xmm6[10],xmm1[11],xmm6[11],xmm1[12],xmm6[12],xmm1[13],xmm6[13],xmm1[14],xmm6[14],xmm1[15],xmm6[15]
8887 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8888 ; SSE-NEXT: punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm6[0],xmm15[1],xmm6[1],xmm15[2],xmm6[2],xmm15[3],xmm6[3],xmm15[4],xmm6[4],xmm15[5],xmm6[5],xmm15[6],xmm6[6],xmm15[7],xmm6[7]
8889 ; SSE-NEXT: movdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8890 ; SSE-NEXT: punpcklwd {{.*#+}} xmm15 = xmm15[0],xmm1[0],xmm15[1],xmm1[1],xmm15[2],xmm1[2],xmm15[3],xmm1[3]
8891 ; SSE-NEXT: movdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8892 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm15[0,1,1,3]
8893 ; SSE-NEXT: packuswb %xmm1, %xmm1
8894 ; SSE-NEXT: pand %xmm13, %xmm1
8895 ; SSE-NEXT: por %xmm2, %xmm1
8896 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
8897 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
8898 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
8899 ; SSE-NEXT: movdqa %xmm1, %xmm2
8900 ; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm6[8],xmm2[9],xmm6[9],xmm2[10],xmm6[10],xmm2[11],xmm6[11],xmm2[12],xmm6[12],xmm2[13],xmm6[13],xmm2[14],xmm6[14],xmm2[15],xmm6[15]
8901 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8902 ; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3],xmm1[4],xmm6[4],xmm1[5],xmm6[5],xmm1[6],xmm6[6],xmm1[7],xmm6[7]
8903 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8904 ; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
8905 ; SSE-NEXT: packuswb %xmm1, %xmm1
8906 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8907 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,2]
8908 ; SSE-NEXT: movdqa %xmm12, %xmm2
8909 ; SSE-NEXT: pandn %xmm1, %xmm2
8910 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
8911 ; SSE-NEXT: movdqa %xmm3, %xmm1
8912 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm6[8],xmm1[9],xmm6[9],xmm1[10],xmm6[10],xmm1[11],xmm6[11],xmm1[12],xmm6[12],xmm1[13],xmm6[13],xmm1[14],xmm6[14],xmm1[15],xmm6[15]
8913 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8914 ; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3],xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]
8915 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8916 ; SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
8917 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8918 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,1,1,3]
8919 ; SSE-NEXT: packuswb %xmm1, %xmm1
8920 ; SSE-NEXT: pand %xmm12, %xmm1
8921 ; SSE-NEXT: por %xmm2, %xmm1
8922 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
8923 ; SSE-NEXT: movdqa %xmm2, %xmm3
8924 ; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm6[8],xmm3[9],xmm6[9],xmm3[10],xmm6[10],xmm3[11],xmm6[11],xmm3[12],xmm6[12],xmm3[13],xmm6[13],xmm3[14],xmm6[14],xmm3[15],xmm6[15]
8925 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8926 ; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3],xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
8927 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8928 ; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
8929 ; SSE-NEXT: packuswb %xmm2, %xmm3
8930 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8931 ; SSE-NEXT: movdqa %xmm9, %xmm2
8932 ; SSE-NEXT: pandn %xmm3, %xmm2
8933 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
8934 ; SSE-NEXT: movdqa %xmm3, %xmm4
8935 ; SSE-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm6[8],xmm4[9],xmm6[9],xmm4[10],xmm6[10],xmm4[11],xmm6[11],xmm4[12],xmm6[12],xmm4[13],xmm6[13],xmm4[14],xmm6[14],xmm4[15],xmm6[15]
8936 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8937 ; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3],xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]
8938 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8939 ; SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
8940 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8941 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,1,1]
8942 ; SSE-NEXT: packuswb %xmm3, %xmm3
8943 ; SSE-NEXT: pand %xmm9, %xmm3
8944 ; SSE-NEXT: por %xmm2, %xmm3
8945 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,2,2,2]
8946 ; SSE-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
8947 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
8948 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8949 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
8950 ; SSE-NEXT: movdqa %xmm14, %xmm0
8951 ; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm6[8],xmm0[9],xmm6[9],xmm0[10],xmm6[10],xmm0[11],xmm6[11],xmm0[12],xmm6[12],xmm0[13],xmm6[13],xmm0[14],xmm6[14],xmm0[15],xmm6[15]
8952 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8953 ; SSE-NEXT: punpcklbw {{.*#+}} xmm14 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3],xmm14[4],xmm6[4],xmm14[5],xmm6[5],xmm14[6],xmm6[6],xmm14[7],xmm6[7]
8954 ; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8955 ; SSE-NEXT: punpcklwd {{.*#+}} xmm14 = xmm14[0],xmm0[0],xmm14[1],xmm0[1],xmm14[2],xmm0[2],xmm14[3],xmm0[3]
8956 ; SSE-NEXT: packuswb %xmm14, %xmm14
8957 ; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8958 ; SSE-NEXT: movdqa %xmm10, %xmm1
8959 ; SSE-NEXT: pandn %xmm14, %xmm1
8960 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
8961 ; SSE-NEXT: movdqa %xmm0, %xmm2
8962 ; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm6[8],xmm2[9],xmm6[9],xmm2[10],xmm6[10],xmm2[11],xmm6[11],xmm2[12],xmm6[12],xmm2[13],xmm6[13],xmm2[14],xmm6[14],xmm2[15],xmm6[15]
8963 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8964 ; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3],xmm0[4],xmm6[4],xmm0[5],xmm6[5],xmm0[6],xmm6[6],xmm0[7],xmm6[7]
8965 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8966 ; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
8967 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8968 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
8969 ; SSE-NEXT: packuswb %xmm0, %xmm0
8970 ; SSE-NEXT: pand %xmm10, %xmm0
8971 ; SSE-NEXT: por %xmm1, %xmm0
8972 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
8973 ; SSE-NEXT: movdqa %xmm14, %xmm1
8974 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm6[8],xmm1[9],xmm6[9],xmm1[10],xmm6[10],xmm1[11],xmm6[11],xmm1[12],xmm6[12],xmm1[13],xmm6[13],xmm1[14],xmm6[14],xmm1[15],xmm6[15]
8975 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8976 ; SSE-NEXT: punpcklbw {{.*#+}} xmm14 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3],xmm14[4],xmm6[4],xmm14[5],xmm6[5],xmm14[6],xmm6[6],xmm14[7],xmm6[7]
8977 ; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8978 ; SSE-NEXT: punpcklwd {{.*#+}} xmm14 = xmm14[0],xmm1[0],xmm14[1],xmm1[1],xmm14[2],xmm1[2],xmm14[3],xmm1[3]
8979 ; SSE-NEXT: packuswb %xmm14, %xmm14
8980 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm14[0,0,2,3]
8981 ; SSE-NEXT: movdqa %xmm13, %xmm2
8982 ; SSE-NEXT: pandn %xmm1, %xmm2
8983 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
8984 ; SSE-NEXT: movdqa %xmm3, %xmm1
8985 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm6[8],xmm1[9],xmm6[9],xmm1[10],xmm6[10],xmm1[11],xmm6[11],xmm1[12],xmm6[12],xmm1[13],xmm6[13],xmm1[14],xmm6[14],xmm1[15],xmm6[15]
8986 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8987 ; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3],xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]
8988 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8989 ; SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
8990 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8991 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,1,1,3]
8992 ; SSE-NEXT: packuswb %xmm1, %xmm1
8993 ; SSE-NEXT: pand %xmm13, %xmm1
8994 ; SSE-NEXT: por %xmm2, %xmm1
8995 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
8996 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
8997 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
8998 ; SSE-NEXT: movdqa %xmm1, %xmm2
8999 ; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm6[8],xmm2[9],xmm6[9],xmm2[10],xmm6[10],xmm2[11],xmm6[11],xmm2[12],xmm6[12],xmm2[13],xmm6[13],xmm2[14],xmm6[14],xmm2[15],xmm6[15]
9000 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9001 ; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3],xmm1[4],xmm6[4],xmm1[5],xmm6[5],xmm1[6],xmm6[6],xmm1[7],xmm6[7]
9002 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9003 ; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
9004 ; SSE-NEXT: packuswb %xmm1, %xmm1
9005 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9006 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,2]
9007 ; SSE-NEXT: movdqa %xmm12, %xmm2
9008 ; SSE-NEXT: pandn %xmm1, %xmm2
9009 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
9010 ; SSE-NEXT: movdqa %xmm3, %xmm1
9011 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm6[8],xmm1[9],xmm6[9],xmm1[10],xmm6[10],xmm1[11],xmm6[11],xmm1[12],xmm6[12],xmm1[13],xmm6[13],xmm1[14],xmm6[14],xmm1[15],xmm6[15]
9012 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9013 ; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3],xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]
9014 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9015 ; SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
9016 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9017 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,1,1,3]
9018 ; SSE-NEXT: packuswb %xmm1, %xmm1
9019 ; SSE-NEXT: pand %xmm12, %xmm1
9020 ; SSE-NEXT: por %xmm2, %xmm1
9021 ; SSE-NEXT: movdqa (%rsp), %xmm2 # 16-byte Reload
9022 ; SSE-NEXT: movdqa %xmm2, %xmm3
9023 ; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm6[8],xmm3[9],xmm6[9],xmm3[10],xmm6[10],xmm3[11],xmm6[11],xmm3[12],xmm6[12],xmm3[13],xmm6[13],xmm3[14],xmm6[14],xmm3[15],xmm6[15]
9024 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9025 ; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3],xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
9026 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9027 ; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
9028 ; SSE-NEXT: packuswb %xmm2, %xmm3
9029 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9030 ; SSE-NEXT: movdqa %xmm9, %xmm2
9031 ; SSE-NEXT: pandn %xmm3, %xmm2
9032 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
9033 ; SSE-NEXT: movdqa %xmm3, %xmm4
9034 ; SSE-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm6[8],xmm4[9],xmm6[9],xmm4[10],xmm6[10],xmm4[11],xmm6[11],xmm4[12],xmm6[12],xmm4[13],xmm6[13],xmm4[14],xmm6[14],xmm4[15],xmm6[15]
9035 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9036 ; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3],xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]
9037 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9038 ; SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
9039 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9040 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,1,1]
9041 ; SSE-NEXT: packuswb %xmm3, %xmm3
9042 ; SSE-NEXT: pand %xmm9, %xmm3
9043 ; SSE-NEXT: por %xmm2, %xmm3
9044 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,2,2,2]
9045 ; SSE-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
9046 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
9047 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9048 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
9049 ; SSE-NEXT: movdqa %xmm8, %xmm0
9050 ; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm6[8],xmm0[9],xmm6[9],xmm0[10],xmm6[10],xmm0[11],xmm6[11],xmm0[12],xmm6[12],xmm0[13],xmm6[13],xmm0[14],xmm6[14],xmm0[15],xmm6[15]
9051 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9052 ; SSE-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3],xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]
9053 ; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9054 ; SSE-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3]
9055 ; SSE-NEXT: packuswb %xmm8, %xmm8
9056 ; SSE-NEXT: movdqa %xmm10, %xmm1
9057 ; SSE-NEXT: pandn %xmm8, %xmm1
9058 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
9059 ; SSE-NEXT: movdqa %xmm0, %xmm2
9060 ; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm6[8],xmm2[9],xmm6[9],xmm2[10],xmm6[10],xmm2[11],xmm6[11],xmm2[12],xmm6[12],xmm2[13],xmm6[13],xmm2[14],xmm6[14],xmm2[15],xmm6[15]
9061 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9062 ; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3],xmm0[4],xmm6[4],xmm0[5],xmm6[5],xmm0[6],xmm6[6],xmm0[7],xmm6[7]
9063 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9064 ; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
9065 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9066 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
9067 ; SSE-NEXT: packuswb %xmm0, %xmm0
9068 ; SSE-NEXT: pand %xmm10, %xmm0
9069 ; SSE-NEXT: por %xmm1, %xmm0
9070 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
9071 ; SSE-NEXT: movdqa %xmm4, %xmm1
9072 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm6[8],xmm1[9],xmm6[9],xmm1[10],xmm6[10],xmm1[11],xmm6[11],xmm1[12],xmm6[12],xmm1[13],xmm6[13],xmm1[14],xmm6[14],xmm1[15],xmm6[15]
9073 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9074 ; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3],xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7]
9075 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9076 ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
9077 ; SSE-NEXT: packuswb %xmm4, %xmm4
9078 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm4[0,0,2,3]
9079 ; SSE-NEXT: movdqa %xmm13, %xmm5
9080 ; SSE-NEXT: pandn %xmm1, %xmm5
9081 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
9082 ; SSE-NEXT: movdqa %xmm2, %xmm1
9083 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm6[8],xmm1[9],xmm6[9],xmm1[10],xmm6[10],xmm1[11],xmm6[11],xmm1[12],xmm6[12],xmm1[13],xmm6[13],xmm1[14],xmm6[14],xmm1[15],xmm6[15]
9084 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9085 ; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3],xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
9086 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9087 ; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
9088 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9089 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,1,1,3]
9090 ; SSE-NEXT: packuswb %xmm1, %xmm1
9091 ; SSE-NEXT: pand %xmm13, %xmm1
9092 ; SSE-NEXT: por %xmm5, %xmm1
9093 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
9094 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
9095 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9096 ; SSE-NEXT: movdqa %xmm1, %xmm2
9097 ; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm6[8],xmm2[9],xmm6[9],xmm2[10],xmm6[10],xmm2[11],xmm6[11],xmm2[12],xmm6[12],xmm2[13],xmm6[13],xmm2[14],xmm6[14],xmm2[15],xmm6[15]
9098 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9099 ; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3],xmm1[4],xmm6[4],xmm1[5],xmm6[5],xmm1[6],xmm6[6],xmm1[7],xmm6[7]
9100 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9101 ; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
9102 ; SSE-NEXT: packuswb %xmm1, %xmm1
9103 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9104 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,2]
9105 ; SSE-NEXT: movdqa %xmm12, %xmm5
9106 ; SSE-NEXT: pandn %xmm1, %xmm5
9107 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
9108 ; SSE-NEXT: movdqa %xmm11, %xmm1
9109 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm6[8],xmm1[9],xmm6[9],xmm1[10],xmm6[10],xmm1[11],xmm6[11],xmm1[12],xmm6[12],xmm1[13],xmm6[13],xmm1[14],xmm6[14],xmm1[15],xmm6[15]
9110 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9111 ; SSE-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm6[0],xmm11[1],xmm6[1],xmm11[2],xmm6[2],xmm11[3],xmm6[3],xmm11[4],xmm6[4],xmm11[5],xmm6[5],xmm11[6],xmm6[6],xmm11[7],xmm6[7]
9112 ; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9113 ; SSE-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm1[0],xmm11[1],xmm1[1],xmm11[2],xmm1[2],xmm11[3],xmm1[3]
9114 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm11[0,1,1,3]
9115 ; SSE-NEXT: packuswb %xmm1, %xmm1
9116 ; SSE-NEXT: pand %xmm12, %xmm1
9117 ; SSE-NEXT: por %xmm5, %xmm1
9118 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
9119 ; SSE-NEXT: movdqa %xmm5, %xmm2
9120 ; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm6[8],xmm2[9],xmm6[9],xmm2[10],xmm6[10],xmm2[11],xmm6[11],xmm2[12],xmm6[12],xmm2[13],xmm6[13],xmm2[14],xmm6[14],xmm2[15],xmm6[15]
9121 ; SSE-NEXT: movdqa %xmm2, %xmm15
9122 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9123 ; SSE-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3],xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]
9124 ; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9125 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
9126 ; SSE-NEXT: movdqa %xmm3, %xmm2
9127 ; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm6[8],xmm2[9],xmm6[9],xmm2[10],xmm6[10],xmm2[11],xmm6[11],xmm2[12],xmm6[12],xmm2[13],xmm6[13],xmm2[14],xmm6[14],xmm2[15],xmm6[15]
9128 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9129 ; SSE-NEXT: movdqa %xmm3, %xmm7
9130 ; SSE-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
9131 ; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9132 ; SSE-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm15[0],xmm5[1],xmm15[1],xmm5[2],xmm15[2],xmm5[3],xmm15[3]
9133 ; SSE-NEXT: packuswb %xmm5, %xmm6
9134 ; SSE-NEXT: movdqa %xmm9, %xmm5
9135 ; SSE-NEXT: pandn %xmm6, %xmm5
9136 ; SSE-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3]
9137 ; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9138 ; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm7[1,1,1,1]
9139 ; SSE-NEXT: packuswb %xmm15, %xmm15
9140 ; SSE-NEXT: pand %xmm9, %xmm15
9141 ; SSE-NEXT: por %xmm5, %xmm15
9142 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm15[2,2,2,2]
9143 ; SSE-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm1[2],xmm5[3],xmm1[3]
9144 ; SSE-NEXT: movsd {{.*#+}} xmm5 = xmm0[0],xmm5[1]
9145 ; SSE-NEXT: movapd %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9146 ; SSE-NEXT: movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
9147 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
9148 ; SSE-NEXT: pand %xmm2, %xmm0
9149 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9150 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
9151 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]
9152 ; SSE-NEXT: packuswb %xmm0, %xmm0
9153 ; SSE-NEXT: movdqa %xmm10, %xmm5
9154 ; SSE-NEXT: pandn %xmm0, %xmm5
9155 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
9156 ; SSE-NEXT: pand %xmm2, %xmm0
9157 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9158 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
9159 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
9160 ; SSE-NEXT: packuswb %xmm0, %xmm0
9161 ; SSE-NEXT: pand %xmm10, %xmm0
9162 ; SSE-NEXT: por %xmm5, %xmm0
9163 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9164 ; SSE-NEXT: pand %xmm2, %xmm1
9165 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9166 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,1,2,0]
9167 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,7,5]
9168 ; SSE-NEXT: packuswb %xmm5, %xmm5
9169 ; SSE-NEXT: movdqa %xmm13, %xmm15
9170 ; SSE-NEXT: pandn %xmm5, %xmm15
9171 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9172 ; SSE-NEXT: pand %xmm2, %xmm1
9173 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9174 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,1,2,0]
9175 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,7,5,6,7]
9176 ; SSE-NEXT: packuswb %xmm5, %xmm5
9177 ; SSE-NEXT: pand %xmm13, %xmm5
9178 ; SSE-NEXT: por %xmm15, %xmm5
9179 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,1,1]
9180 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
9181 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9182 ; SSE-NEXT: pand %xmm2, %xmm1
9183 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9184 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,1,2,0]
9185 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,7,5]
9186 ; SSE-NEXT: packuswb %xmm5, %xmm5
9187 ; SSE-NEXT: movdqa %xmm12, %xmm15
9188 ; SSE-NEXT: pandn %xmm5, %xmm15
9189 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9190 ; SSE-NEXT: pand %xmm2, %xmm1
9191 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9192 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,1,2,0]
9193 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,7,5,6,7]
9194 ; SSE-NEXT: packuswb %xmm5, %xmm5
9195 ; SSE-NEXT: pand %xmm12, %xmm5
9196 ; SSE-NEXT: por %xmm15, %xmm5
9197 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9198 ; SSE-NEXT: pand %xmm2, %xmm1
9199 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9200 ; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm1[0,2,2,3]
9201 ; SSE-NEXT: pshuflw {{.*#+}} xmm15 = xmm15[0,1,1,3,4,5,6,7]
9202 ; SSE-NEXT: packuswb %xmm15, %xmm15
9203 ; SSE-NEXT: movdqa %xmm9, %xmm7
9204 ; SSE-NEXT: pandn %xmm15, %xmm7
9205 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9206 ; SSE-NEXT: pand %xmm2, %xmm1
9207 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9208 ; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm1[0,2,2,3]
9209 ; SSE-NEXT: pshuflw {{.*#+}} xmm15 = xmm15[1,3,2,3,4,5,6,7]
9210 ; SSE-NEXT: packuswb %xmm15, %xmm15
9211 ; SSE-NEXT: pand %xmm9, %xmm15
9212 ; SSE-NEXT: por %xmm7, %xmm15
9213 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm15[2,2,2,2]
9214 ; SSE-NEXT: punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm5[2],xmm7[3],xmm5[3]
9215 ; SSE-NEXT: movsd {{.*#+}} xmm7 = xmm0[0],xmm7[1]
9216 ; SSE-NEXT: movapd %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9217 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
9218 ; SSE-NEXT: pand %xmm2, %xmm0
9219 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9220 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
9221 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]
9222 ; SSE-NEXT: packuswb %xmm0, %xmm0
9223 ; SSE-NEXT: movdqa %xmm10, %xmm5
9224 ; SSE-NEXT: pandn %xmm0, %xmm5
9225 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
9226 ; SSE-NEXT: pand %xmm2, %xmm0
9227 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9228 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
9229 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm0[1,3,2,3,4,5,6,7]
9230 ; SSE-NEXT: packuswb %xmm1, %xmm1
9231 ; SSE-NEXT: pand %xmm10, %xmm1
9232 ; SSE-NEXT: por %xmm5, %xmm1
9233 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
9234 ; SSE-NEXT: pand %xmm2, %xmm0
9235 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9236 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,1,2,0]
9237 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,7,5]
9238 ; SSE-NEXT: packuswb %xmm5, %xmm5
9239 ; SSE-NEXT: movdqa %xmm13, %xmm7
9240 ; SSE-NEXT: pandn %xmm5, %xmm7
9241 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
9242 ; SSE-NEXT: pand %xmm2, %xmm0
9243 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9244 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,1,2,0]
9245 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,7,5,6,7]
9246 ; SSE-NEXT: packuswb %xmm5, %xmm5
9247 ; SSE-NEXT: pand %xmm13, %xmm5
9248 ; SSE-NEXT: por %xmm7, %xmm5
9249 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,1,1]
9250 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]
9251 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
9252 ; SSE-NEXT: pand %xmm2, %xmm0
9253 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9254 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,1,2,0]
9255 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,7,5]
9256 ; SSE-NEXT: packuswb %xmm5, %xmm5
9257 ; SSE-NEXT: movdqa %xmm12, %xmm7
9258 ; SSE-NEXT: pandn %xmm5, %xmm7
9259 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
9260 ; SSE-NEXT: pand %xmm2, %xmm0
9261 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9262 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,1,2,0]
9263 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,7,5,6,7]
9264 ; SSE-NEXT: packuswb %xmm5, %xmm5
9265 ; SSE-NEXT: pand %xmm12, %xmm5
9266 ; SSE-NEXT: por %xmm7, %xmm5
9267 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
9268 ; SSE-NEXT: pand %xmm2, %xmm0
9269 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9270 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm0[0,2,2,3]
9271 ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,1,3,4,5,6,7]
9272 ; SSE-NEXT: packuswb %xmm7, %xmm7
9273 ; SSE-NEXT: movdqa %xmm9, %xmm15
9274 ; SSE-NEXT: pandn %xmm7, %xmm15
9275 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
9276 ; SSE-NEXT: pand %xmm2, %xmm0
9277 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9278 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm0[0,2,2,3]
9279 ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[1,3,2,3,4,5,6,7]
9280 ; SSE-NEXT: packuswb %xmm7, %xmm7
9281 ; SSE-NEXT: pand %xmm9, %xmm7
9282 ; SSE-NEXT: por %xmm15, %xmm7
9283 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[2,2,2,2]
9284 ; SSE-NEXT: punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm5[2],xmm7[3],xmm5[3]
9285 ; SSE-NEXT: movsd {{.*#+}} xmm7 = xmm1[0],xmm7[1]
9286 ; SSE-NEXT: movapd %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9287 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
9288 ; SSE-NEXT: pand %xmm2, %xmm0
9289 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9290 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
9291 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]
9292 ; SSE-NEXT: packuswb %xmm0, %xmm0
9293 ; SSE-NEXT: movdqa %xmm10, %xmm5
9294 ; SSE-NEXT: pandn %xmm0, %xmm5
9295 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
9296 ; SSE-NEXT: pand %xmm2, %xmm0
9297 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9298 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
9299 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
9300 ; SSE-NEXT: packuswb %xmm0, %xmm0
9301 ; SSE-NEXT: pand %xmm10, %xmm0
9302 ; SSE-NEXT: por %xmm5, %xmm0
9303 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9304 ; SSE-NEXT: pand %xmm2, %xmm1
9305 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9306 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,1,2,0]
9307 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,7,5]
9308 ; SSE-NEXT: packuswb %xmm5, %xmm5
9309 ; SSE-NEXT: movdqa %xmm13, %xmm7
9310 ; SSE-NEXT: pandn %xmm5, %xmm7
9311 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9312 ; SSE-NEXT: pand %xmm2, %xmm1
9313 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9314 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,1,2,0]
9315 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,7,5,6,7]
9316 ; SSE-NEXT: packuswb %xmm5, %xmm5
9317 ; SSE-NEXT: pand %xmm13, %xmm5
9318 ; SSE-NEXT: por %xmm7, %xmm5
9319 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,1,1]
9320 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
9321 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9322 ; SSE-NEXT: pand %xmm2, %xmm1
9323 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9324 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,1,2,0]
9325 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,7,5]
9326 ; SSE-NEXT: packuswb %xmm5, %xmm5
9327 ; SSE-NEXT: movdqa %xmm12, %xmm7
9328 ; SSE-NEXT: pandn %xmm5, %xmm7
9329 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9330 ; SSE-NEXT: pand %xmm2, %xmm1
9331 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9332 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,1,2,0]
9333 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,7,5,6,7]
9334 ; SSE-NEXT: packuswb %xmm5, %xmm5
9335 ; SSE-NEXT: pand %xmm12, %xmm5
9336 ; SSE-NEXT: por %xmm7, %xmm5
9337 ; SSE-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
9338 ; SSE-NEXT: pand %xmm2, %xmm1
9339 ; SSE-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill
9340 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm1[0,2,2,3]
9341 ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,1,3,4,5,6,7]
9342 ; SSE-NEXT: packuswb %xmm7, %xmm7
9343 ; SSE-NEXT: movdqa %xmm9, %xmm15
9344 ; SSE-NEXT: pandn %xmm7, %xmm15
9345 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9346 ; SSE-NEXT: pand %xmm2, %xmm1
9347 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9348 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm1[0,2,2,3]
9349 ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[1,3,2,3,4,5,6,7]
9350 ; SSE-NEXT: packuswb %xmm7, %xmm7
9351 ; SSE-NEXT: pand %xmm9, %xmm7
9352 ; SSE-NEXT: por %xmm15, %xmm7
9353 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[2,2,2,2]
9354 ; SSE-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]
9355 ; SSE-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
9356 ; SSE-NEXT: movapd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9357 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
9358 ; SSE-NEXT: pand %xmm2, %xmm0
9359 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9360 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
9361 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]
9362 ; SSE-NEXT: packuswb %xmm0, %xmm0
9363 ; SSE-NEXT: movdqa %xmm10, %xmm5
9364 ; SSE-NEXT: pandn %xmm0, %xmm5
9365 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
9366 ; SSE-NEXT: pand %xmm2, %xmm0
9367 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9368 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
9369 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
9370 ; SSE-NEXT: packuswb %xmm0, %xmm0
9371 ; SSE-NEXT: pand %xmm10, %xmm0
9372 ; SSE-NEXT: por %xmm5, %xmm0
9373 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9374 ; SSE-NEXT: pand %xmm2, %xmm1
9375 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9376 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,1,2,0]
9377 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,7,5]
9378 ; SSE-NEXT: packuswb %xmm5, %xmm5
9379 ; SSE-NEXT: movdqa %xmm13, %xmm7
9380 ; SSE-NEXT: pandn %xmm5, %xmm7
9381 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9382 ; SSE-NEXT: pand %xmm2, %xmm1
9383 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9384 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,1,2,0]
9385 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,7,5,6,7]
9386 ; SSE-NEXT: packuswb %xmm5, %xmm5
9387 ; SSE-NEXT: pand %xmm13, %xmm5
9388 ; SSE-NEXT: por %xmm7, %xmm5
9389 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,1,1]
9390 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
9391 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9392 ; SSE-NEXT: pand %xmm2, %xmm1
9393 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9394 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,1,2,0]
9395 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,7,5]
9396 ; SSE-NEXT: packuswb %xmm5, %xmm5
9397 ; SSE-NEXT: movdqa %xmm12, %xmm7
9398 ; SSE-NEXT: pandn %xmm5, %xmm7
9399 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9400 ; SSE-NEXT: pand %xmm2, %xmm1
9401 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9402 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,1,2,0]
9403 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,7,5,6,7]
9404 ; SSE-NEXT: packuswb %xmm5, %xmm5
9405 ; SSE-NEXT: pand %xmm12, %xmm5
9406 ; SSE-NEXT: por %xmm7, %xmm5
9407 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9408 ; SSE-NEXT: pand %xmm2, %xmm1
9409 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9410 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm1[0,2,2,3]
9411 ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,1,3,4,5,6,7]
9412 ; SSE-NEXT: packuswb %xmm7, %xmm7
9413 ; SSE-NEXT: movdqa %xmm9, %xmm15
9414 ; SSE-NEXT: pandn %xmm7, %xmm15
9415 ; SSE-NEXT: movdqa %xmm3, %xmm1
9416 ; SSE-NEXT: pand %xmm2, %xmm1
9417 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9418 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
9419 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]
9420 ; SSE-NEXT: packuswb %xmm1, %xmm1
9421 ; SSE-NEXT: pand %xmm9, %xmm1
9422 ; SSE-NEXT: por %xmm15, %xmm1
9423 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]
9424 ; SSE-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]
9425 ; SSE-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
9426 ; SSE-NEXT: movapd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9427 ; SSE-NEXT: pshufd $229, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
9428 ; SSE-NEXT: # xmm0 = mem[1,1,2,3]
9429 ; SSE-NEXT: movdqa %xmm10, %xmm1
9430 ; SSE-NEXT: pandn %xmm0, %xmm1
9431 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
9432 ; SSE-NEXT: # xmm0 = mem[3,3,3,3]
9433 ; SSE-NEXT: packuswb %xmm0, %xmm0
9434 ; SSE-NEXT: pand %xmm10, %xmm0
9435 ; SSE-NEXT: por %xmm1, %xmm0
9436 ; SSE-NEXT: movdqa %xmm13, %xmm1
9437 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9438 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
9439 ; SSE-NEXT: # xmm5 = mem[2,2,3,3]
9440 ; SSE-NEXT: packuswb %xmm5, %xmm5
9441 ; SSE-NEXT: pand %xmm13, %xmm5
9442 ; SSE-NEXT: por %xmm1, %xmm5
9443 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm5[1,1,1,1]
9444 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
9445 ; SSE-NEXT: movdqa %xmm12, %xmm1
9446 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9447 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
9448 ; SSE-NEXT: # xmm5 = mem[2,2,3,3]
9449 ; SSE-NEXT: packuswb %xmm5, %xmm5
9450 ; SSE-NEXT: pand %xmm12, %xmm5
9451 ; SSE-NEXT: por %xmm1, %xmm5
9452 ; SSE-NEXT: pshufd $244, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9453 ; SSE-NEXT: # xmm1 = mem[0,1,3,3]
9454 ; SSE-NEXT: movdqa %xmm9, %xmm7
9455 ; SSE-NEXT: pandn %xmm1, %xmm7
9456 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9457 ; SSE-NEXT: # xmm1 = mem[3,3,3,3]
9458 ; SSE-NEXT: packuswb %xmm1, %xmm1
9459 ; SSE-NEXT: pand %xmm9, %xmm1
9460 ; SSE-NEXT: por %xmm7, %xmm1
9461 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]
9462 ; SSE-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]
9463 ; SSE-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
9464 ; SSE-NEXT: movapd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9465 ; SSE-NEXT: pshufd $229, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
9466 ; SSE-NEXT: # xmm0 = mem[1,1,2,3]
9467 ; SSE-NEXT: movdqa %xmm10, %xmm1
9468 ; SSE-NEXT: pandn %xmm0, %xmm1
9469 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
9470 ; SSE-NEXT: # xmm0 = mem[3,3,3,3]
9471 ; SSE-NEXT: packuswb %xmm0, %xmm0
9472 ; SSE-NEXT: pand %xmm10, %xmm0
9473 ; SSE-NEXT: por %xmm1, %xmm0
9474 ; SSE-NEXT: movdqa %xmm13, %xmm1
9475 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9476 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
9477 ; SSE-NEXT: # xmm5 = mem[2,2,3,3]
9478 ; SSE-NEXT: packuswb %xmm5, %xmm5
9479 ; SSE-NEXT: pand %xmm13, %xmm5
9480 ; SSE-NEXT: por %xmm1, %xmm5
9481 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm5[1,1,1,1]
9482 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
9483 ; SSE-NEXT: movdqa %xmm12, %xmm1
9484 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9485 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
9486 ; SSE-NEXT: # xmm5 = mem[2,2,3,3]
9487 ; SSE-NEXT: packuswb %xmm5, %xmm5
9488 ; SSE-NEXT: pand %xmm12, %xmm5
9489 ; SSE-NEXT: por %xmm1, %xmm5
9490 ; SSE-NEXT: pshufd $244, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9491 ; SSE-NEXT: # xmm1 = mem[0,1,3,3]
9492 ; SSE-NEXT: movdqa %xmm9, %xmm7
9493 ; SSE-NEXT: pandn %xmm1, %xmm7
9494 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9495 ; SSE-NEXT: # xmm1 = mem[3,3,3,3]
9496 ; SSE-NEXT: packuswb %xmm1, %xmm1
9497 ; SSE-NEXT: pand %xmm9, %xmm1
9498 ; SSE-NEXT: por %xmm7, %xmm1
9499 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]
9500 ; SSE-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]
9501 ; SSE-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
9502 ; SSE-NEXT: movapd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9503 ; SSE-NEXT: pshufd $229, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
9504 ; SSE-NEXT: # xmm0 = mem[1,1,2,3]
9505 ; SSE-NEXT: movdqa %xmm10, %xmm1
9506 ; SSE-NEXT: pandn %xmm0, %xmm1
9507 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
9508 ; SSE-NEXT: # xmm0 = mem[3,3,3,3]
9509 ; SSE-NEXT: packuswb %xmm0, %xmm0
9510 ; SSE-NEXT: pand %xmm10, %xmm0
9511 ; SSE-NEXT: por %xmm1, %xmm0
9512 ; SSE-NEXT: movdqa %xmm13, %xmm1
9513 ; SSE-NEXT: pandn %xmm14, %xmm1
9514 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
9515 ; SSE-NEXT: # xmm5 = mem[2,2,3,3]
9516 ; SSE-NEXT: packuswb %xmm5, %xmm5
9517 ; SSE-NEXT: pand %xmm13, %xmm5
9518 ; SSE-NEXT: por %xmm1, %xmm5
9519 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm5[1,1,1,1]
9520 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
9521 ; SSE-NEXT: movdqa %xmm12, %xmm1
9522 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9523 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
9524 ; SSE-NEXT: # xmm5 = mem[2,2,3,3]
9525 ; SSE-NEXT: packuswb %xmm5, %xmm5
9526 ; SSE-NEXT: pand %xmm12, %xmm5
9527 ; SSE-NEXT: por %xmm1, %xmm5
9528 ; SSE-NEXT: pshufd $244, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9529 ; SSE-NEXT: # xmm1 = mem[0,1,3,3]
9530 ; SSE-NEXT: movdqa %xmm9, %xmm7
9531 ; SSE-NEXT: pandn %xmm1, %xmm7
9532 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9533 ; SSE-NEXT: # xmm1 = mem[3,3,3,3]
9534 ; SSE-NEXT: packuswb %xmm1, %xmm1
9535 ; SSE-NEXT: pand %xmm9, %xmm1
9536 ; SSE-NEXT: por %xmm7, %xmm1
9537 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]
9538 ; SSE-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]
9539 ; SSE-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
9540 ; SSE-NEXT: movapd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9541 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,2,3]
9542 ; SSE-NEXT: movdqa %xmm10, %xmm1
9543 ; SSE-NEXT: pandn %xmm0, %xmm1
9544 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
9545 ; SSE-NEXT: # xmm0 = mem[3,3,3,3]
9546 ; SSE-NEXT: packuswb %xmm0, %xmm0
9547 ; SSE-NEXT: pand %xmm10, %xmm0
9548 ; SSE-NEXT: por %xmm1, %xmm0
9549 ; SSE-NEXT: movdqa %xmm13, %xmm1
9550 ; SSE-NEXT: pandn %xmm4, %xmm1
9551 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
9552 ; SSE-NEXT: # xmm2 = mem[2,2,3,3]
9553 ; SSE-NEXT: packuswb %xmm2, %xmm2
9554 ; SSE-NEXT: pand %xmm13, %xmm2
9555 ; SSE-NEXT: por %xmm1, %xmm2
9556 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
9557 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
9558 ; SSE-NEXT: movdqa %xmm12, %xmm1
9559 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9560 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm11[2,2,3,3]
9561 ; SSE-NEXT: packuswb %xmm2, %xmm2
9562 ; SSE-NEXT: pand %xmm12, %xmm2
9563 ; SSE-NEXT: por %xmm1, %xmm2
9564 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm6[0,1,3,3]
9565 ; SSE-NEXT: movdqa %xmm9, %xmm3
9566 ; SSE-NEXT: pandn %xmm1, %xmm3
9567 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9568 ; SSE-NEXT: # xmm1 = mem[3,3,3,3]
9569 ; SSE-NEXT: packuswb %xmm1, %xmm1
9570 ; SSE-NEXT: pand %xmm9, %xmm1
9571 ; SSE-NEXT: por %xmm3, %xmm1
9572 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]
9573 ; SSE-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
9574 ; SSE-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
9575 ; SSE-NEXT: movapd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9576 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
9577 ; SSE-NEXT: # xmm7 = mem[3,1,2,3]
9578 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm7[0,1,2,0,4,5,6,7]
9579 ; SSE-NEXT: packuswb %xmm0, %xmm0
9580 ; SSE-NEXT: movdqa %xmm10, %xmm1
9581 ; SSE-NEXT: pandn %xmm0, %xmm1
9582 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
9583 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
9584 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9585 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]
9586 ; SSE-NEXT: packuswb %xmm0, %xmm0
9587 ; SSE-NEXT: pand %xmm10, %xmm0
9588 ; SSE-NEXT: por %xmm1, %xmm0
9589 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9590 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
9591 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9592 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,6]
9593 ; SSE-NEXT: packuswb %xmm1, %xmm1
9594 ; SSE-NEXT: movdqa %xmm13, %xmm2
9595 ; SSE-NEXT: pandn %xmm1, %xmm2
9596 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9597 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
9598 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9599 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
9600 ; SSE-NEXT: packuswb %xmm1, %xmm1
9601 ; SSE-NEXT: pand %xmm13, %xmm1
9602 ; SSE-NEXT: por %xmm2, %xmm1
9603 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
9604 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
9605 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9606 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
9607 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9608 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,6]
9609 ; SSE-NEXT: packuswb %xmm1, %xmm1
9610 ; SSE-NEXT: movdqa %xmm12, %xmm2
9611 ; SSE-NEXT: pandn %xmm1, %xmm2
9612 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9613 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
9614 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9615 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
9616 ; SSE-NEXT: packuswb %xmm1, %xmm1
9617 ; SSE-NEXT: pand %xmm12, %xmm1
9618 ; SSE-NEXT: por %xmm2, %xmm1
9619 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
9620 ; SSE-NEXT: # xmm2 = mem[3,1,2,3]
9621 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9622 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]
9623 ; SSE-NEXT: packuswb %xmm2, %xmm2
9624 ; SSE-NEXT: movdqa %xmm9, %xmm3
9625 ; SSE-NEXT: pandn %xmm2, %xmm3
9626 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
9627 ; SSE-NEXT: # xmm2 = mem[3,1,2,3]
9628 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9629 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]
9630 ; SSE-NEXT: packuswb %xmm2, %xmm2
9631 ; SSE-NEXT: pand %xmm9, %xmm2
9632 ; SSE-NEXT: por %xmm3, %xmm2
9633 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,2,2,2]
9634 ; SSE-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
9635 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
9636 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9637 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload
9638 ; SSE-NEXT: # xmm14 = mem[3,1,2,3]
9639 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm14[0,1,2,0,4,5,6,7]
9640 ; SSE-NEXT: packuswb %xmm0, %xmm0
9641 ; SSE-NEXT: movdqa %xmm10, %xmm1
9642 ; SSE-NEXT: pandn %xmm0, %xmm1
9643 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
9644 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
9645 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9646 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]
9647 ; SSE-NEXT: packuswb %xmm0, %xmm0
9648 ; SSE-NEXT: pand %xmm10, %xmm0
9649 ; SSE-NEXT: por %xmm1, %xmm0
9650 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9651 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
9652 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9653 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,6]
9654 ; SSE-NEXT: packuswb %xmm1, %xmm1
9655 ; SSE-NEXT: movdqa %xmm13, %xmm2
9656 ; SSE-NEXT: pandn %xmm1, %xmm2
9657 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9658 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
9659 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9660 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
9661 ; SSE-NEXT: packuswb %xmm1, %xmm1
9662 ; SSE-NEXT: pand %xmm13, %xmm1
9663 ; SSE-NEXT: por %xmm2, %xmm1
9664 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
9665 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
9666 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9667 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
9668 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9669 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,6]
9670 ; SSE-NEXT: packuswb %xmm1, %xmm1
9671 ; SSE-NEXT: movdqa %xmm12, %xmm2
9672 ; SSE-NEXT: pandn %xmm1, %xmm2
9673 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9674 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
9675 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9676 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
9677 ; SSE-NEXT: packuswb %xmm1, %xmm1
9678 ; SSE-NEXT: pand %xmm12, %xmm1
9679 ; SSE-NEXT: por %xmm2, %xmm1
9680 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
9681 ; SSE-NEXT: # xmm2 = mem[3,1,2,3]
9682 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9683 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]
9684 ; SSE-NEXT: packuswb %xmm2, %xmm2
9685 ; SSE-NEXT: movdqa %xmm9, %xmm3
9686 ; SSE-NEXT: pandn %xmm2, %xmm3
9687 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
9688 ; SSE-NEXT: # xmm11 = mem[3,1,2,3]
9689 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm11[2,0,2,3,4,5,6,7]
9690 ; SSE-NEXT: packuswb %xmm2, %xmm2
9691 ; SSE-NEXT: pand %xmm9, %xmm2
9692 ; SSE-NEXT: por %xmm3, %xmm2
9693 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,2,2,2]
9694 ; SSE-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
9695 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
9696 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9697 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
9698 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
9699 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9700 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]
9701 ; SSE-NEXT: packuswb %xmm0, %xmm0
9702 ; SSE-NEXT: movdqa %xmm10, %xmm1
9703 ; SSE-NEXT: pandn %xmm0, %xmm1
9704 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
9705 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
9706 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9707 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]
9708 ; SSE-NEXT: packuswb %xmm0, %xmm0
9709 ; SSE-NEXT: pand %xmm10, %xmm0
9710 ; SSE-NEXT: por %xmm1, %xmm0
9711 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9712 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
9713 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9714 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,6]
9715 ; SSE-NEXT: packuswb %xmm1, %xmm1
9716 ; SSE-NEXT: movdqa %xmm13, %xmm2
9717 ; SSE-NEXT: pandn %xmm1, %xmm2
9718 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9719 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
9720 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9721 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
9722 ; SSE-NEXT: packuswb %xmm1, %xmm1
9723 ; SSE-NEXT: pand %xmm13, %xmm1
9724 ; SSE-NEXT: por %xmm2, %xmm1
9725 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
9726 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
9727 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9728 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
9729 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9730 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,6]
9731 ; SSE-NEXT: packuswb %xmm1, %xmm1
9732 ; SSE-NEXT: movdqa %xmm12, %xmm2
9733 ; SSE-NEXT: pandn %xmm1, %xmm2
9734 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9735 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
9736 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9737 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
9738 ; SSE-NEXT: packuswb %xmm1, %xmm1
9739 ; SSE-NEXT: pand %xmm12, %xmm1
9740 ; SSE-NEXT: por %xmm2, %xmm1
9741 ; SSE-NEXT: pshufd $231, (%rsp), %xmm2 # 16-byte Folded Reload
9742 ; SSE-NEXT: # xmm2 = mem[3,1,2,3]
9743 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9744 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]
9745 ; SSE-NEXT: packuswb %xmm2, %xmm2
9746 ; SSE-NEXT: movdqa %xmm9, %xmm3
9747 ; SSE-NEXT: pandn %xmm2, %xmm3
9748 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
9749 ; SSE-NEXT: # xmm2 = mem[3,1,2,3]
9750 ; SSE-NEXT: movdqa %xmm2, (%rsp) # 16-byte Spill
9751 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]
9752 ; SSE-NEXT: packuswb %xmm2, %xmm2
9753 ; SSE-NEXT: pand %xmm9, %xmm2
9754 ; SSE-NEXT: por %xmm3, %xmm2
9755 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,2,2,2]
9756 ; SSE-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
9757 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
9758 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9759 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
9760 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
9761 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9762 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]
9763 ; SSE-NEXT: packuswb %xmm0, %xmm0
9764 ; SSE-NEXT: movdqa %xmm10, %xmm1
9765 ; SSE-NEXT: pandn %xmm0, %xmm1
9766 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
9767 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
9768 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9769 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]
9770 ; SSE-NEXT: packuswb %xmm0, %xmm0
9771 ; SSE-NEXT: pand %xmm10, %xmm0
9772 ; SSE-NEXT: por %xmm1, %xmm0
9773 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9774 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
9775 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9776 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,6]
9777 ; SSE-NEXT: packuswb %xmm1, %xmm1
9778 ; SSE-NEXT: movdqa %xmm13, %xmm2
9779 ; SSE-NEXT: pandn %xmm1, %xmm2
9780 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9781 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
9782 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9783 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
9784 ; SSE-NEXT: packuswb %xmm1, %xmm1
9785 ; SSE-NEXT: pand %xmm13, %xmm1
9786 ; SSE-NEXT: por %xmm2, %xmm1
9787 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
9788 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
9789 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9790 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
9791 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9792 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,6]
9793 ; SSE-NEXT: packuswb %xmm1, %xmm1
9794 ; SSE-NEXT: movdqa %xmm12, %xmm2
9795 ; SSE-NEXT: pandn %xmm1, %xmm2
9796 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9797 ; SSE-NEXT: # xmm1 = mem[0,1,1,3]
9798 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9799 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
9800 ; SSE-NEXT: packuswb %xmm1, %xmm1
9801 ; SSE-NEXT: pand %xmm12, %xmm1
9802 ; SSE-NEXT: por %xmm2, %xmm1
9803 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
9804 ; SSE-NEXT: # xmm2 = mem[3,1,2,3]
9805 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9806 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]
9807 ; SSE-NEXT: packuswb %xmm2, %xmm2
9808 ; SSE-NEXT: movdqa %xmm9, %xmm3
9809 ; SSE-NEXT: pandn %xmm2, %xmm3
9810 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
9811 ; SSE-NEXT: # xmm2 = mem[3,1,2,3]
9812 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9813 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]
9814 ; SSE-NEXT: packuswb %xmm2, %xmm2
9815 ; SSE-NEXT: pand %xmm9, %xmm2
9816 ; SSE-NEXT: por %xmm3, %xmm2
9817 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,2,2,2]
9818 ; SSE-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
9819 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
9820 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9821 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
9822 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
9823 ; SSE-NEXT: # xmm2 = xmm2[4],mem[4],xmm2[5],mem[5],xmm2[6],mem[6],xmm2[7],mem[7]
9824 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
9825 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
9826 ; SSE-NEXT: # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
9827 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9828 ; SSE-NEXT: packuswb %xmm2, %xmm2
9829 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9830 ; SSE-NEXT: movdqa %xmm10, %xmm1
9831 ; SSE-NEXT: pandn %xmm2, %xmm1
9832 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
9833 ; SSE-NEXT: packuswb %xmm0, %xmm0
9834 ; SSE-NEXT: pand %xmm10, %xmm0
9835 ; SSE-NEXT: por %xmm1, %xmm0
9836 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9837 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9838 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
9839 ; SSE-NEXT: packuswb %xmm1, %xmm1
9840 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9841 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,2,3]
9842 ; SSE-NEXT: movdqa %xmm13, %xmm2
9843 ; SSE-NEXT: pandn %xmm1, %xmm2
9844 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9845 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9846 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
9847 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9848 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
9849 ; SSE-NEXT: packuswb %xmm1, %xmm1
9850 ; SSE-NEXT: pand %xmm13, %xmm1
9851 ; SSE-NEXT: por %xmm2, %xmm1
9852 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
9853 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
9854 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9855 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9856 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
9857 ; SSE-NEXT: packuswb %xmm1, %xmm1
9858 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9859 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,2]
9860 ; SSE-NEXT: movdqa %xmm12, %xmm2
9861 ; SSE-NEXT: pandn %xmm1, %xmm2
9862 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9863 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9864 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
9865 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9866 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
9867 ; SSE-NEXT: packuswb %xmm1, %xmm1
9868 ; SSE-NEXT: pand %xmm12, %xmm1
9869 ; SSE-NEXT: por %xmm2, %xmm1
9870 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
9871 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
9872 ; SSE-NEXT: # xmm2 = xmm2[4],mem[4],xmm2[5],mem[5],xmm2[6],mem[6],xmm2[7],mem[7]
9873 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
9874 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
9875 ; SSE-NEXT: # xmm4 = xmm4[4],mem[4],xmm4[5],mem[5],xmm4[6],mem[6],xmm4[7],mem[7]
9876 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9877 ; SSE-NEXT: packuswb %xmm2, %xmm3
9878 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9879 ; SSE-NEXT: movdqa %xmm9, %xmm2
9880 ; SSE-NEXT: pandn %xmm3, %xmm2
9881 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,1,1]
9882 ; SSE-NEXT: packuswb %xmm3, %xmm3
9883 ; SSE-NEXT: pand %xmm9, %xmm3
9884 ; SSE-NEXT: por %xmm2, %xmm3
9885 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,2,2,2]
9886 ; SSE-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
9887 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
9888 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9889 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
9890 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
9891 ; SSE-NEXT: # xmm2 = xmm2[4],mem[4],xmm2[5],mem[5],xmm2[6],mem[6],xmm2[7],mem[7]
9892 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
9893 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
9894 ; SSE-NEXT: # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
9895 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9896 ; SSE-NEXT: packuswb %xmm2, %xmm2
9897 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9898 ; SSE-NEXT: movdqa %xmm10, %xmm1
9899 ; SSE-NEXT: pandn %xmm2, %xmm1
9900 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
9901 ; SSE-NEXT: packuswb %xmm0, %xmm0
9902 ; SSE-NEXT: pand %xmm10, %xmm0
9903 ; SSE-NEXT: por %xmm1, %xmm0
9904 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9905 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9906 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
9907 ; SSE-NEXT: packuswb %xmm1, %xmm1
9908 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9909 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,2,3]
9910 ; SSE-NEXT: movdqa %xmm13, %xmm2
9911 ; SSE-NEXT: pandn %xmm1, %xmm2
9912 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9913 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9914 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
9915 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9916 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
9917 ; SSE-NEXT: packuswb %xmm1, %xmm1
9918 ; SSE-NEXT: pand %xmm13, %xmm1
9919 ; SSE-NEXT: por %xmm2, %xmm1
9920 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
9921 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
9922 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9923 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9924 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
9925 ; SSE-NEXT: packuswb %xmm1, %xmm1
9926 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9927 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,2]
9928 ; SSE-NEXT: movdqa %xmm12, %xmm2
9929 ; SSE-NEXT: pandn %xmm1, %xmm2
9930 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9931 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9932 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
9933 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9934 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
9935 ; SSE-NEXT: packuswb %xmm1, %xmm1
9936 ; SSE-NEXT: pand %xmm12, %xmm1
9937 ; SSE-NEXT: por %xmm2, %xmm1
9938 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
9939 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
9940 ; SSE-NEXT: # xmm2 = xmm2[4],mem[4],xmm2[5],mem[5],xmm2[6],mem[6],xmm2[7],mem[7]
9941 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
9942 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
9943 ; SSE-NEXT: # xmm3 = xmm3[4],mem[4],xmm3[5],mem[5],xmm3[6],mem[6],xmm3[7],mem[7]
9944 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9945 ; SSE-NEXT: packuswb %xmm2, %xmm8
9946 ; SSE-NEXT: movdqa %xmm9, %xmm2
9947 ; SSE-NEXT: pandn %xmm8, %xmm2
9948 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,1,1]
9949 ; SSE-NEXT: packuswb %xmm3, %xmm3
9950 ; SSE-NEXT: pand %xmm9, %xmm3
9951 ; SSE-NEXT: por %xmm2, %xmm3
9952 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,2,2,2]
9953 ; SSE-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
9954 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
9955 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9956 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
9957 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
9958 ; SSE-NEXT: # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
9959 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
9960 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
9961 ; SSE-NEXT: # xmm2 = xmm2[4],mem[4],xmm2[5],mem[5],xmm2[6],mem[6],xmm2[7],mem[7]
9962 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9963 ; SSE-NEXT: packuswb %xmm0, %xmm0
9964 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9965 ; SSE-NEXT: movdqa %xmm10, %xmm1
9966 ; SSE-NEXT: pandn %xmm0, %xmm1
9967 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
9968 ; SSE-NEXT: packuswb %xmm0, %xmm0
9969 ; SSE-NEXT: pand %xmm10, %xmm0
9970 ; SSE-NEXT: por %xmm1, %xmm0
9971 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9972 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9973 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
9974 ; SSE-NEXT: packuswb %xmm1, %xmm1
9975 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9976 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,2,3]
9977 ; SSE-NEXT: movdqa %xmm13, %xmm2
9978 ; SSE-NEXT: pandn %xmm1, %xmm2
9979 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9980 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9981 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
9982 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9983 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
9984 ; SSE-NEXT: packuswb %xmm1, %xmm1
9985 ; SSE-NEXT: pand %xmm13, %xmm1
9986 ; SSE-NEXT: por %xmm2, %xmm1
9987 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
9988 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
9989 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9990 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9991 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
9992 ; SSE-NEXT: packuswb %xmm1, %xmm1
9993 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9994 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,2]
9995 ; SSE-NEXT: movdqa %xmm12, %xmm2
9996 ; SSE-NEXT: pandn %xmm1, %xmm2
9997 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
9998 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
9999 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
10000 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10001 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
10002 ; SSE-NEXT: packuswb %xmm1, %xmm1
10003 ; SSE-NEXT: pand %xmm12, %xmm1
10004 ; SSE-NEXT: por %xmm2, %xmm1
10005 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
10006 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
10007 ; SSE-NEXT: # xmm2 = xmm2[4],mem[4],xmm2[5],mem[5],xmm2[6],mem[6],xmm2[7],mem[7]
10008 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10009 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
10010 ; SSE-NEXT: # xmm4 = xmm4[4],mem[4],xmm4[5],mem[5],xmm4[6],mem[6],xmm4[7],mem[7]
10011 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10012 ; SSE-NEXT: packuswb %xmm2, %xmm3
10013 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10014 ; SSE-NEXT: movdqa %xmm9, %xmm2
10015 ; SSE-NEXT: pandn %xmm3, %xmm2
10016 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,1,1]
10017 ; SSE-NEXT: packuswb %xmm3, %xmm3
10018 ; SSE-NEXT: pand %xmm9, %xmm3
10019 ; SSE-NEXT: por %xmm2, %xmm3
10020 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,2,2,2]
10021 ; SSE-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
10022 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
10023 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10024 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
10025 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
10026 ; SSE-NEXT: # xmm2 = xmm2[4],mem[4],xmm2[5],mem[5],xmm2[6],mem[6],xmm2[7],mem[7]
10027 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
10028 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
10029 ; SSE-NEXT: # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
10030 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10031 ; SSE-NEXT: packuswb %xmm2, %xmm2
10032 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10033 ; SSE-NEXT: movdqa %xmm10, %xmm1
10034 ; SSE-NEXT: pandn %xmm2, %xmm1
10035 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
10036 ; SSE-NEXT: packuswb %xmm0, %xmm0
10037 ; SSE-NEXT: pand %xmm10, %xmm0
10038 ; SSE-NEXT: por %xmm1, %xmm0
10039 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
10040 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10041 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
10042 ; SSE-NEXT: packuswb %xmm1, %xmm1
10043 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10044 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,2,3]
10045 ; SSE-NEXT: movdqa %xmm13, %xmm2
10046 ; SSE-NEXT: pandn %xmm1, %xmm2
10047 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
10048 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10049 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
10050 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10051 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]
10052 ; SSE-NEXT: packuswb %xmm1, %xmm1
10053 ; SSE-NEXT: pand %xmm13, %xmm1
10054 ; SSE-NEXT: por %xmm2, %xmm1
10055 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
10056 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
10057 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
10058 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10059 ; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
10060 ; SSE-NEXT: packuswb %xmm1, %xmm1
10061 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10062 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,2]
10063 ; SSE-NEXT: movdqa %xmm12, %xmm2
10064 ; SSE-NEXT: pandn %xmm1, %xmm2
10065 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
10066 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
10067 ; SSE-NEXT: # xmm6 = xmm6[4],mem[4],xmm6[5],mem[5],xmm6[6],mem[6],xmm6[7],mem[7]
10068 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm6[0,1,1,3]
10069 ; SSE-NEXT: packuswb %xmm1, %xmm1
10070 ; SSE-NEXT: pand %xmm12, %xmm1
10071 ; SSE-NEXT: por %xmm2, %xmm1
10072 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
10073 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
10074 ; SSE-NEXT: # xmm2 = xmm2[4],mem[4],xmm2[5],mem[5],xmm2[6],mem[6],xmm2[7],mem[7]
10075 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
10076 ; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
10077 ; SSE-NEXT: # xmm3 = xmm3[4],mem[4],xmm3[5],mem[5],xmm3[6],mem[6],xmm3[7],mem[7]
10078 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10079 ; SSE-NEXT: packuswb %xmm2, %xmm4
10080 ; SSE-NEXT: movdqa %xmm9, %xmm2
10081 ; SSE-NEXT: pandn %xmm4, %xmm2
10082 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,1,1]
10083 ; SSE-NEXT: packuswb %xmm3, %xmm3
10084 ; SSE-NEXT: pand %xmm9, %xmm3
10085 ; SSE-NEXT: por %xmm2, %xmm3
10086 ; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm3[2,2,2,2]
10087 ; SSE-NEXT: punpckhdq {{.*#+}} xmm15 = xmm15[2],xmm1[2],xmm15[3],xmm1[3]
10088 ; SSE-NEXT: movsd {{.*#+}} xmm15 = xmm0[0],xmm15[1]
10089 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm14[0,1,3,1,4,5,6,7]
10090 ; SSE-NEXT: packuswb %xmm0, %xmm0
10091 ; SSE-NEXT: movdqa %xmm10, %xmm1
10092 ; SSE-NEXT: pandn %xmm0, %xmm1
10093 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
10094 ; SSE-NEXT: # xmm0 = mem[3,1,2,3,4,5,6,7]
10095 ; SSE-NEXT: packuswb %xmm0, %xmm0
10096 ; SSE-NEXT: pand %xmm10, %xmm0
10097 ; SSE-NEXT: por %xmm1, %xmm0
10098 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10099 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,4,5,5,7]
10100 ; SSE-NEXT: packuswb %xmm1, %xmm1
10101 ; SSE-NEXT: movdqa %xmm13, %xmm2
10102 ; SSE-NEXT: pandn %xmm1, %xmm2
10103 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10104 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,5,7,6,7]
10105 ; SSE-NEXT: packuswb %xmm1, %xmm1
10106 ; SSE-NEXT: pand %xmm13, %xmm1
10107 ; SSE-NEXT: por %xmm2, %xmm1
10108 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
10109 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
10110 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10111 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,4,5,5,7]
10112 ; SSE-NEXT: packuswb %xmm1, %xmm1
10113 ; SSE-NEXT: movdqa %xmm12, %xmm2
10114 ; SSE-NEXT: pandn %xmm1, %xmm2
10115 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10116 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,5,7,6,7]
10117 ; SSE-NEXT: packuswb %xmm1, %xmm1
10118 ; SSE-NEXT: pand %xmm12, %xmm1
10119 ; SSE-NEXT: por %xmm2, %xmm1
10120 ; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
10121 ; SSE-NEXT: # xmm2 = mem[0,1,3,1,4,5,6,7]
10122 ; SSE-NEXT: packuswb %xmm2, %xmm2
10123 ; SSE-NEXT: movdqa %xmm9, %xmm3
10124 ; SSE-NEXT: pandn %xmm2, %xmm3
10125 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm11[3,1,2,3,4,5,6,7]
10126 ; SSE-NEXT: packuswb %xmm2, %xmm2
10127 ; SSE-NEXT: pand %xmm9, %xmm2
10128 ; SSE-NEXT: por %xmm3, %xmm2
10129 ; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm2[2,2,2,2]
10130 ; SSE-NEXT: punpckhdq {{.*#+}} xmm14 = xmm14[2],xmm1[2],xmm14[3],xmm1[3]
10131 ; SSE-NEXT: movsd {{.*#+}} xmm14 = xmm0[0],xmm14[1]
10132 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm7[0,1,3,1,4,5,6,7]
10133 ; SSE-NEXT: packuswb %xmm0, %xmm0
10134 ; SSE-NEXT: movdqa %xmm10, %xmm1
10135 ; SSE-NEXT: pandn %xmm0, %xmm1
10136 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
10137 ; SSE-NEXT: # xmm0 = mem[3,1,2,3,4,5,6,7]
10138 ; SSE-NEXT: packuswb %xmm0, %xmm0
10139 ; SSE-NEXT: pand %xmm10, %xmm0
10140 ; SSE-NEXT: por %xmm1, %xmm0
10141 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10142 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,4,5,5,7]
10143 ; SSE-NEXT: packuswb %xmm1, %xmm1
10144 ; SSE-NEXT: movdqa %xmm13, %xmm2
10145 ; SSE-NEXT: pandn %xmm1, %xmm2
10146 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10147 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,5,7,6,7]
10148 ; SSE-NEXT: packuswb %xmm1, %xmm1
10149 ; SSE-NEXT: pand %xmm13, %xmm1
10150 ; SSE-NEXT: por %xmm2, %xmm1
10151 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
10152 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
10153 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10154 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,4,5,5,7]
10155 ; SSE-NEXT: packuswb %xmm1, %xmm1
10156 ; SSE-NEXT: movdqa %xmm12, %xmm2
10157 ; SSE-NEXT: pandn %xmm1, %xmm2
10158 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10159 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,5,7,6,7]
10160 ; SSE-NEXT: packuswb %xmm1, %xmm1
10161 ; SSE-NEXT: pand %xmm12, %xmm1
10162 ; SSE-NEXT: por %xmm2, %xmm1
10163 ; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
10164 ; SSE-NEXT: # xmm2 = mem[0,1,3,1,4,5,6,7]
10165 ; SSE-NEXT: packuswb %xmm2, %xmm2
10166 ; SSE-NEXT: movdqa %xmm9, %xmm3
10167 ; SSE-NEXT: pandn %xmm2, %xmm3
10168 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
10169 ; SSE-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7]
10170 ; SSE-NEXT: packuswb %xmm2, %xmm2
10171 ; SSE-NEXT: pand %xmm9, %xmm2
10172 ; SSE-NEXT: por %xmm3, %xmm2
10173 ; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm2[2,2,2,2]
10174 ; SSE-NEXT: punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm1[2],xmm11[3],xmm1[3]
10175 ; SSE-NEXT: movsd {{.*#+}} xmm11 = xmm0[0],xmm11[1]
10176 ; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
10177 ; SSE-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]
10178 ; SSE-NEXT: packuswb %xmm0, %xmm0
10179 ; SSE-NEXT: movdqa %xmm10, %xmm1
10180 ; SSE-NEXT: pandn %xmm0, %xmm1
10181 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
10182 ; SSE-NEXT: # xmm0 = mem[3,1,2,3,4,5,6,7]
10183 ; SSE-NEXT: packuswb %xmm0, %xmm0
10184 ; SSE-NEXT: pand %xmm10, %xmm0
10185 ; SSE-NEXT: por %xmm1, %xmm0
10186 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10187 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,4,5,5,7]
10188 ; SSE-NEXT: packuswb %xmm1, %xmm1
10189 ; SSE-NEXT: movdqa %xmm13, %xmm2
10190 ; SSE-NEXT: pandn %xmm1, %xmm2
10191 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10192 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,5,7,6,7]
10193 ; SSE-NEXT: packuswb %xmm1, %xmm1
10194 ; SSE-NEXT: pand %xmm13, %xmm1
10195 ; SSE-NEXT: por %xmm2, %xmm1
10196 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
10197 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
10198 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10199 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,4,5,5,7]
10200 ; SSE-NEXT: packuswb %xmm1, %xmm1
10201 ; SSE-NEXT: movdqa %xmm12, %xmm2
10202 ; SSE-NEXT: pandn %xmm1, %xmm2
10203 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10204 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,5,7,6,7]
10205 ; SSE-NEXT: packuswb %xmm1, %xmm1
10206 ; SSE-NEXT: pand %xmm12, %xmm1
10207 ; SSE-NEXT: por %xmm2, %xmm1
10208 ; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
10209 ; SSE-NEXT: # xmm2 = mem[0,1,3,1,4,5,6,7]
10210 ; SSE-NEXT: packuswb %xmm2, %xmm2
10211 ; SSE-NEXT: movdqa %xmm9, %xmm3
10212 ; SSE-NEXT: pandn %xmm2, %xmm3
10213 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
10214 ; SSE-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7]
10215 ; SSE-NEXT: packuswb %xmm2, %xmm2
10216 ; SSE-NEXT: pand %xmm9, %xmm2
10217 ; SSE-NEXT: por %xmm3, %xmm2
10218 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm2[2,2,2,2]
10219 ; SSE-NEXT: punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm1[2],xmm7[3],xmm1[3]
10220 ; SSE-NEXT: movsd {{.*#+}} xmm7 = xmm0[0],xmm7[1]
10221 ; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
10222 ; SSE-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]
10223 ; SSE-NEXT: packuswb %xmm0, %xmm0
10224 ; SSE-NEXT: movdqa %xmm10, %xmm1
10225 ; SSE-NEXT: pandn %xmm0, %xmm1
10226 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
10227 ; SSE-NEXT: # xmm0 = mem[3,1,2,3,4,5,6,7]
10228 ; SSE-NEXT: packuswb %xmm0, %xmm0
10229 ; SSE-NEXT: pand %xmm10, %xmm0
10230 ; SSE-NEXT: por %xmm1, %xmm0
10231 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10232 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,4,5,5,7]
10233 ; SSE-NEXT: packuswb %xmm1, %xmm1
10234 ; SSE-NEXT: movdqa %xmm13, %xmm2
10235 ; SSE-NEXT: pandn %xmm1, %xmm2
10236 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10237 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,5,7,6,7]
10238 ; SSE-NEXT: packuswb %xmm1, %xmm1
10239 ; SSE-NEXT: pand %xmm13, %xmm1
10240 ; SSE-NEXT: por %xmm2, %xmm1
10241 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
10242 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
10243 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10244 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,4,5,5,7]
10245 ; SSE-NEXT: packuswb %xmm1, %xmm1
10246 ; SSE-NEXT: movdqa %xmm12, %xmm2
10247 ; SSE-NEXT: pandn %xmm1, %xmm2
10248 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10249 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,5,7,6,7]
10250 ; SSE-NEXT: packuswb %xmm1, %xmm1
10251 ; SSE-NEXT: pand %xmm12, %xmm1
10252 ; SSE-NEXT: por %xmm2, %xmm1
10253 ; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
10254 ; SSE-NEXT: # xmm2 = mem[0,1,3,1,4,5,6,7]
10255 ; SSE-NEXT: packuswb %xmm2, %xmm2
10256 ; SSE-NEXT: movdqa %xmm9, %xmm3
10257 ; SSE-NEXT: pandn %xmm2, %xmm3
10258 ; SSE-NEXT: pshuflw $231, (%rsp), %xmm2 # 16-byte Folded Reload
10259 ; SSE-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7]
10260 ; SSE-NEXT: packuswb %xmm2, %xmm2
10261 ; SSE-NEXT: pand %xmm9, %xmm2
10262 ; SSE-NEXT: por %xmm3, %xmm2
10263 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm2[2,2,2,2]
10264 ; SSE-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm1[2],xmm5[3],xmm1[3]
10265 ; SSE-NEXT: movsd {{.*#+}} xmm5 = xmm0[0],xmm5[1]
10266 ; SSE-NEXT: pshufd $229, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
10267 ; SSE-NEXT: # xmm0 = mem[1,1,2,3]
10268 ; SSE-NEXT: movdqa %xmm10, %xmm1
10269 ; SSE-NEXT: pandn %xmm0, %xmm1
10270 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
10271 ; SSE-NEXT: # xmm0 = mem[3,3,3,3]
10272 ; SSE-NEXT: packuswb %xmm0, %xmm0
10273 ; SSE-NEXT: pand %xmm10, %xmm0
10274 ; SSE-NEXT: por %xmm1, %xmm0
10275 ; SSE-NEXT: movdqa %xmm13, %xmm1
10276 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10277 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
10278 ; SSE-NEXT: # xmm2 = mem[2,2,3,3]
10279 ; SSE-NEXT: packuswb %xmm2, %xmm2
10280 ; SSE-NEXT: pand %xmm13, %xmm2
10281 ; SSE-NEXT: por %xmm1, %xmm2
10282 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
10283 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
10284 ; SSE-NEXT: movdqa %xmm12, %xmm1
10285 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10286 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
10287 ; SSE-NEXT: # xmm2 = mem[2,2,3,3]
10288 ; SSE-NEXT: packuswb %xmm2, %xmm2
10289 ; SSE-NEXT: pand %xmm12, %xmm2
10290 ; SSE-NEXT: por %xmm1, %xmm2
10291 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm8[0,1,3,3]
10292 ; SSE-NEXT: movdqa %xmm9, %xmm3
10293 ; SSE-NEXT: pandn %xmm1, %xmm3
10294 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10295 ; SSE-NEXT: # xmm1 = mem[3,3,3,3]
10296 ; SSE-NEXT: packuswb %xmm1, %xmm1
10297 ; SSE-NEXT: pand %xmm9, %xmm1
10298 ; SSE-NEXT: por %xmm3, %xmm1
10299 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,2,2,2]
10300 ; SSE-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
10301 ; SSE-NEXT: movsd {{.*#+}} xmm3 = xmm0[0],xmm3[1]
10302 ; SSE-NEXT: pshufd $229, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
10303 ; SSE-NEXT: # xmm0 = mem[1,1,2,3]
10304 ; SSE-NEXT: movdqa %xmm10, %xmm1
10305 ; SSE-NEXT: pandn %xmm0, %xmm1
10306 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
10307 ; SSE-NEXT: # xmm0 = mem[3,3,3,3]
10308 ; SSE-NEXT: packuswb %xmm0, %xmm0
10309 ; SSE-NEXT: pand %xmm10, %xmm0
10310 ; SSE-NEXT: por %xmm1, %xmm0
10311 ; SSE-NEXT: movdqa %xmm13, %xmm1
10312 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10313 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
10314 ; SSE-NEXT: # xmm2 = mem[2,2,3,3]
10315 ; SSE-NEXT: packuswb %xmm2, %xmm2
10316 ; SSE-NEXT: pand %xmm13, %xmm2
10317 ; SSE-NEXT: por %xmm1, %xmm2
10318 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
10319 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
10320 ; SSE-NEXT: movdqa %xmm12, %xmm8
10321 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
10322 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
10323 ; SSE-NEXT: # xmm2 = mem[2,2,3,3]
10324 ; SSE-NEXT: packuswb %xmm2, %xmm1
10325 ; SSE-NEXT: pand %xmm12, %xmm1
10326 ; SSE-NEXT: por %xmm8, %xmm1
10327 ; SSE-NEXT: pshufd $244, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
10328 ; SSE-NEXT: # xmm2 = mem[0,1,3,3]
10329 ; SSE-NEXT: movdqa %xmm9, %xmm8
10330 ; SSE-NEXT: pandn %xmm2, %xmm8
10331 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
10332 ; SSE-NEXT: # xmm2 = mem[3,3,3,3]
10333 ; SSE-NEXT: packuswb %xmm2, %xmm2
10334 ; SSE-NEXT: pand %xmm9, %xmm2
10335 ; SSE-NEXT: por %xmm8, %xmm2
10336 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,2,2,2]
10337 ; SSE-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
10338 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
10339 ; SSE-NEXT: pshufd $229, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
10340 ; SSE-NEXT: # xmm0 = mem[1,1,2,3]
10341 ; SSE-NEXT: movdqa %xmm10, %xmm1
10342 ; SSE-NEXT: pandn %xmm0, %xmm1
10343 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
10344 ; SSE-NEXT: # xmm0 = mem[3,3,3,3]
10345 ; SSE-NEXT: packuswb %xmm0, %xmm0
10346 ; SSE-NEXT: pand %xmm10, %xmm0
10347 ; SSE-NEXT: por %xmm1, %xmm0
10348 ; SSE-NEXT: movdqa %xmm13, %xmm1
10349 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10350 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
10351 ; SSE-NEXT: # xmm8 = mem[2,2,3,3]
10352 ; SSE-NEXT: packuswb %xmm8, %xmm8
10353 ; SSE-NEXT: pand %xmm13, %xmm8
10354 ; SSE-NEXT: por %xmm1, %xmm8
10355 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm8[1,1,1,1]
10356 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
10357 ; SSE-NEXT: movdqa %xmm12, %xmm1
10358 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10359 ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm6[2,2,3,3]
10360 ; SSE-NEXT: packuswb %xmm8, %xmm8
10361 ; SSE-NEXT: pand %xmm12, %xmm8
10362 ; SSE-NEXT: por %xmm1, %xmm8
10363 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm4[0,1,3,3]
10364 ; SSE-NEXT: movdqa %xmm9, %xmm4
10365 ; SSE-NEXT: pandn %xmm1, %xmm4
10366 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
10367 ; SSE-NEXT: # xmm1 = mem[3,3,3,3]
10368 ; SSE-NEXT: packuswb %xmm1, %xmm1
10369 ; SSE-NEXT: pand %xmm9, %xmm1
10370 ; SSE-NEXT: por %xmm4, %xmm1
10371 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]
10372 ; SSE-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm8[2],xmm1[3],xmm8[3]
10373 ; SSE-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
10374 ; SSE-NEXT: pshufd $229, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
10375 ; SSE-NEXT: # xmm0 = mem[1,1,2,3]
10376 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
10377 ; SSE-NEXT: # xmm4 = mem[3,3,3,3]
10378 ; SSE-NEXT: packuswb %xmm4, %xmm4
10379 ; SSE-NEXT: pand %xmm10, %xmm4
10380 ; SSE-NEXT: pandn %xmm0, %xmm10
10381 ; SSE-NEXT: por %xmm4, %xmm10
10382 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
10383 ; SSE-NEXT: # xmm0 = mem[2,2,3,3]
10384 ; SSE-NEXT: packuswb %xmm0, %xmm0
10385 ; SSE-NEXT: pand %xmm13, %xmm0
10386 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload
10387 ; SSE-NEXT: por %xmm0, %xmm13
10388 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm13[1,1,1,1]
10389 ; SSE-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm0[0],xmm10[1],xmm0[1]
10390 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
10391 ; SSE-NEXT: # xmm0 = mem[2,2,3,3]
10392 ; SSE-NEXT: packuswb %xmm0, %xmm0
10393 ; SSE-NEXT: pand %xmm12, %xmm0
10394 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
10395 ; SSE-NEXT: por %xmm0, %xmm12
10396 ; SSE-NEXT: pshufd $244, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
10397 ; SSE-NEXT: # xmm0 = mem[0,1,3,3]
10398 ; SSE-NEXT: pshufd $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
10399 ; SSE-NEXT: # xmm4 = mem[3,3,3,3]
10400 ; SSE-NEXT: packuswb %xmm4, %xmm4
10401 ; SSE-NEXT: pand %xmm9, %xmm4
10402 ; SSE-NEXT: pandn %xmm0, %xmm9
10403 ; SSE-NEXT: por %xmm4, %xmm9
10404 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm9[2,2,2,2]
10405 ; SSE-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm12[2],xmm0[3],xmm12[3]
10406 ; SSE-NEXT: movsd {{.*#+}} xmm0 = xmm10[0],xmm0[1]
10407 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10408 ; SSE-NEXT: movaps %xmm4, 32(%rsi)
10409 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10410 ; SSE-NEXT: movaps %xmm4, 48(%rsi)
10411 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10412 ; SSE-NEXT: movaps %xmm4, (%rsi)
10413 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10414 ; SSE-NEXT: movaps %xmm4, 16(%rsi)
10415 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10416 ; SSE-NEXT: movaps %xmm4, 32(%rdx)
10417 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10418 ; SSE-NEXT: movaps %xmm4, 48(%rdx)
10419 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10420 ; SSE-NEXT: movaps %xmm4, (%rdx)
10421 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10422 ; SSE-NEXT: movaps %xmm4, 16(%rdx)
10423 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10424 ; SSE-NEXT: movaps %xmm4, 32(%rcx)
10425 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10426 ; SSE-NEXT: movaps %xmm4, 48(%rcx)
10427 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10428 ; SSE-NEXT: movaps %xmm4, (%rcx)
10429 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10430 ; SSE-NEXT: movaps %xmm4, 16(%rcx)
10431 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10432 ; SSE-NEXT: movaps %xmm4, 32(%r8)
10433 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10434 ; SSE-NEXT: movaps %xmm4, 48(%r8)
10435 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10436 ; SSE-NEXT: movaps %xmm4, (%r8)
10437 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10438 ; SSE-NEXT: movaps %xmm4, 16(%r8)
10439 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10440 ; SSE-NEXT: movaps %xmm4, 32(%r9)
10441 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10442 ; SSE-NEXT: movaps %xmm4, 48(%r9)
10443 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10444 ; SSE-NEXT: movaps %xmm4, (%r9)
10445 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10446 ; SSE-NEXT: movaps %xmm4, 16(%r9)
10447 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
10448 ; SSE-NEXT: movapd %xmm15, 32(%rax)
10449 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10450 ; SSE-NEXT: movaps %xmm4, 48(%rax)
10451 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10452 ; SSE-NEXT: movaps %xmm4, (%rax)
10453 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10454 ; SSE-NEXT: movaps %xmm4, 16(%rax)
10455 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
10456 ; SSE-NEXT: movapd %xmm5, 48(%rax)
10457 ; SSE-NEXT: movapd %xmm7, 32(%rax)
10458 ; SSE-NEXT: movapd %xmm11, 16(%rax)
10459 ; SSE-NEXT: movapd %xmm14, (%rax)
10460 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
10461 ; SSE-NEXT: movapd %xmm0, 48(%rax)
10462 ; SSE-NEXT: movapd %xmm1, 32(%rax)
10463 ; SSE-NEXT: movapd %xmm2, 16(%rax)
10464 ; SSE-NEXT: movapd %xmm3, (%rax)
10465 ; SSE-NEXT: addq $2040, %rsp # imm = 0x7F8
10466 ; SSE-NEXT: retq
10467 ;
10468 ; AVX-LABEL: load_i8_stride8_vf64:
10469 ; AVX: # %bb.0:
10470 ; AVX-NEXT: subq $808, %rsp # imm = 0x328
10471 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm3 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
10472 ; AVX-NEXT: vmovdqa 368(%rdi), %xmm0
10473 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10474 ; AVX-NEXT: vpshufb %xmm3, %xmm0, %xmm0
10475 ; AVX-NEXT: vmovdqa 352(%rdi), %xmm13
10476 ; AVX-NEXT: vpshufb %xmm3, %xmm13, %xmm1
10477 ; AVX-NEXT: vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10478 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
10479 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm1 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
10480 ; AVX-NEXT: vmovdqa 336(%rdi), %xmm15
10481 ; AVX-NEXT: vpshufb %xmm1, %xmm15, %xmm2
10482 ; AVX-NEXT: vmovdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10483 ; AVX-NEXT: vmovdqa 320(%rdi), %xmm4
10484 ; AVX-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10485 ; AVX-NEXT: vpshufb %xmm1, %xmm4, %xmm4
10486 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
10487 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0,1,2,3,4,5],xmm0[6,7]
10488 ; AVX-NEXT: vmovd {{.*#+}} xmm2 = [0,0,0,8,0,0,0,0,0,0,0,0,0,0,0,0]
10489 ; AVX-NEXT: vmovdqa 304(%rdi), %xmm12
10490 ; AVX-NEXT: vpshufb %xmm2, %xmm12, %xmm0
10491 ; AVX-NEXT: vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10492 ; AVX-NEXT: vmovdqa 288(%rdi), %xmm5
10493 ; AVX-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10494 ; AVX-NEXT: vpshufb %xmm2, %xmm5, %xmm5
10495 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
10496 ; AVX-NEXT: vmovd {{.*#+}} xmm9 = [0,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
10497 ; AVX-NEXT: vmovdqa 272(%rdi), %xmm11
10498 ; AVX-NEXT: vpshufb %xmm9, %xmm11, %xmm6
10499 ; AVX-NEXT: vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10500 ; AVX-NEXT: vmovdqa 256(%rdi), %xmm10
10501 ; AVX-NEXT: vpshufb %xmm9, %xmm10, %xmm7
10502 ; AVX-NEXT: vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10503 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
10504 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5,6,7]
10505 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
10506 ; AVX-NEXT: vmovdqa 496(%rdi), %xmm0
10507 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10508 ; AVX-NEXT: vpshufb %xmm3, %xmm0, %xmm5
10509 ; AVX-NEXT: vmovdqa 480(%rdi), %xmm0
10510 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10511 ; AVX-NEXT: vpshufb %xmm3, %xmm0, %xmm6
10512 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
10513 ; AVX-NEXT: vmovdqa 464(%rdi), %xmm0
10514 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10515 ; AVX-NEXT: vpshufb %xmm1, %xmm0, %xmm6
10516 ; AVX-NEXT: vmovdqa 448(%rdi), %xmm0
10517 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10518 ; AVX-NEXT: vpshufb %xmm1, %xmm0, %xmm7
10519 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
10520 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1,2,3,4,5],xmm5[6,7]
10521 ; AVX-NEXT: vmovdqa 432(%rdi), %xmm0
10522 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10523 ; AVX-NEXT: vpshufb %xmm2, %xmm0, %xmm6
10524 ; AVX-NEXT: vmovdqa 416(%rdi), %xmm14
10525 ; AVX-NEXT: vpshufb %xmm2, %xmm14, %xmm7
10526 ; AVX-NEXT: vmovdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10527 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
10528 ; AVX-NEXT: vmovdqa 400(%rdi), %xmm0
10529 ; AVX-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill
10530 ; AVX-NEXT: vpshufb %xmm9, %xmm0, %xmm7
10531 ; AVX-NEXT: vmovdqa 384(%rdi), %xmm0
10532 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10533 ; AVX-NEXT: vpshufb %xmm9, %xmm0, %xmm8
10534 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
10535 ; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3],xmm7[4,5,6,7]
10536 ; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm5
10537 ; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6
10538 ; AVX-NEXT: vblendps {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
10539 ; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
10540 ; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10541 ; AVX-NEXT: vmovdqa 112(%rdi), %xmm0
10542 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10543 ; AVX-NEXT: vpshufb %xmm3, %xmm0, %xmm4
10544 ; AVX-NEXT: vmovdqa 96(%rdi), %xmm0
10545 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10546 ; AVX-NEXT: vpshufb %xmm3, %xmm0, %xmm5
10547 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
10548 ; AVX-NEXT: vmovdqa 80(%rdi), %xmm0
10549 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10550 ; AVX-NEXT: vpshufb %xmm1, %xmm0, %xmm5
10551 ; AVX-NEXT: vmovdqa 64(%rdi), %xmm0
10552 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10553 ; AVX-NEXT: vpshufb %xmm1, %xmm0, %xmm6
10554 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
10555 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3,4,5],xmm4[6,7]
10556 ; AVX-NEXT: vmovdqa 32(%rdi), %xmm0
10557 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10558 ; AVX-NEXT: vmovdqa 48(%rdi), %xmm5
10559 ; AVX-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10560 ; AVX-NEXT: vpshufb %xmm2, %xmm5, %xmm5
10561 ; AVX-NEXT: vpshufb %xmm2, %xmm0, %xmm6
10562 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
10563 ; AVX-NEXT: vmovdqa (%rdi), %xmm0
10564 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10565 ; AVX-NEXT: vmovdqa 16(%rdi), %xmm6
10566 ; AVX-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10567 ; AVX-NEXT: vpshufb %xmm9, %xmm6, %xmm6
10568 ; AVX-NEXT: vpshufb %xmm9, %xmm0, %xmm7
10569 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
10570 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5,6,7]
10571 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
10572 ; AVX-NEXT: vmovdqa 240(%rdi), %xmm0
10573 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10574 ; AVX-NEXT: vpshufb %xmm3, %xmm0, %xmm5
10575 ; AVX-NEXT: vmovdqa 224(%rdi), %xmm0
10576 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10577 ; AVX-NEXT: vpshufb %xmm3, %xmm0, %xmm3
10578 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
10579 ; AVX-NEXT: vmovdqa 208(%rdi), %xmm0
10580 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10581 ; AVX-NEXT: vpshufb %xmm1, %xmm0, %xmm5
10582 ; AVX-NEXT: vmovdqa 192(%rdi), %xmm0
10583 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10584 ; AVX-NEXT: vpshufb %xmm1, %xmm0, %xmm1
10585 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
10586 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm3[6,7]
10587 ; AVX-NEXT: vmovdqa 176(%rdi), %xmm0
10588 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10589 ; AVX-NEXT: vpshufb %xmm2, %xmm0, %xmm3
10590 ; AVX-NEXT: vmovdqa 160(%rdi), %xmm0
10591 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10592 ; AVX-NEXT: vpshufb %xmm2, %xmm0, %xmm2
10593 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
10594 ; AVX-NEXT: vmovdqa 144(%rdi), %xmm0
10595 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10596 ; AVX-NEXT: vpshufb %xmm9, %xmm0, %xmm3
10597 ; AVX-NEXT: vmovdqa 128(%rdi), %xmm0
10598 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10599 ; AVX-NEXT: vpshufb %xmm9, %xmm0, %xmm0
10600 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
10601 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5,6,7]
10602 ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
10603 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
10604 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
10605 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
10606 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10607 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm6 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
10608 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
10609 ; AVX-NEXT: vpshufb %xmm6, %xmm0, %xmm0
10610 ; AVX-NEXT: vpshufb %xmm6, %xmm13, %xmm1
10611 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
10612 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm1 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
10613 ; AVX-NEXT: vpshufb %xmm1, %xmm15, %xmm3
10614 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
10615 ; AVX-NEXT: vpshufb %xmm1, %xmm13, %xmm4
10616 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
10617 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm3[0,1,2,3,4,5],xmm0[6,7]
10618 ; AVX-NEXT: vmovd {{.*#+}} xmm2 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
10619 ; AVX-NEXT: vpshufb %xmm2, %xmm12, %xmm3
10620 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
10621 ; AVX-NEXT: vpshufb %xmm2, %xmm7, %xmm5
10622 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
10623 ; AVX-NEXT: vmovd {{.*#+}} xmm3 = [1,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
10624 ; AVX-NEXT: vpshufb %xmm3, %xmm11, %xmm0
10625 ; AVX-NEXT: vpshufb %xmm3, %xmm10, %xmm15
10626 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
10627 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm5[2,3],xmm0[4,5,6,7]
10628 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm4[4,5,6,7]
10629 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
10630 ; AVX-NEXT: vpshufb %xmm6, %xmm12, %xmm4
10631 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
10632 ; AVX-NEXT: vpshufb %xmm6, %xmm9, %xmm5
10633 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
10634 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
10635 ; AVX-NEXT: vpshufb %xmm1, %xmm10, %xmm5
10636 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
10637 ; AVX-NEXT: vpshufb %xmm1, %xmm11, %xmm15
10638 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm15[0],xmm5[0],xmm15[1],xmm5[1],xmm15[2],xmm5[2],xmm15[3],xmm5[3]
10639 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3,4,5],xmm4[6,7]
10640 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
10641 ; AVX-NEXT: vpshufb %xmm2, %xmm8, %xmm5
10642 ; AVX-NEXT: vpshufb %xmm2, %xmm14, %xmm15
10643 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm15[0],xmm5[0],xmm15[1],xmm5[1],xmm15[2],xmm5[2],xmm15[3],xmm5[3]
10644 ; AVX-NEXT: vmovdqa (%rsp), %xmm14 # 16-byte Reload
10645 ; AVX-NEXT: vpshufb %xmm3, %xmm14, %xmm15
10646 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
10647 ; AVX-NEXT: vpshufb %xmm3, %xmm14, %xmm14
10648 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
10649 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm14[0,1],xmm5[2,3],xmm14[4,5,6,7]
10650 ; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4
10651 ; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm5
10652 ; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
10653 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7]
10654 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10655 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
10656 ; AVX-NEXT: vpshufb %xmm6, %xmm0, %xmm0
10657 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10658 ; AVX-NEXT: vpshufb %xmm6, %xmm4, %xmm4
10659 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
10660 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10661 ; AVX-NEXT: vpshufb %xmm1, %xmm4, %xmm4
10662 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10663 ; AVX-NEXT: vpshufb %xmm1, %xmm5, %xmm5
10664 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
10665 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm4[0,1,2,3,4,5],xmm0[6,7]
10666 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10667 ; AVX-NEXT: vpshufb %xmm2, %xmm4, %xmm4
10668 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10669 ; AVX-NEXT: vpshufb %xmm2, %xmm5, %xmm5
10670 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
10671 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10672 ; AVX-NEXT: vpshufb %xmm3, %xmm5, %xmm5
10673 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
10674 ; AVX-NEXT: vpshufb %xmm3, %xmm14, %xmm14
10675 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm14[0],xmm5[0],xmm14[1],xmm5[1],xmm14[2],xmm5[2],xmm14[3],xmm5[3]
10676 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3],xmm5[4,5,6,7]
10677 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm4[0,1,2,3],xmm0[4,5,6,7]
10678 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10679 ; AVX-NEXT: vpshufb %xmm6, %xmm4, %xmm4
10680 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10681 ; AVX-NEXT: vpshufb %xmm6, %xmm5, %xmm5
10682 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
10683 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10684 ; AVX-NEXT: vpshufb %xmm1, %xmm5, %xmm5
10685 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
10686 ; AVX-NEXT: vpshufb %xmm1, %xmm6, %xmm1
10687 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
10688 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm4[6,7]
10689 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10690 ; AVX-NEXT: vpshufb %xmm2, %xmm4, %xmm4
10691 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10692 ; AVX-NEXT: vpshufb %xmm2, %xmm5, %xmm2
10693 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
10694 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10695 ; AVX-NEXT: vpshufb %xmm3, %xmm4, %xmm4
10696 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10697 ; AVX-NEXT: vpshufb %xmm3, %xmm5, %xmm3
10698 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
10699 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5,6,7]
10700 ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
10701 ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2
10702 ; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
10703 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
10704 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10705 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm0 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
10706 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
10707 ; AVX-NEXT: vpshufb %xmm0, %xmm1, %xmm1
10708 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
10709 ; AVX-NEXT: vpshufb %xmm0, %xmm2, %xmm2
10710 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
10711 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm1 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
10712 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
10713 ; AVX-NEXT: vpshufb %xmm1, %xmm3, %xmm3
10714 ; AVX-NEXT: vpshufb %xmm1, %xmm13, %xmm4
10715 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
10716 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm3[0,1,2,3,4,5],xmm2[6,7]
10717 ; AVX-NEXT: vmovd {{.*#+}} xmm2 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
10718 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
10719 ; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm3
10720 ; AVX-NEXT: vpshufb %xmm2, %xmm7, %xmm5
10721 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
10722 ; AVX-NEXT: vmovd {{.*#+}} xmm3 = [2,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
10723 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
10724 ; AVX-NEXT: vpshufb %xmm3, %xmm7, %xmm6
10725 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
10726 ; AVX-NEXT: vpshufb %xmm3, %xmm13, %xmm14
10727 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
10728 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5,6,7]
10729 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
10730 ; AVX-NEXT: vpshufb %xmm0, %xmm12, %xmm5
10731 ; AVX-NEXT: vpshufb %xmm0, %xmm9, %xmm6
10732 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
10733 ; AVX-NEXT: vpshufb %xmm1, %xmm10, %xmm6
10734 ; AVX-NEXT: vpshufb %xmm1, %xmm11, %xmm14
10735 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
10736 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1,2,3,4,5],xmm5[6,7]
10737 ; AVX-NEXT: vpshufb %xmm2, %xmm8, %xmm6
10738 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
10739 ; AVX-NEXT: vpshufb %xmm2, %xmm9, %xmm14
10740 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
10741 ; AVX-NEXT: vmovdqa (%rsp), %xmm10 # 16-byte Reload
10742 ; AVX-NEXT: vpshufb %xmm3, %xmm10, %xmm14
10743 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
10744 ; AVX-NEXT: vpshufb %xmm3, %xmm11, %xmm15
10745 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
10746 ; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm14[0,1],xmm6[2,3],xmm14[4,5,6,7]
10747 ; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm5
10748 ; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6
10749 ; AVX-NEXT: vblendps {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
10750 ; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
10751 ; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10752 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
10753 ; AVX-NEXT: vpshufb %xmm0, %xmm12, %xmm4
10754 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
10755 ; AVX-NEXT: vpshufb %xmm0, %xmm8, %xmm5
10756 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
10757 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10758 ; AVX-NEXT: vpshufb %xmm1, %xmm5, %xmm5
10759 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
10760 ; AVX-NEXT: vpshufb %xmm1, %xmm6, %xmm6
10761 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
10762 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3,4,5],xmm4[6,7]
10763 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10764 ; AVX-NEXT: vpshufb %xmm2, %xmm5, %xmm5
10765 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
10766 ; AVX-NEXT: vpshufb %xmm2, %xmm6, %xmm6
10767 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
10768 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
10769 ; AVX-NEXT: vpshufb %xmm3, %xmm6, %xmm6
10770 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
10771 ; AVX-NEXT: vpshufb %xmm3, %xmm13, %xmm14
10772 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
10773 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5,6,7]
10774 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
10775 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10776 ; AVX-NEXT: vpshufb %xmm0, %xmm5, %xmm5
10777 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
10778 ; AVX-NEXT: vpshufb %xmm0, %xmm6, %xmm0
10779 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
10780 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10781 ; AVX-NEXT: vpshufb %xmm1, %xmm5, %xmm5
10782 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
10783 ; AVX-NEXT: vpshufb %xmm1, %xmm6, %xmm1
10784 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
10785 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4,5],xmm0[6,7]
10786 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
10787 ; AVX-NEXT: vpshufb %xmm2, %xmm1, %xmm1
10788 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10789 ; AVX-NEXT: vpshufb %xmm2, %xmm5, %xmm2
10790 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
10791 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
10792 ; AVX-NEXT: vpshufb %xmm3, %xmm2, %xmm2
10793 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10794 ; AVX-NEXT: vpshufb %xmm3, %xmm5, %xmm3
10795 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
10796 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5,6,7]
10797 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
10798 ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
10799 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
10800 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
10801 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10802 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm0 = [0,0,3,11,0,0,3,11,0,0,3,11,0,0,3,11]
10803 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
10804 ; AVX-NEXT: vpshufb %xmm0, %xmm13, %xmm1
10805 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
10806 ; AVX-NEXT: vpshufb %xmm0, %xmm2, %xmm2
10807 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
10808 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm1 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
10809 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
10810 ; AVX-NEXT: vpshufb %xmm1, %xmm3, %xmm3
10811 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10812 ; AVX-NEXT: vpshufb %xmm1, %xmm4, %xmm4
10813 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
10814 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm3[0,1,2,3,4,5],xmm2[6,7]
10815 ; AVX-NEXT: vmovd {{.*#+}} xmm2 = [0,0,3,11,0,0,0,0,0,0,0,0,0,0,0,0]
10816 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
10817 ; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm3
10818 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10819 ; AVX-NEXT: vpshufb %xmm2, %xmm5, %xmm5
10820 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
10821 ; AVX-NEXT: vmovd {{.*#+}} xmm3 = [3,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
10822 ; AVX-NEXT: vpshufb %xmm3, %xmm7, %xmm6
10823 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
10824 ; AVX-NEXT: vpshufb %xmm3, %xmm7, %xmm14
10825 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
10826 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5,6,7]
10827 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
10828 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10829 ; AVX-NEXT: vpshufb %xmm0, %xmm5, %xmm5
10830 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
10831 ; AVX-NEXT: vpshufb %xmm0, %xmm6, %xmm6
10832 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
10833 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
10834 ; AVX-NEXT: vpshufb %xmm1, %xmm6, %xmm6
10835 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
10836 ; AVX-NEXT: vpshufb %xmm1, %xmm7, %xmm14
10837 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
10838 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1,2,3,4,5],xmm5[6,7]
10839 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
10840 ; AVX-NEXT: vpshufb %xmm2, %xmm6, %xmm6
10841 ; AVX-NEXT: vpshufb %xmm2, %xmm9, %xmm14
10842 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
10843 ; AVX-NEXT: vpshufb %xmm3, %xmm10, %xmm14
10844 ; AVX-NEXT: vpshufb %xmm3, %xmm11, %xmm15
10845 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
10846 ; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm14[0,1],xmm6[2,3],xmm14[4,5,6,7]
10847 ; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm5
10848 ; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6
10849 ; AVX-NEXT: vblendps {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
10850 ; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
10851 ; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10852 ; AVX-NEXT: vpshufb %xmm0, %xmm12, %xmm4
10853 ; AVX-NEXT: vpshufb %xmm0, %xmm8, %xmm5
10854 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
10855 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
10856 ; AVX-NEXT: vpshufb %xmm1, %xmm9, %xmm5
10857 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
10858 ; AVX-NEXT: vpshufb %xmm1, %xmm10, %xmm6
10859 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
10860 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3,4,5],xmm4[6,7]
10861 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
10862 ; AVX-NEXT: vpshufb %xmm2, %xmm12, %xmm5
10863 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
10864 ; AVX-NEXT: vpshufb %xmm2, %xmm11, %xmm6
10865 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
10866 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
10867 ; AVX-NEXT: vpshufb %xmm3, %xmm6, %xmm6
10868 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
10869 ; AVX-NEXT: vpshufb %xmm3, %xmm7, %xmm14
10870 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
10871 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5,6,7]
10872 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
10873 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10874 ; AVX-NEXT: vpshufb %xmm0, %xmm5, %xmm5
10875 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
10876 ; AVX-NEXT: vpshufb %xmm0, %xmm6, %xmm0
10877 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
10878 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10879 ; AVX-NEXT: vpshufb %xmm1, %xmm5, %xmm5
10880 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
10881 ; AVX-NEXT: vpshufb %xmm1, %xmm6, %xmm1
10882 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
10883 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4,5],xmm0[6,7]
10884 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
10885 ; AVX-NEXT: vpshufb %xmm2, %xmm1, %xmm1
10886 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10887 ; AVX-NEXT: vpshufb %xmm2, %xmm5, %xmm2
10888 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
10889 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
10890 ; AVX-NEXT: vpshufb %xmm3, %xmm2, %xmm2
10891 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10892 ; AVX-NEXT: vpshufb %xmm3, %xmm5, %xmm3
10893 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
10894 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5,6,7]
10895 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
10896 ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
10897 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
10898 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
10899 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10900 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm0 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
10901 ; AVX-NEXT: vpshufb %xmm0, %xmm13, %xmm1
10902 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
10903 ; AVX-NEXT: vpshufb %xmm0, %xmm2, %xmm2
10904 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
10905 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm1 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
10906 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
10907 ; AVX-NEXT: vpshufb %xmm1, %xmm3, %xmm3
10908 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10909 ; AVX-NEXT: vpshufb %xmm1, %xmm4, %xmm4
10910 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
10911 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm3[0,1,2,3,4,5],xmm2[6,7]
10912 ; AVX-NEXT: vmovd {{.*#+}} xmm2 = [0,0,4,12,0,0,0,0,0,0,0,0,0,0,0,0]
10913 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
10914 ; AVX-NEXT: vpshufb %xmm2, %xmm7, %xmm3
10915 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10916 ; AVX-NEXT: vpshufb %xmm2, %xmm5, %xmm5
10917 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
10918 ; AVX-NEXT: vmovd {{.*#+}} xmm3 = [4,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
10919 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
10920 ; AVX-NEXT: vpshufb %xmm3, %xmm6, %xmm6
10921 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
10922 ; AVX-NEXT: vpshufb %xmm3, %xmm8, %xmm14
10923 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
10924 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5,6,7]
10925 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
10926 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10927 ; AVX-NEXT: vpshufb %xmm0, %xmm5, %xmm5
10928 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
10929 ; AVX-NEXT: vpshufb %xmm0, %xmm6, %xmm6
10930 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
10931 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
10932 ; AVX-NEXT: vpshufb %xmm1, %xmm6, %xmm6
10933 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
10934 ; AVX-NEXT: vpshufb %xmm1, %xmm13, %xmm14
10935 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
10936 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1,2,3,4,5],xmm5[6,7]
10937 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
10938 ; AVX-NEXT: vpshufb %xmm2, %xmm6, %xmm6
10939 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
10940 ; AVX-NEXT: vpshufb %xmm2, %xmm13, %xmm14
10941 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
10942 ; AVX-NEXT: vmovdqa (%rsp), %xmm13 # 16-byte Reload
10943 ; AVX-NEXT: vpshufb %xmm3, %xmm13, %xmm14
10944 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
10945 ; AVX-NEXT: vpshufb %xmm3, %xmm13, %xmm15
10946 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
10947 ; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm14[0,1],xmm6[2,3],xmm14[4,5,6,7]
10948 ; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm5
10949 ; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6
10950 ; AVX-NEXT: vblendps {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
10951 ; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
10952 ; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10953 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
10954 ; AVX-NEXT: vpshufb %xmm0, %xmm4, %xmm4
10955 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10956 ; AVX-NEXT: vpshufb %xmm0, %xmm5, %xmm5
10957 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
10958 ; AVX-NEXT: vpshufb %xmm1, %xmm9, %xmm5
10959 ; AVX-NEXT: vpshufb %xmm1, %xmm10, %xmm6
10960 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
10961 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3,4,5],xmm4[6,7]
10962 ; AVX-NEXT: vpshufb %xmm2, %xmm12, %xmm5
10963 ; AVX-NEXT: vpshufb %xmm2, %xmm11, %xmm6
10964 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
10965 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
10966 ; AVX-NEXT: vpshufb %xmm3, %xmm10, %xmm6
10967 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
10968 ; AVX-NEXT: vpshufb %xmm3, %xmm9, %xmm14
10969 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
10970 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5,6,7]
10971 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
10972 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
10973 ; AVX-NEXT: vpshufb %xmm0, %xmm11, %xmm5
10974 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
10975 ; AVX-NEXT: vpshufb %xmm0, %xmm12, %xmm0
10976 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
10977 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
10978 ; AVX-NEXT: vpshufb %xmm1, %xmm13, %xmm5
10979 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
10980 ; AVX-NEXT: vpshufb %xmm1, %xmm6, %xmm1
10981 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
10982 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4,5],xmm0[6,7]
10983 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
10984 ; AVX-NEXT: vpshufb %xmm2, %xmm1, %xmm1
10985 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10986 ; AVX-NEXT: vpshufb %xmm2, %xmm5, %xmm2
10987 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
10988 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
10989 ; AVX-NEXT: vpshufb %xmm3, %xmm2, %xmm2
10990 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
10991 ; AVX-NEXT: vpshufb %xmm3, %xmm5, %xmm3
10992 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
10993 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5,6,7]
10994 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
10995 ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
10996 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
10997 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
10998 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10999 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm0 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
11000 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
11001 ; AVX-NEXT: vpshufb %xmm0, %xmm1, %xmm1
11002 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
11003 ; AVX-NEXT: vpshufb %xmm0, %xmm2, %xmm2
11004 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
11005 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm1 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
11006 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
11007 ; AVX-NEXT: vpshufb %xmm1, %xmm3, %xmm3
11008 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
11009 ; AVX-NEXT: vpshufb %xmm1, %xmm4, %xmm4
11010 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
11011 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm3[0,1,2,3,4,5],xmm2[6,7]
11012 ; AVX-NEXT: vmovd {{.*#+}} xmm2 = [0,0,5,13,0,0,0,0,0,0,0,0,0,0,0,0]
11013 ; AVX-NEXT: vpshufb %xmm2, %xmm7, %xmm3
11014 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11015 ; AVX-NEXT: vpshufb %xmm2, %xmm5, %xmm5
11016 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
11017 ; AVX-NEXT: vmovd {{.*#+}} xmm3 = [5,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
11018 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11019 ; AVX-NEXT: vpshufb %xmm3, %xmm6, %xmm6
11020 ; AVX-NEXT: vpshufb %xmm3, %xmm8, %xmm14
11021 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11022 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5,6,7]
11023 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
11024 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11025 ; AVX-NEXT: vpshufb %xmm0, %xmm5, %xmm5
11026 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11027 ; AVX-NEXT: vpshufb %xmm0, %xmm6, %xmm6
11028 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
11029 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11030 ; AVX-NEXT: vpshufb %xmm1, %xmm6, %xmm6
11031 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
11032 ; AVX-NEXT: vpshufb %xmm1, %xmm7, %xmm14
11033 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11034 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1,2,3,4,5],xmm5[6,7]
11035 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11036 ; AVX-NEXT: vpshufb %xmm2, %xmm6, %xmm6
11037 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
11038 ; AVX-NEXT: vpshufb %xmm2, %xmm7, %xmm14
11039 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11040 ; AVX-NEXT: vmovdqa (%rsp), %xmm7 # 16-byte Reload
11041 ; AVX-NEXT: vpshufb %xmm3, %xmm7, %xmm14
11042 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
11043 ; AVX-NEXT: vpshufb %xmm3, %xmm7, %xmm15
11044 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
11045 ; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm14[0,1],xmm6[2,3],xmm14[4,5,6,7]
11046 ; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm5
11047 ; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6
11048 ; AVX-NEXT: vblendps {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
11049 ; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
11050 ; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11051 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
11052 ; AVX-NEXT: vpshufb %xmm0, %xmm4, %xmm4
11053 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11054 ; AVX-NEXT: vpshufb %xmm0, %xmm5, %xmm5
11055 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
11056 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11057 ; AVX-NEXT: vpshufb %xmm1, %xmm5, %xmm5
11058 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11059 ; AVX-NEXT: vpshufb %xmm1, %xmm6, %xmm6
11060 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
11061 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3,4,5],xmm4[6,7]
11062 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11063 ; AVX-NEXT: vpshufb %xmm2, %xmm5, %xmm5
11064 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11065 ; AVX-NEXT: vpshufb %xmm2, %xmm6, %xmm6
11066 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
11067 ; AVX-NEXT: vpshufb %xmm3, %xmm10, %xmm6
11068 ; AVX-NEXT: vpshufb %xmm3, %xmm9, %xmm14
11069 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11070 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5,6,7]
11071 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
11072 ; AVX-NEXT: vpshufb %xmm0, %xmm11, %xmm5
11073 ; AVX-NEXT: vpshufb %xmm0, %xmm12, %xmm0
11074 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
11075 ; AVX-NEXT: vpshufb %xmm1, %xmm13, %xmm5
11076 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
11077 ; AVX-NEXT: vpshufb %xmm1, %xmm13, %xmm1
11078 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
11079 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4,5],xmm0[6,7]
11080 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
11081 ; AVX-NEXT: vpshufb %xmm2, %xmm7, %xmm1
11082 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
11083 ; AVX-NEXT: vpshufb %xmm2, %xmm8, %xmm2
11084 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
11085 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
11086 ; AVX-NEXT: vpshufb %xmm3, %xmm9, %xmm2
11087 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
11088 ; AVX-NEXT: vpshufb %xmm3, %xmm10, %xmm3
11089 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
11090 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5,6,7]
11091 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
11092 ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
11093 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
11094 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
11095 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11096 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm0 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
11097 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
11098 ; AVX-NEXT: vpshufb %xmm0, %xmm1, %xmm1
11099 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11100 ; AVX-NEXT: vpshufb %xmm0, %xmm11, %xmm2
11101 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
11102 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm1 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
11103 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
11104 ; AVX-NEXT: vpshufb %xmm1, %xmm12, %xmm3
11105 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
11106 ; AVX-NEXT: vpshufb %xmm1, %xmm4, %xmm4
11107 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
11108 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm3[0,1,2,3,4,5],xmm2[6,7]
11109 ; AVX-NEXT: vmovd {{.*#+}} xmm2 = [0,0,6,14,0,0,0,0,0,0,0,0,0,0,0,0]
11110 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
11111 ; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm3
11112 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11113 ; AVX-NEXT: vpshufb %xmm2, %xmm5, %xmm5
11114 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
11115 ; AVX-NEXT: vmovd {{.*#+}} xmm3 = [6,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
11116 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11117 ; AVX-NEXT: vpshufb %xmm3, %xmm6, %xmm6
11118 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
11119 ; AVX-NEXT: vpshufb %xmm3, %xmm14, %xmm14
11120 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11121 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5,6,7]
11122 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
11123 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11124 ; AVX-NEXT: vpshufb %xmm0, %xmm5, %xmm5
11125 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11126 ; AVX-NEXT: vpshufb %xmm0, %xmm6, %xmm6
11127 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
11128 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11129 ; AVX-NEXT: vpshufb %xmm1, %xmm6, %xmm6
11130 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
11131 ; AVX-NEXT: vpshufb %xmm1, %xmm14, %xmm14
11132 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11133 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1,2,3,4,5],xmm5[6,7]
11134 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11135 ; AVX-NEXT: vpshufb %xmm2, %xmm6, %xmm6
11136 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
11137 ; AVX-NEXT: vpshufb %xmm2, %xmm14, %xmm14
11138 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11139 ; AVX-NEXT: vmovdqa (%rsp), %xmm14 # 16-byte Reload
11140 ; AVX-NEXT: vpshufb %xmm3, %xmm14, %xmm14
11141 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
11142 ; AVX-NEXT: vpshufb %xmm3, %xmm15, %xmm15
11143 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
11144 ; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm14[0,1],xmm6[2,3],xmm14[4,5,6,7]
11145 ; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm5
11146 ; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6
11147 ; AVX-NEXT: vblendps {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
11148 ; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
11149 ; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11150 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
11151 ; AVX-NEXT: vpshufb %xmm0, %xmm4, %xmm4
11152 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11153 ; AVX-NEXT: vpshufb %xmm0, %xmm5, %xmm5
11154 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
11155 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11156 ; AVX-NEXT: vpshufb %xmm1, %xmm5, %xmm5
11157 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11158 ; AVX-NEXT: vpshufb %xmm1, %xmm6, %xmm6
11159 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
11160 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3,4,5],xmm4[6,7]
11161 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11162 ; AVX-NEXT: vpshufb %xmm2, %xmm5, %xmm5
11163 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11164 ; AVX-NEXT: vpshufb %xmm2, %xmm6, %xmm6
11165 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
11166 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11167 ; AVX-NEXT: vpshufb %xmm3, %xmm6, %xmm6
11168 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
11169 ; AVX-NEXT: vpshufb %xmm3, %xmm14, %xmm14
11170 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11171 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5,6,7]
11172 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
11173 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11174 ; AVX-NEXT: vpshufb %xmm0, %xmm5, %xmm5
11175 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11176 ; AVX-NEXT: vpshufb %xmm0, %xmm6, %xmm0
11177 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
11178 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11179 ; AVX-NEXT: vpshufb %xmm1, %xmm5, %xmm5
11180 ; AVX-NEXT: vpshufb %xmm1, %xmm13, %xmm1
11181 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
11182 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4,5],xmm0[6,7]
11183 ; AVX-NEXT: vpshufb %xmm2, %xmm7, %xmm1
11184 ; AVX-NEXT: vpshufb %xmm2, %xmm8, %xmm2
11185 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
11186 ; AVX-NEXT: vpshufb %xmm3, %xmm9, %xmm2
11187 ; AVX-NEXT: vpshufb %xmm3, %xmm10, %xmm3
11188 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
11189 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5,6,7]
11190 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
11191 ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
11192 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
11193 ; AVX-NEXT: vblendps {{.*#+}} ymm5 = ymm4[0,1,2,3],ymm0[4,5,6,7]
11194 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm3 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
11195 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
11196 ; AVX-NEXT: vpshufb %xmm3, %xmm0, %xmm0
11197 ; AVX-NEXT: vpshufb %xmm3, %xmm11, %xmm1
11198 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
11199 ; AVX-NEXT: vbroadcastss {{.*#+}} xmm2 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
11200 ; AVX-NEXT: vpshufb %xmm2, %xmm12, %xmm1
11201 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
11202 ; AVX-NEXT: vpshufb %xmm2, %xmm4, %xmm4
11203 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
11204 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm1[0,1,2,3,4,5],xmm0[6,7]
11205 ; AVX-NEXT: vmovd {{.*#+}} xmm1 = [0,0,7,15,0,0,0,0,0,0,0,0,0,0,0,0]
11206 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
11207 ; AVX-NEXT: vpshufb %xmm1, %xmm0, %xmm0
11208 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11209 ; AVX-NEXT: vpshufb %xmm1, %xmm6, %xmm6
11210 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3]
11211 ; AVX-NEXT: vmovd {{.*#+}} xmm0 = [7,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
11212 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11213 ; AVX-NEXT: vpshufb %xmm0, %xmm11, %xmm14
11214 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11215 ; AVX-NEXT: vpshufb %xmm0, %xmm11, %xmm15
11216 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
11217 ; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm14[0,1],xmm6[2,3],xmm14[4,5,6,7]
11218 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm6[0,1,2,3],xmm4[4,5,6,7]
11219 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11220 ; AVX-NEXT: vpshufb %xmm3, %xmm6, %xmm6
11221 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11222 ; AVX-NEXT: vpshufb %xmm3, %xmm11, %xmm14
11223 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11224 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11225 ; AVX-NEXT: vpshufb %xmm2, %xmm11, %xmm14
11226 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11227 ; AVX-NEXT: vpshufb %xmm2, %xmm11, %xmm15
11228 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
11229 ; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm14[0,1,2,3,4,5],xmm6[6,7]
11230 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11231 ; AVX-NEXT: vpshufb %xmm1, %xmm11, %xmm14
11232 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11233 ; AVX-NEXT: vpshufb %xmm1, %xmm11, %xmm15
11234 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
11235 ; AVX-NEXT: vmovdqa (%rsp), %xmm11 # 16-byte Reload
11236 ; AVX-NEXT: vpshufb %xmm0, %xmm11, %xmm15
11237 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11238 ; AVX-NEXT: vpshufb %xmm0, %xmm11, %xmm13
11239 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm15[0],xmm13[1],xmm15[1],xmm13[2],xmm15[2],xmm13[3],xmm15[3]
11240 ; AVX-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0,1],xmm14[2,3],xmm13[4,5,6,7]
11241 ; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6
11242 ; AVX-NEXT: vinsertf128 $1, %xmm13, %ymm0, %ymm13
11243 ; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm13[0,1,2,3,4,5],ymm6[6,7]
11244 ; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm6[4,5,6,7]
11245 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11246 ; AVX-NEXT: vpshufb %xmm3, %xmm6, %xmm6
11247 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11248 ; AVX-NEXT: vpshufb %xmm3, %xmm11, %xmm13
11249 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm13[0],xmm6[0],xmm13[1],xmm6[1],xmm13[2],xmm6[2],xmm13[3],xmm6[3]
11250 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11251 ; AVX-NEXT: vpshufb %xmm2, %xmm11, %xmm13
11252 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11253 ; AVX-NEXT: vpshufb %xmm2, %xmm11, %xmm14
11254 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
11255 ; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm13[0,1,2,3,4,5],xmm6[6,7]
11256 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11257 ; AVX-NEXT: vpshufb %xmm1, %xmm11, %xmm13
11258 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11259 ; AVX-NEXT: vpshufb %xmm1, %xmm11, %xmm14
11260 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
11261 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11262 ; AVX-NEXT: vpshufb %xmm0, %xmm11, %xmm14
11263 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11264 ; AVX-NEXT: vpshufb %xmm0, %xmm11, %xmm15
11265 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
11266 ; AVX-NEXT: vpblendw {{.*#+}} xmm13 = xmm14[0,1],xmm13[2,3],xmm14[4,5,6,7]
11267 ; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm13[0,1,2,3],xmm6[4,5,6,7]
11268 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11269 ; AVX-NEXT: vpshufb %xmm3, %xmm11, %xmm13
11270 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11271 ; AVX-NEXT: vpshufb %xmm3, %xmm11, %xmm3
11272 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm13[0],xmm3[1],xmm13[1],xmm3[2],xmm13[2],xmm3[3],xmm13[3]
11273 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11274 ; AVX-NEXT: vpshufb %xmm2, %xmm11, %xmm12
11275 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11276 ; AVX-NEXT: vpshufb %xmm2, %xmm11, %xmm2
11277 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm12[0],xmm2[1],xmm12[1],xmm2[2],xmm12[2],xmm2[3],xmm12[3]
11278 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5],xmm3[6,7]
11279 ; AVX-NEXT: vpshufb %xmm1, %xmm7, %xmm3
11280 ; AVX-NEXT: vpshufb %xmm1, %xmm8, %xmm1
11281 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
11282 ; AVX-NEXT: vpshufb %xmm0, %xmm9, %xmm3
11283 ; AVX-NEXT: vpshufb %xmm0, %xmm10, %xmm0
11284 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
11285 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
11286 ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm1
11287 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
11288 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
11289 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm6[0,1,2,3],ymm0[4,5,6,7]
11290 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
11291 ; AVX-NEXT: vmovaps %ymm1, (%rsi)
11292 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
11293 ; AVX-NEXT: vmovaps %ymm1, 32(%rsi)
11294 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
11295 ; AVX-NEXT: vmovaps %ymm1, (%rdx)
11296 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
11297 ; AVX-NEXT: vmovaps %ymm1, 32(%rdx)
11298 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
11299 ; AVX-NEXT: vmovaps %ymm1, (%rcx)
11300 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
11301 ; AVX-NEXT: vmovaps %ymm1, 32(%rcx)
11302 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
11303 ; AVX-NEXT: vmovaps %ymm1, (%r8)
11304 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
11305 ; AVX-NEXT: vmovaps %ymm1, 32(%r8)
11306 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
11307 ; AVX-NEXT: vmovaps %ymm1, (%r9)
11308 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
11309 ; AVX-NEXT: vmovaps %ymm1, 32(%r9)
11310 ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax
11311 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
11312 ; AVX-NEXT: vmovaps %ymm1, (%rax)
11313 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
11314 ; AVX-NEXT: vmovaps %ymm1, 32(%rax)
11315 ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax
11316 ; AVX-NEXT: vmovaps %ymm5, (%rax)
11317 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
11318 ; AVX-NEXT: vmovaps %ymm1, 32(%rax)
11319 ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax
11320 ; AVX-NEXT: vmovaps %ymm0, (%rax)
11321 ; AVX-NEXT: vmovaps %ymm4, 32(%rax)
11322 ; AVX-NEXT: addq $808, %rsp # imm = 0x328
11323 ; AVX-NEXT: vzeroupper
11324 ; AVX-NEXT: retq
11325 ;
11326 ; AVX2-LABEL: load_i8_stride8_vf64:
11327 ; AVX2: # %bb.0:
11328 ; AVX2-NEXT: subq $840, %rsp # imm = 0x348
11329 ; AVX2-NEXT: vmovdqa 368(%rdi), %xmm0
11330 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11331 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
11332 ; AVX2-NEXT: vpshufb %xmm2, %xmm0, %xmm0
11333 ; AVX2-NEXT: vmovdqa 352(%rdi), %xmm13
11334 ; AVX2-NEXT: vpshufb %xmm2, %xmm13, %xmm1
11335 ; AVX2-NEXT: vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11336 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
11337 ; AVX2-NEXT: vmovdqa 336(%rdi), %xmm15
11338 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm3 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
11339 ; AVX2-NEXT: vpshufb %xmm3, %xmm15, %xmm1
11340 ; AVX2-NEXT: vmovdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11341 ; AVX2-NEXT: vmovdqa 320(%rdi), %xmm4
11342 ; AVX2-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11343 ; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm4
11344 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
11345 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm1[0,1,2],xmm0[3]
11346 ; AVX2-NEXT: vmovdqa 304(%rdi), %xmm0
11347 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11348 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm9 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
11349 ; AVX2-NEXT: vpshufb %xmm9, %xmm0, %xmm1
11350 ; AVX2-NEXT: vmovdqa 288(%rdi), %xmm11
11351 ; AVX2-NEXT: vpshufb %xmm9, %xmm11, %xmm5
11352 ; AVX2-NEXT: vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11353 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
11354 ; AVX2-NEXT: vmovdqa 272(%rdi), %xmm10
11355 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm1 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
11356 ; AVX2-NEXT: vpshufb %xmm1, %xmm10, %xmm6
11357 ; AVX2-NEXT: vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11358 ; AVX2-NEXT: vmovdqa 256(%rdi), %xmm12
11359 ; AVX2-NEXT: vpshufb %xmm1, %xmm12, %xmm7
11360 ; AVX2-NEXT: vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11361 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
11362 ; AVX2-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
11363 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
11364 ; AVX2-NEXT: vmovdqa 496(%rdi), %xmm0
11365 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11366 ; AVX2-NEXT: vpshufb %xmm2, %xmm0, %xmm5
11367 ; AVX2-NEXT: vmovdqa 480(%rdi), %xmm0
11368 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11369 ; AVX2-NEXT: vpshufb %xmm2, %xmm0, %xmm6
11370 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
11371 ; AVX2-NEXT: vmovdqa 464(%rdi), %xmm0
11372 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11373 ; AVX2-NEXT: vpshufb %xmm3, %xmm0, %xmm6
11374 ; AVX2-NEXT: vmovdqa 448(%rdi), %xmm0
11375 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11376 ; AVX2-NEXT: vpshufb %xmm3, %xmm0, %xmm7
11377 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
11378 ; AVX2-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
11379 ; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
11380 ; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5,6],ymm5[7]
11381 ; AVX2-NEXT: vmovdqa 432(%rdi), %xmm0
11382 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11383 ; AVX2-NEXT: vpshufb %xmm9, %xmm0, %xmm6
11384 ; AVX2-NEXT: vmovdqa 416(%rdi), %xmm0
11385 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11386 ; AVX2-NEXT: vpshufb %xmm9, %xmm0, %xmm7
11387 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
11388 ; AVX2-NEXT: vmovdqa 400(%rdi), %xmm14
11389 ; AVX2-NEXT: vpshufb %xmm1, %xmm14, %xmm7
11390 ; AVX2-NEXT: vmovdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11391 ; AVX2-NEXT: vmovdqa 384(%rdi), %xmm0
11392 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11393 ; AVX2-NEXT: vpshufb %xmm1, %xmm0, %xmm8
11394 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
11395 ; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
11396 ; AVX2-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
11397 ; AVX2-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3,4],ymm6[5],ymm7[6,7]
11398 ; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
11399 ; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
11400 ; AVX2-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11401 ; AVX2-NEXT: vmovdqa 112(%rdi), %xmm8
11402 ; AVX2-NEXT: vpshufb %xmm2, %xmm8, %xmm4
11403 ; AVX2-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11404 ; AVX2-NEXT: vmovdqa 96(%rdi), %xmm0
11405 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11406 ; AVX2-NEXT: vpshufb %xmm2, %xmm0, %xmm5
11407 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
11408 ; AVX2-NEXT: vmovdqa 80(%rdi), %xmm0
11409 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11410 ; AVX2-NEXT: vpshufb %xmm3, %xmm0, %xmm5
11411 ; AVX2-NEXT: vmovdqa 64(%rdi), %xmm0
11412 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11413 ; AVX2-NEXT: vpshufb %xmm3, %xmm0, %xmm6
11414 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
11415 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
11416 ; AVX2-NEXT: vmovdqa 32(%rdi), %xmm0
11417 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11418 ; AVX2-NEXT: vmovdqa 48(%rdi), %xmm5
11419 ; AVX2-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11420 ; AVX2-NEXT: vpshufb %xmm9, %xmm5, %xmm5
11421 ; AVX2-NEXT: vpshufb %xmm9, %xmm0, %xmm6
11422 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
11423 ; AVX2-NEXT: vmovdqa (%rdi), %xmm0
11424 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11425 ; AVX2-NEXT: vmovdqa 16(%rdi), %xmm6
11426 ; AVX2-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11427 ; AVX2-NEXT: vpshufb %xmm1, %xmm6, %xmm6
11428 ; AVX2-NEXT: vpshufb %xmm1, %xmm0, %xmm7
11429 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
11430 ; AVX2-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
11431 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
11432 ; AVX2-NEXT: vmovdqa 240(%rdi), %xmm0
11433 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11434 ; AVX2-NEXT: vpshufb %xmm2, %xmm0, %xmm5
11435 ; AVX2-NEXT: vmovdqa 224(%rdi), %xmm0
11436 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11437 ; AVX2-NEXT: vpshufb %xmm2, %xmm0, %xmm2
11438 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
11439 ; AVX2-NEXT: vmovdqa 208(%rdi), %xmm0
11440 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11441 ; AVX2-NEXT: vpshufb %xmm3, %xmm0, %xmm5
11442 ; AVX2-NEXT: vmovdqa 192(%rdi), %xmm0
11443 ; AVX2-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill
11444 ; AVX2-NEXT: vpshufb %xmm3, %xmm0, %xmm3
11445 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
11446 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
11447 ; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
11448 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
11449 ; AVX2-NEXT: vmovdqa 176(%rdi), %xmm0
11450 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11451 ; AVX2-NEXT: vpshufb %xmm9, %xmm0, %xmm3
11452 ; AVX2-NEXT: vmovdqa 160(%rdi), %xmm0
11453 ; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11454 ; AVX2-NEXT: vpshufb %xmm9, %xmm0, %xmm0
11455 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
11456 ; AVX2-NEXT: vmovdqa 144(%rdi), %xmm3
11457 ; AVX2-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11458 ; AVX2-NEXT: vpshufb %xmm1, %xmm3, %xmm3
11459 ; AVX2-NEXT: vmovdqa 128(%rdi), %xmm5
11460 ; AVX2-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
11461 ; AVX2-NEXT: vpshufb %xmm1, %xmm5, %xmm1
11462 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
11463 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
11464 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
11465 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5],ymm1[6,7]
11466 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
11467 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
11468 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11469 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
11470 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
11471 ; AVX2-NEXT: vpshufb %xmm2, %xmm0, %xmm0
11472 ; AVX2-NEXT: vpshufb %xmm2, %xmm13, %xmm1
11473 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
11474 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm6 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
11475 ; AVX2-NEXT: vpshufb %xmm6, %xmm15, %xmm3
11476 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
11477 ; AVX2-NEXT: vpshufb %xmm6, %xmm13, %xmm4
11478 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
11479 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm3[0,1,2],xmm1[3]
11480 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm3 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
11481 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
11482 ; AVX2-NEXT: vpshufb %xmm3, %xmm0, %xmm1
11483 ; AVX2-NEXT: vpshufb %xmm3, %xmm11, %xmm5
11484 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
11485 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm1 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
11486 ; AVX2-NEXT: vpshufb %xmm1, %xmm10, %xmm0
11487 ; AVX2-NEXT: vpshufb %xmm1, %xmm12, %xmm15
11488 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
11489 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm5[1],xmm0[2,3]
11490 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm4[2,3]
11491 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
11492 ; AVX2-NEXT: vpshufb %xmm2, %xmm9, %xmm4
11493 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
11494 ; AVX2-NEXT: vpshufb %xmm2, %xmm10, %xmm5
11495 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
11496 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11497 ; AVX2-NEXT: vpshufb %xmm6, %xmm11, %xmm5
11498 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
11499 ; AVX2-NEXT: vpshufb %xmm6, %xmm12, %xmm15
11500 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm15[0],xmm5[0],xmm15[1],xmm5[1],xmm15[2],xmm5[2],xmm15[3],xmm5[3]
11501 ; AVX2-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
11502 ; AVX2-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
11503 ; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5,6],ymm4[7]
11504 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11505 ; AVX2-NEXT: vpshufb %xmm3, %xmm5, %xmm5
11506 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
11507 ; AVX2-NEXT: vpshufb %xmm3, %xmm7, %xmm15
11508 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm15[0],xmm5[0],xmm15[1],xmm5[1],xmm15[2],xmm5[2],xmm15[3],xmm5[3]
11509 ; AVX2-NEXT: vpshufb %xmm1, %xmm14, %xmm15
11510 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
11511 ; AVX2-NEXT: vpshufb %xmm1, %xmm7, %xmm14
11512 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
11513 ; AVX2-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
11514 ; AVX2-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
11515 ; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4],ymm5[5],ymm14[6,7]
11516 ; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
11517 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7]
11518 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11519 ; AVX2-NEXT: vpshufb %xmm2, %xmm8, %xmm0
11520 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
11521 ; AVX2-NEXT: vpshufb %xmm2, %xmm8, %xmm4
11522 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
11523 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
11524 ; AVX2-NEXT: vpshufb %xmm6, %xmm7, %xmm4
11525 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11526 ; AVX2-NEXT: vpshufb %xmm6, %xmm5, %xmm5
11527 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
11528 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm4[0,1,2],xmm0[3]
11529 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
11530 ; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm4
11531 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11532 ; AVX2-NEXT: vpshufb %xmm3, %xmm5, %xmm5
11533 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
11534 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11535 ; AVX2-NEXT: vpshufb %xmm1, %xmm5, %xmm5
11536 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
11537 ; AVX2-NEXT: vpshufb %xmm1, %xmm14, %xmm14
11538 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm14[0],xmm5[0],xmm14[1],xmm5[1],xmm14[2],xmm5[2],xmm14[3],xmm5[3]
11539 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0],xmm4[1],xmm5[2,3]
11540 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm4[0,1],xmm0[2,3]
11541 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
11542 ; AVX2-NEXT: vpshufb %xmm2, %xmm4, %xmm4
11543 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11544 ; AVX2-NEXT: vpshufb %xmm2, %xmm5, %xmm2
11545 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
11546 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
11547 ; AVX2-NEXT: vpshufb %xmm6, %xmm4, %xmm4
11548 ; AVX2-NEXT: vmovdqa (%rsp), %xmm5 # 16-byte Reload
11549 ; AVX2-NEXT: vpshufb %xmm6, %xmm5, %xmm5
11550 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
11551 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
11552 ; AVX2-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
11553 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
11554 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
11555 ; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm4
11556 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11557 ; AVX2-NEXT: vpshufb %xmm3, %xmm5, %xmm3
11558 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
11559 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
11560 ; AVX2-NEXT: vpshufb %xmm1, %xmm4, %xmm4
11561 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11562 ; AVX2-NEXT: vpshufb %xmm1, %xmm5, %xmm1
11563 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
11564 ; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
11565 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
11566 ; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5],ymm1[6,7]
11567 ; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
11568 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
11569 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11570 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
11571 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
11572 ; AVX2-NEXT: vpshufb %xmm2, %xmm0, %xmm0
11573 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
11574 ; AVX2-NEXT: vpshufb %xmm2, %xmm1, %xmm1
11575 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
11576 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm3 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
11577 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
11578 ; AVX2-NEXT: vpshufb %xmm3, %xmm1, %xmm1
11579 ; AVX2-NEXT: vpshufb %xmm3, %xmm13, %xmm4
11580 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
11581 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm1[0,1,2],xmm0[3]
11582 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm0 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
11583 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
11584 ; AVX2-NEXT: vpshufb %xmm0, %xmm13, %xmm1
11585 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11586 ; AVX2-NEXT: vpshufb %xmm0, %xmm5, %xmm5
11587 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
11588 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm1 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
11589 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11590 ; AVX2-NEXT: vpshufb %xmm1, %xmm6, %xmm6
11591 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
11592 ; AVX2-NEXT: vpshufb %xmm1, %xmm14, %xmm14
11593 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11594 ; AVX2-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
11595 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
11596 ; AVX2-NEXT: vpshufb %xmm2, %xmm9, %xmm5
11597 ; AVX2-NEXT: vpshufb %xmm2, %xmm10, %xmm6
11598 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
11599 ; AVX2-NEXT: vpshufb %xmm3, %xmm11, %xmm6
11600 ; AVX2-NEXT: vpshufb %xmm3, %xmm12, %xmm14
11601 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11602 ; AVX2-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
11603 ; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
11604 ; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5,6],ymm5[7]
11605 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
11606 ; AVX2-NEXT: vpshufb %xmm0, %xmm9, %xmm6
11607 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
11608 ; AVX2-NEXT: vpshufb %xmm0, %xmm10, %xmm14
11609 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11610 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11611 ; AVX2-NEXT: vpshufb %xmm1, %xmm11, %xmm14
11612 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11613 ; AVX2-NEXT: vpshufb %xmm1, %xmm11, %xmm15
11614 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
11615 ; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
11616 ; AVX2-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
11617 ; AVX2-NEXT: vpblendd {{.*#+}} ymm6 = ymm14[0,1,2,3,4],ymm6[5],ymm14[6,7]
11618 ; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
11619 ; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
11620 ; AVX2-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11621 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
11622 ; AVX2-NEXT: vpshufb %xmm2, %xmm4, %xmm4
11623 ; AVX2-NEXT: vpshufb %xmm2, %xmm8, %xmm5
11624 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
11625 ; AVX2-NEXT: vpshufb %xmm3, %xmm7, %xmm5
11626 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
11627 ; AVX2-NEXT: vpshufb %xmm3, %xmm8, %xmm6
11628 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
11629 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
11630 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11631 ; AVX2-NEXT: vpshufb %xmm0, %xmm5, %xmm5
11632 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
11633 ; AVX2-NEXT: vpshufb %xmm0, %xmm7, %xmm6
11634 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
11635 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11636 ; AVX2-NEXT: vpshufb %xmm1, %xmm6, %xmm6
11637 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11638 ; AVX2-NEXT: vpshufb %xmm1, %xmm11, %xmm14
11639 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11640 ; AVX2-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
11641 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
11642 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11643 ; AVX2-NEXT: vpshufb %xmm2, %xmm5, %xmm5
11644 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11645 ; AVX2-NEXT: vpshufb %xmm2, %xmm6, %xmm2
11646 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
11647 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11648 ; AVX2-NEXT: vpshufb %xmm3, %xmm5, %xmm5
11649 ; AVX2-NEXT: vmovdqa (%rsp), %xmm6 # 16-byte Reload
11650 ; AVX2-NEXT: vpshufb %xmm3, %xmm6, %xmm3
11651 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
11652 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
11653 ; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
11654 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
11655 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
11656 ; AVX2-NEXT: vpshufb %xmm0, %xmm3, %xmm3
11657 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11658 ; AVX2-NEXT: vpshufb %xmm0, %xmm5, %xmm0
11659 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
11660 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
11661 ; AVX2-NEXT: vpshufb %xmm1, %xmm3, %xmm3
11662 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11663 ; AVX2-NEXT: vpshufb %xmm1, %xmm5, %xmm1
11664 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
11665 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
11666 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
11667 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5],ymm1[6,7]
11668 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
11669 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
11670 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11671 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
11672 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11673 ; AVX2-NEXT: vpshufb %xmm2, %xmm11, %xmm0
11674 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
11675 ; AVX2-NEXT: vpshufb %xmm2, %xmm1, %xmm1
11676 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
11677 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm3 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
11678 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
11679 ; AVX2-NEXT: vpshufb %xmm3, %xmm1, %xmm1
11680 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
11681 ; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm4
11682 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
11683 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm1[0,1,2],xmm0[3]
11684 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm0 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
11685 ; AVX2-NEXT: vpshufb %xmm0, %xmm13, %xmm1
11686 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
11687 ; AVX2-NEXT: vpshufb %xmm0, %xmm12, %xmm5
11688 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
11689 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm1 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
11690 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
11691 ; AVX2-NEXT: vpshufb %xmm1, %xmm13, %xmm6
11692 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
11693 ; AVX2-NEXT: vpshufb %xmm1, %xmm14, %xmm14
11694 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11695 ; AVX2-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
11696 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
11697 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11698 ; AVX2-NEXT: vpshufb %xmm2, %xmm5, %xmm5
11699 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11700 ; AVX2-NEXT: vpshufb %xmm2, %xmm6, %xmm6
11701 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
11702 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11703 ; AVX2-NEXT: vpshufb %xmm3, %xmm6, %xmm6
11704 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
11705 ; AVX2-NEXT: vpshufb %xmm3, %xmm14, %xmm14
11706 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11707 ; AVX2-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
11708 ; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
11709 ; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5,6],ymm5[7]
11710 ; AVX2-NEXT: vpshufb %xmm0, %xmm9, %xmm6
11711 ; AVX2-NEXT: vpshufb %xmm0, %xmm10, %xmm14
11712 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11713 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
11714 ; AVX2-NEXT: vpshufb %xmm1, %xmm9, %xmm14
11715 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
11716 ; AVX2-NEXT: vpshufb %xmm1, %xmm10, %xmm15
11717 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
11718 ; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
11719 ; AVX2-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
11720 ; AVX2-NEXT: vpblendd {{.*#+}} ymm6 = ymm14[0,1,2,3,4],ymm6[5],ymm14[6,7]
11721 ; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
11722 ; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
11723 ; AVX2-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11724 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
11725 ; AVX2-NEXT: vpshufb %xmm2, %xmm4, %xmm4
11726 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11727 ; AVX2-NEXT: vpshufb %xmm2, %xmm5, %xmm5
11728 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
11729 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11730 ; AVX2-NEXT: vpshufb %xmm3, %xmm5, %xmm5
11731 ; AVX2-NEXT: vpshufb %xmm3, %xmm8, %xmm6
11732 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
11733 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
11734 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
11735 ; AVX2-NEXT: vpshufb %xmm0, %xmm8, %xmm5
11736 ; AVX2-NEXT: vpshufb %xmm0, %xmm7, %xmm6
11737 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
11738 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11739 ; AVX2-NEXT: vpshufb %xmm1, %xmm6, %xmm6
11740 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
11741 ; AVX2-NEXT: vpshufb %xmm1, %xmm7, %xmm14
11742 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11743 ; AVX2-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
11744 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
11745 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11746 ; AVX2-NEXT: vpshufb %xmm2, %xmm5, %xmm5
11747 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11748 ; AVX2-NEXT: vpshufb %xmm2, %xmm6, %xmm2
11749 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
11750 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11751 ; AVX2-NEXT: vpshufb %xmm3, %xmm5, %xmm5
11752 ; AVX2-NEXT: vmovdqa (%rsp), %xmm6 # 16-byte Reload
11753 ; AVX2-NEXT: vpshufb %xmm3, %xmm6, %xmm3
11754 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
11755 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
11756 ; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
11757 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
11758 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
11759 ; AVX2-NEXT: vpshufb %xmm0, %xmm3, %xmm3
11760 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11761 ; AVX2-NEXT: vpshufb %xmm0, %xmm5, %xmm0
11762 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
11763 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
11764 ; AVX2-NEXT: vpshufb %xmm1, %xmm3, %xmm3
11765 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11766 ; AVX2-NEXT: vpshufb %xmm1, %xmm5, %xmm1
11767 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
11768 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
11769 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
11770 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5],ymm1[6,7]
11771 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
11772 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
11773 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11774 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
11775 ; AVX2-NEXT: vpshufb %xmm2, %xmm11, %xmm0
11776 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
11777 ; AVX2-NEXT: vpshufb %xmm2, %xmm1, %xmm1
11778 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
11779 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm3 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
11780 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
11781 ; AVX2-NEXT: vpshufb %xmm3, %xmm1, %xmm1
11782 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
11783 ; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm4
11784 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
11785 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm1[0,1,2],xmm0[3]
11786 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm0 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
11787 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
11788 ; AVX2-NEXT: vpshufb %xmm0, %xmm1, %xmm1
11789 ; AVX2-NEXT: vpshufb %xmm0, %xmm12, %xmm5
11790 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
11791 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm1 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
11792 ; AVX2-NEXT: vpshufb %xmm1, %xmm13, %xmm6
11793 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
11794 ; AVX2-NEXT: vpshufb %xmm1, %xmm7, %xmm14
11795 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11796 ; AVX2-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
11797 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
11798 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11799 ; AVX2-NEXT: vpshufb %xmm2, %xmm5, %xmm5
11800 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11801 ; AVX2-NEXT: vpshufb %xmm2, %xmm6, %xmm6
11802 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
11803 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11804 ; AVX2-NEXT: vpshufb %xmm3, %xmm6, %xmm6
11805 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
11806 ; AVX2-NEXT: vpshufb %xmm3, %xmm7, %xmm14
11807 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11808 ; AVX2-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
11809 ; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
11810 ; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5,6],ymm5[7]
11811 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11812 ; AVX2-NEXT: vpshufb %xmm0, %xmm6, %xmm6
11813 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
11814 ; AVX2-NEXT: vpshufb %xmm0, %xmm7, %xmm14
11815 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11816 ; AVX2-NEXT: vpshufb %xmm1, %xmm9, %xmm14
11817 ; AVX2-NEXT: vpshufb %xmm1, %xmm10, %xmm15
11818 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
11819 ; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
11820 ; AVX2-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
11821 ; AVX2-NEXT: vpblendd {{.*#+}} ymm6 = ymm14[0,1,2,3,4],ymm6[5],ymm14[6,7]
11822 ; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
11823 ; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
11824 ; AVX2-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11825 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
11826 ; AVX2-NEXT: vpshufb %xmm2, %xmm7, %xmm4
11827 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11828 ; AVX2-NEXT: vpshufb %xmm2, %xmm5, %xmm5
11829 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
11830 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11831 ; AVX2-NEXT: vpshufb %xmm3, %xmm5, %xmm5
11832 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11833 ; AVX2-NEXT: vpshufb %xmm3, %xmm6, %xmm6
11834 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
11835 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
11836 ; AVX2-NEXT: vpshufb %xmm0, %xmm8, %xmm5
11837 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11838 ; AVX2-NEXT: vpshufb %xmm0, %xmm6, %xmm6
11839 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
11840 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
11841 ; AVX2-NEXT: vpshufb %xmm1, %xmm9, %xmm6
11842 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
11843 ; AVX2-NEXT: vpshufb %xmm1, %xmm8, %xmm14
11844 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11845 ; AVX2-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
11846 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
11847 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
11848 ; AVX2-NEXT: vpshufb %xmm2, %xmm10, %xmm5
11849 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11850 ; AVX2-NEXT: vpshufb %xmm2, %xmm11, %xmm2
11851 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
11852 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
11853 ; AVX2-NEXT: vpshufb %xmm3, %xmm12, %xmm5
11854 ; AVX2-NEXT: vmovdqa (%rsp), %xmm13 # 16-byte Reload
11855 ; AVX2-NEXT: vpshufb %xmm3, %xmm13, %xmm3
11856 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
11857 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
11858 ; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
11859 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
11860 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
11861 ; AVX2-NEXT: vpshufb %xmm0, %xmm3, %xmm3
11862 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11863 ; AVX2-NEXT: vpshufb %xmm0, %xmm5, %xmm0
11864 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
11865 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
11866 ; AVX2-NEXT: vpshufb %xmm1, %xmm3, %xmm3
11867 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11868 ; AVX2-NEXT: vpshufb %xmm1, %xmm5, %xmm1
11869 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
11870 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
11871 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
11872 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5],ymm1[6,7]
11873 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
11874 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
11875 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11876 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
11877 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
11878 ; AVX2-NEXT: vpshufb %xmm2, %xmm0, %xmm0
11879 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
11880 ; AVX2-NEXT: vpshufb %xmm2, %xmm1, %xmm1
11881 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
11882 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm3 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
11883 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
11884 ; AVX2-NEXT: vpshufb %xmm3, %xmm1, %xmm1
11885 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
11886 ; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm4
11887 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
11888 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm1[0,1,2],xmm0[3]
11889 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm0 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
11890 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
11891 ; AVX2-NEXT: vpshufb %xmm0, %xmm1, %xmm1
11892 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11893 ; AVX2-NEXT: vpshufb %xmm0, %xmm5, %xmm5
11894 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
11895 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm1 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
11896 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11897 ; AVX2-NEXT: vpshufb %xmm1, %xmm6, %xmm6
11898 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
11899 ; AVX2-NEXT: vpshufb %xmm1, %xmm14, %xmm14
11900 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11901 ; AVX2-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
11902 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
11903 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11904 ; AVX2-NEXT: vpshufb %xmm2, %xmm5, %xmm5
11905 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11906 ; AVX2-NEXT: vpshufb %xmm2, %xmm6, %xmm6
11907 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
11908 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11909 ; AVX2-NEXT: vpshufb %xmm3, %xmm6, %xmm6
11910 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
11911 ; AVX2-NEXT: vpshufb %xmm3, %xmm14, %xmm14
11912 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11913 ; AVX2-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
11914 ; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
11915 ; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5,6],ymm5[7]
11916 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11917 ; AVX2-NEXT: vpshufb %xmm0, %xmm6, %xmm6
11918 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
11919 ; AVX2-NEXT: vpshufb %xmm0, %xmm14, %xmm14
11920 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11921 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
11922 ; AVX2-NEXT: vpshufb %xmm1, %xmm14, %xmm14
11923 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
11924 ; AVX2-NEXT: vpshufb %xmm1, %xmm15, %xmm15
11925 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
11926 ; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
11927 ; AVX2-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
11928 ; AVX2-NEXT: vpblendd {{.*#+}} ymm6 = ymm14[0,1,2,3,4],ymm6[5],ymm14[6,7]
11929 ; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
11930 ; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
11931 ; AVX2-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11932 ; AVX2-NEXT: vpshufb %xmm2, %xmm7, %xmm4
11933 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11934 ; AVX2-NEXT: vpshufb %xmm2, %xmm5, %xmm5
11935 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
11936 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11937 ; AVX2-NEXT: vpshufb %xmm3, %xmm5, %xmm5
11938 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11939 ; AVX2-NEXT: vpshufb %xmm3, %xmm6, %xmm6
11940 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
11941 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
11942 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11943 ; AVX2-NEXT: vpshufb %xmm0, %xmm5, %xmm5
11944 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11945 ; AVX2-NEXT: vpshufb %xmm0, %xmm6, %xmm6
11946 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
11947 ; AVX2-NEXT: vpshufb %xmm1, %xmm9, %xmm6
11948 ; AVX2-NEXT: vpshufb %xmm1, %xmm8, %xmm14
11949 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
11950 ; AVX2-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
11951 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
11952 ; AVX2-NEXT: vpshufb %xmm2, %xmm10, %xmm5
11953 ; AVX2-NEXT: vpshufb %xmm2, %xmm11, %xmm2
11954 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
11955 ; AVX2-NEXT: vpshufb %xmm3, %xmm12, %xmm5
11956 ; AVX2-NEXT: vpshufb %xmm3, %xmm13, %xmm3
11957 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
11958 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
11959 ; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
11960 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
11961 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
11962 ; AVX2-NEXT: vpshufb %xmm0, %xmm13, %xmm3
11963 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
11964 ; AVX2-NEXT: vpshufb %xmm0, %xmm7, %xmm0
11965 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
11966 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
11967 ; AVX2-NEXT: vpshufb %xmm1, %xmm8, %xmm3
11968 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
11969 ; AVX2-NEXT: vpshufb %xmm1, %xmm9, %xmm1
11970 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
11971 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
11972 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
11973 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5],ymm1[6,7]
11974 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
11975 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
11976 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11977 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
11978 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
11979 ; AVX2-NEXT: vpshufb %xmm2, %xmm0, %xmm0
11980 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
11981 ; AVX2-NEXT: vpshufb %xmm2, %xmm10, %xmm1
11982 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
11983 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm3 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
11984 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
11985 ; AVX2-NEXT: vpshufb %xmm3, %xmm11, %xmm1
11986 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
11987 ; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm4
11988 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
11989 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm1[0,1,2],xmm0[3]
11990 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm0 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
11991 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
11992 ; AVX2-NEXT: vpshufb %xmm0, %xmm1, %xmm1
11993 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
11994 ; AVX2-NEXT: vpshufb %xmm0, %xmm5, %xmm5
11995 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
11996 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm1 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
11997 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
11998 ; AVX2-NEXT: vpshufb %xmm1, %xmm6, %xmm6
11999 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
12000 ; AVX2-NEXT: vpshufb %xmm1, %xmm12, %xmm14
12001 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12002 ; AVX2-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
12003 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
12004 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12005 ; AVX2-NEXT: vpshufb %xmm2, %xmm5, %xmm5
12006 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12007 ; AVX2-NEXT: vpshufb %xmm2, %xmm6, %xmm6
12008 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
12009 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12010 ; AVX2-NEXT: vpshufb %xmm3, %xmm6, %xmm6
12011 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
12012 ; AVX2-NEXT: vpshufb %xmm3, %xmm14, %xmm14
12013 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12014 ; AVX2-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
12015 ; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
12016 ; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5,6],ymm5[7]
12017 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12018 ; AVX2-NEXT: vpshufb %xmm0, %xmm6, %xmm6
12019 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
12020 ; AVX2-NEXT: vpshufb %xmm0, %xmm14, %xmm14
12021 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12022 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
12023 ; AVX2-NEXT: vpshufb %xmm1, %xmm14, %xmm14
12024 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
12025 ; AVX2-NEXT: vpshufb %xmm1, %xmm15, %xmm15
12026 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
12027 ; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
12028 ; AVX2-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
12029 ; AVX2-NEXT: vpblendd {{.*#+}} ymm6 = ymm14[0,1,2,3,4],ymm6[5],ymm14[6,7]
12030 ; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
12031 ; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
12032 ; AVX2-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
12033 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
12034 ; AVX2-NEXT: vpshufb %xmm2, %xmm4, %xmm4
12035 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12036 ; AVX2-NEXT: vpshufb %xmm2, %xmm5, %xmm5
12037 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
12038 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12039 ; AVX2-NEXT: vpshufb %xmm3, %xmm5, %xmm5
12040 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12041 ; AVX2-NEXT: vpshufb %xmm3, %xmm6, %xmm6
12042 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
12043 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
12044 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12045 ; AVX2-NEXT: vpshufb %xmm0, %xmm5, %xmm5
12046 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12047 ; AVX2-NEXT: vpshufb %xmm0, %xmm6, %xmm6
12048 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
12049 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12050 ; AVX2-NEXT: vpshufb %xmm1, %xmm6, %xmm6
12051 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
12052 ; AVX2-NEXT: vpshufb %xmm1, %xmm14, %xmm14
12053 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12054 ; AVX2-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
12055 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
12056 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12057 ; AVX2-NEXT: vpshufb %xmm2, %xmm5, %xmm5
12058 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12059 ; AVX2-NEXT: vpshufb %xmm2, %xmm6, %xmm2
12060 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
12061 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12062 ; AVX2-NEXT: vpshufb %xmm3, %xmm5, %xmm5
12063 ; AVX2-NEXT: vmovdqa (%rsp), %xmm6 # 16-byte Reload
12064 ; AVX2-NEXT: vpshufb %xmm3, %xmm6, %xmm3
12065 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
12066 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
12067 ; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
12068 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
12069 ; AVX2-NEXT: vpshufb %xmm0, %xmm13, %xmm3
12070 ; AVX2-NEXT: vmovdqa %xmm13, %xmm5
12071 ; AVX2-NEXT: vpshufb %xmm0, %xmm7, %xmm0
12072 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
12073 ; AVX2-NEXT: vpshufb %xmm1, %xmm8, %xmm3
12074 ; AVX2-NEXT: vpshufb %xmm1, %xmm9, %xmm1
12075 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
12076 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
12077 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
12078 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5],ymm1[6,7]
12079 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
12080 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
12081 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
12082 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm3 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
12083 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
12084 ; AVX2-NEXT: vpshufb %xmm3, %xmm0, %xmm0
12085 ; AVX2-NEXT: vpshufb %xmm3, %xmm10, %xmm1
12086 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
12087 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm0 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
12088 ; AVX2-NEXT: vpshufb %xmm0, %xmm11, %xmm2
12089 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
12090 ; AVX2-NEXT: vpshufb %xmm0, %xmm4, %xmm4
12091 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
12092 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm2[0,1,2],xmm1[3]
12093 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm1 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
12094 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
12095 ; AVX2-NEXT: vpshufb %xmm1, %xmm2, %xmm2
12096 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12097 ; AVX2-NEXT: vpshufb %xmm1, %xmm6, %xmm6
12098 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
12099 ; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
12100 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12101 ; AVX2-NEXT: vpshufb %xmm2, %xmm10, %xmm14
12102 ; AVX2-NEXT: vpshufb %xmm2, %xmm12, %xmm15
12103 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
12104 ; AVX2-NEXT: vpblendd {{.*#+}} xmm6 = xmm14[0],xmm6[1],xmm14[2,3]
12105 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3]
12106 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12107 ; AVX2-NEXT: vpshufb %xmm3, %xmm6, %xmm6
12108 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12109 ; AVX2-NEXT: vpshufb %xmm3, %xmm10, %xmm14
12110 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12111 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12112 ; AVX2-NEXT: vpshufb %xmm0, %xmm10, %xmm14
12113 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12114 ; AVX2-NEXT: vpshufb %xmm0, %xmm10, %xmm15
12115 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
12116 ; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
12117 ; AVX2-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
12118 ; AVX2-NEXT: vpblendd {{.*#+}} ymm6 = ymm14[0,1,2,3,4,5,6],ymm6[7]
12119 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12120 ; AVX2-NEXT: vpshufb %xmm1, %xmm10, %xmm14
12121 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12122 ; AVX2-NEXT: vpshufb %xmm1, %xmm10, %xmm15
12123 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
12124 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12125 ; AVX2-NEXT: vpshufb %xmm2, %xmm10, %xmm15
12126 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12127 ; AVX2-NEXT: vpshufb %xmm2, %xmm10, %xmm13
12128 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm15[0],xmm13[1],xmm15[1],xmm13[2],xmm15[2],xmm13[3],xmm15[3]
12129 ; AVX2-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
12130 ; AVX2-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
12131 ; AVX2-NEXT: vpblendd {{.*#+}} ymm13 = ymm13[0,1,2,3,4],ymm14[5],ymm13[6,7]
12132 ; AVX2-NEXT: vpblendd {{.*#+}} ymm6 = ymm13[0,1,2,3,4,5],ymm6[6,7]
12133 ; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm6[4,5,6,7]
12134 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12135 ; AVX2-NEXT: vpshufb %xmm3, %xmm6, %xmm6
12136 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12137 ; AVX2-NEXT: vpshufb %xmm3, %xmm10, %xmm13
12138 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm13[0],xmm6[0],xmm13[1],xmm6[1],xmm13[2],xmm6[2],xmm13[3],xmm6[3]
12139 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12140 ; AVX2-NEXT: vpshufb %xmm0, %xmm10, %xmm13
12141 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12142 ; AVX2-NEXT: vpshufb %xmm0, %xmm10, %xmm14
12143 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
12144 ; AVX2-NEXT: vpblendd {{.*#+}} xmm6 = xmm13[0,1,2],xmm6[3]
12145 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12146 ; AVX2-NEXT: vpshufb %xmm1, %xmm10, %xmm13
12147 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12148 ; AVX2-NEXT: vpshufb %xmm1, %xmm10, %xmm14
12149 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
12150 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12151 ; AVX2-NEXT: vpshufb %xmm2, %xmm10, %xmm14
12152 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12153 ; AVX2-NEXT: vpshufb %xmm2, %xmm10, %xmm15
12154 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
12155 ; AVX2-NEXT: vpblendd {{.*#+}} xmm13 = xmm14[0],xmm13[1],xmm14[2,3]
12156 ; AVX2-NEXT: vpblendd {{.*#+}} xmm6 = xmm13[0,1],xmm6[2,3]
12157 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12158 ; AVX2-NEXT: vpshufb %xmm3, %xmm10, %xmm13
12159 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12160 ; AVX2-NEXT: vpshufb %xmm3, %xmm10, %xmm3
12161 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm13[0],xmm3[1],xmm13[1],xmm3[2],xmm13[2],xmm3[3],xmm13[3]
12162 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12163 ; AVX2-NEXT: vpshufb %xmm0, %xmm10, %xmm12
12164 ; AVX2-NEXT: vmovdqa (%rsp), %xmm10 # 16-byte Reload
12165 ; AVX2-NEXT: vpshufb %xmm0, %xmm10, %xmm0
12166 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm12[0],xmm0[1],xmm12[1],xmm0[2],xmm12[2],xmm0[3],xmm12[3]
12167 ; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
12168 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
12169 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm3[7]
12170 ; AVX2-NEXT: vpshufb %xmm1, %xmm5, %xmm3
12171 ; AVX2-NEXT: vpshufb %xmm1, %xmm7, %xmm1
12172 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
12173 ; AVX2-NEXT: vpshufb %xmm2, %xmm8, %xmm3
12174 ; AVX2-NEXT: vpshufb %xmm2, %xmm9, %xmm2
12175 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
12176 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
12177 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
12178 ; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4],ymm1[5],ymm2[6,7]
12179 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
12180 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1,2,3],ymm0[4,5,6,7]
12181 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
12182 ; AVX2-NEXT: vmovaps %ymm1, (%rsi)
12183 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
12184 ; AVX2-NEXT: vmovaps %ymm1, 32(%rsi)
12185 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
12186 ; AVX2-NEXT: vmovaps %ymm1, (%rdx)
12187 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
12188 ; AVX2-NEXT: vmovaps %ymm1, 32(%rdx)
12189 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
12190 ; AVX2-NEXT: vmovaps %ymm1, (%rcx)
12191 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
12192 ; AVX2-NEXT: vmovaps %ymm1, 32(%rcx)
12193 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
12194 ; AVX2-NEXT: vmovaps %ymm1, (%r8)
12195 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
12196 ; AVX2-NEXT: vmovaps %ymm1, 32(%r8)
12197 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
12198 ; AVX2-NEXT: vmovaps %ymm1, (%r9)
12199 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
12200 ; AVX2-NEXT: vmovaps %ymm1, 32(%r9)
12201 ; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
12202 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
12203 ; AVX2-NEXT: vmovaps %ymm1, (%rax)
12204 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
12205 ; AVX2-NEXT: vmovaps %ymm1, 32(%rax)
12206 ; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
12207 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
12208 ; AVX2-NEXT: vmovaps %ymm1, (%rax)
12209 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
12210 ; AVX2-NEXT: vmovaps %ymm1, 32(%rax)
12211 ; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
12212 ; AVX2-NEXT: vmovdqa %ymm0, (%rax)
12213 ; AVX2-NEXT: vmovdqa %ymm4, 32(%rax)
12214 ; AVX2-NEXT: addq $840, %rsp # imm = 0x348
12215 ; AVX2-NEXT: vzeroupper
12216 ; AVX2-NEXT: retq
12217 ;
12218 ; AVX2-FP-LABEL: load_i8_stride8_vf64:
12219 ; AVX2-FP: # %bb.0:
12220 ; AVX2-FP-NEXT: subq $840, %rsp # imm = 0x348
12221 ; AVX2-FP-NEXT: vmovdqa 368(%rdi), %xmm0
12222 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12223 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
12224 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm0, %xmm0
12225 ; AVX2-FP-NEXT: vmovdqa 352(%rdi), %xmm13
12226 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm13, %xmm1
12227 ; AVX2-FP-NEXT: vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12228 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
12229 ; AVX2-FP-NEXT: vmovdqa 336(%rdi), %xmm15
12230 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
12231 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm15, %xmm1
12232 ; AVX2-FP-NEXT: vmovdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12233 ; AVX2-FP-NEXT: vmovdqa 320(%rdi), %xmm4
12234 ; AVX2-FP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12235 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm4, %xmm4
12236 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
12237 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm1[0,1,2],xmm0[3]
12238 ; AVX2-FP-NEXT: vmovdqa 304(%rdi), %xmm0
12239 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12240 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm9 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
12241 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm0, %xmm1
12242 ; AVX2-FP-NEXT: vmovdqa 288(%rdi), %xmm11
12243 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm11, %xmm5
12244 ; AVX2-FP-NEXT: vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12245 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
12246 ; AVX2-FP-NEXT: vmovdqa 272(%rdi), %xmm10
12247 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
12248 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm10, %xmm6
12249 ; AVX2-FP-NEXT: vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12250 ; AVX2-FP-NEXT: vmovdqa 256(%rdi), %xmm12
12251 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm12, %xmm7
12252 ; AVX2-FP-NEXT: vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12253 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
12254 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
12255 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
12256 ; AVX2-FP-NEXT: vmovdqa 496(%rdi), %xmm0
12257 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12258 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm0, %xmm5
12259 ; AVX2-FP-NEXT: vmovdqa 480(%rdi), %xmm0
12260 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12261 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm0, %xmm6
12262 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
12263 ; AVX2-FP-NEXT: vmovdqa 464(%rdi), %xmm0
12264 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12265 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm0, %xmm6
12266 ; AVX2-FP-NEXT: vmovdqa 448(%rdi), %xmm0
12267 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12268 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm0, %xmm7
12269 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
12270 ; AVX2-FP-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
12271 ; AVX2-FP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
12272 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5,6],ymm5[7]
12273 ; AVX2-FP-NEXT: vmovdqa 432(%rdi), %xmm0
12274 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12275 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm0, %xmm6
12276 ; AVX2-FP-NEXT: vmovdqa 416(%rdi), %xmm0
12277 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12278 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm0, %xmm7
12279 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
12280 ; AVX2-FP-NEXT: vmovdqa 400(%rdi), %xmm14
12281 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm14, %xmm7
12282 ; AVX2-FP-NEXT: vmovdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12283 ; AVX2-FP-NEXT: vmovdqa 384(%rdi), %xmm0
12284 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12285 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm0, %xmm8
12286 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
12287 ; AVX2-FP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
12288 ; AVX2-FP-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
12289 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3,4],ymm6[5],ymm7[6,7]
12290 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
12291 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
12292 ; AVX2-FP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
12293 ; AVX2-FP-NEXT: vmovdqa 112(%rdi), %xmm8
12294 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm8, %xmm4
12295 ; AVX2-FP-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12296 ; AVX2-FP-NEXT: vmovdqa 96(%rdi), %xmm0
12297 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12298 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm0, %xmm5
12299 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
12300 ; AVX2-FP-NEXT: vmovdqa 80(%rdi), %xmm0
12301 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12302 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm0, %xmm5
12303 ; AVX2-FP-NEXT: vmovdqa 64(%rdi), %xmm0
12304 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12305 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm0, %xmm6
12306 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
12307 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
12308 ; AVX2-FP-NEXT: vmovdqa 32(%rdi), %xmm0
12309 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12310 ; AVX2-FP-NEXT: vmovdqa 48(%rdi), %xmm5
12311 ; AVX2-FP-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12312 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm5, %xmm5
12313 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm0, %xmm6
12314 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
12315 ; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm0
12316 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12317 ; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm6
12318 ; AVX2-FP-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12319 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm6, %xmm6
12320 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm0, %xmm7
12321 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
12322 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
12323 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
12324 ; AVX2-FP-NEXT: vmovdqa 240(%rdi), %xmm0
12325 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12326 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm0, %xmm5
12327 ; AVX2-FP-NEXT: vmovdqa 224(%rdi), %xmm0
12328 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12329 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm0, %xmm2
12330 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
12331 ; AVX2-FP-NEXT: vmovdqa 208(%rdi), %xmm0
12332 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12333 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm0, %xmm5
12334 ; AVX2-FP-NEXT: vmovdqa 192(%rdi), %xmm0
12335 ; AVX2-FP-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill
12336 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm0, %xmm3
12337 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
12338 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
12339 ; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
12340 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
12341 ; AVX2-FP-NEXT: vmovdqa 176(%rdi), %xmm0
12342 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12343 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm0, %xmm3
12344 ; AVX2-FP-NEXT: vmovdqa 160(%rdi), %xmm0
12345 ; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12346 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm0, %xmm0
12347 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
12348 ; AVX2-FP-NEXT: vmovdqa 144(%rdi), %xmm3
12349 ; AVX2-FP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12350 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm3, %xmm3
12351 ; AVX2-FP-NEXT: vmovdqa 128(%rdi), %xmm5
12352 ; AVX2-FP-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
12353 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm5, %xmm1
12354 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
12355 ; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
12356 ; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
12357 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5],ymm1[6,7]
12358 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
12359 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
12360 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
12361 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
12362 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
12363 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm0, %xmm0
12364 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm13, %xmm1
12365 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
12366 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm6 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
12367 ; AVX2-FP-NEXT: vpshufb %xmm6, %xmm15, %xmm3
12368 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
12369 ; AVX2-FP-NEXT: vpshufb %xmm6, %xmm13, %xmm4
12370 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
12371 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm3[0,1,2],xmm1[3]
12372 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
12373 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
12374 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm0, %xmm1
12375 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm11, %xmm5
12376 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
12377 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
12378 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm10, %xmm0
12379 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm12, %xmm15
12380 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
12381 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm5[1],xmm0[2,3]
12382 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm4[2,3]
12383 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
12384 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm9, %xmm4
12385 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12386 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm10, %xmm5
12387 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
12388 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
12389 ; AVX2-FP-NEXT: vpshufb %xmm6, %xmm11, %xmm5
12390 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
12391 ; AVX2-FP-NEXT: vpshufb %xmm6, %xmm12, %xmm15
12392 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm15[0],xmm5[0],xmm15[1],xmm5[1],xmm15[2],xmm5[2],xmm15[3],xmm5[3]
12393 ; AVX2-FP-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
12394 ; AVX2-FP-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
12395 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5,6],ymm4[7]
12396 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12397 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm5, %xmm5
12398 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
12399 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm7, %xmm15
12400 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm15[0],xmm5[0],xmm15[1],xmm5[1],xmm15[2],xmm5[2],xmm15[3],xmm5[3]
12401 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm14, %xmm15
12402 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
12403 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm7, %xmm14
12404 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
12405 ; AVX2-FP-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
12406 ; AVX2-FP-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
12407 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4],ymm5[5],ymm14[6,7]
12408 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
12409 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7]
12410 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
12411 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm8, %xmm0
12412 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
12413 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm8, %xmm4
12414 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
12415 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
12416 ; AVX2-FP-NEXT: vpshufb %xmm6, %xmm7, %xmm4
12417 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12418 ; AVX2-FP-NEXT: vpshufb %xmm6, %xmm5, %xmm5
12419 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
12420 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm4[0,1,2],xmm0[3]
12421 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
12422 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm4, %xmm4
12423 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12424 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm5, %xmm5
12425 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
12426 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12427 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm5, %xmm5
12428 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
12429 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm14, %xmm14
12430 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm14[0],xmm5[0],xmm14[1],xmm5[1],xmm14[2],xmm5[2],xmm14[3],xmm5[3]
12431 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0],xmm4[1],xmm5[2,3]
12432 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm4[0,1],xmm0[2,3]
12433 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
12434 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm4, %xmm4
12435 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12436 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm5, %xmm2
12437 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
12438 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
12439 ; AVX2-FP-NEXT: vpshufb %xmm6, %xmm4, %xmm4
12440 ; AVX2-FP-NEXT: vmovdqa (%rsp), %xmm5 # 16-byte Reload
12441 ; AVX2-FP-NEXT: vpshufb %xmm6, %xmm5, %xmm5
12442 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
12443 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
12444 ; AVX2-FP-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
12445 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
12446 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
12447 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm4, %xmm4
12448 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12449 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm5, %xmm3
12450 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
12451 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
12452 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm4, %xmm4
12453 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12454 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm5, %xmm1
12455 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
12456 ; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
12457 ; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
12458 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5],ymm1[6,7]
12459 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
12460 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
12461 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
12462 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
12463 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
12464 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm0, %xmm0
12465 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
12466 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm1, %xmm1
12467 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
12468 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
12469 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
12470 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
12471 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm13, %xmm4
12472 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
12473 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm1[0,1,2],xmm0[3]
12474 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
12475 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
12476 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm13, %xmm1
12477 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12478 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm5, %xmm5
12479 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
12480 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
12481 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12482 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm6, %xmm6
12483 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
12484 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm14, %xmm14
12485 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12486 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
12487 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
12488 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm9, %xmm5
12489 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm10, %xmm6
12490 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
12491 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm11, %xmm6
12492 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm12, %xmm14
12493 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12494 ; AVX2-FP-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
12495 ; AVX2-FP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
12496 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5,6],ymm5[7]
12497 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
12498 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm9, %xmm6
12499 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12500 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm10, %xmm14
12501 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12502 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
12503 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm11, %xmm14
12504 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
12505 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm11, %xmm15
12506 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
12507 ; AVX2-FP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
12508 ; AVX2-FP-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
12509 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm6 = ymm14[0,1,2,3,4],ymm6[5],ymm14[6,7]
12510 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
12511 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
12512 ; AVX2-FP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
12513 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
12514 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm4, %xmm4
12515 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm8, %xmm5
12516 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
12517 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm7, %xmm5
12518 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
12519 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm8, %xmm6
12520 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
12521 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
12522 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12523 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm5, %xmm5
12524 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
12525 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm7, %xmm6
12526 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
12527 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12528 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm6, %xmm6
12529 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
12530 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm11, %xmm14
12531 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12532 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
12533 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
12534 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12535 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm5, %xmm5
12536 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12537 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm6, %xmm2
12538 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
12539 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12540 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm5, %xmm5
12541 ; AVX2-FP-NEXT: vmovdqa (%rsp), %xmm6 # 16-byte Reload
12542 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm6, %xmm3
12543 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
12544 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
12545 ; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
12546 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
12547 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
12548 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm3, %xmm3
12549 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12550 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm5, %xmm0
12551 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
12552 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
12553 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm3, %xmm3
12554 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12555 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm5, %xmm1
12556 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
12557 ; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
12558 ; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
12559 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5],ymm1[6,7]
12560 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
12561 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
12562 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
12563 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
12564 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
12565 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm11, %xmm0
12566 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
12567 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm1, %xmm1
12568 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
12569 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
12570 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
12571 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
12572 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
12573 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm4, %xmm4
12574 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
12575 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm1[0,1,2],xmm0[3]
12576 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
12577 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm13, %xmm1
12578 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
12579 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm12, %xmm5
12580 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
12581 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
12582 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
12583 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm13, %xmm6
12584 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
12585 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm14, %xmm14
12586 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12587 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
12588 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
12589 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12590 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm5, %xmm5
12591 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12592 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm6, %xmm6
12593 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
12594 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12595 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm6, %xmm6
12596 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
12597 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm14, %xmm14
12598 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12599 ; AVX2-FP-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
12600 ; AVX2-FP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
12601 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5,6],ymm5[7]
12602 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm9, %xmm6
12603 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm10, %xmm14
12604 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12605 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
12606 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm9, %xmm14
12607 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12608 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm10, %xmm15
12609 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
12610 ; AVX2-FP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
12611 ; AVX2-FP-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
12612 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm6 = ymm14[0,1,2,3,4],ymm6[5],ymm14[6,7]
12613 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
12614 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
12615 ; AVX2-FP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
12616 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
12617 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm4, %xmm4
12618 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12619 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm5, %xmm5
12620 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
12621 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12622 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm5, %xmm5
12623 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm8, %xmm6
12624 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
12625 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
12626 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
12627 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm8, %xmm5
12628 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm7, %xmm6
12629 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
12630 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12631 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm6, %xmm6
12632 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
12633 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm7, %xmm14
12634 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12635 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
12636 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
12637 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12638 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm5, %xmm5
12639 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12640 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm6, %xmm2
12641 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
12642 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12643 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm5, %xmm5
12644 ; AVX2-FP-NEXT: vmovdqa (%rsp), %xmm6 # 16-byte Reload
12645 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm6, %xmm3
12646 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
12647 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
12648 ; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
12649 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
12650 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
12651 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm3, %xmm3
12652 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12653 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm5, %xmm0
12654 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
12655 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
12656 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm3, %xmm3
12657 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12658 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm5, %xmm1
12659 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
12660 ; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
12661 ; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
12662 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5],ymm1[6,7]
12663 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
12664 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
12665 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
12666 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
12667 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm11, %xmm0
12668 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
12669 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm1, %xmm1
12670 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
12671 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
12672 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
12673 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
12674 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
12675 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm4, %xmm4
12676 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
12677 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm1[0,1,2],xmm0[3]
12678 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
12679 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
12680 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm1, %xmm1
12681 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm12, %xmm5
12682 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
12683 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
12684 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm13, %xmm6
12685 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
12686 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm7, %xmm14
12687 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12688 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
12689 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
12690 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12691 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm5, %xmm5
12692 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12693 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm6, %xmm6
12694 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
12695 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12696 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm6, %xmm6
12697 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
12698 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm7, %xmm14
12699 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12700 ; AVX2-FP-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
12701 ; AVX2-FP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
12702 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5,6],ymm5[7]
12703 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12704 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm6, %xmm6
12705 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
12706 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm7, %xmm14
12707 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12708 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm9, %xmm14
12709 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm10, %xmm15
12710 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
12711 ; AVX2-FP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
12712 ; AVX2-FP-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
12713 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm6 = ymm14[0,1,2,3,4],ymm6[5],ymm14[6,7]
12714 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
12715 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
12716 ; AVX2-FP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
12717 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
12718 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm7, %xmm4
12719 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12720 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm5, %xmm5
12721 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
12722 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12723 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm5, %xmm5
12724 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12725 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm6, %xmm6
12726 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
12727 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
12728 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm8, %xmm5
12729 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12730 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm6, %xmm6
12731 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
12732 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
12733 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm9, %xmm6
12734 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
12735 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm8, %xmm14
12736 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12737 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
12738 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
12739 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12740 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm10, %xmm5
12741 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
12742 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm11, %xmm2
12743 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
12744 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
12745 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm12, %xmm5
12746 ; AVX2-FP-NEXT: vmovdqa (%rsp), %xmm13 # 16-byte Reload
12747 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm13, %xmm3
12748 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
12749 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
12750 ; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
12751 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
12752 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
12753 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm3, %xmm3
12754 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12755 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm5, %xmm0
12756 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
12757 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
12758 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm3, %xmm3
12759 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12760 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm5, %xmm1
12761 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
12762 ; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
12763 ; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
12764 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5],ymm1[6,7]
12765 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
12766 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
12767 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
12768 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
12769 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
12770 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm0, %xmm0
12771 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
12772 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm1, %xmm1
12773 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
12774 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
12775 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
12776 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
12777 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
12778 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm4, %xmm4
12779 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
12780 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm1[0,1,2],xmm0[3]
12781 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
12782 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
12783 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm1, %xmm1
12784 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12785 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm5, %xmm5
12786 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
12787 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
12788 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12789 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm6, %xmm6
12790 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
12791 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm14, %xmm14
12792 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12793 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
12794 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
12795 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12796 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm5, %xmm5
12797 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12798 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm6, %xmm6
12799 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
12800 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12801 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm6, %xmm6
12802 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
12803 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm14, %xmm14
12804 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12805 ; AVX2-FP-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
12806 ; AVX2-FP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
12807 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5,6],ymm5[7]
12808 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12809 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm6, %xmm6
12810 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
12811 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm14, %xmm14
12812 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12813 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
12814 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm14, %xmm14
12815 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
12816 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm15, %xmm15
12817 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
12818 ; AVX2-FP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
12819 ; AVX2-FP-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
12820 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm6 = ymm14[0,1,2,3,4],ymm6[5],ymm14[6,7]
12821 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
12822 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
12823 ; AVX2-FP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
12824 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm7, %xmm4
12825 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12826 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm5, %xmm5
12827 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
12828 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12829 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm5, %xmm5
12830 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12831 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm6, %xmm6
12832 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
12833 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
12834 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12835 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm5, %xmm5
12836 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12837 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm6, %xmm6
12838 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
12839 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm9, %xmm6
12840 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm8, %xmm14
12841 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12842 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
12843 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
12844 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm10, %xmm5
12845 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm11, %xmm2
12846 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
12847 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm12, %xmm5
12848 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm13, %xmm3
12849 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
12850 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
12851 ; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
12852 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
12853 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
12854 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm13, %xmm3
12855 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
12856 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm7, %xmm0
12857 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
12858 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
12859 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm8, %xmm3
12860 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
12861 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm9, %xmm1
12862 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
12863 ; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
12864 ; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
12865 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5],ymm1[6,7]
12866 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
12867 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
12868 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
12869 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
12870 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
12871 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm0, %xmm0
12872 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12873 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm10, %xmm1
12874 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
12875 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
12876 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
12877 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm11, %xmm1
12878 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
12879 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm4, %xmm4
12880 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
12881 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm1[0,1,2],xmm0[3]
12882 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
12883 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
12884 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm1, %xmm1
12885 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12886 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm5, %xmm5
12887 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
12888 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
12889 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12890 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm6, %xmm6
12891 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
12892 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm12, %xmm14
12893 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12894 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
12895 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
12896 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12897 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm5, %xmm5
12898 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12899 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm6, %xmm6
12900 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
12901 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12902 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm6, %xmm6
12903 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
12904 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm14, %xmm14
12905 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12906 ; AVX2-FP-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
12907 ; AVX2-FP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
12908 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5,6],ymm5[7]
12909 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12910 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm6, %xmm6
12911 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
12912 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm14, %xmm14
12913 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12914 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
12915 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm14, %xmm14
12916 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
12917 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm15, %xmm15
12918 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
12919 ; AVX2-FP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
12920 ; AVX2-FP-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
12921 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm6 = ymm14[0,1,2,3,4],ymm6[5],ymm14[6,7]
12922 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
12923 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
12924 ; AVX2-FP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
12925 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
12926 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm4, %xmm4
12927 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12928 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm5, %xmm5
12929 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
12930 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12931 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm5, %xmm5
12932 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12933 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm6, %xmm6
12934 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
12935 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
12936 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12937 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm5, %xmm5
12938 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12939 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm6, %xmm6
12940 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
12941 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12942 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm6, %xmm6
12943 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
12944 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm14, %xmm14
12945 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
12946 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3]
12947 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
12948 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12949 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm5, %xmm5
12950 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12951 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm6, %xmm2
12952 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
12953 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
12954 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm5, %xmm5
12955 ; AVX2-FP-NEXT: vmovdqa (%rsp), %xmm6 # 16-byte Reload
12956 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm6, %xmm3
12957 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
12958 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
12959 ; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
12960 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
12961 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm13, %xmm3
12962 ; AVX2-FP-NEXT: vmovdqa %xmm13, %xmm5
12963 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm7, %xmm0
12964 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
12965 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm8, %xmm3
12966 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm9, %xmm1
12967 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
12968 ; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
12969 ; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
12970 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5],ymm1[6,7]
12971 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
12972 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
12973 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
12974 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
12975 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
12976 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm0, %xmm0
12977 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm10, %xmm1
12978 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
12979 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
12980 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm11, %xmm2
12981 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
12982 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm4, %xmm4
12983 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
12984 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm2[0,1,2],xmm1[3]
12985 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
12986 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
12987 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm2, %xmm2
12988 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12989 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm6, %xmm6
12990 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
12991 ; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
12992 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
12993 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm10, %xmm14
12994 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm12, %xmm15
12995 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
12996 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm6 = xmm14[0],xmm6[1],xmm14[2,3]
12997 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3]
12998 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
12999 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm6, %xmm6
13000 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
13001 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm10, %xmm14
13002 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
13003 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
13004 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm10, %xmm14
13005 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
13006 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm10, %xmm15
13007 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
13008 ; AVX2-FP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
13009 ; AVX2-FP-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
13010 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm6 = ymm14[0,1,2,3,4,5,6],ymm6[7]
13011 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
13012 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm10, %xmm14
13013 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
13014 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm10, %xmm15
13015 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
13016 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
13017 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm10, %xmm15
13018 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
13019 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm10, %xmm13
13020 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm15[0],xmm13[1],xmm15[1],xmm13[2],xmm15[2],xmm13[3],xmm15[3]
13021 ; AVX2-FP-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
13022 ; AVX2-FP-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
13023 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm13 = ymm13[0,1,2,3,4],ymm14[5],ymm13[6,7]
13024 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm6 = ymm13[0,1,2,3,4,5],ymm6[6,7]
13025 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm6[4,5,6,7]
13026 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
13027 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm6, %xmm6
13028 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
13029 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm10, %xmm13
13030 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm13[0],xmm6[0],xmm13[1],xmm6[1],xmm13[2],xmm6[2],xmm13[3],xmm6[3]
13031 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
13032 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm10, %xmm13
13033 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
13034 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm10, %xmm14
13035 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
13036 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm6 = xmm13[0,1,2],xmm6[3]
13037 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
13038 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm10, %xmm13
13039 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
13040 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm10, %xmm14
13041 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
13042 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
13043 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm10, %xmm14
13044 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
13045 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm10, %xmm15
13046 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
13047 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm13 = xmm14[0],xmm13[1],xmm14[2,3]
13048 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm6 = xmm13[0,1],xmm6[2,3]
13049 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
13050 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm10, %xmm13
13051 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
13052 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm10, %xmm3
13053 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm13[0],xmm3[1],xmm13[1],xmm3[2],xmm13[2],xmm3[3],xmm13[3]
13054 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
13055 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm10, %xmm12
13056 ; AVX2-FP-NEXT: vmovdqa (%rsp), %xmm10 # 16-byte Reload
13057 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm10, %xmm0
13058 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm12[0],xmm0[1],xmm12[1],xmm0[2],xmm12[2],xmm0[3],xmm12[3]
13059 ; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
13060 ; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
13061 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm3[7]
13062 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm5, %xmm3
13063 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm7, %xmm1
13064 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
13065 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm8, %xmm3
13066 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm9, %xmm2
13067 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
13068 ; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
13069 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
13070 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4],ymm1[5],ymm2[6,7]
13071 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
13072 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1,2,3],ymm0[4,5,6,7]
13073 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
13074 ; AVX2-FP-NEXT: vmovaps %ymm1, (%rsi)
13075 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
13076 ; AVX2-FP-NEXT: vmovaps %ymm1, 32(%rsi)
13077 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
13078 ; AVX2-FP-NEXT: vmovaps %ymm1, (%rdx)
13079 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
13080 ; AVX2-FP-NEXT: vmovaps %ymm1, 32(%rdx)
13081 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
13082 ; AVX2-FP-NEXT: vmovaps %ymm1, (%rcx)
13083 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
13084 ; AVX2-FP-NEXT: vmovaps %ymm1, 32(%rcx)
13085 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
13086 ; AVX2-FP-NEXT: vmovaps %ymm1, (%r8)
13087 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
13088 ; AVX2-FP-NEXT: vmovaps %ymm1, 32(%r8)
13089 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
13090 ; AVX2-FP-NEXT: vmovaps %ymm1, (%r9)
13091 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
13092 ; AVX2-FP-NEXT: vmovaps %ymm1, 32(%r9)
13093 ; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %rax
13094 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
13095 ; AVX2-FP-NEXT: vmovaps %ymm1, (%rax)
13096 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
13097 ; AVX2-FP-NEXT: vmovaps %ymm1, 32(%rax)
13098 ; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %rax
13099 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
13100 ; AVX2-FP-NEXT: vmovaps %ymm1, (%rax)
13101 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
13102 ; AVX2-FP-NEXT: vmovaps %ymm1, 32(%rax)
13103 ; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %rax
13104 ; AVX2-FP-NEXT: vmovdqa %ymm0, (%rax)
13105 ; AVX2-FP-NEXT: vmovdqa %ymm4, 32(%rax)
13106 ; AVX2-FP-NEXT: addq $840, %rsp # imm = 0x348
13107 ; AVX2-FP-NEXT: vzeroupper
13108 ; AVX2-FP-NEXT: retq
13109 ;
13110 ; AVX2-FCP-LABEL: load_i8_stride8_vf64:
13111 ; AVX2-FCP: # %bb.0:
13112 ; AVX2-FCP-NEXT: subq $904, %rsp # imm = 0x388
13113 ; AVX2-FCP-NEXT: vmovdqa 368(%rdi), %xmm0
13114 ; AVX2-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13115 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
13116 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm0, %xmm0
13117 ; AVX2-FCP-NEXT: vmovdqa 352(%rdi), %xmm2
13118 ; AVX2-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13119 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm2, %xmm2
13120 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
13121 ; AVX2-FCP-NEXT: vmovdqa 336(%rdi), %xmm15
13122 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
13123 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm15, %xmm2
13124 ; AVX2-FCP-NEXT: vmovdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13125 ; AVX2-FCP-NEXT: vmovdqa 320(%rdi), %xmm4
13126 ; AVX2-FCP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13127 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm4, %xmm4
13128 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
13129 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
13130 ; AVX2-FCP-NEXT: vmovdqa 304(%rdi), %xmm14
13131 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
13132 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm14, %xmm4
13133 ; AVX2-FCP-NEXT: vmovdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13134 ; AVX2-FCP-NEXT: vmovdqa 288(%rdi), %xmm5
13135 ; AVX2-FCP-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13136 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm5, %xmm5
13137 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
13138 ; AVX2-FCP-NEXT: vmovdqa 272(%rdi), %xmm5
13139 ; AVX2-FCP-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13140 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm8 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
13141 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm5, %xmm5
13142 ; AVX2-FCP-NEXT: vmovdqa 256(%rdi), %xmm6
13143 ; AVX2-FCP-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13144 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm6, %xmm6
13145 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
13146 ; AVX2-FCP-NEXT: vmovdqa 448(%rdi), %ymm6
13147 ; AVX2-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13148 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0],xmm4[1],xmm5[2,3]
13149 ; AVX2-FCP-NEXT: vmovdqa 480(%rdi), %ymm5
13150 ; AVX2-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13151 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm4[0,1],xmm0[2,3]
13152 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,2,2,3,0,2,4,6]
13153 ; AVX2-FCP-NEXT: vpermd %ymm5, %ymm0, %ymm4
13154 ; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13155 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28]
13156 ; AVX2-FCP-NEXT: vpermd %ymm6, %ymm0, %ymm5
13157 ; AVX2-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13158 ; AVX2-FCP-NEXT: vpbroadcastd {{.*#+}} ymm7 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
13159 ; AVX2-FCP-NEXT: vpshufb %ymm7, %ymm5, %ymm5
13160 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm5[0,1,2,3,4,5,6],ymm4[7]
13161 ; AVX2-FCP-NEXT: vmovdqa 384(%rdi), %ymm5
13162 ; AVX2-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13163 ; AVX2-FCP-NEXT: vmovdqa 416(%rdi), %ymm4
13164 ; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13165 ; AVX2-FCP-NEXT: vpermd %ymm4, %ymm0, %ymm13
13166 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm11 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u]
13167 ; AVX2-FCP-NEXT: vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13168 ; AVX2-FCP-NEXT: vpermd %ymm5, %ymm0, %ymm5
13169 ; AVX2-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13170 ; AVX2-FCP-NEXT: vpbroadcastd {{.*#+}} ymm6 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
13171 ; AVX2-FCP-NEXT: vpshufb %ymm6, %ymm5, %ymm12
13172 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2,3,4],ymm11[5],ymm12[6,7]
13173 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3,4,5],ymm10[6,7]
13174 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3],ymm10[4,5,6,7]
13175 ; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13176 ; AVX2-FCP-NEXT: vmovdqa 112(%rdi), %xmm4
13177 ; AVX2-FCP-NEXT: vmovdqa %xmm4, (%rsp) # 16-byte Spill
13178 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm4, %xmm9
13179 ; AVX2-FCP-NEXT: vmovdqa 96(%rdi), %xmm4
13180 ; AVX2-FCP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13181 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm4, %xmm1
13182 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm9[0],xmm1[1],xmm9[1],xmm1[2],xmm9[2],xmm1[3],xmm9[3]
13183 ; AVX2-FCP-NEXT: vmovdqa 80(%rdi), %xmm4
13184 ; AVX2-FCP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13185 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm4, %xmm9
13186 ; AVX2-FCP-NEXT: vmovdqa 64(%rdi), %xmm4
13187 ; AVX2-FCP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13188 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm4, %xmm3
13189 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
13190 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
13191 ; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %xmm4
13192 ; AVX2-FCP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13193 ; AVX2-FCP-NEXT: vmovdqa 48(%rdi), %xmm3
13194 ; AVX2-FCP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13195 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm3, %xmm3
13196 ; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm4, %xmm2
13197 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
13198 ; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm4
13199 ; AVX2-FCP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13200 ; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm3
13201 ; AVX2-FCP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13202 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm3, %xmm3
13203 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm4, %xmm8
13204 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
13205 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2,3]
13206 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm2[0,1],xmm1[2,3]
13207 ; AVX2-FCP-NEXT: vmovdqa 192(%rdi), %ymm2
13208 ; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13209 ; AVX2-FCP-NEXT: vmovdqa 224(%rdi), %ymm1
13210 ; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13211 ; AVX2-FCP-NEXT: vpermd %ymm1, %ymm0, %ymm1
13212 ; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13213 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28]
13214 ; AVX2-FCP-NEXT: vpermd %ymm2, %ymm0, %ymm2
13215 ; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13216 ; AVX2-FCP-NEXT: vpshufb %ymm7, %ymm2, %ymm10
13217 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5,6],ymm1[7]
13218 ; AVX2-FCP-NEXT: vmovdqa 128(%rdi), %ymm1
13219 ; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13220 ; AVX2-FCP-NEXT: vmovdqa 160(%rdi), %ymm2
13221 ; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13222 ; AVX2-FCP-NEXT: vpermd %ymm2, %ymm0, %ymm2
13223 ; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13224 ; AVX2-FCP-NEXT: vpermd %ymm1, %ymm0, %ymm1
13225 ; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13226 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u]
13227 ; AVX2-FCP-NEXT: vpshufb %ymm6, %ymm1, %ymm12
13228 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm12[0,1,2,3,4],ymm0[5],ymm12[6,7]
13229 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm10[6,7]
13230 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm0[4,5,6,7]
13231 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13232 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm8 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
13233 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
13234 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm6, %xmm0
13235 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
13236 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm7, %xmm10
13237 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm10[0],xmm0[0],xmm10[1],xmm0[1],xmm10[2],xmm0[2],xmm10[3],xmm0[3]
13238 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
13239 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm15, %xmm12
13240 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
13241 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm2, %xmm11
13242 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
13243 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm11[0,1,2],xmm0[3]
13244 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
13245 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm14, %xmm12
13246 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
13247 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm15, %xmm14
13248 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm14[0],xmm12[0],xmm14[1],xmm12[1],xmm14[2],xmm12[2],xmm14[3],xmm12[3]
13249 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm14 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
13250 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
13251 ; AVX2-FCP-NEXT: vpshufb %xmm14, %xmm0, %xmm0
13252 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
13253 ; AVX2-FCP-NEXT: vpshufb %xmm14, %xmm4, %xmm9
13254 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm9[0],xmm0[0],xmm9[1],xmm0[1],xmm9[2],xmm0[2],xmm9[3],xmm0[3]
13255 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm12[1],xmm0[2,3]
13256 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
13257 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
13258 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29]
13259 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
13260 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u]
13261 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm9[0,1,2,3,4,5,6],ymm1[7]
13262 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm9 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u,u,u,u,u]
13263 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
13264 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u,u,u,u,u,u,u,u,u]
13265 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm12[0,1,2,3,4],ymm9[5],ymm12[6,7]
13266 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm9[0,1,2,3,4,5],ymm1[6,7]
13267 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
13268 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13269 ; AVX2-FCP-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload
13270 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm0, %xmm0
13271 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
13272 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm13, %xmm1
13273 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
13274 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
13275 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm1, %xmm1
13276 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
13277 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm5, %xmm8
13278 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm8[0],xmm1[0],xmm8[1],xmm1[1],xmm8[2],xmm1[2],xmm8[3],xmm1[3]
13279 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
13280 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
13281 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm1, %xmm1
13282 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
13283 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm8, %xmm8
13284 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm8[0],xmm1[0],xmm8[1],xmm1[1],xmm8[2],xmm1[2],xmm8[3],xmm1[3]
13285 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
13286 ; AVX2-FCP-NEXT: vpshufb %xmm14, %xmm8, %xmm8
13287 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
13288 ; AVX2-FCP-NEXT: vpshufb %xmm14, %xmm9, %xmm9
13289 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
13290 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm8[0],xmm1[1],xmm8[2,3]
13291 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
13292 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
13293 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29]
13294 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
13295 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm8 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u]
13296 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm8[0,1,2,3,4,5,6],ymm1[7]
13297 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
13298 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm8 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u,u,u,u,u]
13299 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
13300 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u,u,u,u,u,u,u,u,u]
13301 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
13302 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm8[0,1,2,3,4,5],ymm1[6,7]
13303 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
13304 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13305 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm8 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
13306 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm6, %xmm0
13307 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm7, %xmm1
13308 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
13309 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
13310 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
13311 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm6, %xmm9
13312 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm2, %xmm10
13313 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
13314 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm9[0,1,2],xmm0[3]
13315 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm9 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
13316 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
13317 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm2, %xmm10
13318 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm15, %xmm11
13319 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]
13320 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
13321 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
13322 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm7, %xmm12
13323 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm4, %xmm14
13324 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm14[0],xmm12[0],xmm14[1],xmm12[1],xmm14[2],xmm12[2],xmm14[3],xmm12[3]
13325 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm10 = xmm12[0],xmm10[1],xmm12[2,3]
13326 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
13327 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm10 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,22,26,30]
13328 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
13329 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,22,26,30,u,u,u,u]
13330 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3,4,5,6],ymm10[7]
13331 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
13332 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,22,26,30,u,u,u,u,u,u,u,u]
13333 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
13334 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm14 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,22,26,30,u,u,u,u,u,u,u,u,u,u,u,u]
13335 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm14[0,1,2,3,4],ymm12[5],ymm14[6,7]
13336 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3,4,5],ymm10[6,7]
13337 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm10[4,5,6,7]
13338 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13339 ; AVX2-FCP-NEXT: vmovdqa (%rsp), %xmm4 # 16-byte Reload
13340 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm4, %xmm0
13341 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm13, %xmm8
13342 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3]
13343 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
13344 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm13, %xmm8
13345 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm5, %xmm1
13346 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3]
13347 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
13348 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
13349 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm15, %xmm1
13350 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
13351 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm3, %xmm8
13352 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm8[0],xmm1[0],xmm8[1],xmm1[1],xmm8[2],xmm1[2],xmm8[3],xmm1[3]
13353 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
13354 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm5, %xmm8
13355 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
13356 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm5, %xmm9
13357 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
13358 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm8[0],xmm1[1],xmm8[2,3]
13359 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
13360 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
13361 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,22,26,30]
13362 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
13363 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm8 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,22,26,30,u,u,u,u]
13364 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm8[0,1,2,3,4,5,6],ymm1[7]
13365 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
13366 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm8 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,22,26,30,u,u,u,u,u,u,u,u]
13367 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
13368 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm9 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,22,26,30,u,u,u,u,u,u,u,u,u,u,u,u]
13369 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
13370 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm8[0,1,2,3,4,5],ymm1[6,7]
13371 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
13372 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13373 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm5 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
13374 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
13375 ; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm0, %xmm0
13376 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
13377 ; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm1, %xmm1
13378 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
13379 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
13380 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm6, %xmm9
13381 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
13382 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm5, %xmm10
13383 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
13384 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm9[0,1,2],xmm0[3]
13385 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm9 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
13386 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm2, %xmm10
13387 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
13388 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm8, %xmm11
13389 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]
13390 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
13391 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm7, %xmm12
13392 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
13393 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm2, %xmm14
13394 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm14[0],xmm12[0],xmm14[1],xmm12[1],xmm14[2],xmm12[2],xmm14[3],xmm12[3]
13395 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm10 = xmm12[0],xmm10[1],xmm12[2,3]
13396 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
13397 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
13398 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm10 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,19,23,27,31]
13399 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
13400 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,19,23,27,31,u,u,u,u]
13401 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3,4,5,6],ymm10[7]
13402 ; AVX2-FCP-NEXT: vpbroadcastd {{.*#+}} ymm14 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
13403 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
13404 ; AVX2-FCP-NEXT: vpshufb %ymm14, %ymm6, %ymm7
13405 ; AVX2-FCP-NEXT: vpbroadcastd {{.*#+}} ymm2 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
13406 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
13407 ; AVX2-FCP-NEXT: vpshufb %ymm2, %ymm6, %ymm6
13408 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4],ymm7[5],ymm6[6,7]
13409 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],ymm10[6,7]
13410 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm6[4,5,6,7]
13411 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13412 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm6 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
13413 ; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm4, %xmm0
13414 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
13415 ; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm4, %xmm6
13416 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3]
13417 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm13, %xmm6
13418 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
13419 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm13, %xmm1
13420 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3]
13421 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
13422 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm15, %xmm1
13423 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm3, %xmm6
13424 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm6[0],xmm1[0],xmm6[1],xmm1[1],xmm6[2],xmm1[2],xmm6[3],xmm1[3]
13425 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
13426 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm3, %xmm6
13427 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
13428 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm3, %xmm7
13429 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
13430 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm6[0],xmm1[1],xmm6[2,3]
13431 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
13432 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
13433 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,19,23,27,31]
13434 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
13435 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,19,23,27,31,u,u,u,u]
13436 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
13437 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
13438 ; AVX2-FCP-NEXT: vpshufb %ymm14, %ymm3, %ymm3
13439 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
13440 ; AVX2-FCP-NEXT: vpshufb %ymm2, %ymm4, %ymm2
13441 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5],ymm2[6,7]
13442 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
13443 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
13444 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13445 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
13446 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
13447 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm0, %xmm0
13448 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
13449 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm14, %xmm1
13450 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
13451 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm6 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
13452 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
13453 ; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm1, %xmm1
13454 ; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm5, %xmm2
13455 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
13456 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
13457 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm7 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
13458 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
13459 ; AVX2-FCP-NEXT: vpshufb %xmm7, %xmm1, %xmm1
13460 ; AVX2-FCP-NEXT: vpshufb %xmm7, %xmm8, %xmm2
13461 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
13462 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm8 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
13463 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
13464 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm2, %xmm2
13465 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
13466 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm3, %xmm3
13467 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
13468 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2,3]
13469 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm1[0,1],xmm0[2,3]
13470 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [1,3,2,3,1,3,5,7]
13471 ; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
13472 ; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13473 ; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
13474 ; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13475 ; AVX2-FCP-NEXT: vpbroadcastd {{.*#+}} ymm15 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
13476 ; AVX2-FCP-NEXT: vpshufb %ymm15, %ymm1, %ymm3
13477 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u]
13478 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm4[0,1,2,3,4,5,6],ymm3[7]
13479 ; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
13480 ; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13481 ; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
13482 ; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13483 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm11 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u]
13484 ; AVX2-FCP-NEXT: vpbroadcastd {{.*#+}} ymm1 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
13485 ; AVX2-FCP-NEXT: vpshufb %ymm1, %ymm2, %ymm12
13486 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2,3,4],ymm11[5],ymm12[6,7]
13487 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3,4,5],ymm10[6,7]
13488 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm9[0,1,2,3],ymm10[4,5,6,7]
13489 ; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13490 ; AVX2-FCP-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload
13491 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm3 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
13492 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm9
13493 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
13494 ; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm5
13495 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm9[0],xmm5[1],xmm9[1],xmm5[2],xmm9[2],xmm5[3],xmm9[3]
13496 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
13497 ; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm2, %xmm9
13498 ; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm13, %xmm6
13499 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm9[0],xmm6[1],xmm9[1],xmm6[2],xmm9[2],xmm6[3],xmm9[3]
13500 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3]
13501 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
13502 ; AVX2-FCP-NEXT: vpshufb %xmm7, %xmm3, %xmm6
13503 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
13504 ; AVX2-FCP-NEXT: vpshufb %xmm7, %xmm4, %xmm7
13505 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
13506 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
13507 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm4, %xmm7
13508 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
13509 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm9, %xmm8
13510 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
13511 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0],xmm6[1],xmm7[2,3]
13512 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm6[0,1],xmm5[2,3]
13513 ; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
13514 ; AVX2-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13515 ; AVX2-FCP-NEXT: vpshufb %ymm15, %ymm5, %ymm7
13516 ; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
13517 ; AVX2-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13518 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm9 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u]
13519 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5,6],ymm7[7]
13520 ; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
13521 ; AVX2-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13522 ; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
13523 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13524 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm10 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u]
13525 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm11 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u,u,u,u,u]
13526 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3,4],ymm10[5],ymm11[6,7]
13527 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4,5],ymm9[6,7]
13528 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm9[4,5,6,7]
13529 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13530 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm8 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
13531 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
13532 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm0, %xmm9
13533 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm14, %xmm10
13534 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
13535 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
13536 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
13537 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm5, %xmm11
13538 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
13539 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm6, %xmm12
13540 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
13541 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm11[0,1,2],xmm9[3]
13542 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
13543 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
13544 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm0, %xmm12
13545 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
13546 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm7, %xmm13
13547 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
13548 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm13 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
13549 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
13550 ; AVX2-FCP-NEXT: vpshufb %xmm13, %xmm0, %xmm14
13551 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
13552 ; AVX2-FCP-NEXT: vpshufb %xmm13, %xmm0, %xmm15
13553 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
13554 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm14[0],xmm12[1],xmm14[2,3]
13555 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm12[0,1],xmm9[2,3]
13556 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
13557 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29]
13558 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
13559 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm14 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u]
13560 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm14[0,1,2,3,4,5,6],ymm12[7]
13561 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
13562 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm14 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u,u,u,u,u]
13563 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
13564 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm15 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u,u,u,u,u,u,u,u,u]
13565 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
13566 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm14[0,1,2,3,4,5],ymm12[6,7]
13567 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3],ymm12[4,5,6,7]
13568 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13569 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm1, %xmm9
13570 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
13571 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm0, %xmm8
13572 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
13573 ; AVX2-FCP-NEXT: vmovdqa %xmm2, %xmm1
13574 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm2, %xmm9
13575 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
13576 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm2, %xmm10
13577 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
13578 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
13579 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm3, %xmm9
13580 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
13581 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm3, %xmm10
13582 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
13583 ; AVX2-FCP-NEXT: vpshufb %xmm13, %xmm4, %xmm10
13584 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
13585 ; AVX2-FCP-NEXT: vpshufb %xmm13, %xmm4, %xmm11
13586 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]
13587 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0],xmm9[1],xmm10[2,3]
13588 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
13589 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
13590 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29]
13591 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
13592 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm10 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u]
13593 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4,5,6],ymm9[7]
13594 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
13595 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm10 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u,u,u,u,u]
13596 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
13597 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm11 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u,u,u,u,u,u,u,u,u]
13598 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3,4],ymm10[5],ymm11[6,7]
13599 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4,5],ymm9[6,7]
13600 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm9[4,5,6,7]
13601 ; AVX2-FCP-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13602 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm8 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
13603 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
13604 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm9, %xmm9
13605 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
13606 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm10, %xmm10
13607 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
13608 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm10 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
13609 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm5, %xmm11
13610 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm6, %xmm12
13611 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
13612 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm11[0,1,2],xmm9[3]
13613 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
13614 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
13615 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm6, %xmm12
13616 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm7, %xmm13
13617 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
13618 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm13 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
13619 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
13620 ; AVX2-FCP-NEXT: vpshufb %xmm13, %xmm7, %xmm14
13621 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
13622 ; AVX2-FCP-NEXT: vpshufb %xmm13, %xmm5, %xmm15
13623 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
13624 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm14[0],xmm12[1],xmm14[2,3]
13625 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm12[0,1],xmm9[2,3]
13626 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
13627 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,22,26,30]
13628 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
13629 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm14 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,22,26,30,u,u,u,u]
13630 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm14[0,1,2,3,4,5,6],ymm12[7]
13631 ; AVX2-FCP-NEXT: vpbroadcastd {{.*#+}} ymm5 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
13632 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
13633 ; AVX2-FCP-NEXT: vpshufb %ymm5, %ymm14, %ymm14
13634 ; AVX2-FCP-NEXT: vpbroadcastd {{.*#+}} ymm0 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
13635 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
13636 ; AVX2-FCP-NEXT: vpshufb %ymm0, %ymm15, %ymm15
13637 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
13638 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm14[0,1,2,3,4,5],ymm12[6,7]
13639 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm12[4,5,6,7]
13640 ; AVX2-FCP-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13641 ; AVX2-FCP-NEXT: vmovdqa (%rsp), %xmm9 # 16-byte Reload
13642 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm9, %xmm9
13643 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
13644 ; AVX2-FCP-NEXT: vpshufb %xmm8, %xmm0, %xmm8
13645 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
13646 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm1, %xmm9
13647 ; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm2, %xmm10
13648 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
13649 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
13650 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
13651 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm0, %xmm9
13652 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm3, %xmm10
13653 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
13654 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
13655 ; AVX2-FCP-NEXT: vpshufb %xmm13, %xmm0, %xmm10
13656 ; AVX2-FCP-NEXT: vpshufb %xmm13, %xmm4, %xmm11
13657 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]
13658 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0],xmm9[1],xmm10[2,3]
13659 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
13660 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
13661 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm9 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,22,26,30]
13662 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
13663 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm10 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,22,26,30,u,u,u,u]
13664 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4,5,6],ymm9[7]
13665 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
13666 ; AVX2-FCP-NEXT: vpshufb %ymm5, %ymm1, %ymm10
13667 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
13668 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm11 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,22,26,30,u,u,u,u,u,u,u,u,u,u,u,u]
13669 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3,4],ymm10[5],ymm11[6,7]
13670 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4,5],ymm9[6,7]
13671 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm8[0,1,2,3],ymm9[4,5,6,7]
13672 ; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
13673 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm9 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
13674 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
13675 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm1, %xmm10
13676 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
13677 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm1, %xmm11
13678 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]
13679 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
13680 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
13681 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm1, %xmm12
13682 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
13683 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm1, %xmm13
13684 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
13685 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm10 = xmm12[0,1,2],xmm10[3]
13686 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm12 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
13687 ; AVX2-FCP-NEXT: vpshufb %xmm12, %xmm6, %xmm13
13688 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
13689 ; AVX2-FCP-NEXT: vpshufb %xmm12, %xmm1, %xmm14
13690 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
13691 ; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm14 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
13692 ; AVX2-FCP-NEXT: vpshufb %xmm14, %xmm7, %xmm15
13693 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
13694 ; AVX2-FCP-NEXT: vpshufb %xmm14, %xmm1, %xmm8
13695 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm15[0],xmm8[1],xmm15[1],xmm8[2],xmm15[2],xmm8[3],xmm15[3]
13696 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm8[0],xmm13[1],xmm8[2,3]
13697 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm8[0,1],xmm10[2,3]
13698 ; AVX2-FCP-NEXT: vpbroadcastd {{.*#+}} ymm5 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
13699 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
13700 ; AVX2-FCP-NEXT: vpshufb %ymm5, %ymm1, %ymm2
13701 ; AVX2-FCP-NEXT: vpbroadcastd {{.*#+}} ymm6 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
13702 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
13703 ; AVX2-FCP-NEXT: vpshufb %ymm6, %ymm1, %ymm1
13704 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm2[7]
13705 ; AVX2-FCP-NEXT: vpbroadcastd {{.*#+}} ymm7 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
13706 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
13707 ; AVX2-FCP-NEXT: vpshufb %ymm7, %ymm2, %ymm2
13708 ; AVX2-FCP-NEXT: vpbroadcastd {{.*#+}} ymm10 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
13709 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
13710 ; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm3, %ymm3
13711 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
13712 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
13713 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm8[0,1,2,3],ymm1[4,5,6,7]
13714 ; AVX2-FCP-NEXT: vmovdqa (%rsp), %xmm2 # 16-byte Reload
13715 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm2, %xmm2
13716 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
13717 ; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm3, %xmm3
13718 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
13719 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
13720 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm3, %xmm3
13721 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
13722 ; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm4, %xmm4
13723 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
13724 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3]
13725 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
13726 ; AVX2-FCP-NEXT: vpshufb %xmm12, %xmm3, %xmm3
13727 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
13728 ; AVX2-FCP-NEXT: vpshufb %xmm12, %xmm4, %xmm4
13729 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
13730 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
13731 ; AVX2-FCP-NEXT: vpshufb %xmm14, %xmm4, %xmm4
13732 ; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
13733 ; AVX2-FCP-NEXT: vpshufb %xmm14, %xmm8, %xmm8
13734 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm8[0],xmm4[0],xmm8[1],xmm4[1],xmm8[2],xmm4[2],xmm8[3],xmm4[3]
13735 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3]
13736 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]
13737 ; AVX2-FCP-NEXT: vpshufb %ymm5, %ymm0, %ymm3
13738 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
13739 ; AVX2-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm4
13740 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5,6],ymm3[7]
13741 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
13742 ; AVX2-FCP-NEXT: vpshufb %ymm7, %ymm0, %ymm4
13743 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
13744 ; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm0, %ymm0
13745 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5],ymm0[6,7]
13746 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm3[6,7]
13747 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
13748 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
13749 ; AVX2-FCP-NEXT: vmovaps %ymm2, (%rsi)
13750 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
13751 ; AVX2-FCP-NEXT: vmovaps %ymm2, 32(%rsi)
13752 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
13753 ; AVX2-FCP-NEXT: vmovaps %ymm2, (%rdx)
13754 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
13755 ; AVX2-FCP-NEXT: vmovaps %ymm2, 32(%rdx)
13756 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
13757 ; AVX2-FCP-NEXT: vmovaps %ymm2, (%rcx)
13758 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
13759 ; AVX2-FCP-NEXT: vmovaps %ymm2, 32(%rcx)
13760 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
13761 ; AVX2-FCP-NEXT: vmovaps %ymm2, (%r8)
13762 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
13763 ; AVX2-FCP-NEXT: vmovaps %ymm2, 32(%r8)
13764 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
13765 ; AVX2-FCP-NEXT: vmovaps %ymm2, (%r9)
13766 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
13767 ; AVX2-FCP-NEXT: vmovaps %ymm2, 32(%r9)
13768 ; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
13769 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
13770 ; AVX2-FCP-NEXT: vmovaps %ymm2, (%rax)
13771 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
13772 ; AVX2-FCP-NEXT: vmovaps %ymm2, 32(%rax)
13773 ; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
13774 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
13775 ; AVX2-FCP-NEXT: vmovaps %ymm2, (%rax)
13776 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
13777 ; AVX2-FCP-NEXT: vmovaps %ymm2, 32(%rax)
13778 ; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
13779 ; AVX2-FCP-NEXT: vmovdqa %ymm0, (%rax)
13780 ; AVX2-FCP-NEXT: vmovdqa %ymm1, 32(%rax)
13781 ; AVX2-FCP-NEXT: addq $904, %rsp # imm = 0x388
13782 ; AVX2-FCP-NEXT: vzeroupper
13783 ; AVX2-FCP-NEXT: retq
13784 ;
13785 ; AVX512-LABEL: load_i8_stride8_vf64:
13786 ; AVX512: # %bb.0:
13787 ; AVX512-NEXT: subq $520, %rsp # imm = 0x208
13788 ; AVX512-NEXT: vmovdqa64 256(%rdi), %zmm17
13789 ; AVX512-NEXT: vmovdqa64 384(%rdi), %zmm0
13790 ; AVX512-NEXT: vpmovqb %zmm0, %xmm2
13791 ; AVX512-NEXT: vmovdqa 496(%rdi), %xmm7
13792 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
13793 ; AVX512-NEXT: vpshufb %xmm0, %xmm7, %xmm1
13794 ; AVX512-NEXT: vmovdqa 480(%rdi), %xmm8
13795 ; AVX512-NEXT: vpshufb %xmm0, %xmm8, %xmm3
13796 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
13797 ; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm3
13798 ; AVX512-NEXT: vmovdqa 464(%rdi), %xmm11
13799 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
13800 ; AVX512-NEXT: vpshufb %xmm1, %xmm11, %xmm4
13801 ; AVX512-NEXT: vmovdqa 448(%rdi), %xmm15
13802 ; AVX512-NEXT: vpshufb %xmm1, %xmm15, %xmm5
13803 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
13804 ; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
13805 ; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5,6],ymm3[7]
13806 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
13807 ; AVX512-NEXT: vmovdqa 384(%rdi), %ymm4
13808 ; AVX512-NEXT: vpmovqb %ymm4, %xmm4
13809 ; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
13810 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4],ymm2[5],ymm4[6,7]
13811 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
13812 ; AVX512-NEXT: vmovdqa 368(%rdi), %xmm4
13813 ; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm3
13814 ; AVX512-NEXT: vmovdqa64 %xmm4, %xmm18
13815 ; AVX512-NEXT: vmovdqa 352(%rdi), %xmm14
13816 ; AVX512-NEXT: vpshufb %xmm0, %xmm14, %xmm4
13817 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
13818 ; AVX512-NEXT: vmovdqa 336(%rdi), %xmm4
13819 ; AVX512-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13820 ; AVX512-NEXT: vpshufb %xmm1, %xmm4, %xmm4
13821 ; AVX512-NEXT: vmovdqa 320(%rdi), %xmm6
13822 ; AVX512-NEXT: vpshufb %xmm1, %xmm6, %xmm5
13823 ; AVX512-NEXT: vmovdqa64 %xmm6, %xmm19
13824 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
13825 ; AVX512-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1,2],xmm3[3]
13826 ; AVX512-NEXT: vpmovqb %zmm17, %xmm4
13827 ; AVX512-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]
13828 ; AVX512-NEXT: vinserti32x4 $2, %xmm3, %zmm0, %zmm3
13829 ; AVX512-NEXT: movb $-64, %al
13830 ; AVX512-NEXT: kmovw %eax, %k1
13831 ; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm3 {%k1}
13832 ; AVX512-NEXT: vmovdqa 240(%rdi), %xmm4
13833 ; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm2
13834 ; AVX512-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13835 ; AVX512-NEXT: vmovdqa64 %xmm4, %xmm25
13836 ; AVX512-NEXT: vmovdqa 224(%rdi), %xmm4
13837 ; AVX512-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13838 ; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm4
13839 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
13840 ; AVX512-NEXT: vmovdqa 208(%rdi), %xmm5
13841 ; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm4
13842 ; AVX512-NEXT: vmovdqa64 %xmm5, %xmm29
13843 ; AVX512-NEXT: vmovdqa 192(%rdi), %xmm10
13844 ; AVX512-NEXT: vpshufb %xmm1, %xmm10, %xmm5
13845 ; AVX512-NEXT: vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13846 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
13847 ; AVX512-NEXT: vmovdqa64 128(%rdi), %zmm5
13848 ; AVX512-NEXT: vpmovqb %zmm5, %xmm5
13849 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
13850 ; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
13851 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
13852 ; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm4
13853 ; AVX512-NEXT: vmovdqa 128(%rdi), %ymm5
13854 ; AVX512-NEXT: vpmovqb %ymm5, %xmm5
13855 ; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
13856 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
13857 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
13858 ; AVX512-NEXT: vmovdqa 112(%rdi), %xmm4
13859 ; AVX512-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13860 ; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm4
13861 ; AVX512-NEXT: vmovdqa 96(%rdi), %xmm5
13862 ; AVX512-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13863 ; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm0
13864 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
13865 ; AVX512-NEXT: vmovdqa 80(%rdi), %xmm5
13866 ; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm4
13867 ; AVX512-NEXT: vmovdqa64 %xmm5, %xmm30
13868 ; AVX512-NEXT: vmovdqa 64(%rdi), %xmm5
13869 ; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm1
13870 ; AVX512-NEXT: vmovdqa64 %xmm5, %xmm28
13871 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
13872 ; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
13873 ; AVX512-NEXT: vmovdqa64 (%rdi), %zmm16
13874 ; AVX512-NEXT: vpmovqb %zmm16, %xmm1
13875 ; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
13876 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
13877 ; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm3, %zmm0
13878 ; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
13879 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm9 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
13880 ; AVX512-NEXT: vpshufb %xmm9, %xmm7, %xmm0
13881 ; AVX512-NEXT: vmovdqa64 %xmm7, %xmm24
13882 ; AVX512-NEXT: vpshufb %xmm9, %xmm8, %xmm2
13883 ; AVX512-NEXT: vmovdqa64 %xmm8, %xmm21
13884 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
13885 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm8 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
13886 ; AVX512-NEXT: vpshufb %xmm8, %xmm11, %xmm3
13887 ; AVX512-NEXT: vmovdqa64 %xmm11, %xmm27
13888 ; AVX512-NEXT: vpshufb %xmm8, %xmm15, %xmm4
13889 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
13890 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
13891 ; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
13892 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
13893 ; AVX512-NEXT: vmovdqa 416(%rdi), %xmm0
13894 ; AVX512-NEXT: vmovdqa 432(%rdi), %xmm11
13895 ; AVX512-NEXT: vmovd {{.*#+}} xmm3 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
13896 ; AVX512-NEXT: vpshufb %xmm3, %xmm11, %xmm2
13897 ; AVX512-NEXT: vpshufb %xmm3, %xmm0, %xmm5
13898 ; AVX512-NEXT: vmovdqa64 %xmm0, %xmm22
13899 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
13900 ; AVX512-NEXT: vmovdqa 384(%rdi), %xmm0
13901 ; AVX512-NEXT: vmovdqa 400(%rdi), %xmm12
13902 ; AVX512-NEXT: vmovd {{.*#+}} xmm2 = [1,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
13903 ; AVX512-NEXT: vpshufb %xmm2, %xmm12, %xmm6
13904 ; AVX512-NEXT: vpshufb %xmm2, %xmm0, %xmm7
13905 ; AVX512-NEXT: vmovdqa64 %xmm0, %xmm31
13906 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
13907 ; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
13908 ; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
13909 ; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4],ymm5[5],ymm6[6,7]
13910 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
13911 ; AVX512-NEXT: vmovdqa64 %xmm18, %xmm0
13912 ; AVX512-NEXT: vpshufb %xmm9, %xmm0, %xmm5
13913 ; AVX512-NEXT: vmovdqa64 %xmm18, %xmm23
13914 ; AVX512-NEXT: vpshufb %xmm9, %xmm14, %xmm6
13915 ; AVX512-NEXT: vmovdqa64 %xmm14, %xmm26
13916 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
13917 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
13918 ; AVX512-NEXT: vpshufb %xmm8, %xmm0, %xmm6
13919 ; AVX512-NEXT: vmovdqa64 %xmm19, %xmm14
13920 ; AVX512-NEXT: vpshufb %xmm8, %xmm14, %xmm7
13921 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
13922 ; AVX512-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3]
13923 ; AVX512-NEXT: vpsrlq $8, %zmm17, %zmm6
13924 ; AVX512-NEXT: vpmovqb %zmm6, %xmm6
13925 ; AVX512-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]
13926 ; AVX512-NEXT: vinserti32x4 $2, %xmm5, %zmm0, %zmm19
13927 ; AVX512-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm19 {%k1}
13928 ; AVX512-NEXT: vmovdqa64 %xmm25, %xmm0
13929 ; AVX512-NEXT: vpshufb %xmm9, %xmm0, %xmm4
13930 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
13931 ; AVX512-NEXT: vpshufb %xmm9, %xmm0, %xmm5
13932 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
13933 ; AVX512-NEXT: vmovdqa64 %xmm29, %xmm0
13934 ; AVX512-NEXT: vpshufb %xmm8, %xmm0, %xmm5
13935 ; AVX512-NEXT: vpshufb %xmm8, %xmm10, %xmm6
13936 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
13937 ; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
13938 ; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
13939 ; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm4[7]
13940 ; AVX512-NEXT: vmovdqa 160(%rdi), %xmm0
13941 ; AVX512-NEXT: vmovdqa 176(%rdi), %xmm5
13942 ; AVX512-NEXT: vpshufb %xmm3, %xmm5, %xmm4
13943 ; AVX512-NEXT: vmovdqa64 %xmm5, %xmm25
13944 ; AVX512-NEXT: vpshufb %xmm3, %xmm0, %xmm3
13945 ; AVX512-NEXT: vmovdqa64 %xmm0, %xmm20
13946 ; AVX512-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13947 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
13948 ; AVX512-NEXT: vmovdqa 128(%rdi), %xmm4
13949 ; AVX512-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13950 ; AVX512-NEXT: vmovdqa 144(%rdi), %xmm0
13951 ; AVX512-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13952 ; AVX512-NEXT: vpshufb %xmm2, %xmm0, %xmm0
13953 ; AVX512-NEXT: vpshufb %xmm2, %xmm4, %xmm2
13954 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
13955 ; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm2
13956 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
13957 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5],ymm0[6,7]
13958 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
13959 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
13960 ; AVX512-NEXT: vpshufb %xmm9, %xmm1, %xmm1
13961 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
13962 ; AVX512-NEXT: vpshufb %xmm9, %xmm2, %xmm2
13963 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
13964 ; AVX512-NEXT: vmovdqa64 %xmm30, %xmm7
13965 ; AVX512-NEXT: vpshufb %xmm8, %xmm7, %xmm2
13966 ; AVX512-NEXT: vmovdqa64 %xmm28, %xmm3
13967 ; AVX512-NEXT: vpshufb %xmm8, %xmm3, %xmm3
13968 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
13969 ; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
13970 ; AVX512-NEXT: vpsrlq $8, %zmm16, %zmm2
13971 ; AVX512-NEXT: vpmovqb %zmm2, %xmm2
13972 ; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
13973 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
13974 ; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm19, %zmm0
13975 ; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
13976 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm4 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
13977 ; AVX512-NEXT: vmovdqa64 %xmm24, %xmm13
13978 ; AVX512-NEXT: vmovdqa64 %xmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13979 ; AVX512-NEXT: vpshufb %xmm4, %xmm13, %xmm1
13980 ; AVX512-NEXT: vmovdqa64 %xmm21, %xmm10
13981 ; AVX512-NEXT: vmovdqa64 %xmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
13982 ; AVX512-NEXT: vpshufb %xmm4, %xmm10, %xmm2
13983 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
13984 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
13985 ; AVX512-NEXT: vmovdqa64 %xmm27, %xmm0
13986 ; AVX512-NEXT: vpshufb %xmm1, %xmm0, %xmm3
13987 ; AVX512-NEXT: vpshufb %xmm1, %xmm15, %xmm8
13988 ; AVX512-NEXT: vmovdqa64 %xmm15, %xmm18
13989 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
13990 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
13991 ; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
13992 ; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm3[0,1,2,3,4,5,6],ymm2[7]
13993 ; AVX512-NEXT: vmovd {{.*#+}} xmm3 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
13994 ; AVX512-NEXT: vpshufb %xmm3, %xmm11, %xmm2
13995 ; AVX512-NEXT: vmovdqa64 %xmm11, %xmm24
13996 ; AVX512-NEXT: vmovdqa64 %xmm22, %xmm11
13997 ; AVX512-NEXT: vpshufb %xmm3, %xmm11, %xmm9
13998 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm2[0],xmm9[1],xmm2[1],xmm9[2],xmm2[2],xmm9[3],xmm2[3]
13999 ; AVX512-NEXT: vmovd {{.*#+}} xmm2 = [2,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
14000 ; AVX512-NEXT: vpshufb %xmm2, %xmm12, %xmm0
14001 ; AVX512-NEXT: vmovdqa64 %xmm12, %xmm30
14002 ; AVX512-NEXT: vmovdqa64 %xmm31, %xmm5
14003 ; AVX512-NEXT: vpshufb %xmm2, %xmm5, %xmm15
14004 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
14005 ; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
14006 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
14007 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm9[5],ymm0[6,7]
14008 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm8[6,7]
14009 ; AVX512-NEXT: vmovdqa64 %xmm23, %xmm12
14010 ; AVX512-NEXT: vmovdqa64 %xmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
14011 ; AVX512-NEXT: vpshufb %xmm4, %xmm12, %xmm8
14012 ; AVX512-NEXT: vmovdqa64 %xmm26, %xmm5
14013 ; AVX512-NEXT: vpshufb %xmm4, %xmm5, %xmm9
14014 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
14015 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
14016 ; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm9
14017 ; AVX512-NEXT: vpshufb %xmm1, %xmm14, %xmm15
14018 ; AVX512-NEXT: vmovdqa64 %xmm14, %xmm21
14019 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
14020 ; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
14021 ; AVX512-NEXT: vpsrlq $16, %zmm17, %zmm9
14022 ; AVX512-NEXT: vpmovqb %zmm9, %xmm9
14023 ; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
14024 ; AVX512-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
14025 ; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm8 {%k1}
14026 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
14027 ; AVX512-NEXT: vpshufb %xmm4, %xmm0, %xmm0
14028 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
14029 ; AVX512-NEXT: vpshufb %xmm4, %xmm14, %xmm9
14030 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm9[0],xmm0[0],xmm9[1],xmm0[1],xmm9[2],xmm0[2],xmm9[3],xmm0[3]
14031 ; AVX512-NEXT: vmovdqa64 %xmm29, %xmm6
14032 ; AVX512-NEXT: vpshufb %xmm1, %xmm6, %xmm9
14033 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
14034 ; AVX512-NEXT: vpshufb %xmm1, %xmm15, %xmm15
14035 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
14036 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
14037 ; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
14038 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3,4,5,6],ymm0[7]
14039 ; AVX512-NEXT: vmovdqa64 %xmm25, %xmm9
14040 ; AVX512-NEXT: vpshufb %xmm3, %xmm9, %xmm9
14041 ; AVX512-NEXT: vmovdqa64 %xmm20, %xmm15
14042 ; AVX512-NEXT: vpshufb %xmm3, %xmm15, %xmm3
14043 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
14044 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
14045 ; AVX512-NEXT: vpshufb %xmm2, %xmm9, %xmm9
14046 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
14047 ; AVX512-NEXT: vpshufb %xmm2, %xmm15, %xmm2
14048 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
14049 ; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
14050 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
14051 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5],ymm2[6,7]
14052 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
14053 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
14054 ; AVX512-NEXT: vpshufb %xmm4, %xmm2, %xmm2
14055 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
14056 ; AVX512-NEXT: vpshufb %xmm4, %xmm3, %xmm3
14057 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
14058 ; AVX512-NEXT: vpshufb %xmm1, %xmm7, %xmm3
14059 ; AVX512-NEXT: vmovdqa64 %xmm7, %xmm29
14060 ; AVX512-NEXT: vmovdqa64 %xmm28, %xmm7
14061 ; AVX512-NEXT: vmovdqa64 %xmm28, (%rsp) # 16-byte Spill
14062 ; AVX512-NEXT: vpshufb %xmm1, %xmm7, %xmm1
14063 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
14064 ; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3]
14065 ; AVX512-NEXT: vpsrlq $16, %zmm16, %zmm2
14066 ; AVX512-NEXT: vpmovqb %zmm2, %xmm2
14067 ; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
14068 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
14069 ; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm0
14070 ; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
14071 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,3,11,0,0,3,11,0,0,3,11,0,0,3,11]
14072 ; AVX512-NEXT: vpshufb %xmm0, %xmm13, %xmm1
14073 ; AVX512-NEXT: vpshufb %xmm0, %xmm10, %xmm2
14074 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
14075 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
14076 ; AVX512-NEXT: vmovdqa64 %xmm27, %xmm10
14077 ; AVX512-NEXT: vmovdqa64 %xmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
14078 ; AVX512-NEXT: vpshufb %xmm1, %xmm10, %xmm3
14079 ; AVX512-NEXT: vmovdqa64 %xmm18, %xmm4
14080 ; AVX512-NEXT: vpshufb %xmm1, %xmm4, %xmm4
14081 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
14082 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
14083 ; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
14084 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
14085 ; AVX512-NEXT: vmovd {{.*#+}} xmm2 = [0,0,3,11,0,0,0,0,0,0,0,0,0,0,0,0]
14086 ; AVX512-NEXT: vmovdqa64 %xmm24, %xmm3
14087 ; AVX512-NEXT: vpshufb %xmm2, %xmm3, %xmm3
14088 ; AVX512-NEXT: vpshufb %xmm2, %xmm11, %xmm8
14089 ; AVX512-NEXT: vmovdqa64 %xmm22, %xmm28
14090 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
14091 ; AVX512-NEXT: vmovd {{.*#+}} xmm3 = [3,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
14092 ; AVX512-NEXT: vmovdqa64 %xmm30, %xmm9
14093 ; AVX512-NEXT: vpshufb %xmm3, %xmm9, %xmm9
14094 ; AVX512-NEXT: vmovdqa64 %xmm31, %xmm11
14095 ; AVX512-NEXT: vpshufb %xmm3, %xmm11, %xmm15
14096 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
14097 ; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
14098 ; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
14099 ; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
14100 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
14101 ; AVX512-NEXT: vpshufb %xmm0, %xmm12, %xmm8
14102 ; AVX512-NEXT: vmovdqa64 %xmm26, %xmm13
14103 ; AVX512-NEXT: vmovdqa64 %xmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
14104 ; AVX512-NEXT: vpshufb %xmm0, %xmm13, %xmm9
14105 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
14106 ; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm9
14107 ; AVX512-NEXT: vmovdqa64 %xmm21, %xmm5
14108 ; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm15
14109 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
14110 ; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
14111 ; AVX512-NEXT: vpsrlq $24, %zmm17, %zmm9
14112 ; AVX512-NEXT: vpmovqb %zmm9, %xmm9
14113 ; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
14114 ; AVX512-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
14115 ; AVX512-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
14116 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
14117 ; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm4
14118 ; AVX512-NEXT: vpshufb %xmm0, %xmm14, %xmm9
14119 ; AVX512-NEXT: vmovdqa64 %xmm14, %xmm22
14120 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
14121 ; AVX512-NEXT: vpshufb %xmm1, %xmm6, %xmm9
14122 ; AVX512-NEXT: vmovdqa64 %xmm6, %xmm26
14123 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
14124 ; AVX512-NEXT: vpshufb %xmm1, %xmm6, %xmm15
14125 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
14126 ; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
14127 ; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
14128 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
14129 ; AVX512-NEXT: vmovdqa64 %xmm25, %xmm14
14130 ; AVX512-NEXT: vpshufb %xmm2, %xmm14, %xmm9
14131 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
14132 ; AVX512-NEXT: vpshufb %xmm2, %xmm6, %xmm2
14133 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
14134 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
14135 ; AVX512-NEXT: vpshufb %xmm3, %xmm6, %xmm9
14136 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
14137 ; AVX512-NEXT: vpshufb %xmm3, %xmm6, %xmm3
14138 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
14139 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
14140 ; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
14141 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
14142 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
14143 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
14144 ; AVX512-NEXT: vpshufb %xmm0, %xmm3, %xmm3
14145 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
14146 ; AVX512-NEXT: vpshufb %xmm0, %xmm6, %xmm0
14147 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
14148 ; AVX512-NEXT: vmovdqa64 %xmm29, %xmm3
14149 ; AVX512-NEXT: vpshufb %xmm1, %xmm3, %xmm3
14150 ; AVX512-NEXT: vpshufb %xmm1, %xmm7, %xmm1
14151 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
14152 ; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
14153 ; AVX512-NEXT: vpsrlq $24, %zmm16, %zmm1
14154 ; AVX512-NEXT: vpmovqb %zmm1, %xmm1
14155 ; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
14156 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
14157 ; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm0
14158 ; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
14159 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
14160 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
14161 ; AVX512-NEXT: vpshufb %xmm0, %xmm1, %xmm1
14162 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
14163 ; AVX512-NEXT: vpshufb %xmm0, %xmm2, %xmm2
14164 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
14165 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
14166 ; AVX512-NEXT: vpshufb %xmm1, %xmm10, %xmm3
14167 ; AVX512-NEXT: vmovdqa64 %xmm18, %xmm10
14168 ; AVX512-NEXT: vmovdqa64 %xmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
14169 ; AVX512-NEXT: vpshufb %xmm1, %xmm10, %xmm4
14170 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
14171 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
14172 ; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
14173 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
14174 ; AVX512-NEXT: vmovd {{.*#+}} xmm2 = [0,0,4,12,0,0,0,0,0,0,0,0,0,0,0,0]
14175 ; AVX512-NEXT: vmovdqa64 %xmm24, %xmm11
14176 ; AVX512-NEXT: vpshufb %xmm2, %xmm11, %xmm3
14177 ; AVX512-NEXT: vmovdqa64 %xmm28, %xmm7
14178 ; AVX512-NEXT: vpshufb %xmm2, %xmm7, %xmm8
14179 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
14180 ; AVX512-NEXT: vmovd {{.*#+}} xmm3 = [4,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
14181 ; AVX512-NEXT: vmovdqa64 %xmm30, %xmm7
14182 ; AVX512-NEXT: vpshufb %xmm3, %xmm7, %xmm9
14183 ; AVX512-NEXT: vmovdqa64 %xmm31, %xmm12
14184 ; AVX512-NEXT: vpshufb %xmm3, %xmm12, %xmm15
14185 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
14186 ; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
14187 ; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
14188 ; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
14189 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
14190 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
14191 ; AVX512-NEXT: vpshufb %xmm0, %xmm7, %xmm8
14192 ; AVX512-NEXT: vpshufb %xmm0, %xmm13, %xmm9
14193 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
14194 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
14195 ; AVX512-NEXT: vpshufb %xmm1, %xmm7, %xmm9
14196 ; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm15
14197 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
14198 ; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
14199 ; AVX512-NEXT: vpsrlq $32, %zmm17, %zmm9
14200 ; AVX512-NEXT: vpmovqb %zmm9, %xmm9
14201 ; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
14202 ; AVX512-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
14203 ; AVX512-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
14204 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
14205 ; AVX512-NEXT: vpshufb %xmm0, %xmm13, %xmm4
14206 ; AVX512-NEXT: vmovdqa64 %xmm22, %xmm5
14207 ; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm9
14208 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
14209 ; AVX512-NEXT: vmovdqa64 %xmm26, %xmm5
14210 ; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm9
14211 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
14212 ; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm15
14213 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
14214 ; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
14215 ; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
14216 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
14217 ; AVX512-NEXT: vpshufb %xmm2, %xmm14, %xmm9
14218 ; AVX512-NEXT: vmovdqa64 %xmm25, %xmm20
14219 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
14220 ; AVX512-NEXT: vpshufb %xmm2, %xmm5, %xmm2
14221 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
14222 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
14223 ; AVX512-NEXT: vpshufb %xmm3, %xmm5, %xmm9
14224 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
14225 ; AVX512-NEXT: vpshufb %xmm3, %xmm14, %xmm3
14226 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
14227 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
14228 ; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
14229 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
14230 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
14231 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
14232 ; AVX512-NEXT: vpshufb %xmm0, %xmm7, %xmm3
14233 ; AVX512-NEXT: vpshufb %xmm0, %xmm6, %xmm0
14234 ; AVX512-NEXT: vmovdqa64 %xmm6, %xmm25
14235 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
14236 ; AVX512-NEXT: vmovdqa64 %xmm29, %xmm3
14237 ; AVX512-NEXT: vpshufb %xmm1, %xmm3, %xmm3
14238 ; AVX512-NEXT: vmovdqa (%rsp), %xmm4 # 16-byte Reload
14239 ; AVX512-NEXT: vpshufb %xmm1, %xmm4, %xmm1
14240 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
14241 ; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
14242 ; AVX512-NEXT: vpsrlq $32, %zmm16, %zmm1
14243 ; AVX512-NEXT: vmovdqa64 %zmm16, %zmm18
14244 ; AVX512-NEXT: vpmovqb %zmm1, %xmm1
14245 ; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
14246 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
14247 ; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm0
14248 ; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
14249 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
14250 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
14251 ; AVX512-NEXT: vpshufb %xmm0, %xmm1, %xmm1
14252 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
14253 ; AVX512-NEXT: vpshufb %xmm0, %xmm2, %xmm2
14254 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
14255 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
14256 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
14257 ; AVX512-NEXT: vpshufb %xmm1, %xmm3, %xmm3
14258 ; AVX512-NEXT: vpshufb %xmm1, %xmm10, %xmm4
14259 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
14260 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
14261 ; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
14262 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
14263 ; AVX512-NEXT: vmovd {{.*#+}} xmm2 = [0,0,5,13,0,0,0,0,0,0,0,0,0,0,0,0]
14264 ; AVX512-NEXT: vpshufb %xmm2, %xmm11, %xmm3
14265 ; AVX512-NEXT: vmovdqa64 %xmm24, %xmm27
14266 ; AVX512-NEXT: vmovdqa64 %xmm28, %xmm5
14267 ; AVX512-NEXT: vpshufb %xmm2, %xmm5, %xmm8
14268 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
14269 ; AVX512-NEXT: vmovd {{.*#+}} xmm3 = [5,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
14270 ; AVX512-NEXT: vmovdqa64 %xmm30, %xmm10
14271 ; AVX512-NEXT: vpshufb %xmm3, %xmm10, %xmm9
14272 ; AVX512-NEXT: vpshufb %xmm3, %xmm12, %xmm15
14273 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
14274 ; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
14275 ; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
14276 ; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
14277 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
14278 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
14279 ; AVX512-NEXT: vpshufb %xmm0, %xmm6, %xmm8
14280 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
14281 ; AVX512-NEXT: vpshufb %xmm0, %xmm6, %xmm9
14282 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
14283 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
14284 ; AVX512-NEXT: vpshufb %xmm1, %xmm6, %xmm9
14285 ; AVX512-NEXT: vmovdqa64 %xmm21, %xmm11
14286 ; AVX512-NEXT: vpshufb %xmm1, %xmm11, %xmm15
14287 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
14288 ; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
14289 ; AVX512-NEXT: vpsrlq $40, %zmm17, %zmm9
14290 ; AVX512-NEXT: vmovdqa64 %zmm17, %zmm23
14291 ; AVX512-NEXT: vpmovqb %zmm9, %xmm9
14292 ; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
14293 ; AVX512-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
14294 ; AVX512-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
14295 ; AVX512-NEXT: vpshufb %xmm0, %xmm13, %xmm4
14296 ; AVX512-NEXT: vmovdqa64 %xmm13, %xmm30
14297 ; AVX512-NEXT: vmovdqa64 %xmm22, %xmm6
14298 ; AVX512-NEXT: vpshufb %xmm0, %xmm6, %xmm9
14299 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
14300 ; AVX512-NEXT: vmovdqa64 %xmm26, %xmm13
14301 ; AVX512-NEXT: vpshufb %xmm1, %xmm13, %xmm9
14302 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
14303 ; AVX512-NEXT: vpshufb %xmm1, %xmm6, %xmm15
14304 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
14305 ; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
14306 ; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
14307 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
14308 ; AVX512-NEXT: vmovdqa64 %xmm20, %xmm9
14309 ; AVX512-NEXT: vpshufb %xmm2, %xmm9, %xmm9
14310 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
14311 ; AVX512-NEXT: vpshufb %xmm2, %xmm12, %xmm2
14312 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
14313 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
14314 ; AVX512-NEXT: vpshufb %xmm3, %xmm15, %xmm9
14315 ; AVX512-NEXT: vmovdqa64 %xmm15, %xmm28
14316 ; AVX512-NEXT: vpshufb %xmm3, %xmm14, %xmm3
14317 ; AVX512-NEXT: vmovdqa64 %xmm14, %xmm16
14318 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
14319 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
14320 ; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
14321 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
14322 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
14323 ; AVX512-NEXT: vpshufb %xmm0, %xmm7, %xmm3
14324 ; AVX512-NEXT: vmovdqa64 %xmm7, %xmm24
14325 ; AVX512-NEXT: vmovdqa64 %xmm25, %xmm4
14326 ; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm0
14327 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
14328 ; AVX512-NEXT: vmovdqa64 %xmm29, %xmm7
14329 ; AVX512-NEXT: vpshufb %xmm1, %xmm7, %xmm3
14330 ; AVX512-NEXT: vmovdqa (%rsp), %xmm4 # 16-byte Reload
14331 ; AVX512-NEXT: vpshufb %xmm1, %xmm4, %xmm1
14332 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
14333 ; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
14334 ; AVX512-NEXT: vpsrlq $40, %zmm18, %zmm1
14335 ; AVX512-NEXT: vmovdqa64 %zmm18, %zmm26
14336 ; AVX512-NEXT: vpmovqb %zmm1, %xmm1
14337 ; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
14338 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
14339 ; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm0
14340 ; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
14341 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
14342 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
14343 ; AVX512-NEXT: vpshufb %xmm0, %xmm2, %xmm1
14344 ; AVX512-NEXT: vmovdqa64 %xmm2, %xmm29
14345 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
14346 ; AVX512-NEXT: vpshufb %xmm0, %xmm2, %xmm2
14347 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
14348 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
14349 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
14350 ; AVX512-NEXT: vpshufb %xmm1, %xmm3, %xmm3
14351 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
14352 ; AVX512-NEXT: vpshufb %xmm1, %xmm4, %xmm4
14353 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
14354 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
14355 ; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
14356 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
14357 ; AVX512-NEXT: vmovd {{.*#+}} xmm2 = [0,0,6,14,0,0,0,0,0,0,0,0,0,0,0,0]
14358 ; AVX512-NEXT: vmovdqa64 %xmm27, %xmm3
14359 ; AVX512-NEXT: vpshufb %xmm2, %xmm3, %xmm3
14360 ; AVX512-NEXT: vpshufb %xmm2, %xmm5, %xmm8
14361 ; AVX512-NEXT: vmovdqa64 %xmm5, %xmm21
14362 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
14363 ; AVX512-NEXT: vmovd {{.*#+}} xmm3 = [6,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
14364 ; AVX512-NEXT: vpshufb %xmm3, %xmm10, %xmm9
14365 ; AVX512-NEXT: vmovdqa64 %xmm10, %xmm18
14366 ; AVX512-NEXT: vmovdqa64 %xmm31, %xmm5
14367 ; AVX512-NEXT: vpshufb %xmm3, %xmm5, %xmm15
14368 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
14369 ; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
14370 ; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
14371 ; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
14372 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
14373 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
14374 ; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm8
14375 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
14376 ; AVX512-NEXT: vpshufb %xmm0, %xmm9, %xmm9
14377 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
14378 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
14379 ; AVX512-NEXT: vpshufb %xmm1, %xmm10, %xmm9
14380 ; AVX512-NEXT: vpshufb %xmm1, %xmm11, %xmm15
14381 ; AVX512-NEXT: vmovdqa64 %xmm11, %xmm22
14382 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
14383 ; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
14384 ; AVX512-NEXT: vpsrlq $48, %zmm17, %zmm9
14385 ; AVX512-NEXT: vpmovqb %zmm9, %xmm9
14386 ; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
14387 ; AVX512-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
14388 ; AVX512-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
14389 ; AVX512-NEXT: vmovdqa64 %xmm30, %xmm4
14390 ; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm4
14391 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
14392 ; AVX512-NEXT: vpshufb %xmm0, %xmm9, %xmm9
14393 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
14394 ; AVX512-NEXT: vpshufb %xmm1, %xmm13, %xmm9
14395 ; AVX512-NEXT: vmovdqa64 %xmm13, %xmm19
14396 ; AVX512-NEXT: vpshufb %xmm1, %xmm6, %xmm15
14397 ; AVX512-NEXT: vmovdqa64 %xmm6, %xmm17
14398 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
14399 ; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
14400 ; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
14401 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
14402 ; AVX512-NEXT: vmovdqa64 %xmm20, %xmm6
14403 ; AVX512-NEXT: vpshufb %xmm2, %xmm6, %xmm9
14404 ; AVX512-NEXT: vpshufb %xmm2, %xmm12, %xmm2
14405 ; AVX512-NEXT: vmovdqa64 %xmm12, %xmm20
14406 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
14407 ; AVX512-NEXT: vmovdqa64 %xmm28, %xmm14
14408 ; AVX512-NEXT: vpshufb %xmm3, %xmm14, %xmm9
14409 ; AVX512-NEXT: vmovdqa64 %xmm16, %xmm13
14410 ; AVX512-NEXT: vpshufb %xmm3, %xmm13, %xmm3
14411 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
14412 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
14413 ; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
14414 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
14415 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
14416 ; AVX512-NEXT: vmovdqa64 %xmm24, %xmm15
14417 ; AVX512-NEXT: vpshufb %xmm0, %xmm15, %xmm3
14418 ; AVX512-NEXT: vmovdqa64 %xmm25, %xmm12
14419 ; AVX512-NEXT: vpshufb %xmm0, %xmm12, %xmm0
14420 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
14421 ; AVX512-NEXT: vpshufb %xmm1, %xmm7, %xmm3
14422 ; AVX512-NEXT: vmovdqa64 %xmm7, %xmm16
14423 ; AVX512-NEXT: vmovdqa (%rsp), %xmm7 # 16-byte Reload
14424 ; AVX512-NEXT: vpshufb %xmm1, %xmm7, %xmm1
14425 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
14426 ; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
14427 ; AVX512-NEXT: vpsrlq $48, %zmm26, %zmm1
14428 ; AVX512-NEXT: vpmovqb %zmm1, %xmm1
14429 ; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
14430 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
14431 ; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm24
14432 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
14433 ; AVX512-NEXT: vmovdqa64 %xmm29, %xmm1
14434 ; AVX512-NEXT: vpshufb %xmm0, %xmm1, %xmm1
14435 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
14436 ; AVX512-NEXT: vpshufb %xmm0, %xmm2, %xmm2
14437 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
14438 ; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
14439 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
14440 ; AVX512-NEXT: vpshufb %xmm1, %xmm3, %xmm3
14441 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
14442 ; AVX512-NEXT: vpshufb %xmm1, %xmm4, %xmm4
14443 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
14444 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
14445 ; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
14446 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
14447 ; AVX512-NEXT: vmovd {{.*#+}} xmm2 = [0,0,7,15,0,0,0,0,0,0,0,0,0,0,0,0]
14448 ; AVX512-NEXT: vmovdqa64 %xmm27, %xmm3
14449 ; AVX512-NEXT: vpshufb %xmm2, %xmm3, %xmm3
14450 ; AVX512-NEXT: vmovdqa64 %xmm21, %xmm8
14451 ; AVX512-NEXT: vpshufb %xmm2, %xmm8, %xmm8
14452 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
14453 ; AVX512-NEXT: vmovd {{.*#+}} xmm3 = [7,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
14454 ; AVX512-NEXT: vmovdqa64 %xmm18, %xmm9
14455 ; AVX512-NEXT: vpshufb %xmm3, %xmm9, %xmm9
14456 ; AVX512-NEXT: vmovdqa64 %xmm31, %xmm11
14457 ; AVX512-NEXT: vpshufb %xmm3, %xmm11, %xmm11
14458 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
14459 ; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
14460 ; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
14461 ; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
14462 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
14463 ; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm8
14464 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
14465 ; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm9
14466 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
14467 ; AVX512-NEXT: vpshufb %xmm1, %xmm10, %xmm9
14468 ; AVX512-NEXT: vmovdqa64 %xmm22, %xmm5
14469 ; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm11
14470 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
14471 ; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
14472 ; AVX512-NEXT: vpsrlq $56, %zmm23, %zmm9
14473 ; AVX512-NEXT: vpmovqb %zmm9, %xmm9
14474 ; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
14475 ; AVX512-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
14476 ; AVX512-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
14477 ; AVX512-NEXT: vmovdqa64 %xmm30, %xmm4
14478 ; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm4
14479 ; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
14480 ; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm9
14481 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
14482 ; AVX512-NEXT: vmovdqa64 %xmm19, %xmm5
14483 ; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm9
14484 ; AVX512-NEXT: vmovdqa64 %xmm17, %xmm5
14485 ; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm11
14486 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
14487 ; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
14488 ; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
14489 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
14490 ; AVX512-NEXT: vpshufb %xmm2, %xmm6, %xmm6
14491 ; AVX512-NEXT: vmovdqa64 %xmm20, %xmm5
14492 ; AVX512-NEXT: vpshufb %xmm2, %xmm5, %xmm2
14493 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3]
14494 ; AVX512-NEXT: vpshufb %xmm3, %xmm14, %xmm5
14495 ; AVX512-NEXT: vpshufb %xmm3, %xmm13, %xmm3
14496 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
14497 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
14498 ; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
14499 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
14500 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
14501 ; AVX512-NEXT: vpshufb %xmm0, %xmm15, %xmm3
14502 ; AVX512-NEXT: vpshufb %xmm0, %xmm12, %xmm0
14503 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
14504 ; AVX512-NEXT: vmovdqa64 %xmm16, %xmm3
14505 ; AVX512-NEXT: vpshufb %xmm1, %xmm3, %xmm3
14506 ; AVX512-NEXT: vpshufb %xmm1, %xmm7, %xmm1
14507 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
14508 ; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
14509 ; AVX512-NEXT: vpsrlq $56, %zmm26, %zmm1
14510 ; AVX512-NEXT: vpmovqb %zmm1, %xmm1
14511 ; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
14512 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
14513 ; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm0
14514 ; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
14515 ; AVX512-NEXT: vmovaps %zmm1, (%rsi)
14516 ; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
14517 ; AVX512-NEXT: vmovaps %zmm1, (%rdx)
14518 ; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
14519 ; AVX512-NEXT: vmovaps %zmm1, (%rcx)
14520 ; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
14521 ; AVX512-NEXT: vmovaps %zmm1, (%r8)
14522 ; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
14523 ; AVX512-NEXT: vmovaps %zmm1, (%r9)
14524 ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
14525 ; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
14526 ; AVX512-NEXT: vmovaps %zmm1, (%rax)
14527 ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
14528 ; AVX512-NEXT: vmovdqa64 %zmm24, (%rax)
14529 ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
14530 ; AVX512-NEXT: vmovdqa64 %zmm0, (%rax)
14531 ; AVX512-NEXT: addq $520, %rsp # imm = 0x208
14532 ; AVX512-NEXT: vzeroupper
14533 ; AVX512-NEXT: retq
14534 ;
14535 ; AVX512-FCP-LABEL: load_i8_stride8_vf64:
14536 ; AVX512-FCP: # %bb.0:
14537 ; AVX512-FCP-NEXT: subq $440, %rsp # imm = 0x1B8
14538 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm14 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
14539 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,2,2,3,0,2,4,6]
14540 ; AVX512-FCP-NEXT: vmovdqa 480(%rdi), %ymm1
14541 ; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
14542 ; AVX512-FCP-NEXT: vpermd %ymm1, %ymm0, %ymm3
14543 ; AVX512-FCP-NEXT: vpshufb %ymm14, %ymm3, %ymm1
14544 ; AVX512-FCP-NEXT: vmovdqa64 %ymm3, %ymm19
14545 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm9 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
14546 ; AVX512-FCP-NEXT: vmovdqa 448(%rdi), %ymm2
14547 ; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
14548 ; AVX512-FCP-NEXT: vpermd %ymm2, %ymm0, %ymm4
14549 ; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm4, %ymm2
14550 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
14551 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm11 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
14552 ; AVX512-FCP-NEXT: vmovdqa 416(%rdi), %ymm3
14553 ; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
14554 ; AVX512-FCP-NEXT: vpermd %ymm3, %ymm0, %ymm12
14555 ; AVX512-FCP-NEXT: vpshufb %ymm11, %ymm12, %ymm3
14556 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm10 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
14557 ; AVX512-FCP-NEXT: vmovdqa 384(%rdi), %ymm5
14558 ; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
14559 ; AVX512-FCP-NEXT: vpermd %ymm5, %ymm0, %ymm13
14560 ; AVX512-FCP-NEXT: vpshufb %ymm10, %ymm13, %ymm5
14561 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3,4],ymm3[5],ymm5[6,7]
14562 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
14563 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm3 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
14564 ; AVX512-FCP-NEXT: vmovdqa 368(%rdi), %xmm2
14565 ; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm5
14566 ; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm30
14567 ; AVX512-FCP-NEXT: vmovdqa 352(%rdi), %xmm2
14568 ; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm6
14569 ; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm25
14570 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
14571 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm5 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
14572 ; AVX512-FCP-NEXT: vmovdqa 336(%rdi), %xmm2
14573 ; AVX512-FCP-NEXT: vpshufb %xmm5, %xmm2, %xmm7
14574 ; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm24
14575 ; AVX512-FCP-NEXT: vmovdqa 320(%rdi), %xmm15
14576 ; AVX512-FCP-NEXT: vpshufb %xmm5, %xmm15, %xmm8
14577 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
14578 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1,2],xmm6[3]
14579 ; AVX512-FCP-NEXT: vmovdqa64 256(%rdi), %zmm28
14580 ; AVX512-FCP-NEXT: vpmovqb %zmm28, %xmm7
14581 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3]
14582 ; AVX512-FCP-NEXT: vinserti32x4 $2, %xmm6, %zmm0, %zmm16
14583 ; AVX512-FCP-NEXT: movb $-64, %al
14584 ; AVX512-FCP-NEXT: kmovw %eax, %k1
14585 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm16 {%k1}
14586 ; AVX512-FCP-NEXT: vmovdqa 224(%rdi), %ymm1
14587 ; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
14588 ; AVX512-FCP-NEXT: vpermd %ymm1, %ymm0, %ymm2
14589 ; AVX512-FCP-NEXT: vpshufb %ymm14, %ymm2, %ymm1
14590 ; AVX512-FCP-NEXT: vmovdqa64 %ymm2, %ymm23
14591 ; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm2
14592 ; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
14593 ; AVX512-FCP-NEXT: vpermd %ymm2, %ymm0, %ymm2
14594 ; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm2, %ymm6
14595 ; AVX512-FCP-NEXT: vmovdqa64 %ymm2, %ymm27
14596 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5,6],ymm1[7]
14597 ; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %ymm1
14598 ; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
14599 ; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm2
14600 ; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
14601 ; AVX512-FCP-NEXT: vpermd %ymm1, %ymm0, %ymm1
14602 ; AVX512-FCP-NEXT: vpshufb %ymm11, %ymm1, %ymm7
14603 ; AVX512-FCP-NEXT: vmovdqa64 %ymm1, %ymm18
14604 ; AVX512-FCP-NEXT: vpermd %ymm2, %ymm0, %ymm0
14605 ; AVX512-FCP-NEXT: vpshufb %ymm10, %ymm0, %ymm8
14606 ; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm17
14607 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3,4],ymm7[5],ymm8[6,7]
14608 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm7[0,1,2,3,4,5],ymm6[6,7]
14609 ; AVX512-FCP-NEXT: vmovdqa 112(%rdi), %xmm10
14610 ; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm10, %xmm7
14611 ; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %xmm0
14612 ; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm0, %xmm3
14613 ; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm20
14614 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm7[0],xmm3[1],xmm7[1],xmm3[2],xmm7[2],xmm3[3],xmm7[3]
14615 ; AVX512-FCP-NEXT: vmovdqa 80(%rdi), %xmm2
14616 ; AVX512-FCP-NEXT: vpshufb %xmm5, %xmm2, %xmm0
14617 ; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm31
14618 ; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %xmm2
14619 ; AVX512-FCP-NEXT: vpshufb %xmm5, %xmm2, %xmm5
14620 ; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm22
14621 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
14622 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[3]
14623 ; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm29
14624 ; AVX512-FCP-NEXT: vpmovqb %zmm29, %xmm3
14625 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
14626 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
14627 ; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm16, %zmm0
14628 ; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
14629 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm6 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
14630 ; AVX512-FCP-NEXT: vmovdqa64 %ymm19, %ymm11
14631 ; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm11, %ymm0
14632 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm7 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
14633 ; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm4, %ymm1
14634 ; AVX512-FCP-NEXT: vmovdqa64 %ymm4, %ymm26
14635 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6],ymm0[7]
14636 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u,u,u,u,u]
14637 ; AVX512-FCP-NEXT: vmovdqa64 %ymm12, %ymm19
14638 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm2 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
14639 ; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm13, %ymm3
14640 ; AVX512-FCP-NEXT: vmovdqa64 %ymm13, %ymm21
14641 ; AVX512-FCP-NEXT: vmovdqa64 %ymm2, %ymm16
14642 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4],ymm1[5],ymm3[6,7]
14643 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3,4,5],ymm0[6,7]
14644 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm1 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
14645 ; AVX512-FCP-NEXT: vmovdqa64 %xmm30, %xmm13
14646 ; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm13, %xmm3
14647 ; AVX512-FCP-NEXT: vmovdqa64 %xmm25, %xmm14
14648 ; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm14, %xmm5
14649 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
14650 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm5 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
14651 ; AVX512-FCP-NEXT: vmovdqa64 %xmm24, %xmm9
14652 ; AVX512-FCP-NEXT: vpshufb %xmm5, %xmm9, %xmm0
14653 ; AVX512-FCP-NEXT: vmovdqa %xmm15, %xmm12
14654 ; AVX512-FCP-NEXT: vpshufb %xmm5, %xmm15, %xmm15
14655 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
14656 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[3]
14657 ; AVX512-FCP-NEXT: vpsrlq $8, %zmm28, %zmm3
14658 ; AVX512-FCP-NEXT: vpmovqb %zmm3, %xmm3
14659 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
14660 ; AVX512-FCP-NEXT: vinserti32x4 $2, %xmm0, %zmm0, %zmm0
14661 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 {%k1}
14662 ; AVX512-FCP-NEXT: vmovdqa64 %ymm23, %ymm8
14663 ; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm8, %ymm2
14664 ; AVX512-FCP-NEXT: vmovdqa64 %ymm27, %ymm4
14665 ; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm4, %ymm3
14666 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
14667 ; AVX512-FCP-NEXT: vmovdqa64 %ymm18, %ymm7
14668 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u,u,u,u,u]
14669 ; AVX512-FCP-NEXT: vmovdqa64 %ymm17, %ymm6
14670 ; AVX512-FCP-NEXT: vmovdqa64 %ymm16, %ymm15
14671 ; AVX512-FCP-NEXT: vpshufb %ymm15, %ymm6, %ymm15
14672 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm15[0,1,2,3,4],ymm3[5],ymm15[6,7]
14673 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
14674 ; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm10, %xmm3
14675 ; AVX512-FCP-NEXT: vmovdqa64 %xmm10, %xmm16
14676 ; AVX512-FCP-NEXT: vmovdqa64 %xmm20, %xmm10
14677 ; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm10, %xmm1
14678 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
14679 ; AVX512-FCP-NEXT: vmovdqa64 %xmm31, %xmm15
14680 ; AVX512-FCP-NEXT: vpshufb %xmm5, %xmm15, %xmm3
14681 ; AVX512-FCP-NEXT: vmovdqa64 %xmm22, %xmm15
14682 ; AVX512-FCP-NEXT: vpshufb %xmm5, %xmm15, %xmm5
14683 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
14684 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
14685 ; AVX512-FCP-NEXT: vpsrlq $8, %zmm29, %zmm3
14686 ; AVX512-FCP-NEXT: vpmovqb %zmm3, %xmm3
14687 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
14688 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
14689 ; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm0
14690 ; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
14691 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm1 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
14692 ; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm11, %ymm0
14693 ; AVX512-FCP-NEXT: vmovdqa64 %ymm11, %ymm20
14694 ; AVX512-FCP-NEXT: vmovdqa64 %ymm1, %ymm23
14695 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm2 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
14696 ; AVX512-FCP-NEXT: vmovdqa64 %ymm26, %ymm11
14697 ; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm11, %ymm1
14698 ; AVX512-FCP-NEXT: vmovdqa64 %ymm2, %ymm24
14699 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6],ymm0[7]
14700 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm2 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
14701 ; AVX512-FCP-NEXT: vmovdqa64 %ymm19, %ymm1
14702 ; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm1, %ymm1
14703 ; AVX512-FCP-NEXT: vmovdqa64 %ymm2, %ymm25
14704 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm3 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
14705 ; AVX512-FCP-NEXT: vmovdqa64 %ymm21, %ymm2
14706 ; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm2, %ymm2
14707 ; AVX512-FCP-NEXT: vmovdqa64 %ymm3, %ymm26
14708 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4],ymm1[5],ymm2[6,7]
14709 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
14710 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm1 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
14711 ; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm13, %xmm2
14712 ; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm14, %xmm3
14713 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
14714 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm3 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
14715 ; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm9, %xmm5
14716 ; AVX512-FCP-NEXT: vmovdqa64 %xmm9, %xmm17
14717 ; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm12, %xmm15
14718 ; AVX512-FCP-NEXT: vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
14719 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm15[0],xmm5[0],xmm15[1],xmm5[1],xmm15[2],xmm5[2],xmm15[3],xmm5[3]
14720 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1,2],xmm2[3]
14721 ; AVX512-FCP-NEXT: vpsrlq $16, %zmm28, %zmm5
14722 ; AVX512-FCP-NEXT: vpmovqb %zmm5, %xmm5
14723 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
14724 ; AVX512-FCP-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm2
14725 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm2 {%k1}
14726 ; AVX512-FCP-NEXT: vmovdqa64 %ymm23, %ymm0
14727 ; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm8, %ymm0
14728 ; AVX512-FCP-NEXT: vmovdqa %ymm8, %ymm9
14729 ; AVX512-FCP-NEXT: vmovdqa64 %ymm24, %ymm5
14730 ; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm4, %ymm5
14731 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3,4,5,6],ymm0[7]
14732 ; AVX512-FCP-NEXT: vmovdqa64 %ymm25, %ymm4
14733 ; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm7, %ymm5
14734 ; AVX512-FCP-NEXT: vmovdqa64 %ymm18, %ymm25
14735 ; AVX512-FCP-NEXT: vmovdqa64 %ymm26, %ymm4
14736 ; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm6, %ymm15
14737 ; AVX512-FCP-NEXT: vmovdqa64 %ymm6, %ymm26
14738 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm15[0,1,2,3,4],ymm5[5],ymm15[6,7]
14739 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3,4,5],ymm0[6,7]
14740 ; AVX512-FCP-NEXT: vmovdqa64 %xmm16, %xmm6
14741 ; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm6, %xmm5
14742 ; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm10, %xmm1
14743 ; AVX512-FCP-NEXT: vmovdqa64 %xmm10, %xmm16
14744 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
14745 ; AVX512-FCP-NEXT: vmovdqa64 %xmm31, %xmm7
14746 ; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm7, %xmm5
14747 ; AVX512-FCP-NEXT: vmovdqa64 %xmm22, %xmm8
14748 ; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm8, %xmm3
14749 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
14750 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
14751 ; AVX512-FCP-NEXT: vpsrlq $16, %zmm29, %zmm3
14752 ; AVX512-FCP-NEXT: vpmovqb %zmm3, %xmm3
14753 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
14754 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
14755 ; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm0
14756 ; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
14757 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm1 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
14758 ; AVX512-FCP-NEXT: vmovdqa64 %ymm20, %ymm0
14759 ; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm0, %ymm0
14760 ; AVX512-FCP-NEXT: vmovdqa64 %ymm1, %ymm18
14761 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm2 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
14762 ; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm11, %ymm1
14763 ; AVX512-FCP-NEXT: vmovdqa %ymm2, %ymm11
14764 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6],ymm0[7]
14765 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm10 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
14766 ; AVX512-FCP-NEXT: vmovdqa64 %ymm19, %ymm1
14767 ; AVX512-FCP-NEXT: vpshufb %ymm10, %ymm1, %ymm1
14768 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm4 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
14769 ; AVX512-FCP-NEXT: vmovdqa64 %ymm21, %ymm2
14770 ; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm2, %ymm2
14771 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4],ymm1[5],ymm2[6,7]
14772 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
14773 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm1 = [0,0,3,11,0,0,3,11,0,0,3,11,0,0,3,11]
14774 ; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm13, %xmm2
14775 ; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm14, %xmm3
14776 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
14777 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm3 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
14778 ; AVX512-FCP-NEXT: vmovdqa64 %xmm17, %xmm15
14779 ; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm15, %xmm5
14780 ; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm12, %xmm12
14781 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm12[0],xmm5[0],xmm12[1],xmm5[1],xmm12[2],xmm5[2],xmm12[3],xmm5[3]
14782 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1,2],xmm2[3]
14783 ; AVX512-FCP-NEXT: vpsrlq $24, %zmm28, %zmm5
14784 ; AVX512-FCP-NEXT: vmovdqa64 %zmm28, %zmm24
14785 ; AVX512-FCP-NEXT: vpmovqb %zmm5, %xmm5
14786 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
14787 ; AVX512-FCP-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm2
14788 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm2 {%k1}
14789 ; AVX512-FCP-NEXT: vmovdqa64 %ymm18, %ymm0
14790 ; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm9, %ymm0
14791 ; AVX512-FCP-NEXT: vmovdqa64 %ymm27, %ymm5
14792 ; AVX512-FCP-NEXT: vpshufb %ymm11, %ymm5, %ymm5
14793 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3,4,5,6],ymm0[7]
14794 ; AVX512-FCP-NEXT: vmovdqa64 %ymm25, %ymm5
14795 ; AVX512-FCP-NEXT: vpshufb %ymm10, %ymm5, %ymm5
14796 ; AVX512-FCP-NEXT: vmovdqa64 %ymm26, %ymm9
14797 ; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm9, %ymm4
14798 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4],ymm5[5],ymm4[6,7]
14799 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
14800 ; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm6, %xmm4
14801 ; AVX512-FCP-NEXT: vmovdqa64 %xmm6, %xmm21
14802 ; AVX512-FCP-NEXT: vmovdqa64 %xmm16, %xmm5
14803 ; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm5, %xmm1
14804 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
14805 ; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm7, %xmm4
14806 ; AVX512-FCP-NEXT: vmovdqa64 %xmm31, %xmm16
14807 ; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm8, %xmm3
14808 ; AVX512-FCP-NEXT: vmovdqa64 %xmm22, %xmm6
14809 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
14810 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
14811 ; AVX512-FCP-NEXT: vpsrlq $24, %zmm29, %zmm3
14812 ; AVX512-FCP-NEXT: vpmovqb %zmm3, %xmm3
14813 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
14814 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
14815 ; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm0
14816 ; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
14817 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [1,3,2,3,1,3,5,7]
14818 ; AVX512-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
14819 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm17 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
14820 ; AVX512-FCP-NEXT: vmovdqa64 %ymm17, %ymm1
14821 ; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm2, %ymm1
14822 ; AVX512-FCP-NEXT: vmovdqa64 %ymm2, %ymm28
14823 ; AVX512-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 32-byte Folded Reload
14824 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm18 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
14825 ; AVX512-FCP-NEXT: vmovdqa64 %ymm18, %ymm2
14826 ; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm4, %ymm2
14827 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
14828 ; AVX512-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
14829 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm7 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
14830 ; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm2, %ymm9
14831 ; AVX512-FCP-NEXT: vmovdqa64 %ymm2, %ymm19
14832 ; AVX512-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
14833 ; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
14834 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm8 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
14835 ; AVX512-FCP-NEXT: vpshufb %ymm8, %ymm2, %ymm10
14836 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4],ymm9[5],ymm10[6,7]
14837 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm9[0,1,2,3,4,5],ymm1[6,7]
14838 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm11 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
14839 ; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm13, %xmm9
14840 ; AVX512-FCP-NEXT: vmovdqa64 %xmm30, %xmm20
14841 ; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm14, %xmm10
14842 ; AVX512-FCP-NEXT: vmovdqa64 %xmm14, %xmm22
14843 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
14844 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm12 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
14845 ; AVX512-FCP-NEXT: vpshufb %xmm12, %xmm15, %xmm10
14846 ; AVX512-FCP-NEXT: vmovdqa64 %xmm15, %xmm23
14847 ; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
14848 ; AVX512-FCP-NEXT: vpshufb %xmm12, %xmm2, %xmm13
14849 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm13[0],xmm10[0],xmm13[1],xmm10[1],xmm13[2],xmm10[2],xmm13[3],xmm10[3]
14850 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
14851 ; AVX512-FCP-NEXT: vpsrlq $32, %zmm24, %zmm10
14852 ; AVX512-FCP-NEXT: vmovdqa64 %zmm24, %zmm25
14853 ; AVX512-FCP-NEXT: vpmovqb %zmm10, %xmm10
14854 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
14855 ; AVX512-FCP-NEXT: vinserti32x4 $2, %xmm9, %zmm0, %zmm13
14856 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm13 {%k1}
14857 ; AVX512-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
14858 ; AVX512-FCP-NEXT: vmovdqa64 %ymm17, %ymm1
14859 ; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm2, %ymm1
14860 ; AVX512-FCP-NEXT: vmovdqa64 %ymm2, %ymm27
14861 ; AVX512-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
14862 ; AVX512-FCP-NEXT: vmovdqa64 %ymm18, %ymm2
14863 ; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm3, %ymm15
14864 ; AVX512-FCP-NEXT: vmovdqa64 %ymm3, %ymm26
14865 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm15 = ymm15[0,1,2,3,4,5,6],ymm1[7]
14866 ; AVX512-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
14867 ; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm2, %ymm1
14868 ; AVX512-FCP-NEXT: vmovdqa64 %ymm2, %ymm31
14869 ; AVX512-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
14870 ; AVX512-FCP-NEXT: vpshufb %ymm8, %ymm0, %ymm14
14871 ; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm30
14872 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm14[0,1,2,3,4],ymm1[5],ymm14[6,7]
14873 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm15[6,7]
14874 ; AVX512-FCP-NEXT: vmovdqa64 %xmm21, %xmm10
14875 ; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm10, %xmm14
14876 ; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm5, %xmm11
14877 ; AVX512-FCP-NEXT: vmovdqa64 %xmm5, %xmm24
14878 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm14[0],xmm11[1],xmm14[1],xmm11[2],xmm14[2],xmm11[3],xmm14[3]
14879 ; AVX512-FCP-NEXT: vmovdqa64 %xmm16, %xmm8
14880 ; AVX512-FCP-NEXT: vpshufb %xmm12, %xmm8, %xmm14
14881 ; AVX512-FCP-NEXT: vpshufb %xmm12, %xmm6, %xmm12
14882 ; AVX512-FCP-NEXT: vmovdqa64 %xmm6, %xmm21
14883 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm14[0],xmm12[1],xmm14[1],xmm12[2],xmm14[2],xmm12[3],xmm14[3]
14884 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1,2],xmm11[3]
14885 ; AVX512-FCP-NEXT: vpsrlq $32, %zmm29, %zmm12
14886 ; AVX512-FCP-NEXT: vpmovqb %zmm12, %xmm12
14887 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1],xmm11[2,3]
14888 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3],ymm1[4,5,6,7]
14889 ; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm13, %zmm0
14890 ; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
14891 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm3 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
14892 ; AVX512-FCP-NEXT: vmovdqa64 %ymm28, %ymm0
14893 ; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm0, %ymm1
14894 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm5 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
14895 ; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm4, %ymm11
14896 ; AVX512-FCP-NEXT: vmovdqa64 %ymm4, %ymm16
14897 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3,4,5,6],ymm1[7]
14898 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm6 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
14899 ; AVX512-FCP-NEXT: vmovdqa64 %ymm19, %ymm0
14900 ; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm11
14901 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm18 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
14902 ; AVX512-FCP-NEXT: vmovdqa64 %ymm18, %ymm7
14903 ; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
14904 ; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm2, %ymm12
14905 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2,3,4],ymm11[5],ymm12[6,7]
14906 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3,4,5],ymm1[6,7]
14907 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm11 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
14908 ; AVX512-FCP-NEXT: vmovdqa64 %xmm20, %xmm2
14909 ; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm2, %xmm12
14910 ; AVX512-FCP-NEXT: vmovdqa64 %xmm20, %xmm19
14911 ; AVX512-FCP-NEXT: vmovdqa64 %xmm22, %xmm4
14912 ; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm4, %xmm13
14913 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
14914 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm13 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
14915 ; AVX512-FCP-NEXT: vmovdqa64 %xmm23, %xmm9
14916 ; AVX512-FCP-NEXT: vpshufb %xmm13, %xmm9, %xmm14
14917 ; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
14918 ; AVX512-FCP-NEXT: vpshufb %xmm13, %xmm7, %xmm15
14919 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
14920 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm14[0,1,2],xmm12[3]
14921 ; AVX512-FCP-NEXT: vpsrlq $40, %zmm25, %zmm14
14922 ; AVX512-FCP-NEXT: vpmovqb %zmm14, %xmm14
14923 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm14[0,1],xmm12[2,3]
14924 ; AVX512-FCP-NEXT: vinserti32x4 $2, %xmm12, %zmm0, %zmm12
14925 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm12 {%k1}
14926 ; AVX512-FCP-NEXT: vmovdqa64 %ymm27, %ymm1
14927 ; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm1, %ymm1
14928 ; AVX512-FCP-NEXT: vmovdqa64 %ymm26, %ymm2
14929 ; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm2, %ymm14
14930 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm14[0,1,2,3,4,5,6],ymm1[7]
14931 ; AVX512-FCP-NEXT: vmovdqa64 %ymm31, %ymm3
14932 ; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm14
14933 ; AVX512-FCP-NEXT: vmovdqa64 %ymm18, %ymm3
14934 ; AVX512-FCP-NEXT: vmovdqa64 %ymm30, %ymm2
14935 ; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm2, %ymm15
14936 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
14937 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm14[0,1,2,3,4,5],ymm1[6,7]
14938 ; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm10, %xmm14
14939 ; AVX512-FCP-NEXT: vmovdqa64 %xmm10, %xmm20
14940 ; AVX512-FCP-NEXT: vmovdqa64 %xmm24, %xmm2
14941 ; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm2, %xmm11
14942 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm14[0],xmm11[1],xmm14[1],xmm11[2],xmm14[2],xmm11[3],xmm14[3]
14943 ; AVX512-FCP-NEXT: vpshufb %xmm13, %xmm8, %xmm14
14944 ; AVX512-FCP-NEXT: vmovdqa64 %xmm8, %xmm22
14945 ; AVX512-FCP-NEXT: vmovdqa64 %xmm21, %xmm2
14946 ; AVX512-FCP-NEXT: vpshufb %xmm13, %xmm2, %xmm13
14947 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
14948 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm13[0,1,2],xmm11[3]
14949 ; AVX512-FCP-NEXT: vpsrlq $40, %zmm29, %zmm13
14950 ; AVX512-FCP-NEXT: vmovdqa64 %zmm29, %zmm18
14951 ; AVX512-FCP-NEXT: vpmovqb %zmm13, %xmm13
14952 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm13[0,1],xmm11[2,3]
14953 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3],ymm1[4,5,6,7]
14954 ; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm12, %zmm29
14955 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm3 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
14956 ; AVX512-FCP-NEXT: vmovdqa64 %ymm28, %ymm10
14957 ; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm10, %ymm1
14958 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm21 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
14959 ; AVX512-FCP-NEXT: vmovdqa64 %ymm16, %ymm5
14960 ; AVX512-FCP-NEXT: vmovdqa64 %ymm21, %ymm6
14961 ; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm5, %ymm11
14962 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3,4,5,6],ymm1[7]
14963 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm6 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
14964 ; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm11
14965 ; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm17
14966 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm8 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
14967 ; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
14968 ; AVX512-FCP-NEXT: vpshufb %ymm8, %ymm0, %ymm12
14969 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2,3,4],ymm11[5],ymm12[6,7]
14970 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3,4,5],ymm1[6,7]
14971 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm11 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
14972 ; AVX512-FCP-NEXT: vmovdqa64 %xmm19, %xmm0
14973 ; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm0, %xmm12
14974 ; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm4, %xmm13
14975 ; AVX512-FCP-NEXT: vmovdqa64 %xmm4, %xmm23
14976 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
14977 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm13 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
14978 ; AVX512-FCP-NEXT: vpshufb %xmm13, %xmm9, %xmm14
14979 ; AVX512-FCP-NEXT: vmovdqa64 %xmm9, %xmm28
14980 ; AVX512-FCP-NEXT: vpshufb %xmm13, %xmm7, %xmm15
14981 ; AVX512-FCP-NEXT: vmovdqa64 %xmm7, %xmm16
14982 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
14983 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm14[0,1,2],xmm12[3]
14984 ; AVX512-FCP-NEXT: vpsrlq $48, %zmm25, %zmm14
14985 ; AVX512-FCP-NEXT: vpmovqb %zmm14, %xmm14
14986 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm14[0,1],xmm12[2,3]
14987 ; AVX512-FCP-NEXT: vinserti32x4 $2, %xmm12, %zmm0, %zmm12
14988 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm12 {%k1}
14989 ; AVX512-FCP-NEXT: vmovdqa64 %ymm27, %ymm0
14990 ; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm0, %ymm1
14991 ; AVX512-FCP-NEXT: vmovdqa64 %ymm26, %ymm9
14992 ; AVX512-FCP-NEXT: vmovdqa64 %ymm21, %ymm3
14993 ; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm9, %ymm14
14994 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm14[0,1,2,3,4,5,6],ymm1[7]
14995 ; AVX512-FCP-NEXT: vmovdqa64 %ymm31, %ymm3
14996 ; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm14
14997 ; AVX512-FCP-NEXT: vmovdqa64 %ymm30, %ymm3
14998 ; AVX512-FCP-NEXT: vpshufb %ymm8, %ymm3, %ymm15
14999 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
15000 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm14[0,1,2,3,4,5],ymm1[6,7]
15001 ; AVX512-FCP-NEXT: vmovdqa64 %xmm20, %xmm8
15002 ; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm8, %xmm14
15003 ; AVX512-FCP-NEXT: vmovdqa64 %xmm24, %xmm15
15004 ; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm15, %xmm11
15005 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm14[0],xmm11[1],xmm14[1],xmm11[2],xmm14[2],xmm11[3],xmm14[3]
15006 ; AVX512-FCP-NEXT: vmovdqa64 %xmm22, %xmm3
15007 ; AVX512-FCP-NEXT: vpshufb %xmm13, %xmm3, %xmm14
15008 ; AVX512-FCP-NEXT: vpshufb %xmm13, %xmm2, %xmm13
15009 ; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm20
15010 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
15011 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm13[0,1,2],xmm11[3]
15012 ; AVX512-FCP-NEXT: vpsrlq $48, %zmm18, %zmm13
15013 ; AVX512-FCP-NEXT: vpmovqb %zmm13, %xmm13
15014 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm13[0,1],xmm11[2,3]
15015 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3],ymm1[4,5,6,7]
15016 ; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm12, %zmm21
15017 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm13 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
15018 ; AVX512-FCP-NEXT: vpshufb %ymm13, %ymm10, %ymm6
15019 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm14 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
15020 ; AVX512-FCP-NEXT: vpshufb %ymm14, %ymm5, %ymm5
15021 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm6[7]
15022 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm1 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
15023 ; AVX512-FCP-NEXT: vmovdqa64 %ymm17, %ymm2
15024 ; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm2, %ymm3
15025 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm7 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
15026 ; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
15027 ; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm2, %ymm4
15028 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4],ymm3[5],ymm4[6,7]
15029 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm5[6,7]
15030 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm4 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
15031 ; AVX512-FCP-NEXT: vmovdqa64 %xmm19, %xmm2
15032 ; AVX512-FCP-NEXT: vpshufb %xmm4, %xmm2, %xmm5
15033 ; AVX512-FCP-NEXT: vmovdqa64 %xmm23, %xmm2
15034 ; AVX512-FCP-NEXT: vpshufb %xmm4, %xmm2, %xmm6
15035 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
15036 ; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm6 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
15037 ; AVX512-FCP-NEXT: vmovdqa64 %xmm28, %xmm2
15038 ; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm2, %xmm11
15039 ; AVX512-FCP-NEXT: vmovdqa64 %xmm16, %xmm2
15040 ; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm2, %xmm12
15041 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
15042 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm5 = xmm11[0,1,2],xmm5[3]
15043 ; AVX512-FCP-NEXT: vpsrlq $56, %zmm25, %zmm11
15044 ; AVX512-FCP-NEXT: vpmovqb %zmm11, %xmm11
15045 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm5 = xmm11[0,1],xmm5[2,3]
15046 ; AVX512-FCP-NEXT: vinserti32x4 $2, %xmm5, %zmm0, %zmm5
15047 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm5 {%k1}
15048 ; AVX512-FCP-NEXT: vpshufb %ymm13, %ymm0, %ymm3
15049 ; AVX512-FCP-NEXT: vpshufb %ymm14, %ymm9, %ymm9
15050 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm9[0,1,2,3,4,5,6],ymm3[7]
15051 ; AVX512-FCP-NEXT: vmovdqa64 %ymm31, %ymm0
15052 ; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm0, %ymm2
15053 ; AVX512-FCP-NEXT: vmovdqa64 %ymm30, %ymm0
15054 ; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm0, %ymm0
15055 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5],ymm0[6,7]
15056 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm3[6,7]
15057 ; AVX512-FCP-NEXT: vpshufb %xmm4, %xmm8, %xmm2
15058 ; AVX512-FCP-NEXT: vpshufb %xmm4, %xmm15, %xmm3
15059 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
15060 ; AVX512-FCP-NEXT: vmovdqa64 %xmm22, %xmm1
15061 ; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm1, %xmm3
15062 ; AVX512-FCP-NEXT: vmovdqa64 %xmm20, %xmm1
15063 ; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm1, %xmm4
15064 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
15065 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3]
15066 ; AVX512-FCP-NEXT: vpsrlq $56, %zmm18, %zmm3
15067 ; AVX512-FCP-NEXT: vpmovqb %zmm3, %xmm3
15068 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]
15069 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
15070 ; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm5, %zmm0
15071 ; AVX512-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
15072 ; AVX512-FCP-NEXT: vmovaps %zmm1, (%rsi)
15073 ; AVX512-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
15074 ; AVX512-FCP-NEXT: vmovaps %zmm1, (%rdx)
15075 ; AVX512-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
15076 ; AVX512-FCP-NEXT: vmovaps %zmm1, (%rcx)
15077 ; AVX512-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
15078 ; AVX512-FCP-NEXT: vmovaps %zmm1, (%r8)
15079 ; AVX512-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
15080 ; AVX512-FCP-NEXT: vmovaps %zmm1, (%r9)
15081 ; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
15082 ; AVX512-FCP-NEXT: vmovdqa64 %zmm29, (%rax)
15083 ; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
15084 ; AVX512-FCP-NEXT: vmovdqa64 %zmm21, (%rax)
15085 ; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
15086 ; AVX512-FCP-NEXT: vmovdqa64 %zmm0, (%rax)
15087 ; AVX512-FCP-NEXT: addq $440, %rsp # imm = 0x1B8
15088 ; AVX512-FCP-NEXT: vzeroupper
15089 ; AVX512-FCP-NEXT: retq
15090 ;
15091 ; AVX512DQ-LABEL: load_i8_stride8_vf64:
15092 ; AVX512DQ: # %bb.0:
15093 ; AVX512DQ-NEXT: subq $520, %rsp # imm = 0x208
15094 ; AVX512DQ-NEXT: vmovdqa64 256(%rdi), %zmm17
15095 ; AVX512DQ-NEXT: vmovdqa64 384(%rdi), %zmm0
15096 ; AVX512DQ-NEXT: vpmovqb %zmm0, %xmm2
15097 ; AVX512DQ-NEXT: vmovdqa 496(%rdi), %xmm7
15098 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
15099 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm7, %xmm1
15100 ; AVX512DQ-NEXT: vmovdqa 480(%rdi), %xmm8
15101 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm8, %xmm3
15102 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
15103 ; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm3
15104 ; AVX512DQ-NEXT: vmovdqa 464(%rdi), %xmm11
15105 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm1 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
15106 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm11, %xmm4
15107 ; AVX512DQ-NEXT: vmovdqa 448(%rdi), %xmm15
15108 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm15, %xmm5
15109 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
15110 ; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
15111 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5,6],ymm3[7]
15112 ; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
15113 ; AVX512DQ-NEXT: vmovdqa 384(%rdi), %ymm4
15114 ; AVX512DQ-NEXT: vpmovqb %ymm4, %xmm4
15115 ; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
15116 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4],ymm2[5],ymm4[6,7]
15117 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
15118 ; AVX512DQ-NEXT: vmovdqa 368(%rdi), %xmm4
15119 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm3
15120 ; AVX512DQ-NEXT: vmovdqa64 %xmm4, %xmm18
15121 ; AVX512DQ-NEXT: vmovdqa 352(%rdi), %xmm14
15122 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm14, %xmm4
15123 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
15124 ; AVX512DQ-NEXT: vmovdqa 336(%rdi), %xmm4
15125 ; AVX512DQ-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
15126 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm4, %xmm4
15127 ; AVX512DQ-NEXT: vmovdqa 320(%rdi), %xmm6
15128 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm6, %xmm5
15129 ; AVX512DQ-NEXT: vmovdqa64 %xmm6, %xmm19
15130 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
15131 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1,2],xmm3[3]
15132 ; AVX512DQ-NEXT: vpmovqb %zmm17, %xmm4
15133 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]
15134 ; AVX512DQ-NEXT: vinserti32x4 $2, %xmm3, %zmm0, %zmm3
15135 ; AVX512DQ-NEXT: movb $-64, %al
15136 ; AVX512DQ-NEXT: kmovw %eax, %k1
15137 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm3 {%k1}
15138 ; AVX512DQ-NEXT: vmovdqa 240(%rdi), %xmm4
15139 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm2
15140 ; AVX512DQ-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
15141 ; AVX512DQ-NEXT: vmovdqa64 %xmm4, %xmm25
15142 ; AVX512DQ-NEXT: vmovdqa 224(%rdi), %xmm4
15143 ; AVX512DQ-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
15144 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm4
15145 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
15146 ; AVX512DQ-NEXT: vmovdqa 208(%rdi), %xmm5
15147 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm4
15148 ; AVX512DQ-NEXT: vmovdqa64 %xmm5, %xmm29
15149 ; AVX512DQ-NEXT: vmovdqa 192(%rdi), %xmm10
15150 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm10, %xmm5
15151 ; AVX512DQ-NEXT: vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
15152 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
15153 ; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %zmm5
15154 ; AVX512DQ-NEXT: vpmovqb %zmm5, %xmm5
15155 ; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
15156 ; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
15157 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
15158 ; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm4
15159 ; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm5
15160 ; AVX512DQ-NEXT: vpmovqb %ymm5, %xmm5
15161 ; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
15162 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
15163 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
15164 ; AVX512DQ-NEXT: vmovdqa 112(%rdi), %xmm4
15165 ; AVX512DQ-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
15166 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm4
15167 ; AVX512DQ-NEXT: vmovdqa 96(%rdi), %xmm5
15168 ; AVX512DQ-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
15169 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm0
15170 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
15171 ; AVX512DQ-NEXT: vmovdqa 80(%rdi), %xmm5
15172 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm4
15173 ; AVX512DQ-NEXT: vmovdqa64 %xmm5, %xmm30
15174 ; AVX512DQ-NEXT: vmovdqa 64(%rdi), %xmm5
15175 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm1
15176 ; AVX512DQ-NEXT: vmovdqa64 %xmm5, %xmm28
15177 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
15178 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
15179 ; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm16
15180 ; AVX512DQ-NEXT: vpmovqb %zmm16, %xmm1
15181 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
15182 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
15183 ; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm3, %zmm0
15184 ; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
15185 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm9 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
15186 ; AVX512DQ-NEXT: vpshufb %xmm9, %xmm7, %xmm0
15187 ; AVX512DQ-NEXT: vmovdqa64 %xmm7, %xmm24
15188 ; AVX512DQ-NEXT: vpshufb %xmm9, %xmm8, %xmm2
15189 ; AVX512DQ-NEXT: vmovdqa64 %xmm8, %xmm21
15190 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
15191 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm8 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
15192 ; AVX512DQ-NEXT: vpshufb %xmm8, %xmm11, %xmm3
15193 ; AVX512DQ-NEXT: vmovdqa64 %xmm11, %xmm27
15194 ; AVX512DQ-NEXT: vpshufb %xmm8, %xmm15, %xmm4
15195 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
15196 ; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
15197 ; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
15198 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
15199 ; AVX512DQ-NEXT: vmovdqa 416(%rdi), %xmm0
15200 ; AVX512DQ-NEXT: vmovdqa 432(%rdi), %xmm11
15201 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm3 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
15202 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm11, %xmm2
15203 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm0, %xmm5
15204 ; AVX512DQ-NEXT: vmovdqa64 %xmm0, %xmm22
15205 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
15206 ; AVX512DQ-NEXT: vmovdqa 384(%rdi), %xmm0
15207 ; AVX512DQ-NEXT: vmovdqa 400(%rdi), %xmm12
15208 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm2 = [1,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
15209 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm12, %xmm6
15210 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm0, %xmm7
15211 ; AVX512DQ-NEXT: vmovdqa64 %xmm0, %xmm31
15212 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
15213 ; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
15214 ; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
15215 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4],ymm5[5],ymm6[6,7]
15216 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
15217 ; AVX512DQ-NEXT: vmovdqa64 %xmm18, %xmm0
15218 ; AVX512DQ-NEXT: vpshufb %xmm9, %xmm0, %xmm5
15219 ; AVX512DQ-NEXT: vmovdqa64 %xmm18, %xmm23
15220 ; AVX512DQ-NEXT: vpshufb %xmm9, %xmm14, %xmm6
15221 ; AVX512DQ-NEXT: vmovdqa64 %xmm14, %xmm26
15222 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
15223 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
15224 ; AVX512DQ-NEXT: vpshufb %xmm8, %xmm0, %xmm6
15225 ; AVX512DQ-NEXT: vmovdqa64 %xmm19, %xmm14
15226 ; AVX512DQ-NEXT: vpshufb %xmm8, %xmm14, %xmm7
15227 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
15228 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3]
15229 ; AVX512DQ-NEXT: vpsrlq $8, %zmm17, %zmm6
15230 ; AVX512DQ-NEXT: vpmovqb %zmm6, %xmm6
15231 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]
15232 ; AVX512DQ-NEXT: vinserti32x4 $2, %xmm5, %zmm0, %zmm19
15233 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm19 {%k1}
15234 ; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm0
15235 ; AVX512DQ-NEXT: vpshufb %xmm9, %xmm0, %xmm4
15236 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
15237 ; AVX512DQ-NEXT: vpshufb %xmm9, %xmm0, %xmm5
15238 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
15239 ; AVX512DQ-NEXT: vmovdqa64 %xmm29, %xmm0
15240 ; AVX512DQ-NEXT: vpshufb %xmm8, %xmm0, %xmm5
15241 ; AVX512DQ-NEXT: vpshufb %xmm8, %xmm10, %xmm6
15242 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
15243 ; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
15244 ; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
15245 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm4[7]
15246 ; AVX512DQ-NEXT: vmovdqa 160(%rdi), %xmm0
15247 ; AVX512DQ-NEXT: vmovdqa 176(%rdi), %xmm5
15248 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm5, %xmm4
15249 ; AVX512DQ-NEXT: vmovdqa64 %xmm5, %xmm25
15250 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm0, %xmm3
15251 ; AVX512DQ-NEXT: vmovdqa64 %xmm0, %xmm20
15252 ; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
15253 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
15254 ; AVX512DQ-NEXT: vmovdqa 128(%rdi), %xmm4
15255 ; AVX512DQ-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
15256 ; AVX512DQ-NEXT: vmovdqa 144(%rdi), %xmm0
15257 ; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
15258 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm0, %xmm0
15259 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm4, %xmm2
15260 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
15261 ; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm2
15262 ; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
15263 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5],ymm0[6,7]
15264 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
15265 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
15266 ; AVX512DQ-NEXT: vpshufb %xmm9, %xmm1, %xmm1
15267 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
15268 ; AVX512DQ-NEXT: vpshufb %xmm9, %xmm2, %xmm2
15269 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
15270 ; AVX512DQ-NEXT: vmovdqa64 %xmm30, %xmm7
15271 ; AVX512DQ-NEXT: vpshufb %xmm8, %xmm7, %xmm2
15272 ; AVX512DQ-NEXT: vmovdqa64 %xmm28, %xmm3
15273 ; AVX512DQ-NEXT: vpshufb %xmm8, %xmm3, %xmm3
15274 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
15275 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
15276 ; AVX512DQ-NEXT: vpsrlq $8, %zmm16, %zmm2
15277 ; AVX512DQ-NEXT: vpmovqb %zmm2, %xmm2
15278 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
15279 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
15280 ; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm19, %zmm0
15281 ; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
15282 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm4 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
15283 ; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm13
15284 ; AVX512DQ-NEXT: vmovdqa64 %xmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
15285 ; AVX512DQ-NEXT: vpshufb %xmm4, %xmm13, %xmm1
15286 ; AVX512DQ-NEXT: vmovdqa64 %xmm21, %xmm10
15287 ; AVX512DQ-NEXT: vmovdqa64 %xmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
15288 ; AVX512DQ-NEXT: vpshufb %xmm4, %xmm10, %xmm2
15289 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
15290 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm1 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
15291 ; AVX512DQ-NEXT: vmovdqa64 %xmm27, %xmm0
15292 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm0, %xmm3
15293 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm15, %xmm8
15294 ; AVX512DQ-NEXT: vmovdqa64 %xmm15, %xmm18
15295 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
15296 ; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
15297 ; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
15298 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm3[0,1,2,3,4,5,6],ymm2[7]
15299 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm3 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
15300 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm11, %xmm2
15301 ; AVX512DQ-NEXT: vmovdqa64 %xmm11, %xmm24
15302 ; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm11
15303 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm11, %xmm9
15304 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm2[0],xmm9[1],xmm2[1],xmm9[2],xmm2[2],xmm9[3],xmm2[3]
15305 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm2 = [2,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
15306 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm12, %xmm0
15307 ; AVX512DQ-NEXT: vmovdqa64 %xmm12, %xmm30
15308 ; AVX512DQ-NEXT: vmovdqa64 %xmm31, %xmm5
15309 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm5, %xmm15
15310 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
15311 ; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
15312 ; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
15313 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm9[5],ymm0[6,7]
15314 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm8[6,7]
15315 ; AVX512DQ-NEXT: vmovdqa64 %xmm23, %xmm12
15316 ; AVX512DQ-NEXT: vmovdqa64 %xmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
15317 ; AVX512DQ-NEXT: vpshufb %xmm4, %xmm12, %xmm8
15318 ; AVX512DQ-NEXT: vmovdqa64 %xmm26, %xmm5
15319 ; AVX512DQ-NEXT: vpshufb %xmm4, %xmm5, %xmm9
15320 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
15321 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
15322 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm9
15323 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm14, %xmm15
15324 ; AVX512DQ-NEXT: vmovdqa64 %xmm14, %xmm21
15325 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
15326 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
15327 ; AVX512DQ-NEXT: vpsrlq $16, %zmm17, %zmm9
15328 ; AVX512DQ-NEXT: vpmovqb %zmm9, %xmm9
15329 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
15330 ; AVX512DQ-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
15331 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm8 {%k1}
15332 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
15333 ; AVX512DQ-NEXT: vpshufb %xmm4, %xmm0, %xmm0
15334 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
15335 ; AVX512DQ-NEXT: vpshufb %xmm4, %xmm14, %xmm9
15336 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm9[0],xmm0[0],xmm9[1],xmm0[1],xmm9[2],xmm0[2],xmm9[3],xmm0[3]
15337 ; AVX512DQ-NEXT: vmovdqa64 %xmm29, %xmm6
15338 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm6, %xmm9
15339 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
15340 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm15, %xmm15
15341 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
15342 ; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
15343 ; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
15344 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3,4,5,6],ymm0[7]
15345 ; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm9
15346 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm9, %xmm9
15347 ; AVX512DQ-NEXT: vmovdqa64 %xmm20, %xmm15
15348 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm15, %xmm3
15349 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
15350 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
15351 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm9, %xmm9
15352 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
15353 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm15, %xmm2
15354 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
15355 ; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
15356 ; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
15357 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5],ymm2[6,7]
15358 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
15359 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
15360 ; AVX512DQ-NEXT: vpshufb %xmm4, %xmm2, %xmm2
15361 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
15362 ; AVX512DQ-NEXT: vpshufb %xmm4, %xmm3, %xmm3
15363 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
15364 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm7, %xmm3
15365 ; AVX512DQ-NEXT: vmovdqa64 %xmm7, %xmm29
15366 ; AVX512DQ-NEXT: vmovdqa64 %xmm28, %xmm7
15367 ; AVX512DQ-NEXT: vmovdqa64 %xmm28, (%rsp) # 16-byte Spill
15368 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm7, %xmm1
15369 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
15370 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3]
15371 ; AVX512DQ-NEXT: vpsrlq $16, %zmm16, %zmm2
15372 ; AVX512DQ-NEXT: vpmovqb %zmm2, %xmm2
15373 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
15374 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
15375 ; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm0
15376 ; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
15377 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,3,11,0,0,3,11,0,0,3,11,0,0,3,11]
15378 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm13, %xmm1
15379 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm10, %xmm2
15380 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
15381 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm1 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
15382 ; AVX512DQ-NEXT: vmovdqa64 %xmm27, %xmm10
15383 ; AVX512DQ-NEXT: vmovdqa64 %xmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
15384 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm10, %xmm3
15385 ; AVX512DQ-NEXT: vmovdqa64 %xmm18, %xmm4
15386 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm4, %xmm4
15387 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
15388 ; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
15389 ; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
15390 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
15391 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm2 = [0,0,3,11,0,0,0,0,0,0,0,0,0,0,0,0]
15392 ; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm3
15393 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm3, %xmm3
15394 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm11, %xmm8
15395 ; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm28
15396 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
15397 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm3 = [3,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
15398 ; AVX512DQ-NEXT: vmovdqa64 %xmm30, %xmm9
15399 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm9, %xmm9
15400 ; AVX512DQ-NEXT: vmovdqa64 %xmm31, %xmm11
15401 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm11, %xmm15
15402 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
15403 ; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
15404 ; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
15405 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
15406 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
15407 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm12, %xmm8
15408 ; AVX512DQ-NEXT: vmovdqa64 %xmm26, %xmm13
15409 ; AVX512DQ-NEXT: vmovdqa64 %xmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
15410 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm13, %xmm9
15411 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
15412 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm9
15413 ; AVX512DQ-NEXT: vmovdqa64 %xmm21, %xmm5
15414 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm15
15415 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
15416 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
15417 ; AVX512DQ-NEXT: vpsrlq $24, %zmm17, %zmm9
15418 ; AVX512DQ-NEXT: vpmovqb %zmm9, %xmm9
15419 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
15420 ; AVX512DQ-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
15421 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
15422 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
15423 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm4
15424 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm14, %xmm9
15425 ; AVX512DQ-NEXT: vmovdqa64 %xmm14, %xmm22
15426 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
15427 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm6, %xmm9
15428 ; AVX512DQ-NEXT: vmovdqa64 %xmm6, %xmm26
15429 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
15430 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm6, %xmm15
15431 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
15432 ; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
15433 ; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
15434 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
15435 ; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm14
15436 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm14, %xmm9
15437 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
15438 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm6, %xmm2
15439 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
15440 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
15441 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm6, %xmm9
15442 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
15443 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm6, %xmm3
15444 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
15445 ; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
15446 ; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
15447 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
15448 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
15449 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
15450 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm3, %xmm3
15451 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
15452 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm6, %xmm0
15453 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
15454 ; AVX512DQ-NEXT: vmovdqa64 %xmm29, %xmm3
15455 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm3
15456 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm7, %xmm1
15457 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
15458 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
15459 ; AVX512DQ-NEXT: vpsrlq $24, %zmm16, %zmm1
15460 ; AVX512DQ-NEXT: vpmovqb %zmm1, %xmm1
15461 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
15462 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
15463 ; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm0
15464 ; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
15465 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
15466 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
15467 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm1, %xmm1
15468 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
15469 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm2, %xmm2
15470 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
15471 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm1 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
15472 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm10, %xmm3
15473 ; AVX512DQ-NEXT: vmovdqa64 %xmm18, %xmm10
15474 ; AVX512DQ-NEXT: vmovdqa64 %xmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
15475 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm10, %xmm4
15476 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
15477 ; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
15478 ; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
15479 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
15480 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm2 = [0,0,4,12,0,0,0,0,0,0,0,0,0,0,0,0]
15481 ; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm11
15482 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm11, %xmm3
15483 ; AVX512DQ-NEXT: vmovdqa64 %xmm28, %xmm7
15484 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm7, %xmm8
15485 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
15486 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm3 = [4,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
15487 ; AVX512DQ-NEXT: vmovdqa64 %xmm30, %xmm7
15488 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm7, %xmm9
15489 ; AVX512DQ-NEXT: vmovdqa64 %xmm31, %xmm12
15490 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm12, %xmm15
15491 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
15492 ; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
15493 ; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
15494 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
15495 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
15496 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
15497 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm7, %xmm8
15498 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm13, %xmm9
15499 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
15500 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
15501 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm7, %xmm9
15502 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm15
15503 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
15504 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
15505 ; AVX512DQ-NEXT: vpsrlq $32, %zmm17, %zmm9
15506 ; AVX512DQ-NEXT: vpmovqb %zmm9, %xmm9
15507 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
15508 ; AVX512DQ-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
15509 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
15510 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
15511 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm13, %xmm4
15512 ; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm5
15513 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm9
15514 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
15515 ; AVX512DQ-NEXT: vmovdqa64 %xmm26, %xmm5
15516 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm9
15517 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
15518 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm15
15519 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
15520 ; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
15521 ; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
15522 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
15523 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm14, %xmm9
15524 ; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm20
15525 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
15526 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm5, %xmm2
15527 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
15528 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
15529 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm5, %xmm9
15530 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
15531 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm14, %xmm3
15532 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
15533 ; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
15534 ; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
15535 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
15536 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
15537 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
15538 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm7, %xmm3
15539 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm6, %xmm0
15540 ; AVX512DQ-NEXT: vmovdqa64 %xmm6, %xmm25
15541 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
15542 ; AVX512DQ-NEXT: vmovdqa64 %xmm29, %xmm3
15543 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm3
15544 ; AVX512DQ-NEXT: vmovdqa (%rsp), %xmm4 # 16-byte Reload
15545 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm4, %xmm1
15546 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
15547 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
15548 ; AVX512DQ-NEXT: vpsrlq $32, %zmm16, %zmm1
15549 ; AVX512DQ-NEXT: vmovdqa64 %zmm16, %zmm18
15550 ; AVX512DQ-NEXT: vpmovqb %zmm1, %xmm1
15551 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
15552 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
15553 ; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm0
15554 ; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
15555 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
15556 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
15557 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm1, %xmm1
15558 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
15559 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm2, %xmm2
15560 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
15561 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm1 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
15562 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
15563 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm3
15564 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm10, %xmm4
15565 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
15566 ; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
15567 ; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
15568 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
15569 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm2 = [0,0,5,13,0,0,0,0,0,0,0,0,0,0,0,0]
15570 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm11, %xmm3
15571 ; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm27
15572 ; AVX512DQ-NEXT: vmovdqa64 %xmm28, %xmm5
15573 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm5, %xmm8
15574 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
15575 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm3 = [5,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
15576 ; AVX512DQ-NEXT: vmovdqa64 %xmm30, %xmm10
15577 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm10, %xmm9
15578 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm12, %xmm15
15579 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
15580 ; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
15581 ; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
15582 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
15583 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
15584 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
15585 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm6, %xmm8
15586 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
15587 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm6, %xmm9
15588 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
15589 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
15590 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm6, %xmm9
15591 ; AVX512DQ-NEXT: vmovdqa64 %xmm21, %xmm11
15592 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm11, %xmm15
15593 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
15594 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
15595 ; AVX512DQ-NEXT: vpsrlq $40, %zmm17, %zmm9
15596 ; AVX512DQ-NEXT: vmovdqa64 %zmm17, %zmm23
15597 ; AVX512DQ-NEXT: vpmovqb %zmm9, %xmm9
15598 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
15599 ; AVX512DQ-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
15600 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
15601 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm13, %xmm4
15602 ; AVX512DQ-NEXT: vmovdqa64 %xmm13, %xmm30
15603 ; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm6
15604 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm6, %xmm9
15605 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
15606 ; AVX512DQ-NEXT: vmovdqa64 %xmm26, %xmm13
15607 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm13, %xmm9
15608 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
15609 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm6, %xmm15
15610 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
15611 ; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
15612 ; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
15613 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
15614 ; AVX512DQ-NEXT: vmovdqa64 %xmm20, %xmm9
15615 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm9, %xmm9
15616 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
15617 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm12, %xmm2
15618 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
15619 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
15620 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm15, %xmm9
15621 ; AVX512DQ-NEXT: vmovdqa64 %xmm15, %xmm28
15622 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm14, %xmm3
15623 ; AVX512DQ-NEXT: vmovdqa64 %xmm14, %xmm16
15624 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
15625 ; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
15626 ; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
15627 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
15628 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
15629 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm7, %xmm3
15630 ; AVX512DQ-NEXT: vmovdqa64 %xmm7, %xmm24
15631 ; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm4
15632 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm0
15633 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
15634 ; AVX512DQ-NEXT: vmovdqa64 %xmm29, %xmm7
15635 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm7, %xmm3
15636 ; AVX512DQ-NEXT: vmovdqa (%rsp), %xmm4 # 16-byte Reload
15637 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm4, %xmm1
15638 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
15639 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
15640 ; AVX512DQ-NEXT: vpsrlq $40, %zmm18, %zmm1
15641 ; AVX512DQ-NEXT: vmovdqa64 %zmm18, %zmm26
15642 ; AVX512DQ-NEXT: vpmovqb %zmm1, %xmm1
15643 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
15644 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
15645 ; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm0
15646 ; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
15647 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
15648 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
15649 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm2, %xmm1
15650 ; AVX512DQ-NEXT: vmovdqa64 %xmm2, %xmm29
15651 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
15652 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm2, %xmm2
15653 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
15654 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm1 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
15655 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
15656 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm3
15657 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
15658 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm4, %xmm4
15659 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
15660 ; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
15661 ; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
15662 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
15663 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm2 = [0,0,6,14,0,0,0,0,0,0,0,0,0,0,0,0]
15664 ; AVX512DQ-NEXT: vmovdqa64 %xmm27, %xmm3
15665 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm3, %xmm3
15666 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm5, %xmm8
15667 ; AVX512DQ-NEXT: vmovdqa64 %xmm5, %xmm21
15668 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
15669 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm3 = [6,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
15670 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm10, %xmm9
15671 ; AVX512DQ-NEXT: vmovdqa64 %xmm10, %xmm18
15672 ; AVX512DQ-NEXT: vmovdqa64 %xmm31, %xmm5
15673 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm5, %xmm15
15674 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
15675 ; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
15676 ; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
15677 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
15678 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
15679 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
15680 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm8
15681 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
15682 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm9, %xmm9
15683 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
15684 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
15685 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm10, %xmm9
15686 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm11, %xmm15
15687 ; AVX512DQ-NEXT: vmovdqa64 %xmm11, %xmm22
15688 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
15689 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
15690 ; AVX512DQ-NEXT: vpsrlq $48, %zmm17, %zmm9
15691 ; AVX512DQ-NEXT: vpmovqb %zmm9, %xmm9
15692 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
15693 ; AVX512DQ-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
15694 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
15695 ; AVX512DQ-NEXT: vmovdqa64 %xmm30, %xmm4
15696 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm4
15697 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
15698 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm9, %xmm9
15699 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
15700 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm13, %xmm9
15701 ; AVX512DQ-NEXT: vmovdqa64 %xmm13, %xmm19
15702 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm6, %xmm15
15703 ; AVX512DQ-NEXT: vmovdqa64 %xmm6, %xmm17
15704 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
15705 ; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
15706 ; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
15707 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
15708 ; AVX512DQ-NEXT: vmovdqa64 %xmm20, %xmm6
15709 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm6, %xmm9
15710 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm12, %xmm2
15711 ; AVX512DQ-NEXT: vmovdqa64 %xmm12, %xmm20
15712 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
15713 ; AVX512DQ-NEXT: vmovdqa64 %xmm28, %xmm14
15714 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm14, %xmm9
15715 ; AVX512DQ-NEXT: vmovdqa64 %xmm16, %xmm13
15716 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm13, %xmm3
15717 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
15718 ; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
15719 ; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
15720 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
15721 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
15722 ; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm15
15723 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm15, %xmm3
15724 ; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm12
15725 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm12, %xmm0
15726 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
15727 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm7, %xmm3
15728 ; AVX512DQ-NEXT: vmovdqa64 %xmm7, %xmm16
15729 ; AVX512DQ-NEXT: vmovdqa (%rsp), %xmm7 # 16-byte Reload
15730 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm7, %xmm1
15731 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
15732 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
15733 ; AVX512DQ-NEXT: vpsrlq $48, %zmm26, %zmm1
15734 ; AVX512DQ-NEXT: vpmovqb %zmm1, %xmm1
15735 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
15736 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
15737 ; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm24
15738 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
15739 ; AVX512DQ-NEXT: vmovdqa64 %xmm29, %xmm1
15740 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm1, %xmm1
15741 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
15742 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm2, %xmm2
15743 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
15744 ; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm1 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
15745 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
15746 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm3
15747 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
15748 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm4, %xmm4
15749 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
15750 ; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
15751 ; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
15752 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
15753 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm2 = [0,0,7,15,0,0,0,0,0,0,0,0,0,0,0,0]
15754 ; AVX512DQ-NEXT: vmovdqa64 %xmm27, %xmm3
15755 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm3, %xmm3
15756 ; AVX512DQ-NEXT: vmovdqa64 %xmm21, %xmm8
15757 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm8, %xmm8
15758 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
15759 ; AVX512DQ-NEXT: vmovd {{.*#+}} xmm3 = [7,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
15760 ; AVX512DQ-NEXT: vmovdqa64 %xmm18, %xmm9
15761 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm9, %xmm9
15762 ; AVX512DQ-NEXT: vmovdqa64 %xmm31, %xmm11
15763 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm11, %xmm11
15764 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
15765 ; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
15766 ; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
15767 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
15768 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
15769 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm8
15770 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
15771 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm9
15772 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
15773 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm10, %xmm9
15774 ; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm5
15775 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm11
15776 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
15777 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
15778 ; AVX512DQ-NEXT: vpsrlq $56, %zmm23, %zmm9
15779 ; AVX512DQ-NEXT: vpmovqb %zmm9, %xmm9
15780 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
15781 ; AVX512DQ-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
15782 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
15783 ; AVX512DQ-NEXT: vmovdqa64 %xmm30, %xmm4
15784 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm4
15785 ; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
15786 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm9
15787 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
15788 ; AVX512DQ-NEXT: vmovdqa64 %xmm19, %xmm5
15789 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm9
15790 ; AVX512DQ-NEXT: vmovdqa64 %xmm17, %xmm5
15791 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm11
15792 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
15793 ; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
15794 ; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
15795 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
15796 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm6, %xmm6
15797 ; AVX512DQ-NEXT: vmovdqa64 %xmm20, %xmm5
15798 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm5, %xmm2
15799 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3]
15800 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm14, %xmm5
15801 ; AVX512DQ-NEXT: vpshufb %xmm3, %xmm13, %xmm3
15802 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
15803 ; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
15804 ; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
15805 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
15806 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
15807 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm15, %xmm3
15808 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm12, %xmm0
15809 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
15810 ; AVX512DQ-NEXT: vmovdqa64 %xmm16, %xmm3
15811 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm3
15812 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm7, %xmm1
15813 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
15814 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
15815 ; AVX512DQ-NEXT: vpsrlq $56, %zmm26, %zmm1
15816 ; AVX512DQ-NEXT: vpmovqb %zmm1, %xmm1
15817 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
15818 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
15819 ; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm0
15820 ; AVX512DQ-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
15821 ; AVX512DQ-NEXT: vmovaps %zmm1, (%rsi)
15822 ; AVX512DQ-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
15823 ; AVX512DQ-NEXT: vmovaps %zmm1, (%rdx)
15824 ; AVX512DQ-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
15825 ; AVX512DQ-NEXT: vmovaps %zmm1, (%rcx)
15826 ; AVX512DQ-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
15827 ; AVX512DQ-NEXT: vmovaps %zmm1, (%r8)
15828 ; AVX512DQ-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
15829 ; AVX512DQ-NEXT: vmovaps %zmm1, (%r9)
15830 ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax
15831 ; AVX512DQ-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
15832 ; AVX512DQ-NEXT: vmovaps %zmm1, (%rax)
15833 ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax
15834 ; AVX512DQ-NEXT: vmovdqa64 %zmm24, (%rax)
15835 ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax
15836 ; AVX512DQ-NEXT: vmovdqa64 %zmm0, (%rax)
15837 ; AVX512DQ-NEXT: addq $520, %rsp # imm = 0x208
15838 ; AVX512DQ-NEXT: vzeroupper
15839 ; AVX512DQ-NEXT: retq
15840 ;
15841 ; AVX512DQ-FCP-LABEL: load_i8_stride8_vf64:
15842 ; AVX512DQ-FCP: # %bb.0:
15843 ; AVX512DQ-FCP-NEXT: subq $440, %rsp # imm = 0x1B8
15844 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm14 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
15845 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,2,2,3,0,2,4,6]
15846 ; AVX512DQ-FCP-NEXT: vmovdqa 480(%rdi), %ymm1
15847 ; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
15848 ; AVX512DQ-FCP-NEXT: vpermd %ymm1, %ymm0, %ymm3
15849 ; AVX512DQ-FCP-NEXT: vpshufb %ymm14, %ymm3, %ymm1
15850 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm3, %ymm19
15851 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm9 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
15852 ; AVX512DQ-FCP-NEXT: vmovdqa 448(%rdi), %ymm2
15853 ; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
15854 ; AVX512DQ-FCP-NEXT: vpermd %ymm2, %ymm0, %ymm4
15855 ; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm4, %ymm2
15856 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
15857 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm11 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
15858 ; AVX512DQ-FCP-NEXT: vmovdqa 416(%rdi), %ymm3
15859 ; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
15860 ; AVX512DQ-FCP-NEXT: vpermd %ymm3, %ymm0, %ymm12
15861 ; AVX512DQ-FCP-NEXT: vpshufb %ymm11, %ymm12, %ymm3
15862 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm10 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
15863 ; AVX512DQ-FCP-NEXT: vmovdqa 384(%rdi), %ymm5
15864 ; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
15865 ; AVX512DQ-FCP-NEXT: vpermd %ymm5, %ymm0, %ymm13
15866 ; AVX512DQ-FCP-NEXT: vpshufb %ymm10, %ymm13, %ymm5
15867 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3,4],ymm3[5],ymm5[6,7]
15868 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
15869 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm3 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
15870 ; AVX512DQ-FCP-NEXT: vmovdqa 368(%rdi), %xmm2
15871 ; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm5
15872 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm30
15873 ; AVX512DQ-FCP-NEXT: vmovdqa 352(%rdi), %xmm2
15874 ; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm6
15875 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm25
15876 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
15877 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm5 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
15878 ; AVX512DQ-FCP-NEXT: vmovdqa 336(%rdi), %xmm2
15879 ; AVX512DQ-FCP-NEXT: vpshufb %xmm5, %xmm2, %xmm7
15880 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm24
15881 ; AVX512DQ-FCP-NEXT: vmovdqa 320(%rdi), %xmm15
15882 ; AVX512DQ-FCP-NEXT: vpshufb %xmm5, %xmm15, %xmm8
15883 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
15884 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1,2],xmm6[3]
15885 ; AVX512DQ-FCP-NEXT: vmovdqa64 256(%rdi), %zmm28
15886 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm28, %xmm7
15887 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3]
15888 ; AVX512DQ-FCP-NEXT: vinserti32x4 $2, %xmm6, %zmm0, %zmm16
15889 ; AVX512DQ-FCP-NEXT: movb $-64, %al
15890 ; AVX512DQ-FCP-NEXT: kmovw %eax, %k1
15891 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm16 {%k1}
15892 ; AVX512DQ-FCP-NEXT: vmovdqa 224(%rdi), %ymm1
15893 ; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
15894 ; AVX512DQ-FCP-NEXT: vpermd %ymm1, %ymm0, %ymm2
15895 ; AVX512DQ-FCP-NEXT: vpshufb %ymm14, %ymm2, %ymm1
15896 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm2, %ymm23
15897 ; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm2
15898 ; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
15899 ; AVX512DQ-FCP-NEXT: vpermd %ymm2, %ymm0, %ymm2
15900 ; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm2, %ymm6
15901 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm2, %ymm27
15902 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5,6],ymm1[7]
15903 ; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %ymm1
15904 ; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
15905 ; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm2
15906 ; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
15907 ; AVX512DQ-FCP-NEXT: vpermd %ymm1, %ymm0, %ymm1
15908 ; AVX512DQ-FCP-NEXT: vpshufb %ymm11, %ymm1, %ymm7
15909 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, %ymm18
15910 ; AVX512DQ-FCP-NEXT: vpermd %ymm2, %ymm0, %ymm0
15911 ; AVX512DQ-FCP-NEXT: vpshufb %ymm10, %ymm0, %ymm8
15912 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm17
15913 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3,4],ymm7[5],ymm8[6,7]
15914 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm7[0,1,2,3,4,5],ymm6[6,7]
15915 ; AVX512DQ-FCP-NEXT: vmovdqa 112(%rdi), %xmm10
15916 ; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm10, %xmm7
15917 ; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %xmm0
15918 ; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm0, %xmm3
15919 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm20
15920 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm7[0],xmm3[1],xmm7[1],xmm3[2],xmm7[2],xmm3[3],xmm7[3]
15921 ; AVX512DQ-FCP-NEXT: vmovdqa 80(%rdi), %xmm2
15922 ; AVX512DQ-FCP-NEXT: vpshufb %xmm5, %xmm2, %xmm0
15923 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm31
15924 ; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %xmm2
15925 ; AVX512DQ-FCP-NEXT: vpshufb %xmm5, %xmm2, %xmm5
15926 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm22
15927 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
15928 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[3]
15929 ; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm29
15930 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm29, %xmm3
15931 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
15932 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
15933 ; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm16, %zmm0
15934 ; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
15935 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm6 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
15936 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm19, %ymm11
15937 ; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm11, %ymm0
15938 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm7 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
15939 ; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm4, %ymm1
15940 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm4, %ymm26
15941 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6],ymm0[7]
15942 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u,u,u,u,u]
15943 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm12, %ymm19
15944 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm2 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
15945 ; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm13, %ymm3
15946 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm13, %ymm21
15947 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm2, %ymm16
15948 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4],ymm1[5],ymm3[6,7]
15949 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3,4,5],ymm0[6,7]
15950 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm1 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
15951 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm30, %xmm13
15952 ; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm13, %xmm3
15953 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm25, %xmm14
15954 ; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm14, %xmm5
15955 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
15956 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm5 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
15957 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm24, %xmm9
15958 ; AVX512DQ-FCP-NEXT: vpshufb %xmm5, %xmm9, %xmm0
15959 ; AVX512DQ-FCP-NEXT: vmovdqa %xmm15, %xmm12
15960 ; AVX512DQ-FCP-NEXT: vpshufb %xmm5, %xmm15, %xmm15
15961 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
15962 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[3]
15963 ; AVX512DQ-FCP-NEXT: vpsrlq $8, %zmm28, %zmm3
15964 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm3, %xmm3
15965 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
15966 ; AVX512DQ-FCP-NEXT: vinserti32x4 $2, %xmm0, %zmm0, %zmm0
15967 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 {%k1}
15968 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm23, %ymm8
15969 ; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm8, %ymm2
15970 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm27, %ymm4
15971 ; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm4, %ymm3
15972 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
15973 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm18, %ymm7
15974 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,21,25,29,u,u,u,u,u,u,u,u]
15975 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm17, %ymm6
15976 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm16, %ymm15
15977 ; AVX512DQ-FCP-NEXT: vpshufb %ymm15, %ymm6, %ymm15
15978 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm15[0,1,2,3,4],ymm3[5],ymm15[6,7]
15979 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
15980 ; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm10, %xmm3
15981 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm10, %xmm16
15982 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm20, %xmm10
15983 ; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm10, %xmm1
15984 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
15985 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm31, %xmm15
15986 ; AVX512DQ-FCP-NEXT: vpshufb %xmm5, %xmm15, %xmm3
15987 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm22, %xmm15
15988 ; AVX512DQ-FCP-NEXT: vpshufb %xmm5, %xmm15, %xmm5
15989 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
15990 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
15991 ; AVX512DQ-FCP-NEXT: vpsrlq $8, %zmm29, %zmm3
15992 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm3, %xmm3
15993 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
15994 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
15995 ; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm0
15996 ; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
15997 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm1 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
15998 ; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm11, %ymm0
15999 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm11, %ymm20
16000 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, %ymm23
16001 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm2 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
16002 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm26, %ymm11
16003 ; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm11, %ymm1
16004 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm2, %ymm24
16005 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6],ymm0[7]
16006 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm2 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
16007 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm19, %ymm1
16008 ; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm1, %ymm1
16009 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm2, %ymm25
16010 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm3 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
16011 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm21, %ymm2
16012 ; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm2, %ymm2
16013 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm3, %ymm26
16014 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4],ymm1[5],ymm2[6,7]
16015 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
16016 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm1 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
16017 ; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm13, %xmm2
16018 ; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm14, %xmm3
16019 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
16020 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm3 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
16021 ; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm9, %xmm5
16022 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm9, %xmm17
16023 ; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm12, %xmm15
16024 ; AVX512DQ-FCP-NEXT: vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
16025 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm15[0],xmm5[0],xmm15[1],xmm5[1],xmm15[2],xmm5[2],xmm15[3],xmm5[3]
16026 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1,2],xmm2[3]
16027 ; AVX512DQ-FCP-NEXT: vpsrlq $16, %zmm28, %zmm5
16028 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm5, %xmm5
16029 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
16030 ; AVX512DQ-FCP-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm2
16031 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm2 {%k1}
16032 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm23, %ymm0
16033 ; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm8, %ymm0
16034 ; AVX512DQ-FCP-NEXT: vmovdqa %ymm8, %ymm9
16035 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm24, %ymm5
16036 ; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm4, %ymm5
16037 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3,4,5,6],ymm0[7]
16038 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm25, %ymm4
16039 ; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm7, %ymm5
16040 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm18, %ymm25
16041 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm26, %ymm4
16042 ; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm6, %ymm15
16043 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm6, %ymm26
16044 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm15[0,1,2,3,4],ymm5[5],ymm15[6,7]
16045 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3,4,5],ymm0[6,7]
16046 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm16, %xmm6
16047 ; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm6, %xmm5
16048 ; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm10, %xmm1
16049 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm10, %xmm16
16050 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
16051 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm31, %xmm7
16052 ; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm7, %xmm5
16053 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm22, %xmm8
16054 ; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm8, %xmm3
16055 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
16056 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
16057 ; AVX512DQ-FCP-NEXT: vpsrlq $16, %zmm29, %zmm3
16058 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm3, %xmm3
16059 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
16060 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
16061 ; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm0
16062 ; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
16063 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm1 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
16064 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm20, %ymm0
16065 ; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm0, %ymm0
16066 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, %ymm18
16067 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm2 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
16068 ; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm11, %ymm1
16069 ; AVX512DQ-FCP-NEXT: vmovdqa %ymm2, %ymm11
16070 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6],ymm0[7]
16071 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm10 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
16072 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm19, %ymm1
16073 ; AVX512DQ-FCP-NEXT: vpshufb %ymm10, %ymm1, %ymm1
16074 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm4 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
16075 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm21, %ymm2
16076 ; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm2, %ymm2
16077 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4],ymm1[5],ymm2[6,7]
16078 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
16079 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm1 = [0,0,3,11,0,0,3,11,0,0,3,11,0,0,3,11]
16080 ; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm13, %xmm2
16081 ; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm14, %xmm3
16082 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
16083 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm3 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
16084 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm17, %xmm15
16085 ; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm15, %xmm5
16086 ; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm12, %xmm12
16087 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm12[0],xmm5[0],xmm12[1],xmm5[1],xmm12[2],xmm5[2],xmm12[3],xmm5[3]
16088 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1,2],xmm2[3]
16089 ; AVX512DQ-FCP-NEXT: vpsrlq $24, %zmm28, %zmm5
16090 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm28, %zmm24
16091 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm5, %xmm5
16092 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
16093 ; AVX512DQ-FCP-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm2
16094 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm2 {%k1}
16095 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm18, %ymm0
16096 ; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm9, %ymm0
16097 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm27, %ymm5
16098 ; AVX512DQ-FCP-NEXT: vpshufb %ymm11, %ymm5, %ymm5
16099 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3,4,5,6],ymm0[7]
16100 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm25, %ymm5
16101 ; AVX512DQ-FCP-NEXT: vpshufb %ymm10, %ymm5, %ymm5
16102 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm26, %ymm9
16103 ; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm9, %ymm4
16104 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4],ymm5[5],ymm4[6,7]
16105 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
16106 ; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm6, %xmm4
16107 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm6, %xmm21
16108 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm16, %xmm5
16109 ; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm5, %xmm1
16110 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
16111 ; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm7, %xmm4
16112 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm31, %xmm16
16113 ; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm8, %xmm3
16114 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm22, %xmm6
16115 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
16116 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
16117 ; AVX512DQ-FCP-NEXT: vpsrlq $24, %zmm29, %zmm3
16118 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm3, %xmm3
16119 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
16120 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
16121 ; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm0
16122 ; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
16123 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [1,3,2,3,1,3,5,7]
16124 ; AVX512DQ-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
16125 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm17 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
16126 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm17, %ymm1
16127 ; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm2, %ymm1
16128 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm2, %ymm28
16129 ; AVX512DQ-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 32-byte Folded Reload
16130 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm18 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
16131 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm18, %ymm2
16132 ; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm4, %ymm2
16133 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
16134 ; AVX512DQ-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
16135 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm7 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
16136 ; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm2, %ymm9
16137 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm2, %ymm19
16138 ; AVX512DQ-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
16139 ; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
16140 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm8 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
16141 ; AVX512DQ-FCP-NEXT: vpshufb %ymm8, %ymm2, %ymm10
16142 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4],ymm9[5],ymm10[6,7]
16143 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm9[0,1,2,3,4,5],ymm1[6,7]
16144 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm11 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
16145 ; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm13, %xmm9
16146 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm30, %xmm20
16147 ; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm14, %xmm10
16148 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm14, %xmm22
16149 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
16150 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm12 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
16151 ; AVX512DQ-FCP-NEXT: vpshufb %xmm12, %xmm15, %xmm10
16152 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm15, %xmm23
16153 ; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
16154 ; AVX512DQ-FCP-NEXT: vpshufb %xmm12, %xmm2, %xmm13
16155 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm13[0],xmm10[0],xmm13[1],xmm10[1],xmm13[2],xmm10[2],xmm13[3],xmm10[3]
16156 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
16157 ; AVX512DQ-FCP-NEXT: vpsrlq $32, %zmm24, %zmm10
16158 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm24, %zmm25
16159 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm10, %xmm10
16160 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
16161 ; AVX512DQ-FCP-NEXT: vinserti32x4 $2, %xmm9, %zmm0, %zmm13
16162 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm13 {%k1}
16163 ; AVX512DQ-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
16164 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm17, %ymm1
16165 ; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm2, %ymm1
16166 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm2, %ymm27
16167 ; AVX512DQ-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
16168 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm18, %ymm2
16169 ; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm3, %ymm15
16170 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm3, %ymm26
16171 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm15 = ymm15[0,1,2,3,4,5,6],ymm1[7]
16172 ; AVX512DQ-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
16173 ; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm2, %ymm1
16174 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm2, %ymm31
16175 ; AVX512DQ-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
16176 ; AVX512DQ-FCP-NEXT: vpshufb %ymm8, %ymm0, %ymm14
16177 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm30
16178 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm14[0,1,2,3,4],ymm1[5],ymm14[6,7]
16179 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm15[6,7]
16180 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm21, %xmm10
16181 ; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm10, %xmm14
16182 ; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm5, %xmm11
16183 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm5, %xmm24
16184 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm14[0],xmm11[1],xmm14[1],xmm11[2],xmm14[2],xmm11[3],xmm14[3]
16185 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm16, %xmm8
16186 ; AVX512DQ-FCP-NEXT: vpshufb %xmm12, %xmm8, %xmm14
16187 ; AVX512DQ-FCP-NEXT: vpshufb %xmm12, %xmm6, %xmm12
16188 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm6, %xmm21
16189 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm14[0],xmm12[1],xmm14[1],xmm12[2],xmm14[2],xmm12[3],xmm14[3]
16190 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1,2],xmm11[3]
16191 ; AVX512DQ-FCP-NEXT: vpsrlq $32, %zmm29, %zmm12
16192 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm12, %xmm12
16193 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1],xmm11[2,3]
16194 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3],ymm1[4,5,6,7]
16195 ; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm13, %zmm0
16196 ; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
16197 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm3 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
16198 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm28, %ymm0
16199 ; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm0, %ymm1
16200 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm5 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
16201 ; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm4, %ymm11
16202 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm4, %ymm16
16203 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3,4,5,6],ymm1[7]
16204 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm6 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
16205 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm19, %ymm0
16206 ; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm11
16207 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm18 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
16208 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm18, %ymm7
16209 ; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
16210 ; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm2, %ymm12
16211 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2,3,4],ymm11[5],ymm12[6,7]
16212 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3,4,5],ymm1[6,7]
16213 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm11 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
16214 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm20, %xmm2
16215 ; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm2, %xmm12
16216 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm20, %xmm19
16217 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm22, %xmm4
16218 ; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm4, %xmm13
16219 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
16220 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm13 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
16221 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm23, %xmm9
16222 ; AVX512DQ-FCP-NEXT: vpshufb %xmm13, %xmm9, %xmm14
16223 ; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
16224 ; AVX512DQ-FCP-NEXT: vpshufb %xmm13, %xmm7, %xmm15
16225 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
16226 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm14[0,1,2],xmm12[3]
16227 ; AVX512DQ-FCP-NEXT: vpsrlq $40, %zmm25, %zmm14
16228 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm14, %xmm14
16229 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm14[0,1],xmm12[2,3]
16230 ; AVX512DQ-FCP-NEXT: vinserti32x4 $2, %xmm12, %zmm0, %zmm12
16231 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm12 {%k1}
16232 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm27, %ymm1
16233 ; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm1, %ymm1
16234 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm26, %ymm2
16235 ; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm2, %ymm14
16236 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm14[0,1,2,3,4,5,6],ymm1[7]
16237 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm31, %ymm3
16238 ; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm14
16239 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm18, %ymm3
16240 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm30, %ymm2
16241 ; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm2, %ymm15
16242 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
16243 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm14[0,1,2,3,4,5],ymm1[6,7]
16244 ; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm10, %xmm14
16245 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm10, %xmm20
16246 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm24, %xmm2
16247 ; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm2, %xmm11
16248 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm14[0],xmm11[1],xmm14[1],xmm11[2],xmm14[2],xmm11[3],xmm14[3]
16249 ; AVX512DQ-FCP-NEXT: vpshufb %xmm13, %xmm8, %xmm14
16250 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm8, %xmm22
16251 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm21, %xmm2
16252 ; AVX512DQ-FCP-NEXT: vpshufb %xmm13, %xmm2, %xmm13
16253 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
16254 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm13[0,1,2],xmm11[3]
16255 ; AVX512DQ-FCP-NEXT: vpsrlq $40, %zmm29, %zmm13
16256 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm29, %zmm18
16257 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm13, %xmm13
16258 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm13[0,1],xmm11[2,3]
16259 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3],ymm1[4,5,6,7]
16260 ; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm12, %zmm29
16261 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm3 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
16262 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm28, %ymm10
16263 ; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm10, %ymm1
16264 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm21 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
16265 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm16, %ymm5
16266 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm21, %ymm6
16267 ; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm5, %ymm11
16268 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3,4,5,6],ymm1[7]
16269 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm6 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
16270 ; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm11
16271 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm17
16272 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm8 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
16273 ; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
16274 ; AVX512DQ-FCP-NEXT: vpshufb %ymm8, %ymm0, %ymm12
16275 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2,3,4],ymm11[5],ymm12[6,7]
16276 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3,4,5],ymm1[6,7]
16277 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm11 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
16278 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm19, %xmm0
16279 ; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm0, %xmm12
16280 ; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm4, %xmm13
16281 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm4, %xmm23
16282 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
16283 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm13 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
16284 ; AVX512DQ-FCP-NEXT: vpshufb %xmm13, %xmm9, %xmm14
16285 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm9, %xmm28
16286 ; AVX512DQ-FCP-NEXT: vpshufb %xmm13, %xmm7, %xmm15
16287 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm7, %xmm16
16288 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
16289 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm14[0,1,2],xmm12[3]
16290 ; AVX512DQ-FCP-NEXT: vpsrlq $48, %zmm25, %zmm14
16291 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm14, %xmm14
16292 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm14[0,1],xmm12[2,3]
16293 ; AVX512DQ-FCP-NEXT: vinserti32x4 $2, %xmm12, %zmm0, %zmm12
16294 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm12 {%k1}
16295 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm27, %ymm0
16296 ; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm0, %ymm1
16297 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm26, %ymm9
16298 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm21, %ymm3
16299 ; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm9, %ymm14
16300 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm14[0,1,2,3,4,5,6],ymm1[7]
16301 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm31, %ymm3
16302 ; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm14
16303 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm30, %ymm3
16304 ; AVX512DQ-FCP-NEXT: vpshufb %ymm8, %ymm3, %ymm15
16305 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
16306 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm14[0,1,2,3,4,5],ymm1[6,7]
16307 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm20, %xmm8
16308 ; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm8, %xmm14
16309 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm24, %xmm15
16310 ; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm15, %xmm11
16311 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm14[0],xmm11[1],xmm14[1],xmm11[2],xmm14[2],xmm11[3],xmm14[3]
16312 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm22, %xmm3
16313 ; AVX512DQ-FCP-NEXT: vpshufb %xmm13, %xmm3, %xmm14
16314 ; AVX512DQ-FCP-NEXT: vpshufb %xmm13, %xmm2, %xmm13
16315 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm20
16316 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
16317 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm13[0,1,2],xmm11[3]
16318 ; AVX512DQ-FCP-NEXT: vpsrlq $48, %zmm18, %zmm13
16319 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm13, %xmm13
16320 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm13[0,1],xmm11[2,3]
16321 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3],ymm1[4,5,6,7]
16322 ; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm12, %zmm21
16323 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm13 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
16324 ; AVX512DQ-FCP-NEXT: vpshufb %ymm13, %ymm10, %ymm6
16325 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm14 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
16326 ; AVX512DQ-FCP-NEXT: vpshufb %ymm14, %ymm5, %ymm5
16327 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm6[7]
16328 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm1 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
16329 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm17, %ymm2
16330 ; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm2, %ymm3
16331 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm7 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
16332 ; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
16333 ; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm2, %ymm4
16334 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4],ymm3[5],ymm4[6,7]
16335 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm5[6,7]
16336 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm4 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
16337 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm19, %xmm2
16338 ; AVX512DQ-FCP-NEXT: vpshufb %xmm4, %xmm2, %xmm5
16339 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm23, %xmm2
16340 ; AVX512DQ-FCP-NEXT: vpshufb %xmm4, %xmm2, %xmm6
16341 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
16342 ; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm6 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
16343 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm28, %xmm2
16344 ; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm2, %xmm11
16345 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm16, %xmm2
16346 ; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm2, %xmm12
16347 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
16348 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm5 = xmm11[0,1,2],xmm5[3]
16349 ; AVX512DQ-FCP-NEXT: vpsrlq $56, %zmm25, %zmm11
16350 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm11, %xmm11
16351 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm5 = xmm11[0,1],xmm5[2,3]
16352 ; AVX512DQ-FCP-NEXT: vinserti32x4 $2, %xmm5, %zmm0, %zmm5
16353 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm5 {%k1}
16354 ; AVX512DQ-FCP-NEXT: vpshufb %ymm13, %ymm0, %ymm3
16355 ; AVX512DQ-FCP-NEXT: vpshufb %ymm14, %ymm9, %ymm9
16356 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm9[0,1,2,3,4,5,6],ymm3[7]
16357 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm31, %ymm0
16358 ; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm0, %ymm2
16359 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm30, %ymm0
16360 ; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm0, %ymm0
16361 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5],ymm0[6,7]
16362 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm3[6,7]
16363 ; AVX512DQ-FCP-NEXT: vpshufb %xmm4, %xmm8, %xmm2
16364 ; AVX512DQ-FCP-NEXT: vpshufb %xmm4, %xmm15, %xmm3
16365 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
16366 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm22, %xmm1
16367 ; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm1, %xmm3
16368 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm20, %xmm1
16369 ; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm1, %xmm4
16370 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
16371 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3]
16372 ; AVX512DQ-FCP-NEXT: vpsrlq $56, %zmm18, %zmm3
16373 ; AVX512DQ-FCP-NEXT: vpmovqb %zmm3, %xmm3
16374 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]
16375 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
16376 ; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm5, %zmm0
16377 ; AVX512DQ-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
16378 ; AVX512DQ-FCP-NEXT: vmovaps %zmm1, (%rsi)
16379 ; AVX512DQ-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
16380 ; AVX512DQ-FCP-NEXT: vmovaps %zmm1, (%rdx)
16381 ; AVX512DQ-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
16382 ; AVX512DQ-FCP-NEXT: vmovaps %zmm1, (%rcx)
16383 ; AVX512DQ-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
16384 ; AVX512DQ-FCP-NEXT: vmovaps %zmm1, (%r8)
16385 ; AVX512DQ-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
16386 ; AVX512DQ-FCP-NEXT: vmovaps %zmm1, (%r9)
16387 ; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
16388 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm29, (%rax)
16389 ; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
16390 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm21, (%rax)
16391 ; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
16392 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, (%rax)
16393 ; AVX512DQ-FCP-NEXT: addq $440, %rsp # imm = 0x1B8
16394 ; AVX512DQ-FCP-NEXT: vzeroupper
16395 ; AVX512DQ-FCP-NEXT: retq
16396 ;
16397 ; AVX512BW-LABEL: load_i8_stride8_vf64:
16398 ; AVX512BW: # %bb.0:
16399 ; AVX512BW-NEXT: subq $744, %rsp # imm = 0x2E8
16400 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm5
16401 ; AVX512BW-NEXT: vmovdqa64 128(%rdi), %zmm0
16402 ; AVX512BW-NEXT: vpmovqb %zmm0, %xmm0
16403 ; AVX512BW-NEXT: vmovdqa64 256(%rdi), %zmm1
16404 ; AVX512BW-NEXT: vmovdqa64 384(%rdi), %zmm2
16405 ; AVX512BW-NEXT: vpmovqb %zmm2, %xmm2
16406 ; AVX512BW-NEXT: vmovdqa64 496(%rdi), %xmm24
16407 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm12 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
16408 ; AVX512BW-NEXT: vpshufb %xmm12, %xmm24, %xmm3
16409 ; AVX512BW-NEXT: vmovdqa64 480(%rdi), %xmm25
16410 ; AVX512BW-NEXT: vpshufb %xmm12, %xmm25, %xmm4
16411 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
16412 ; AVX512BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
16413 ; AVX512BW-NEXT: vmovdqa64 464(%rdi), %xmm26
16414 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm19 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
16415 ; AVX512BW-NEXT: vpshufb %xmm19, %xmm26, %xmm4
16416 ; AVX512BW-NEXT: vmovdqa64 448(%rdi), %xmm30
16417 ; AVX512BW-NEXT: vpshufb %xmm19, %xmm30, %xmm6
16418 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
16419 ; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
16420 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5,6],ymm3[7]
16421 ; AVX512BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
16422 ; AVX512BW-NEXT: vmovdqa 384(%rdi), %ymm4
16423 ; AVX512BW-NEXT: vpmovqb %ymm4, %xmm4
16424 ; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
16425 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4],ymm2[5],ymm4[6,7]
16426 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm7 = ymm2[0,1,2,3,4,5],ymm3[6,7]
16427 ; AVX512BW-NEXT: vmovdqa64 368(%rdi), %xmm31
16428 ; AVX512BW-NEXT: vpshufb %xmm12, %xmm31, %xmm4
16429 ; AVX512BW-NEXT: vmovdqa64 %xmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
16430 ; AVX512BW-NEXT: vmovdqa64 352(%rdi), %xmm27
16431 ; AVX512BW-NEXT: vpshufb %xmm12, %xmm27, %xmm6
16432 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
16433 ; AVX512BW-NEXT: vmovdqa64 336(%rdi), %xmm22
16434 ; AVX512BW-NEXT: vpshufb %xmm19, %xmm22, %xmm11
16435 ; AVX512BW-NEXT: vmovdqa64 %xmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
16436 ; AVX512BW-NEXT: vmovdqa 320(%rdi), %xmm9
16437 ; AVX512BW-NEXT: vpshufb %xmm19, %xmm9, %xmm15
16438 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm15[0],xmm11[0],xmm15[1],xmm11[1],xmm15[2],xmm11[2],xmm15[3],xmm11[3]
16439 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1,2],xmm10[3]
16440 ; AVX512BW-NEXT: vpmovqb %zmm1, %xmm11
16441 ; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm15
16442 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3]
16443 ; AVX512BW-NEXT: vinserti32x4 $2, %xmm10, %zmm0, %zmm20
16444 ; AVX512BW-NEXT: movb $-64, %al
16445 ; AVX512BW-NEXT: kmovd %eax, %k1
16446 ; AVX512BW-NEXT: vinserti64x4 $1, %ymm7, %zmm0, %zmm20 {%k1}
16447 ; AVX512BW-NEXT: vmovdqa64 240(%rdi), %xmm28
16448 ; AVX512BW-NEXT: vpshufb %xmm12, %xmm28, %xmm7
16449 ; AVX512BW-NEXT: vmovdqa64 224(%rdi), %xmm18
16450 ; AVX512BW-NEXT: vpshufb %xmm12, %xmm18, %xmm10
16451 ; AVX512BW-NEXT: vmovdqa64 %xmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
16452 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm10[0],xmm7[0],xmm10[1],xmm7[1],xmm10[2],xmm7[2],xmm10[3],xmm7[3]
16453 ; AVX512BW-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
16454 ; AVX512BW-NEXT: vmovdqa64 208(%rdi), %xmm17
16455 ; AVX512BW-NEXT: vpshufb %xmm19, %xmm17, %xmm10
16456 ; AVX512BW-NEXT: vmovdqa64 %xmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
16457 ; AVX512BW-NEXT: vmovdqa 192(%rdi), %xmm8
16458 ; AVX512BW-NEXT: vpshufb %xmm19, %xmm8, %xmm16
16459 ; AVX512BW-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
16460 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm16[0],xmm10[0],xmm16[1],xmm10[1],xmm16[2],xmm10[2],xmm16[3],xmm10[3]
16461 ; AVX512BW-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
16462 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm7 = ymm10[0,1,2,3,4,5,6],ymm7[7]
16463 ; AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
16464 ; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm10
16465 ; AVX512BW-NEXT: vpmovqb %ymm10, %xmm10
16466 ; AVX512BW-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
16467 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm10[0,1,2,3,4],ymm0[5],ymm10[6,7]
16468 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm7[6,7]
16469 ; AVX512BW-NEXT: vmovdqa 112(%rdi), %xmm10
16470 ; AVX512BW-NEXT: vpshufb %xmm12, %xmm10, %xmm16
16471 ; AVX512BW-NEXT: vmovdqa 96(%rdi), %xmm14
16472 ; AVX512BW-NEXT: vpshufb %xmm12, %xmm14, %xmm12
16473 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm12[0],xmm16[0],xmm12[1],xmm16[1],xmm12[2],xmm16[2],xmm12[3],xmm16[3]
16474 ; AVX512BW-NEXT: vmovdqa 80(%rdi), %xmm2
16475 ; AVX512BW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
16476 ; AVX512BW-NEXT: vpshufb %xmm19, %xmm2, %xmm21
16477 ; AVX512BW-NEXT: vmovdqa 64(%rdi), %xmm2
16478 ; AVX512BW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
16479 ; AVX512BW-NEXT: vpshufb %xmm19, %xmm2, %xmm19
16480 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm21[0],xmm19[1],xmm21[1],xmm19[2],xmm21[2],xmm19[3],xmm21[3]
16481 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
16482 ; AVX512BW-NEXT: vpmovqb %zmm5, %xmm13
16483 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
16484 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
16485 ; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm20, %zmm0
16486 ; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
16487 ; AVX512BW-NEXT: vmovdqa 160(%rdi), %xmm6
16488 ; AVX512BW-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
16489 ; AVX512BW-NEXT: vmovdqa 384(%rdi), %xmm7
16490 ; AVX512BW-NEXT: vmovdqa64 400(%rdi), %xmm21
16491 ; AVX512BW-NEXT: vmovdqa64 416(%rdi), %xmm23
16492 ; AVX512BW-NEXT: vmovdqa64 432(%rdi), %xmm29
16493 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
16494 ; AVX512BW-NEXT: vmovdqa64 %xmm24, %xmm19
16495 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm24, %xmm1
16496 ; AVX512BW-NEXT: vmovdqa64 %xmm25, %xmm11
16497 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm25, %xmm13
16498 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3]
16499 ; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
16500 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm13 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
16501 ; AVX512BW-NEXT: vmovdqa64 %xmm26, %xmm12
16502 ; AVX512BW-NEXT: vpshufb %xmm13, %xmm26, %xmm24
16503 ; AVX512BW-NEXT: vmovdqa64 %xmm30, %xmm16
16504 ; AVX512BW-NEXT: vmovdqa64 %xmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
16505 ; AVX512BW-NEXT: vpshufb %xmm13, %xmm30, %xmm25
16506 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm24 = xmm25[0],xmm24[0],xmm25[1],xmm24[1],xmm25[2],xmm24[2],xmm25[3],xmm24[3]
16507 ; AVX512BW-NEXT: vinserti32x4 $1, %xmm24, %ymm0, %ymm2
16508 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
16509 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm26 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
16510 ; AVX512BW-NEXT: vpshufb %xmm26, %xmm29, %xmm24
16511 ; AVX512BW-NEXT: vpshufb %xmm26, %xmm23, %xmm25
16512 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm24 = xmm25[0],xmm24[0],xmm25[1],xmm24[1],xmm25[2],xmm24[2],xmm25[3],xmm24[3]
16513 ; AVX512BW-NEXT: vinserti32x4 $1, %xmm24, %ymm0, %ymm2
16514 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm30 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
16515 ; AVX512BW-NEXT: vpshufb %xmm30, %xmm21, %xmm24
16516 ; AVX512BW-NEXT: vpshufb %xmm30, %xmm7, %xmm25
16517 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm24 = xmm25[0],xmm24[0],xmm25[1],xmm24[1],xmm25[2],xmm24[2],xmm25[3],xmm24[3]
16518 ; AVX512BW-NEXT: vinserti32x4 $1, %xmm24, %ymm0, %ymm3
16519 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
16520 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
16521 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm31, %xmm2
16522 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm27, %xmm3
16523 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
16524 ; AVX512BW-NEXT: vpshufb %xmm13, %xmm22, %xmm3
16525 ; AVX512BW-NEXT: vpshufb %xmm13, %xmm9, %xmm24
16526 ; AVX512BW-NEXT: vmovdqa64 %xmm9, %xmm31
16527 ; AVX512BW-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
16528 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm24[0],xmm3[0],xmm24[1],xmm3[1],xmm24[2],xmm3[2],xmm24[3],xmm3[3]
16529 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3]
16530 ; AVX512BW-NEXT: vpsrlq $8, %zmm15, %zmm3
16531 ; AVX512BW-NEXT: vmovdqa64 %zmm15, %zmm22
16532 ; AVX512BW-NEXT: vpmovqb %zmm3, %xmm3
16533 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]
16534 ; AVX512BW-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm2
16535 ; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm2 {%k1}
16536 ; AVX512BW-NEXT: vmovdqa64 %xmm28, %xmm9
16537 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm28, %xmm1
16538 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm18, %xmm3
16539 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
16540 ; AVX512BW-NEXT: vpshufb %xmm13, %xmm17, %xmm3
16541 ; AVX512BW-NEXT: vpshufb %xmm13, %xmm8, %xmm24
16542 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm24[0],xmm3[0],xmm24[1],xmm3[1],xmm24[2],xmm3[2],xmm24[3],xmm3[3]
16543 ; AVX512BW-NEXT: vmovdqa 176(%rdi), %xmm0
16544 ; AVX512BW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
16545 ; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
16546 ; AVX512BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
16547 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
16548 ; AVX512BW-NEXT: vpshufb %xmm26, %xmm0, %xmm3
16549 ; AVX512BW-NEXT: vpshufb %xmm26, %xmm6, %xmm25
16550 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm25[0],xmm3[0],xmm25[1],xmm3[1],xmm25[2],xmm3[2],xmm25[3],xmm3[3]
16551 ; AVX512BW-NEXT: vmovdqa 128(%rdi), %xmm6
16552 ; AVX512BW-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
16553 ; AVX512BW-NEXT: vmovdqa 144(%rdi), %xmm0
16554 ; AVX512BW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
16555 ; AVX512BW-NEXT: vpshufb %xmm30, %xmm0, %xmm0
16556 ; AVX512BW-NEXT: vpshufb %xmm30, %xmm6, %xmm30
16557 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm30[0],xmm0[0],xmm30[1],xmm0[1],xmm30[2],xmm0[2],xmm30[3],xmm0[3]
16558 ; AVX512BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
16559 ; AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
16560 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5],ymm0[6,7]
16561 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
16562 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm10, %xmm1
16563 ; AVX512BW-NEXT: vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
16564 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm14, %xmm3
16565 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
16566 ; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
16567 ; AVX512BW-NEXT: vpshufb %xmm13, %xmm15, %xmm3
16568 ; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm28 # 16-byte Reload
16569 ; AVX512BW-NEXT: vpshufb %xmm13, %xmm28, %xmm4
16570 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
16571 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
16572 ; AVX512BW-NEXT: vpsrlq $8, %zmm5, %zmm3
16573 ; AVX512BW-NEXT: vpmovqb %zmm3, %xmm3
16574 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
16575 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
16576 ; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm0
16577 ; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
16578 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm1 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
16579 ; AVX512BW-NEXT: vpshufb %xmm1, %xmm19, %xmm2
16580 ; AVX512BW-NEXT: vmovdqa64 %xmm19, %xmm20
16581 ; AVX512BW-NEXT: vpshufb %xmm1, %xmm11, %xmm3
16582 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
16583 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
16584 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm12, %xmm4
16585 ; AVX512BW-NEXT: vmovdqa64 %xmm12, %xmm25
16586 ; AVX512BW-NEXT: vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
16587 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm16, %xmm13
16588 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
16589 ; AVX512BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
16590 ; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
16591 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
16592 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
16593 ; AVX512BW-NEXT: vmovdqa64 %xmm29, %xmm24
16594 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm29, %xmm13
16595 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm23, %xmm30
16596 ; AVX512BW-NEXT: vmovdqa64 %xmm23, %xmm16
16597 ; AVX512BW-NEXT: vmovdqa64 %xmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
16598 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
16599 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm30 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
16600 ; AVX512BW-NEXT: vmovdqa64 %xmm21, %xmm18
16601 ; AVX512BW-NEXT: vpshufb %xmm30, %xmm21, %xmm0
16602 ; AVX512BW-NEXT: vmovdqa64 %xmm7, %xmm17
16603 ; AVX512BW-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
16604 ; AVX512BW-NEXT: vpshufb %xmm30, %xmm7, %xmm19
16605 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm19[0],xmm0[0],xmm19[1],xmm0[1],xmm19[2],xmm0[2],xmm19[3],xmm0[3]
16606 ; AVX512BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
16607 ; AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
16608 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm13[5],ymm0[6,7]
16609 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
16610 ; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
16611 ; AVX512BW-NEXT: vpshufb %xmm1, %xmm12, %xmm2
16612 ; AVX512BW-NEXT: vpshufb %xmm1, %xmm27, %xmm13
16613 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm13[0],xmm2[0],xmm13[1],xmm2[1],xmm13[2],xmm2[2],xmm13[3],xmm2[3]
16614 ; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
16615 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm8, %xmm13
16616 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm31, %xmm19
16617 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
16618 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm13[0,1,2],xmm2[3]
16619 ; AVX512BW-NEXT: vmovdqa64 %zmm22, %zmm6
16620 ; AVX512BW-NEXT: vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
16621 ; AVX512BW-NEXT: vpsrlq $16, %zmm22, %zmm13
16622 ; AVX512BW-NEXT: vpmovqb %zmm13, %xmm13
16623 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm13[0,1],xmm2[2,3]
16624 ; AVX512BW-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm2
16625 ; AVX512BW-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm2 {%k1}
16626 ; AVX512BW-NEXT: vpshufb %xmm1, %xmm9, %xmm0
16627 ; AVX512BW-NEXT: vmovdqa64 %xmm9, %xmm26
16628 ; AVX512BW-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
16629 ; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm21 # 16-byte Reload
16630 ; AVX512BW-NEXT: vpshufb %xmm1, %xmm21, %xmm13
16631 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm13[0],xmm0[0],xmm13[1],xmm0[1],xmm13[2],xmm0[2],xmm13[3],xmm0[3]
16632 ; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm22 # 16-byte Reload
16633 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm22, %xmm13
16634 ; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm29 # 16-byte Reload
16635 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm29, %xmm19
16636 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
16637 ; AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
16638 ; AVX512BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
16639 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3,4,5,6],ymm0[7]
16640 ; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
16641 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm9, %xmm13
16642 ; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
16643 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm7, %xmm4
16644 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm13[0],xmm4[1],xmm13[1],xmm4[2],xmm13[2],xmm4[3],xmm13[3]
16645 ; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm31 # 16-byte Reload
16646 ; AVX512BW-NEXT: vpshufb %xmm30, %xmm31, %xmm13
16647 ; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm23 # 16-byte Reload
16648 ; AVX512BW-NEXT: vpshufb %xmm30, %xmm23, %xmm19
16649 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
16650 ; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
16651 ; AVX512BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
16652 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm13[0,1,2,3,4],ymm4[5],ymm13[6,7]
16653 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
16654 ; AVX512BW-NEXT: vpshufb %xmm1, %xmm10, %xmm4
16655 ; AVX512BW-NEXT: vpshufb %xmm1, %xmm14, %xmm1
16656 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
16657 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm15, %xmm4
16658 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm28, %xmm3
16659 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
16660 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
16661 ; AVX512BW-NEXT: vpsrlq $16, %zmm5, %zmm3
16662 ; AVX512BW-NEXT: vmovdqa64 %zmm5, %zmm28
16663 ; AVX512BW-NEXT: vpmovqb %zmm3, %xmm3
16664 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
16665 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
16666 ; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm0
16667 ; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
16668 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
16669 ; AVX512BW-NEXT: vmovdqa64 %xmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
16670 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm20, %xmm1
16671 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm11, %xmm2
16672 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
16673 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm2 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
16674 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm25, %xmm3
16675 ; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
16676 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm4, %xmm4
16677 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
16678 ; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
16679 ; AVX512BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
16680 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
16681 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
16682 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm24, %xmm4
16683 ; AVX512BW-NEXT: vmovdqa64 %xmm24, (%rsp) # 16-byte Spill
16684 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm16, %xmm19
16685 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm19[0],xmm4[0],xmm19[1],xmm4[1],xmm19[2],xmm4[2],xmm19[3],xmm4[3]
16686 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm19 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
16687 ; AVX512BW-NEXT: vpshufb %xmm19, %xmm18, %xmm30
16688 ; AVX512BW-NEXT: vpshufb %xmm19, %xmm17, %xmm13
16689 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm30[0],xmm13[1],xmm30[1],xmm13[2],xmm30[2],xmm13[3],xmm30[3]
16690 ; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
16691 ; AVX512BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
16692 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm13[0,1,2,3,4],ymm4[5],ymm13[6,7]
16693 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
16694 ; AVX512BW-NEXT: vmovdqa %xmm12, %xmm10
16695 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm12, %xmm4
16696 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm27, %xmm13
16697 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
16698 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm8, %xmm13
16699 ; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
16700 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm8, %xmm30
16701 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
16702 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm13[0,1,2],xmm4[3]
16703 ; AVX512BW-NEXT: vpsrlq $24, %zmm6, %zmm13
16704 ; AVX512BW-NEXT: vpmovqb %zmm13, %xmm13
16705 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm13[0,1],xmm4[2,3]
16706 ; AVX512BW-NEXT: vinserti32x4 $2, %xmm4, %zmm0, %zmm4
16707 ; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm4 {%k1}
16708 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm26, %xmm1
16709 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm21, %xmm13
16710 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3]
16711 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm22, %xmm13
16712 ; AVX512BW-NEXT: vmovdqa64 %xmm22, %xmm21
16713 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm29, %xmm30
16714 ; AVX512BW-NEXT: vmovdqa64 %xmm29, %xmm22
16715 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
16716 ; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
16717 ; AVX512BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
16718 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm13[0,1,2,3,4,5,6],ymm1[7]
16719 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm9, %xmm13
16720 ; AVX512BW-NEXT: vmovdqa64 %xmm9, %xmm25
16721 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm7, %xmm3
16722 ; AVX512BW-NEXT: vmovdqa %xmm7, %xmm9
16723 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm13[0],xmm3[1],xmm13[1],xmm3[2],xmm13[2],xmm3[3],xmm13[3]
16724 ; AVX512BW-NEXT: vpshufb %xmm19, %xmm31, %xmm13
16725 ; AVX512BW-NEXT: vpshufb %xmm19, %xmm23, %xmm19
16726 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
16727 ; AVX512BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
16728 ; AVX512BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
16729 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm13[0,1,2,3,4],ymm3[5],ymm13[6,7]
16730 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
16731 ; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
16732 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm5, %xmm3
16733 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm14, %xmm0
16734 ; AVX512BW-NEXT: vmovdqa64 %xmm14, %xmm23
16735 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
16736 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm15, %xmm3
16737 ; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm29 # 16-byte Reload
16738 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm29, %xmm2
16739 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
16740 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
16741 ; AVX512BW-NEXT: vmovdqa64 %zmm28, %zmm17
16742 ; AVX512BW-NEXT: vpsrlq $24, %zmm28, %zmm2
16743 ; AVX512BW-NEXT: vpmovqb %zmm2, %xmm2
16744 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
16745 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
16746 ; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm4, %zmm0
16747 ; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
16748 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
16749 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm20, %xmm1
16750 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm11, %xmm3
16751 ; AVX512BW-NEXT: vmovdqa64 %xmm11, %xmm28
16752 ; AVX512BW-NEXT: vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
16753 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
16754 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
16755 ; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
16756 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm12, %xmm4
16757 ; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm16 # 16-byte Reload
16758 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm16, %xmm13
16759 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
16760 ; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
16761 ; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
16762 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
16763 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
16764 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm24, %xmm13
16765 ; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm26 # 16-byte Reload
16766 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm26, %xmm19
16767 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
16768 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm19 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
16769 ; AVX512BW-NEXT: vpshufb %xmm19, %xmm18, %xmm30
16770 ; AVX512BW-NEXT: vmovdqa64 %xmm18, %xmm24
16771 ; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm18 # 16-byte Reload
16772 ; AVX512BW-NEXT: vpshufb %xmm19, %xmm18, %xmm2
16773 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm30[0],xmm2[1],xmm30[1],xmm2[2],xmm30[2],xmm2[3],xmm30[3]
16774 ; AVX512BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
16775 ; AVX512BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
16776 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm13[5],ymm2[6,7]
16777 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
16778 ; AVX512BW-NEXT: vmovdqa %xmm10, %xmm6
16779 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm10, %xmm2
16780 ; AVX512BW-NEXT: vmovdqa64 %xmm27, %xmm7
16781 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm27, %xmm13
16782 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm13[0],xmm2[0],xmm13[1],xmm2[1],xmm13[2],xmm2[2],xmm13[3],xmm2[3]
16783 ; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
16784 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm10, %xmm13
16785 ; AVX512BW-NEXT: vmovdqa64 %xmm8, %xmm27
16786 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm8, %xmm30
16787 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
16788 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm13[0,1,2],xmm2[3]
16789 ; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
16790 ; AVX512BW-NEXT: vpsrlq $32, %zmm10, %zmm13
16791 ; AVX512BW-NEXT: vpmovqb %zmm13, %xmm13
16792 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm13[0,1],xmm2[2,3]
16793 ; AVX512BW-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm2
16794 ; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm2 {%k1}
16795 ; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
16796 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm14, %xmm1
16797 ; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
16798 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm11, %xmm13
16799 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3]
16800 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm21, %xmm13
16801 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm22, %xmm30
16802 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
16803 ; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
16804 ; AVX512BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
16805 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm13[0,1,2,3,4,5,6],ymm1[7]
16806 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm25, %xmm13
16807 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm9, %xmm4
16808 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm13[0],xmm4[1],xmm13[1],xmm4[2],xmm13[2],xmm4[3],xmm13[3]
16809 ; AVX512BW-NEXT: vpshufb %xmm19, %xmm31, %xmm13
16810 ; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
16811 ; AVX512BW-NEXT: vpshufb %xmm19, %xmm8, %xmm19
16812 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
16813 ; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
16814 ; AVX512BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
16815 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm13[0,1,2,3,4],ymm4[5],ymm13[6,7]
16816 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
16817 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm5, %xmm4
16818 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm23, %xmm0
16819 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
16820 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm15, %xmm4
16821 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm29, %xmm3
16822 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
16823 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[3]
16824 ; AVX512BW-NEXT: vmovdqa64 %zmm17, %zmm20
16825 ; AVX512BW-NEXT: vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
16826 ; AVX512BW-NEXT: vpsrlq $32, %zmm17, %zmm3
16827 ; AVX512BW-NEXT: vpmovqb %zmm3, %xmm3
16828 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
16829 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
16830 ; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm0
16831 ; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
16832 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
16833 ; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
16834 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm1, %xmm1
16835 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm28, %xmm2
16836 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
16837 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm2 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
16838 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm12, %xmm3
16839 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm16, %xmm4
16840 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
16841 ; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
16842 ; AVX512BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
16843 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
16844 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
16845 ; AVX512BW-NEXT: vmovdqa (%rsp), %xmm15 # 16-byte Reload
16846 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm15, %xmm4
16847 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm26, %xmm13
16848 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
16849 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm13 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
16850 ; AVX512BW-NEXT: vpshufb %xmm13, %xmm24, %xmm19
16851 ; AVX512BW-NEXT: vmovdqa64 %xmm24, %xmm26
16852 ; AVX512BW-NEXT: vmovdqa64 %xmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
16853 ; AVX512BW-NEXT: vpshufb %xmm13, %xmm18, %xmm30
16854 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm19 = xmm30[0],xmm19[0],xmm30[1],xmm19[1],xmm30[2],xmm19[2],xmm30[3],xmm19[3]
16855 ; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
16856 ; AVX512BW-NEXT: vinserti32x4 $1, %xmm19, %ymm0, %ymm5
16857 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
16858 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
16859 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm6, %xmm4
16860 ; AVX512BW-NEXT: vmovdqa64 %xmm6, %xmm24
16861 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm7, %xmm5
16862 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
16863 ; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
16864 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm5, %xmm5
16865 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm27, %xmm19
16866 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
16867 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
16868 ; AVX512BW-NEXT: vpsrlq $40, %zmm10, %zmm5
16869 ; AVX512BW-NEXT: vpmovqb %zmm5, %xmm5
16870 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
16871 ; AVX512BW-NEXT: vinserti32x4 $2, %xmm4, %zmm0, %zmm4
16872 ; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm4 {%k1}
16873 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm14, %xmm1
16874 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm11, %xmm5
16875 ; AVX512BW-NEXT: vmovdqa64 %xmm11, %xmm17
16876 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
16877 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm21, %xmm5
16878 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm22, %xmm19
16879 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
16880 ; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
16881 ; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
16882 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
16883 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm25, %xmm5
16884 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm9, %xmm3
16885 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
16886 ; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm31 # 16-byte Reload
16887 ; AVX512BW-NEXT: vpshufb %xmm13, %xmm31, %xmm5
16888 ; AVX512BW-NEXT: vpshufb %xmm13, %xmm8, %xmm13
16889 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]
16890 ; AVX512BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
16891 ; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
16892 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3,4],ymm3[5],ymm5[6,7]
16893 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
16894 ; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
16895 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm11, %xmm3
16896 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm23, %xmm0
16897 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
16898 ; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
16899 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm6, %xmm3
16900 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm29, %xmm2
16901 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
16902 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
16903 ; AVX512BW-NEXT: vpsrlq $40, %zmm20, %zmm2
16904 ; AVX512BW-NEXT: vpmovqb %zmm2, %xmm2
16905 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
16906 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
16907 ; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm4, %zmm0
16908 ; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
16909 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
16910 ; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm28 # 16-byte Reload
16911 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm28, %xmm1
16912 ; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
16913 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm8, %xmm2
16914 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
16915 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm2 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
16916 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm12, %xmm4
16917 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm16, %xmm5
16918 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
16919 ; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
16920 ; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
16921 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
16922 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
16923 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm15, %xmm5
16924 ; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
16925 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm15, %xmm13
16926 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]
16927 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm13 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
16928 ; AVX512BW-NEXT: vpshufb %xmm13, %xmm26, %xmm19
16929 ; AVX512BW-NEXT: vpshufb %xmm13, %xmm18, %xmm30
16930 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm19 = xmm30[0],xmm19[0],xmm30[1],xmm19[1],xmm30[2],xmm19[2],xmm30[3],xmm19[3]
16931 ; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
16932 ; AVX512BW-NEXT: vinserti32x4 $1, %xmm19, %ymm0, %ymm3
16933 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5],ymm3[6,7]
16934 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
16935 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm24, %xmm3
16936 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm7, %xmm5
16937 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
16938 ; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm27 # 16-byte Reload
16939 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm27, %xmm5
16940 ; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm30 # 16-byte Reload
16941 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm30, %xmm19
16942 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
16943 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm3 = xmm5[0,1,2],xmm3[3]
16944 ; AVX512BW-NEXT: vpsrlq $48, %zmm10, %zmm5
16945 ; AVX512BW-NEXT: vpmovqb %zmm5, %xmm5
16946 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm3 = xmm5[0,1],xmm3[2,3]
16947 ; AVX512BW-NEXT: vinserti32x4 $2, %xmm3, %zmm0, %zmm3
16948 ; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm3 {%k1}
16949 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm14, %xmm1
16950 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm17, %xmm5
16951 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
16952 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm21, %xmm5
16953 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm22, %xmm19
16954 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
16955 ; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
16956 ; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
16957 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
16958 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm25, %xmm5
16959 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm9, %xmm4
16960 ; AVX512BW-NEXT: vmovdqa64 %xmm9, %xmm18
16961 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
16962 ; AVX512BW-NEXT: vpshufb %xmm13, %xmm31, %xmm5
16963 ; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm20 # 16-byte Reload
16964 ; AVX512BW-NEXT: vpshufb %xmm13, %xmm20, %xmm13
16965 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]
16966 ; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
16967 ; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
16968 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
16969 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
16970 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm11, %xmm4
16971 ; AVX512BW-NEXT: vpshufb %xmm0, %xmm23, %xmm0
16972 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
16973 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm6, %xmm4
16974 ; AVX512BW-NEXT: vmovdqa64 %xmm6, %xmm26
16975 ; AVX512BW-NEXT: vpshufb %xmm2, %xmm29, %xmm2
16976 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
16977 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
16978 ; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
16979 ; AVX512BW-NEXT: vpsrlq $48, %zmm19, %zmm2
16980 ; AVX512BW-NEXT: vpmovqb %zmm2, %xmm2
16981 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
16982 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
16983 ; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm3, %zmm0
16984 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm1 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
16985 ; AVX512BW-NEXT: vpshufb %xmm1, %xmm28, %xmm2
16986 ; AVX512BW-NEXT: vpshufb %xmm1, %xmm8, %xmm3
16987 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
16988 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
16989 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm12, %xmm4
16990 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm16, %xmm5
16991 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
16992 ; AVX512BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
16993 ; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
16994 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
16995 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
16996 ; AVX512BW-NEXT: vmovdqa (%rsp), %xmm5 # 16-byte Reload
16997 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm5, %xmm5
16998 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm15, %xmm8
16999 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3]
17000 ; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm8 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
17001 ; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
17002 ; AVX512BW-NEXT: vpshufb %xmm8, %xmm6, %xmm9
17003 ; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
17004 ; AVX512BW-NEXT: vpshufb %xmm8, %xmm6, %xmm13
17005 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm13[0],xmm9[0],xmm13[1],xmm9[1],xmm13[2],xmm9[2],xmm13[3],xmm9[3]
17006 ; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
17007 ; AVX512BW-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
17008 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4],ymm5[5],ymm9[6,7]
17009 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3,4,5],ymm2[6,7]
17010 ; AVX512BW-NEXT: vpshufb %xmm1, %xmm24, %xmm5
17011 ; AVX512BW-NEXT: vpshufb %xmm1, %xmm7, %xmm9
17012 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
17013 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm27, %xmm9
17014 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm30, %xmm6
17015 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm9[0],xmm6[1],xmm9[1],xmm6[2],xmm9[2],xmm6[3],xmm9[3]
17016 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3]
17017 ; AVX512BW-NEXT: vpsrlq $56, %zmm10, %zmm6
17018 ; AVX512BW-NEXT: vpmovqb %zmm6, %xmm6
17019 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]
17020 ; AVX512BW-NEXT: vinserti32x4 $2, %xmm5, %zmm0, %zmm5
17021 ; AVX512BW-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm5 {%k1}
17022 ; AVX512BW-NEXT: vpshufb %xmm1, %xmm14, %xmm2
17023 ; AVX512BW-NEXT: vpshufb %xmm1, %xmm17, %xmm6
17024 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
17025 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm21, %xmm6
17026 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm22, %xmm9
17027 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm9[0],xmm6[0],xmm9[1],xmm6[1],xmm9[2],xmm6[2],xmm9[3],xmm6[3]
17028 ; AVX512BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
17029 ; AVX512BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
17030 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3,4,5,6],ymm2[7]
17031 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm25, %xmm6
17032 ; AVX512BW-NEXT: vpshufb %xmm4, %xmm18, %xmm4
17033 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
17034 ; AVX512BW-NEXT: vpshufb %xmm8, %xmm31, %xmm6
17035 ; AVX512BW-NEXT: vpshufb %xmm8, %xmm20, %xmm8
17036 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]
17037 ; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
17038 ; AVX512BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
17039 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm6[0,1,2,3,4],ymm4[5],ymm6[6,7]
17040 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
17041 ; AVX512BW-NEXT: vpshufb %xmm1, %xmm11, %xmm4
17042 ; AVX512BW-NEXT: vpshufb %xmm1, %xmm23, %xmm1
17043 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
17044 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm26, %xmm4
17045 ; AVX512BW-NEXT: vpshufb %xmm3, %xmm29, %xmm3
17046 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
17047 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
17048 ; AVX512BW-NEXT: vpsrlq $56, %zmm19, %zmm3
17049 ; AVX512BW-NEXT: vpmovqb %zmm3, %xmm3
17050 ; AVX512BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
17051 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
17052 ; AVX512BW-NEXT: vinserti64x4 $0, %ymm1, %zmm5, %zmm1
17053 ; AVX512BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
17054 ; AVX512BW-NEXT: vmovaps %zmm2, (%rsi)
17055 ; AVX512BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
17056 ; AVX512BW-NEXT: vmovaps %zmm2, (%rdx)
17057 ; AVX512BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
17058 ; AVX512BW-NEXT: vmovaps %zmm2, (%rcx)
17059 ; AVX512BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
17060 ; AVX512BW-NEXT: vmovaps %zmm2, (%r8)
17061 ; AVX512BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
17062 ; AVX512BW-NEXT: vmovaps %zmm2, (%r9)
17063 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
17064 ; AVX512BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
17065 ; AVX512BW-NEXT: vmovaps %zmm2, (%rax)
17066 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
17067 ; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rax)
17068 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
17069 ; AVX512BW-NEXT: vmovdqa64 %zmm1, (%rax)
17070 ; AVX512BW-NEXT: addq $744, %rsp # imm = 0x2E8
17071 ; AVX512BW-NEXT: vzeroupper
17072 ; AVX512BW-NEXT: retq
17073 ;
17074 ; AVX512BW-FCP-LABEL: load_i8_stride8_vf64:
17075 ; AVX512BW-FCP: # %bb.0:
17076 ; AVX512BW-FCP-NEXT: subq $328, %rsp # imm = 0x148
17077 ; AVX512BW-FCP-NEXT: vmovdqa64 256(%rdi), %zmm18
17078 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm11 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
17079 ; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,2,2,3,0,2,4,6]
17080 ; AVX512BW-FCP-NEXT: vmovdqa 480(%rdi), %ymm1
17081 ; AVX512BW-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
17082 ; AVX512BW-FCP-NEXT: vpermd %ymm1, %ymm0, %ymm30
17083 ; AVX512BW-FCP-NEXT: vpshufb %ymm11, %ymm30, %ymm1
17084 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm8 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
17085 ; AVX512BW-FCP-NEXT: vmovdqa 448(%rdi), %ymm2
17086 ; AVX512BW-FCP-NEXT: vmovdqu %ymm2, (%rsp) # 32-byte Spill
17087 ; AVX512BW-FCP-NEXT: vpermd %ymm2, %ymm0, %ymm31
17088 ; AVX512BW-FCP-NEXT: vpshufb %ymm8, %ymm31, %ymm2
17089 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
17090 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm9 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
17091 ; AVX512BW-FCP-NEXT: vmovdqa 416(%rdi), %ymm2
17092 ; AVX512BW-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
17093 ; AVX512BW-FCP-NEXT: vpermd %ymm2, %ymm0, %ymm19
17094 ; AVX512BW-FCP-NEXT: vpshufb %ymm9, %ymm19, %ymm2
17095 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm10 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
17096 ; AVX512BW-FCP-NEXT: vmovdqa64 384(%rdi), %ymm29
17097 ; AVX512BW-FCP-NEXT: vpermd %ymm29, %ymm0, %ymm14
17098 ; AVX512BW-FCP-NEXT: vpshufb %ymm10, %ymm14, %ymm3
17099 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
17100 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
17101 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm7 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
17102 ; AVX512BW-FCP-NEXT: vmovdqa64 368(%rdi), %xmm21
17103 ; AVX512BW-FCP-NEXT: vpshufb %xmm7, %xmm21, %xmm2
17104 ; AVX512BW-FCP-NEXT: vmovdqa 352(%rdi), %xmm4
17105 ; AVX512BW-FCP-NEXT: vpshufb %xmm7, %xmm4, %xmm3
17106 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
17107 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm23 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
17108 ; AVX512BW-FCP-NEXT: vmovdqa 336(%rdi), %xmm12
17109 ; AVX512BW-FCP-NEXT: vpshufb %xmm23, %xmm12, %xmm5
17110 ; AVX512BW-FCP-NEXT: vmovdqa64 320(%rdi), %xmm28
17111 ; AVX512BW-FCP-NEXT: vpshufb %xmm23, %xmm28, %xmm6
17112 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
17113 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1,2],xmm2[3]
17114 ; AVX512BW-FCP-NEXT: vpmovqb %zmm18, %xmm5
17115 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
17116 ; AVX512BW-FCP-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm20
17117 ; AVX512BW-FCP-NEXT: movb $-64, %al
17118 ; AVX512BW-FCP-NEXT: kmovd %eax, %k1
17119 ; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm20 {%k1}
17120 ; AVX512BW-FCP-NEXT: vmovdqa 224(%rdi), %ymm1
17121 ; AVX512BW-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
17122 ; AVX512BW-FCP-NEXT: vpermd %ymm1, %ymm0, %ymm17
17123 ; AVX512BW-FCP-NEXT: vpshufb %ymm11, %ymm17, %ymm1
17124 ; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm2
17125 ; AVX512BW-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
17126 ; AVX512BW-FCP-NEXT: vpermd %ymm2, %ymm0, %ymm3
17127 ; AVX512BW-FCP-NEXT: vpshufb %ymm8, %ymm3, %ymm5
17128 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm1[7]
17129 ; AVX512BW-FCP-NEXT: vmovdqa 160(%rdi), %ymm1
17130 ; AVX512BW-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
17131 ; AVX512BW-FCP-NEXT: vpermd %ymm1, %ymm0, %ymm2
17132 ; AVX512BW-FCP-NEXT: vpshufb %ymm9, %ymm2, %ymm6
17133 ; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm27
17134 ; AVX512BW-FCP-NEXT: vpermd %ymm27, %ymm0, %ymm16
17135 ; AVX512BW-FCP-NEXT: vpshufb %ymm10, %ymm16, %ymm8
17136 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm8[0,1,2,3,4],ymm6[5],ymm8[6,7]
17137 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1,2,3,4,5],ymm5[6,7]
17138 ; AVX512BW-FCP-NEXT: vmovdqa64 112(%rdi), %xmm26
17139 ; AVX512BW-FCP-NEXT: vpshufb %xmm7, %xmm26, %xmm8
17140 ; AVX512BW-FCP-NEXT: vmovdqa64 96(%rdi), %xmm24
17141 ; AVX512BW-FCP-NEXT: vpshufb %xmm7, %xmm24, %xmm7
17142 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
17143 ; AVX512BW-FCP-NEXT: vmovdqa64 80(%rdi), %xmm22
17144 ; AVX512BW-FCP-NEXT: vpshufb %xmm23, %xmm22, %xmm25
17145 ; AVX512BW-FCP-NEXT: vmovdqa 64(%rdi), %xmm8
17146 ; AVX512BW-FCP-NEXT: vpshufb %xmm23, %xmm8, %xmm23
17147 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm23[0],xmm25[0],xmm23[1],xmm25[1],xmm23[2],xmm25[2],xmm23[3],xmm25[3]
17148 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm15 = xmm15[0,1,2],xmm10[3]
17149 ; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm10
17150 ; AVX512BW-FCP-NEXT: vpmovqb %zmm10, %xmm13
17151 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm13 = xmm13[0,1],xmm15[2,3]
17152 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3],ymm0[4,5,6,7]
17153 ; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm20, %zmm0
17154 ; AVX512BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
17155 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm11 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
17156 ; AVX512BW-FCP-NEXT: vpshufb %ymm11, %ymm30, %ymm0
17157 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm6 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
17158 ; AVX512BW-FCP-NEXT: vpshufb %ymm6, %ymm31, %ymm13
17159 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3,4,5,6],ymm0[7]
17160 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm9 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
17161 ; AVX512BW-FCP-NEXT: vpshufb %ymm9, %ymm19, %ymm13
17162 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm1 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
17163 ; AVX512BW-FCP-NEXT: vpshufb %ymm1, %ymm14, %ymm15
17164 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm13 = ymm15[0,1,2,3,4],ymm13[5],ymm15[6,7]
17165 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3,4,5],ymm0[6,7]
17166 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm13 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
17167 ; AVX512BW-FCP-NEXT: vmovdqa64 %xmm21, %xmm5
17168 ; AVX512BW-FCP-NEXT: vpshufb %xmm13, %xmm21, %xmm15
17169 ; AVX512BW-FCP-NEXT: vpshufb %xmm13, %xmm4, %xmm23
17170 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm23[0],xmm15[0],xmm23[1],xmm15[1],xmm23[2],xmm15[2],xmm23[3],xmm15[3]
17171 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm23 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
17172 ; AVX512BW-FCP-NEXT: vmovdqa %xmm12, %xmm7
17173 ; AVX512BW-FCP-NEXT: vpshufb %xmm23, %xmm12, %xmm25
17174 ; AVX512BW-FCP-NEXT: vpshufb %xmm23, %xmm28, %xmm20
17175 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm20[0],xmm25[0],xmm20[1],xmm25[1],xmm20[2],xmm25[2],xmm20[3],xmm25[3]
17176 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm12[0,1,2],xmm15[3]
17177 ; AVX512BW-FCP-NEXT: vpsrlq $8, %zmm18, %zmm15
17178 ; AVX512BW-FCP-NEXT: vpmovqb %zmm15, %xmm15
17179 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm15[0,1],xmm12[2,3]
17180 ; AVX512BW-FCP-NEXT: vinserti32x4 $2, %xmm12, %zmm0, %zmm12
17181 ; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm12 {%k1}
17182 ; AVX512BW-FCP-NEXT: vpshufb %ymm11, %ymm17, %ymm0
17183 ; AVX512BW-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm15
17184 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm15[0,1,2,3,4,5,6],ymm0[7]
17185 ; AVX512BW-FCP-NEXT: vpshufb %ymm9, %ymm2, %ymm15
17186 ; AVX512BW-FCP-NEXT: vpshufb %ymm1, %ymm16, %ymm11
17187 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3,4],ymm15[5],ymm11[6,7]
17188 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3,4,5],ymm0[6,7]
17189 ; AVX512BW-FCP-NEXT: vpshufb %xmm13, %xmm26, %xmm11
17190 ; AVX512BW-FCP-NEXT: vpshufb %xmm13, %xmm24, %xmm13
17191 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1],xmm13[2],xmm11[2],xmm13[3],xmm11[3]
17192 ; AVX512BW-FCP-NEXT: vpshufb %xmm23, %xmm22, %xmm13
17193 ; AVX512BW-FCP-NEXT: vpshufb %xmm23, %xmm8, %xmm15
17194 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm15[0],xmm13[0],xmm15[1],xmm13[1],xmm15[2],xmm13[2],xmm15[3],xmm13[3]
17195 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm13[0,1,2],xmm11[3]
17196 ; AVX512BW-FCP-NEXT: vpsrlq $8, %zmm10, %zmm13
17197 ; AVX512BW-FCP-NEXT: vpmovqb %zmm13, %xmm13
17198 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm13[0,1],xmm11[2,3]
17199 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3],ymm0[4,5,6,7]
17200 ; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm12, %zmm0
17201 ; AVX512BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
17202 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm21 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
17203 ; AVX512BW-FCP-NEXT: vpshufb %ymm21, %ymm30, %ymm0
17204 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm25 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
17205 ; AVX512BW-FCP-NEXT: vpshufb %ymm25, %ymm31, %ymm11
17206 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3,4,5,6],ymm0[7]
17207 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm9 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
17208 ; AVX512BW-FCP-NEXT: vpshufb %ymm9, %ymm19, %ymm11
17209 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm6 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
17210 ; AVX512BW-FCP-NEXT: vpshufb %ymm6, %ymm14, %ymm12
17211 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2,3,4],ymm11[5],ymm12[6,7]
17212 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3,4,5],ymm0[6,7]
17213 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
17214 ; AVX512BW-FCP-NEXT: vpshufb %xmm11, %xmm5, %xmm12
17215 ; AVX512BW-FCP-NEXT: vpshufb %xmm11, %xmm4, %xmm13
17216 ; AVX512BW-FCP-NEXT: vmovdqa %xmm4, %xmm1
17217 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
17218 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm13 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
17219 ; AVX512BW-FCP-NEXT: vpshufb %xmm13, %xmm7, %xmm15
17220 ; AVX512BW-FCP-NEXT: vmovdqa64 %xmm7, %xmm23
17221 ; AVX512BW-FCP-NEXT: vpshufb %xmm13, %xmm28, %xmm20
17222 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm20[0],xmm15[0],xmm20[1],xmm15[1],xmm20[2],xmm15[2],xmm20[3],xmm15[3]
17223 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm15[0,1,2],xmm12[3]
17224 ; AVX512BW-FCP-NEXT: vpsrlq $16, %zmm18, %zmm15
17225 ; AVX512BW-FCP-NEXT: vpmovqb %zmm15, %xmm15
17226 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm15[0,1],xmm12[2,3]
17227 ; AVX512BW-FCP-NEXT: vinserti32x4 $2, %xmm12, %zmm0, %zmm12
17228 ; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm12 {%k1}
17229 ; AVX512BW-FCP-NEXT: vpshufb %ymm21, %ymm17, %ymm0
17230 ; AVX512BW-FCP-NEXT: vpshufb %ymm25, %ymm3, %ymm15
17231 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm15[0,1,2,3,4,5,6],ymm0[7]
17232 ; AVX512BW-FCP-NEXT: vpshufb %ymm9, %ymm2, %ymm15
17233 ; AVX512BW-FCP-NEXT: vpshufb %ymm6, %ymm16, %ymm9
17234 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4],ymm15[5],ymm9[6,7]
17235 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3,4,5],ymm0[6,7]
17236 ; AVX512BW-FCP-NEXT: vpshufb %xmm11, %xmm26, %xmm9
17237 ; AVX512BW-FCP-NEXT: vpshufb %xmm11, %xmm24, %xmm11
17238 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
17239 ; AVX512BW-FCP-NEXT: vpshufb %xmm13, %xmm22, %xmm11
17240 ; AVX512BW-FCP-NEXT: vpshufb %xmm13, %xmm8, %xmm13
17241 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1],xmm13[2],xmm11[2],xmm13[3],xmm11[3]
17242 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm11[0,1,2],xmm9[3]
17243 ; AVX512BW-FCP-NEXT: vpsrlq $16, %zmm10, %zmm11
17244 ; AVX512BW-FCP-NEXT: vpmovqb %zmm11, %xmm11
17245 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm11[0,1],xmm9[2,3]
17246 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3],ymm0[4,5,6,7]
17247 ; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm12, %zmm0
17248 ; AVX512BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
17249 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm6 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
17250 ; AVX512BW-FCP-NEXT: vpshufb %ymm6, %ymm30, %ymm0
17251 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm4 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
17252 ; AVX512BW-FCP-NEXT: vpshufb %ymm4, %ymm31, %ymm9
17253 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3,4,5,6],ymm0[7]
17254 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm31 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
17255 ; AVX512BW-FCP-NEXT: vpshufb %ymm31, %ymm19, %ymm9
17256 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm30 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
17257 ; AVX512BW-FCP-NEXT: vpshufb %ymm30, %ymm14, %ymm11
17258 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3,4],ymm9[5],ymm11[6,7]
17259 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3,4,5],ymm0[6,7]
17260 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm9 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
17261 ; AVX512BW-FCP-NEXT: vmovdqa64 %xmm5, %xmm19
17262 ; AVX512BW-FCP-NEXT: vpshufb %xmm9, %xmm5, %xmm11
17263 ; AVX512BW-FCP-NEXT: vmovdqa64 %xmm1, %xmm20
17264 ; AVX512BW-FCP-NEXT: vpshufb %xmm9, %xmm1, %xmm12
17265 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
17266 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm12 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
17267 ; AVX512BW-FCP-NEXT: vpshufb %xmm12, %xmm7, %xmm14
17268 ; AVX512BW-FCP-NEXT: vpshufb %xmm12, %xmm28, %xmm15
17269 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
17270 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm14[0,1,2],xmm11[3]
17271 ; AVX512BW-FCP-NEXT: vpsrlq $24, %zmm18, %zmm14
17272 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm18, %zmm25
17273 ; AVX512BW-FCP-NEXT: vpmovqb %zmm14, %xmm14
17274 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm14[0,1],xmm11[2,3]
17275 ; AVX512BW-FCP-NEXT: vinserti32x4 $2, %xmm11, %zmm0, %zmm11
17276 ; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm11 {%k1}
17277 ; AVX512BW-FCP-NEXT: vpshufb %ymm6, %ymm17, %ymm0
17278 ; AVX512BW-FCP-NEXT: vpshufb %ymm4, %ymm3, %ymm3
17279 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5,6],ymm0[7]
17280 ; AVX512BW-FCP-NEXT: vpshufb %ymm31, %ymm2, %ymm2
17281 ; AVX512BW-FCP-NEXT: vpshufb %ymm30, %ymm16, %ymm1
17282 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5],ymm1[6,7]
17283 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
17284 ; AVX512BW-FCP-NEXT: vpshufb %xmm9, %xmm26, %xmm1
17285 ; AVX512BW-FCP-NEXT: vpshufb %xmm9, %xmm24, %xmm2
17286 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
17287 ; AVX512BW-FCP-NEXT: vpshufb %xmm12, %xmm22, %xmm2
17288 ; AVX512BW-FCP-NEXT: vpshufb %xmm12, %xmm8, %xmm3
17289 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
17290 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
17291 ; AVX512BW-FCP-NEXT: vpsrlq $24, %zmm10, %zmm2
17292 ; AVX512BW-FCP-NEXT: vpmovqb %zmm2, %xmm2
17293 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
17294 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
17295 ; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm11, %zmm0
17296 ; AVX512BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
17297 ; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [1,3,2,3,1,3,5,7]
17298 ; AVX512BW-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm6 # 32-byte Folded Reload
17299 ; AVX512BW-FCP-NEXT: vpermd (%rsp), %ymm3, %ymm7 # 32-byte Folded Reload
17300 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm13 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
17301 ; AVX512BW-FCP-NEXT: vpshufb %ymm13, %ymm6, %ymm2
17302 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm1 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
17303 ; AVX512BW-FCP-NEXT: vpshufb %ymm1, %ymm7, %ymm9
17304 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5,6],ymm2[7]
17305 ; AVX512BW-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm5 # 32-byte Folded Reload
17306 ; AVX512BW-FCP-NEXT: vpermd %ymm29, %ymm3, %ymm14
17307 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm0 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
17308 ; AVX512BW-FCP-NEXT: vpshufb %ymm0, %ymm5, %ymm11
17309 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm4 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
17310 ; AVX512BW-FCP-NEXT: vpshufb %ymm4, %ymm14, %ymm12
17311 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2,3,4],ymm11[5],ymm12[6,7]
17312 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3,4,5],ymm9[6,7]
17313 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
17314 ; AVX512BW-FCP-NEXT: vpshufb %xmm11, %xmm19, %xmm12
17315 ; AVX512BW-FCP-NEXT: vmovdqa64 %xmm19, %xmm16
17316 ; AVX512BW-FCP-NEXT: vpshufb %xmm11, %xmm20, %xmm15
17317 ; AVX512BW-FCP-NEXT: vmovdqa64 %xmm20, %xmm17
17318 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm15[0],xmm12[0],xmm15[1],xmm12[1],xmm15[2],xmm12[2],xmm15[3],xmm12[3]
17319 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm20 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
17320 ; AVX512BW-FCP-NEXT: vpshufb %xmm20, %xmm23, %xmm15
17321 ; AVX512BW-FCP-NEXT: vpshufb %xmm20, %xmm28, %xmm29
17322 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm29[0],xmm15[0],xmm29[1],xmm15[1],xmm29[2],xmm15[2],xmm29[3],xmm15[3]
17323 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm15[0,1,2],xmm12[3]
17324 ; AVX512BW-FCP-NEXT: vpsrlq $32, %zmm18, %zmm15
17325 ; AVX512BW-FCP-NEXT: vpmovqb %zmm15, %xmm15
17326 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm15[0,1],xmm12[2,3]
17327 ; AVX512BW-FCP-NEXT: vinserti32x4 $2, %xmm12, %zmm0, %zmm12
17328 ; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm9, %zmm0, %zmm12 {%k1}
17329 ; AVX512BW-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm19 # 32-byte Folded Reload
17330 ; AVX512BW-FCP-NEXT: vpshufb %ymm13, %ymm19, %ymm9
17331 ; AVX512BW-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm18 # 32-byte Folded Reload
17332 ; AVX512BW-FCP-NEXT: vpshufb %ymm1, %ymm18, %ymm15
17333 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm15[0,1,2,3,4,5,6],ymm9[7]
17334 ; AVX512BW-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm15 # 32-byte Folded Reload
17335 ; AVX512BW-FCP-NEXT: vpermd %ymm27, %ymm3, %ymm21
17336 ; AVX512BW-FCP-NEXT: vpshufb %ymm0, %ymm15, %ymm3
17337 ; AVX512BW-FCP-NEXT: vpshufb %ymm4, %ymm21, %ymm13
17338 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm13[0,1,2,3,4],ymm3[5],ymm13[6,7]
17339 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm9[6,7]
17340 ; AVX512BW-FCP-NEXT: vpshufb %xmm11, %xmm26, %xmm9
17341 ; AVX512BW-FCP-NEXT: vpshufb %xmm11, %xmm24, %xmm11
17342 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
17343 ; AVX512BW-FCP-NEXT: vpshufb %xmm20, %xmm22, %xmm11
17344 ; AVX512BW-FCP-NEXT: vpshufb %xmm20, %xmm8, %xmm13
17345 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1],xmm13[2],xmm11[2],xmm13[3],xmm11[3]
17346 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm11[0,1,2],xmm9[3]
17347 ; AVX512BW-FCP-NEXT: vpsrlq $32, %zmm10, %zmm11
17348 ; AVX512BW-FCP-NEXT: vpmovqb %zmm11, %xmm11
17349 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm11[0,1],xmm9[2,3]
17350 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm9[0,1,2,3],ymm3[4,5,6,7]
17351 ; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm3, %zmm12, %zmm29
17352 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm4 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
17353 ; AVX512BW-FCP-NEXT: vpshufb %ymm4, %ymm6, %ymm9
17354 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm0 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
17355 ; AVX512BW-FCP-NEXT: vpshufb %ymm0, %ymm7, %ymm11
17356 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3,4,5,6],ymm9[7]
17357 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm2 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
17358 ; AVX512BW-FCP-NEXT: vpshufb %ymm2, %ymm5, %ymm11
17359 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm1 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
17360 ; AVX512BW-FCP-NEXT: vpshufb %ymm1, %ymm14, %ymm12
17361 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2,3,4],ymm11[5],ymm12[6,7]
17362 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3,4,5],ymm9[6,7]
17363 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
17364 ; AVX512BW-FCP-NEXT: vpshufb %xmm11, %xmm16, %xmm12
17365 ; AVX512BW-FCP-NEXT: vpshufb %xmm11, %xmm17, %xmm13
17366 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
17367 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm13 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
17368 ; AVX512BW-FCP-NEXT: vpshufb %xmm13, %xmm23, %xmm20
17369 ; AVX512BW-FCP-NEXT: vpshufb %xmm13, %xmm28, %xmm27
17370 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm27[0],xmm20[0],xmm27[1],xmm20[1],xmm27[2],xmm20[2],xmm27[3],xmm20[3]
17371 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1,2],xmm12[3]
17372 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm25, %zmm27
17373 ; AVX512BW-FCP-NEXT: vpsrlq $40, %zmm25, %zmm12
17374 ; AVX512BW-FCP-NEXT: vpmovqb %zmm12, %xmm12
17375 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm3 = xmm12[0,1],xmm3[2,3]
17376 ; AVX512BW-FCP-NEXT: vinserti32x4 $2, %xmm3, %zmm0, %zmm3
17377 ; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm9, %zmm0, %zmm3 {%k1}
17378 ; AVX512BW-FCP-NEXT: vpshufb %ymm4, %ymm19, %ymm9
17379 ; AVX512BW-FCP-NEXT: vpshufb %ymm0, %ymm18, %ymm12
17380 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm12[0,1,2,3,4,5,6],ymm9[7]
17381 ; AVX512BW-FCP-NEXT: vpshufb %ymm2, %ymm15, %ymm12
17382 ; AVX512BW-FCP-NEXT: vpshufb %ymm1, %ymm21, %ymm4
17383 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4],ymm12[5],ymm4[6,7]
17384 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],ymm9[6,7]
17385 ; AVX512BW-FCP-NEXT: vpshufb %xmm11, %xmm26, %xmm9
17386 ; AVX512BW-FCP-NEXT: vpshufb %xmm11, %xmm24, %xmm11
17387 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
17388 ; AVX512BW-FCP-NEXT: vpshufb %xmm13, %xmm22, %xmm11
17389 ; AVX512BW-FCP-NEXT: vpshufb %xmm13, %xmm8, %xmm12
17390 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
17391 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm11[0,1,2],xmm9[3]
17392 ; AVX512BW-FCP-NEXT: vpsrlq $40, %zmm10, %zmm11
17393 ; AVX512BW-FCP-NEXT: vpmovqb %zmm11, %xmm11
17394 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm11[0,1],xmm9[2,3]
17395 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3],ymm4[4,5,6,7]
17396 ; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm3, %zmm20
17397 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm0 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
17398 ; AVX512BW-FCP-NEXT: vpshufb %ymm0, %ymm6, %ymm3
17399 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm1 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
17400 ; AVX512BW-FCP-NEXT: vpshufb %ymm1, %ymm7, %ymm4
17401 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5,6],ymm3[7]
17402 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm2 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
17403 ; AVX512BW-FCP-NEXT: vpshufb %ymm2, %ymm5, %ymm4
17404 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm25 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
17405 ; AVX512BW-FCP-NEXT: vpshufb %ymm25, %ymm14, %ymm9
17406 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4],ymm4[5],ymm9[6,7]
17407 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
17408 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm4 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
17409 ; AVX512BW-FCP-NEXT: vpshufb %xmm4, %xmm16, %xmm9
17410 ; AVX512BW-FCP-NEXT: vpshufb %xmm4, %xmm17, %xmm11
17411 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
17412 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
17413 ; AVX512BW-FCP-NEXT: vpshufb %xmm11, %xmm23, %xmm12
17414 ; AVX512BW-FCP-NEXT: vpshufb %xmm11, %xmm28, %xmm13
17415 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
17416 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm12[0,1,2],xmm9[3]
17417 ; AVX512BW-FCP-NEXT: vpsrlq $48, %zmm27, %zmm12
17418 ; AVX512BW-FCP-NEXT: vpmovqb %zmm12, %xmm12
17419 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm12[0,1],xmm9[2,3]
17420 ; AVX512BW-FCP-NEXT: vinserti32x4 $2, %xmm9, %zmm0, %zmm9
17421 ; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm9 {%k1}
17422 ; AVX512BW-FCP-NEXT: vpshufb %ymm0, %ymm19, %ymm3
17423 ; AVX512BW-FCP-NEXT: vpshufb %ymm1, %ymm18, %ymm12
17424 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm12[0,1,2,3,4,5,6],ymm3[7]
17425 ; AVX512BW-FCP-NEXT: vpshufb %ymm2, %ymm15, %ymm12
17426 ; AVX512BW-FCP-NEXT: vpshufb %ymm25, %ymm21, %ymm13
17427 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3,4],ymm12[5],ymm13[6,7]
17428 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm12[0,1,2,3,4,5],ymm3[6,7]
17429 ; AVX512BW-FCP-NEXT: vpshufb %xmm4, %xmm26, %xmm12
17430 ; AVX512BW-FCP-NEXT: vpshufb %xmm4, %xmm24, %xmm4
17431 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm12[0],xmm4[1],xmm12[1],xmm4[2],xmm12[2],xmm4[3],xmm12[3]
17432 ; AVX512BW-FCP-NEXT: vpshufb %xmm11, %xmm22, %xmm12
17433 ; AVX512BW-FCP-NEXT: vpshufb %xmm11, %xmm8, %xmm11
17434 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
17435 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm11[0,1,2],xmm4[3]
17436 ; AVX512BW-FCP-NEXT: vpsrlq $48, %zmm10, %zmm11
17437 ; AVX512BW-FCP-NEXT: vpmovqb %zmm11, %xmm11
17438 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm11[0,1],xmm4[2,3]
17439 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
17440 ; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm3, %zmm9, %zmm3
17441 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm12 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
17442 ; AVX512BW-FCP-NEXT: vpshufb %ymm12, %ymm6, %ymm0
17443 ; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm6 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
17444 ; AVX512BW-FCP-NEXT: vpshufb %ymm6, %ymm7, %ymm1
17445 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6],ymm0[7]
17446 ; AVX512BW-FCP-NEXT: vpshufb %ymm31, %ymm5, %ymm1
17447 ; AVX512BW-FCP-NEXT: vpshufb %ymm30, %ymm14, %ymm2
17448 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4],ymm1[5],ymm2[6,7]
17449 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
17450 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
17451 ; AVX512BW-FCP-NEXT: vpshufb %xmm1, %xmm16, %xmm2
17452 ; AVX512BW-FCP-NEXT: vpshufb %xmm1, %xmm17, %xmm4
17453 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
17454 ; AVX512BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm4 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
17455 ; AVX512BW-FCP-NEXT: vpshufb %xmm4, %xmm23, %xmm9
17456 ; AVX512BW-FCP-NEXT: vpshufb %xmm4, %xmm28, %xmm11
17457 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
17458 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm9[0,1,2],xmm2[3]
17459 ; AVX512BW-FCP-NEXT: vpsrlq $56, %zmm27, %zmm9
17460 ; AVX512BW-FCP-NEXT: vpmovqb %zmm9, %xmm9
17461 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm9[0,1],xmm2[2,3]
17462 ; AVX512BW-FCP-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm2
17463 ; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm2 {%k1}
17464 ; AVX512BW-FCP-NEXT: vpshufb %ymm12, %ymm19, %ymm0
17465 ; AVX512BW-FCP-NEXT: vpshufb %ymm6, %ymm18, %ymm9
17466 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3,4,5,6],ymm0[7]
17467 ; AVX512BW-FCP-NEXT: vpshufb %ymm31, %ymm15, %ymm9
17468 ; AVX512BW-FCP-NEXT: vpshufb %ymm30, %ymm21, %ymm11
17469 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3,4],ymm9[5],ymm11[6,7]
17470 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3,4,5],ymm0[6,7]
17471 ; AVX512BW-FCP-NEXT: vpshufb %xmm1, %xmm26, %xmm5
17472 ; AVX512BW-FCP-NEXT: vpshufb %xmm1, %xmm24, %xmm1
17473 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
17474 ; AVX512BW-FCP-NEXT: vpshufb %xmm4, %xmm22, %xmm5
17475 ; AVX512BW-FCP-NEXT: vpshufb %xmm4, %xmm8, %xmm4
17476 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
17477 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm4[0,1,2],xmm1[3]
17478 ; AVX512BW-FCP-NEXT: vpsrlq $56, %zmm10, %zmm4
17479 ; AVX512BW-FCP-NEXT: vpmovqb %zmm4, %xmm4
17480 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm4[0,1],xmm1[2,3]
17481 ; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
17482 ; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm0
17483 ; AVX512BW-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
17484 ; AVX512BW-FCP-NEXT: vmovaps %zmm1, (%rsi)
17485 ; AVX512BW-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
17486 ; AVX512BW-FCP-NEXT: vmovaps %zmm1, (%rdx)
17487 ; AVX512BW-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
17488 ; AVX512BW-FCP-NEXT: vmovaps %zmm1, (%rcx)
17489 ; AVX512BW-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
17490 ; AVX512BW-FCP-NEXT: vmovaps %zmm1, (%r8)
17491 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm29, (%r9)
17492 ; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
17493 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm20, (%rax)
17494 ; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
17495 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm3, (%rax)
17496 ; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
17497 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, (%rax)
17498 ; AVX512BW-FCP-NEXT: addq $328, %rsp # imm = 0x148
17499 ; AVX512BW-FCP-NEXT: vzeroupper
17500 ; AVX512BW-FCP-NEXT: retq
17501 ;
17502 ; AVX512DQ-BW-LABEL: load_i8_stride8_vf64:
17503 ; AVX512DQ-BW: # %bb.0:
17504 ; AVX512DQ-BW-NEXT: subq $744, %rsp # imm = 0x2E8
17505 ; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm5
17506 ; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %zmm0
17507 ; AVX512DQ-BW-NEXT: vpmovqb %zmm0, %xmm0
17508 ; AVX512DQ-BW-NEXT: vmovdqa64 256(%rdi), %zmm1
17509 ; AVX512DQ-BW-NEXT: vmovdqa64 384(%rdi), %zmm2
17510 ; AVX512DQ-BW-NEXT: vpmovqb %zmm2, %xmm2
17511 ; AVX512DQ-BW-NEXT: vmovdqa64 496(%rdi), %xmm24
17512 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm12 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
17513 ; AVX512DQ-BW-NEXT: vpshufb %xmm12, %xmm24, %xmm3
17514 ; AVX512DQ-BW-NEXT: vmovdqa64 480(%rdi), %xmm25
17515 ; AVX512DQ-BW-NEXT: vpshufb %xmm12, %xmm25, %xmm4
17516 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
17517 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
17518 ; AVX512DQ-BW-NEXT: vmovdqa64 464(%rdi), %xmm26
17519 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm19 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
17520 ; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm26, %xmm4
17521 ; AVX512DQ-BW-NEXT: vmovdqa64 448(%rdi), %xmm30
17522 ; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm30, %xmm6
17523 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
17524 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
17525 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5,6],ymm3[7]
17526 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
17527 ; AVX512DQ-BW-NEXT: vmovdqa 384(%rdi), %ymm4
17528 ; AVX512DQ-BW-NEXT: vpmovqb %ymm4, %xmm4
17529 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
17530 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4],ymm2[5],ymm4[6,7]
17531 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm7 = ymm2[0,1,2,3,4,5],ymm3[6,7]
17532 ; AVX512DQ-BW-NEXT: vmovdqa64 368(%rdi), %xmm31
17533 ; AVX512DQ-BW-NEXT: vpshufb %xmm12, %xmm31, %xmm4
17534 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
17535 ; AVX512DQ-BW-NEXT: vmovdqa64 352(%rdi), %xmm27
17536 ; AVX512DQ-BW-NEXT: vpshufb %xmm12, %xmm27, %xmm6
17537 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
17538 ; AVX512DQ-BW-NEXT: vmovdqa64 336(%rdi), %xmm22
17539 ; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm22, %xmm11
17540 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
17541 ; AVX512DQ-BW-NEXT: vmovdqa 320(%rdi), %xmm9
17542 ; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm9, %xmm15
17543 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm15[0],xmm11[0],xmm15[1],xmm11[1],xmm15[2],xmm11[2],xmm15[3],xmm11[3]
17544 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1,2],xmm10[3]
17545 ; AVX512DQ-BW-NEXT: vpmovqb %zmm1, %xmm11
17546 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, %zmm15
17547 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3]
17548 ; AVX512DQ-BW-NEXT: vinserti32x4 $2, %xmm10, %zmm0, %zmm20
17549 ; AVX512DQ-BW-NEXT: movb $-64, %al
17550 ; AVX512DQ-BW-NEXT: kmovd %eax, %k1
17551 ; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm7, %zmm0, %zmm20 {%k1}
17552 ; AVX512DQ-BW-NEXT: vmovdqa64 240(%rdi), %xmm28
17553 ; AVX512DQ-BW-NEXT: vpshufb %xmm12, %xmm28, %xmm7
17554 ; AVX512DQ-BW-NEXT: vmovdqa64 224(%rdi), %xmm18
17555 ; AVX512DQ-BW-NEXT: vpshufb %xmm12, %xmm18, %xmm10
17556 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
17557 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm10[0],xmm7[0],xmm10[1],xmm7[1],xmm10[2],xmm7[2],xmm10[3],xmm7[3]
17558 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
17559 ; AVX512DQ-BW-NEXT: vmovdqa64 208(%rdi), %xmm17
17560 ; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm17, %xmm10
17561 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
17562 ; AVX512DQ-BW-NEXT: vmovdqa 192(%rdi), %xmm8
17563 ; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm8, %xmm16
17564 ; AVX512DQ-BW-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
17565 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm16[0],xmm10[0],xmm16[1],xmm10[1],xmm16[2],xmm10[2],xmm16[3],xmm10[3]
17566 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
17567 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm7 = ymm10[0,1,2,3,4,5,6],ymm7[7]
17568 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
17569 ; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm10
17570 ; AVX512DQ-BW-NEXT: vpmovqb %ymm10, %xmm10
17571 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
17572 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm10[0,1,2,3,4],ymm0[5],ymm10[6,7]
17573 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm7[6,7]
17574 ; AVX512DQ-BW-NEXT: vmovdqa 112(%rdi), %xmm10
17575 ; AVX512DQ-BW-NEXT: vpshufb %xmm12, %xmm10, %xmm16
17576 ; AVX512DQ-BW-NEXT: vmovdqa 96(%rdi), %xmm14
17577 ; AVX512DQ-BW-NEXT: vpshufb %xmm12, %xmm14, %xmm12
17578 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm12[0],xmm16[0],xmm12[1],xmm16[1],xmm12[2],xmm16[2],xmm12[3],xmm16[3]
17579 ; AVX512DQ-BW-NEXT: vmovdqa 80(%rdi), %xmm2
17580 ; AVX512DQ-BW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
17581 ; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm2, %xmm21
17582 ; AVX512DQ-BW-NEXT: vmovdqa 64(%rdi), %xmm2
17583 ; AVX512DQ-BW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
17584 ; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm2, %xmm19
17585 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm21[0],xmm19[1],xmm21[1],xmm19[2],xmm21[2],xmm19[3],xmm21[3]
17586 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
17587 ; AVX512DQ-BW-NEXT: vpmovqb %zmm5, %xmm13
17588 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
17589 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
17590 ; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm20, %zmm0
17591 ; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
17592 ; AVX512DQ-BW-NEXT: vmovdqa 160(%rdi), %xmm6
17593 ; AVX512DQ-BW-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
17594 ; AVX512DQ-BW-NEXT: vmovdqa 384(%rdi), %xmm7
17595 ; AVX512DQ-BW-NEXT: vmovdqa64 400(%rdi), %xmm21
17596 ; AVX512DQ-BW-NEXT: vmovdqa64 416(%rdi), %xmm23
17597 ; AVX512DQ-BW-NEXT: vmovdqa64 432(%rdi), %xmm29
17598 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
17599 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm24, %xmm19
17600 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm24, %xmm1
17601 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm25, %xmm11
17602 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm25, %xmm13
17603 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3]
17604 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
17605 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm13 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
17606 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm26, %xmm12
17607 ; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm26, %xmm24
17608 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm30, %xmm16
17609 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
17610 ; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm30, %xmm25
17611 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm24 = xmm25[0],xmm24[0],xmm25[1],xmm24[1],xmm25[2],xmm24[2],xmm25[3],xmm24[3]
17612 ; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm24, %ymm0, %ymm2
17613 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
17614 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm26 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
17615 ; AVX512DQ-BW-NEXT: vpshufb %xmm26, %xmm29, %xmm24
17616 ; AVX512DQ-BW-NEXT: vpshufb %xmm26, %xmm23, %xmm25
17617 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm24 = xmm25[0],xmm24[0],xmm25[1],xmm24[1],xmm25[2],xmm24[2],xmm25[3],xmm24[3]
17618 ; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm24, %ymm0, %ymm2
17619 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm30 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
17620 ; AVX512DQ-BW-NEXT: vpshufb %xmm30, %xmm21, %xmm24
17621 ; AVX512DQ-BW-NEXT: vpshufb %xmm30, %xmm7, %xmm25
17622 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm24 = xmm25[0],xmm24[0],xmm25[1],xmm24[1],xmm25[2],xmm24[2],xmm25[3],xmm24[3]
17623 ; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm24, %ymm0, %ymm3
17624 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
17625 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
17626 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm31, %xmm2
17627 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm27, %xmm3
17628 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
17629 ; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm22, %xmm3
17630 ; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm9, %xmm24
17631 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm9, %xmm31
17632 ; AVX512DQ-BW-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
17633 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm24[0],xmm3[0],xmm24[1],xmm3[1],xmm24[2],xmm3[2],xmm24[3],xmm3[3]
17634 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3]
17635 ; AVX512DQ-BW-NEXT: vpsrlq $8, %zmm15, %zmm3
17636 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm15, %zmm22
17637 ; AVX512DQ-BW-NEXT: vpmovqb %zmm3, %xmm3
17638 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]
17639 ; AVX512DQ-BW-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm2
17640 ; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm2 {%k1}
17641 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm28, %xmm9
17642 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm28, %xmm1
17643 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm18, %xmm3
17644 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
17645 ; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm17, %xmm3
17646 ; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm8, %xmm24
17647 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm24[0],xmm3[0],xmm24[1],xmm3[1],xmm24[2],xmm3[2],xmm24[3],xmm3[3]
17648 ; AVX512DQ-BW-NEXT: vmovdqa 176(%rdi), %xmm0
17649 ; AVX512DQ-BW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
17650 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
17651 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
17652 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
17653 ; AVX512DQ-BW-NEXT: vpshufb %xmm26, %xmm0, %xmm3
17654 ; AVX512DQ-BW-NEXT: vpshufb %xmm26, %xmm6, %xmm25
17655 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm25[0],xmm3[0],xmm25[1],xmm3[1],xmm25[2],xmm3[2],xmm25[3],xmm3[3]
17656 ; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %xmm6
17657 ; AVX512DQ-BW-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
17658 ; AVX512DQ-BW-NEXT: vmovdqa 144(%rdi), %xmm0
17659 ; AVX512DQ-BW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
17660 ; AVX512DQ-BW-NEXT: vpshufb %xmm30, %xmm0, %xmm0
17661 ; AVX512DQ-BW-NEXT: vpshufb %xmm30, %xmm6, %xmm30
17662 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm30[0],xmm0[0],xmm30[1],xmm0[1],xmm30[2],xmm0[2],xmm30[3],xmm0[3]
17663 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
17664 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
17665 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5],ymm0[6,7]
17666 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
17667 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm10, %xmm1
17668 ; AVX512DQ-BW-NEXT: vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
17669 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm14, %xmm3
17670 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
17671 ; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
17672 ; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm15, %xmm3
17673 ; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm28 # 16-byte Reload
17674 ; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm28, %xmm4
17675 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
17676 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
17677 ; AVX512DQ-BW-NEXT: vpsrlq $8, %zmm5, %zmm3
17678 ; AVX512DQ-BW-NEXT: vpmovqb %zmm3, %xmm3
17679 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
17680 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
17681 ; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm0
17682 ; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
17683 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm1 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
17684 ; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm19, %xmm2
17685 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm19, %xmm20
17686 ; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm11, %xmm3
17687 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
17688 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
17689 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm12, %xmm4
17690 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm12, %xmm25
17691 ; AVX512DQ-BW-NEXT: vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
17692 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm16, %xmm13
17693 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
17694 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
17695 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
17696 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
17697 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
17698 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm29, %xmm24
17699 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm29, %xmm13
17700 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm23, %xmm30
17701 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm23, %xmm16
17702 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
17703 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
17704 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm30 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
17705 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm21, %xmm18
17706 ; AVX512DQ-BW-NEXT: vpshufb %xmm30, %xmm21, %xmm0
17707 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm7, %xmm17
17708 ; AVX512DQ-BW-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
17709 ; AVX512DQ-BW-NEXT: vpshufb %xmm30, %xmm7, %xmm19
17710 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm19[0],xmm0[0],xmm19[1],xmm0[1],xmm19[2],xmm0[2],xmm19[3],xmm0[3]
17711 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
17712 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
17713 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm13[5],ymm0[6,7]
17714 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
17715 ; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
17716 ; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm12, %xmm2
17717 ; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm27, %xmm13
17718 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm13[0],xmm2[0],xmm13[1],xmm2[1],xmm13[2],xmm2[2],xmm13[3],xmm2[3]
17719 ; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
17720 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm8, %xmm13
17721 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm31, %xmm19
17722 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
17723 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm13[0,1,2],xmm2[3]
17724 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm22, %zmm6
17725 ; AVX512DQ-BW-NEXT: vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
17726 ; AVX512DQ-BW-NEXT: vpsrlq $16, %zmm22, %zmm13
17727 ; AVX512DQ-BW-NEXT: vpmovqb %zmm13, %xmm13
17728 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm13[0,1],xmm2[2,3]
17729 ; AVX512DQ-BW-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm2
17730 ; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm2 {%k1}
17731 ; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm9, %xmm0
17732 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm9, %xmm26
17733 ; AVX512DQ-BW-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
17734 ; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm21 # 16-byte Reload
17735 ; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm21, %xmm13
17736 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm13[0],xmm0[0],xmm13[1],xmm0[1],xmm13[2],xmm0[2],xmm13[3],xmm0[3]
17737 ; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm22 # 16-byte Reload
17738 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm22, %xmm13
17739 ; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm29 # 16-byte Reload
17740 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm29, %xmm19
17741 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
17742 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
17743 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
17744 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3,4,5,6],ymm0[7]
17745 ; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
17746 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm9, %xmm13
17747 ; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
17748 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm7, %xmm4
17749 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm13[0],xmm4[1],xmm13[1],xmm4[2],xmm13[2],xmm4[3],xmm13[3]
17750 ; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm31 # 16-byte Reload
17751 ; AVX512DQ-BW-NEXT: vpshufb %xmm30, %xmm31, %xmm13
17752 ; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm23 # 16-byte Reload
17753 ; AVX512DQ-BW-NEXT: vpshufb %xmm30, %xmm23, %xmm19
17754 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
17755 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
17756 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
17757 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm13[0,1,2,3,4],ymm4[5],ymm13[6,7]
17758 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
17759 ; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm10, %xmm4
17760 ; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm14, %xmm1
17761 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
17762 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm15, %xmm4
17763 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm28, %xmm3
17764 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
17765 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
17766 ; AVX512DQ-BW-NEXT: vpsrlq $16, %zmm5, %zmm3
17767 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, %zmm28
17768 ; AVX512DQ-BW-NEXT: vpmovqb %zmm3, %xmm3
17769 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
17770 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
17771 ; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm0
17772 ; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
17773 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
17774 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
17775 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm20, %xmm1
17776 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm11, %xmm2
17777 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
17778 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm2 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
17779 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm25, %xmm3
17780 ; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
17781 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm4, %xmm4
17782 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
17783 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
17784 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
17785 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
17786 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
17787 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm24, %xmm4
17788 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm24, (%rsp) # 16-byte Spill
17789 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm16, %xmm19
17790 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm19[0],xmm4[0],xmm19[1],xmm4[1],xmm19[2],xmm4[2],xmm19[3],xmm4[3]
17791 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm19 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
17792 ; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm18, %xmm30
17793 ; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm17, %xmm13
17794 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm30[0],xmm13[1],xmm30[1],xmm13[2],xmm30[2],xmm13[3],xmm30[3]
17795 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
17796 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
17797 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm13[0,1,2,3,4],ymm4[5],ymm13[6,7]
17798 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
17799 ; AVX512DQ-BW-NEXT: vmovdqa %xmm12, %xmm10
17800 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm12, %xmm4
17801 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm27, %xmm13
17802 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
17803 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm8, %xmm13
17804 ; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
17805 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm8, %xmm30
17806 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
17807 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm13[0,1,2],xmm4[3]
17808 ; AVX512DQ-BW-NEXT: vpsrlq $24, %zmm6, %zmm13
17809 ; AVX512DQ-BW-NEXT: vpmovqb %zmm13, %xmm13
17810 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm13[0,1],xmm4[2,3]
17811 ; AVX512DQ-BW-NEXT: vinserti32x4 $2, %xmm4, %zmm0, %zmm4
17812 ; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm4 {%k1}
17813 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm26, %xmm1
17814 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm21, %xmm13
17815 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3]
17816 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm22, %xmm13
17817 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm22, %xmm21
17818 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm29, %xmm30
17819 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm29, %xmm22
17820 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
17821 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
17822 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
17823 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm13[0,1,2,3,4,5,6],ymm1[7]
17824 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm9, %xmm13
17825 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm9, %xmm25
17826 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm7, %xmm3
17827 ; AVX512DQ-BW-NEXT: vmovdqa %xmm7, %xmm9
17828 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm13[0],xmm3[1],xmm13[1],xmm3[2],xmm13[2],xmm3[3],xmm13[3]
17829 ; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm31, %xmm13
17830 ; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm23, %xmm19
17831 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
17832 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
17833 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
17834 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm13[0,1,2,3,4],ymm3[5],ymm13[6,7]
17835 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
17836 ; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
17837 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm5, %xmm3
17838 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm14, %xmm0
17839 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm14, %xmm23
17840 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
17841 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm15, %xmm3
17842 ; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm29 # 16-byte Reload
17843 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm29, %xmm2
17844 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
17845 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
17846 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm28, %zmm17
17847 ; AVX512DQ-BW-NEXT: vpsrlq $24, %zmm28, %zmm2
17848 ; AVX512DQ-BW-NEXT: vpmovqb %zmm2, %xmm2
17849 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
17850 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
17851 ; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm4, %zmm0
17852 ; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
17853 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
17854 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm20, %xmm1
17855 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm11, %xmm3
17856 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm11, %xmm28
17857 ; AVX512DQ-BW-NEXT: vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
17858 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
17859 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
17860 ; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
17861 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm12, %xmm4
17862 ; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm16 # 16-byte Reload
17863 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm16, %xmm13
17864 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
17865 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
17866 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
17867 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
17868 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
17869 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm24, %xmm13
17870 ; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm26 # 16-byte Reload
17871 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm26, %xmm19
17872 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
17873 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm19 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
17874 ; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm18, %xmm30
17875 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm18, %xmm24
17876 ; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm18 # 16-byte Reload
17877 ; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm18, %xmm2
17878 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm30[0],xmm2[1],xmm30[1],xmm2[2],xmm30[2],xmm2[3],xmm30[3]
17879 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
17880 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
17881 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm13[5],ymm2[6,7]
17882 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
17883 ; AVX512DQ-BW-NEXT: vmovdqa %xmm10, %xmm6
17884 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm10, %xmm2
17885 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm27, %xmm7
17886 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm27, %xmm13
17887 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm13[0],xmm2[0],xmm13[1],xmm2[1],xmm13[2],xmm2[2],xmm13[3],xmm2[3]
17888 ; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
17889 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm10, %xmm13
17890 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm8, %xmm27
17891 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm8, %xmm30
17892 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
17893 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm13[0,1,2],xmm2[3]
17894 ; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
17895 ; AVX512DQ-BW-NEXT: vpsrlq $32, %zmm10, %zmm13
17896 ; AVX512DQ-BW-NEXT: vpmovqb %zmm13, %xmm13
17897 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm13[0,1],xmm2[2,3]
17898 ; AVX512DQ-BW-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm2
17899 ; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm2 {%k1}
17900 ; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
17901 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm14, %xmm1
17902 ; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
17903 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm11, %xmm13
17904 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3]
17905 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm21, %xmm13
17906 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm22, %xmm30
17907 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
17908 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
17909 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
17910 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm13[0,1,2,3,4,5,6],ymm1[7]
17911 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm25, %xmm13
17912 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm9, %xmm4
17913 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm13[0],xmm4[1],xmm13[1],xmm4[2],xmm13[2],xmm4[3],xmm13[3]
17914 ; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm31, %xmm13
17915 ; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
17916 ; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm8, %xmm19
17917 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
17918 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
17919 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
17920 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm13[0,1,2,3,4],ymm4[5],ymm13[6,7]
17921 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
17922 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm5, %xmm4
17923 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm23, %xmm0
17924 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
17925 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm15, %xmm4
17926 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm29, %xmm3
17927 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
17928 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[3]
17929 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm17, %zmm20
17930 ; AVX512DQ-BW-NEXT: vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
17931 ; AVX512DQ-BW-NEXT: vpsrlq $32, %zmm17, %zmm3
17932 ; AVX512DQ-BW-NEXT: vpmovqb %zmm3, %xmm3
17933 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
17934 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
17935 ; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm0
17936 ; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
17937 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
17938 ; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
17939 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm1, %xmm1
17940 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm28, %xmm2
17941 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
17942 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm2 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
17943 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm12, %xmm3
17944 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm16, %xmm4
17945 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
17946 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
17947 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
17948 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
17949 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
17950 ; AVX512DQ-BW-NEXT: vmovdqa (%rsp), %xmm15 # 16-byte Reload
17951 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm15, %xmm4
17952 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm26, %xmm13
17953 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
17954 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm13 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
17955 ; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm24, %xmm19
17956 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm24, %xmm26
17957 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
17958 ; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm18, %xmm30
17959 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm19 = xmm30[0],xmm19[0],xmm30[1],xmm19[1],xmm30[2],xmm19[2],xmm30[3],xmm19[3]
17960 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
17961 ; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm19, %ymm0, %ymm5
17962 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
17963 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
17964 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm6, %xmm4
17965 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm6, %xmm24
17966 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm7, %xmm5
17967 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
17968 ; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
17969 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm5, %xmm5
17970 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm27, %xmm19
17971 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
17972 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
17973 ; AVX512DQ-BW-NEXT: vpsrlq $40, %zmm10, %zmm5
17974 ; AVX512DQ-BW-NEXT: vpmovqb %zmm5, %xmm5
17975 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
17976 ; AVX512DQ-BW-NEXT: vinserti32x4 $2, %xmm4, %zmm0, %zmm4
17977 ; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm4 {%k1}
17978 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm14, %xmm1
17979 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm11, %xmm5
17980 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm11, %xmm17
17981 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
17982 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm21, %xmm5
17983 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm22, %xmm19
17984 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
17985 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
17986 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
17987 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
17988 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm25, %xmm5
17989 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm9, %xmm3
17990 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
17991 ; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm31 # 16-byte Reload
17992 ; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm31, %xmm5
17993 ; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm8, %xmm13
17994 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]
17995 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
17996 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
17997 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3,4],ymm3[5],ymm5[6,7]
17998 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
17999 ; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
18000 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm11, %xmm3
18001 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm23, %xmm0
18002 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
18003 ; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
18004 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm6, %xmm3
18005 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm29, %xmm2
18006 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
18007 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
18008 ; AVX512DQ-BW-NEXT: vpsrlq $40, %zmm20, %zmm2
18009 ; AVX512DQ-BW-NEXT: vpmovqb %zmm2, %xmm2
18010 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
18011 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
18012 ; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm4, %zmm0
18013 ; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
18014 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
18015 ; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm28 # 16-byte Reload
18016 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm28, %xmm1
18017 ; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
18018 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm8, %xmm2
18019 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
18020 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm2 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
18021 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm12, %xmm4
18022 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm16, %xmm5
18023 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
18024 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
18025 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
18026 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
18027 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
18028 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm15, %xmm5
18029 ; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
18030 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm15, %xmm13
18031 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]
18032 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm13 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
18033 ; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm26, %xmm19
18034 ; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm18, %xmm30
18035 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm19 = xmm30[0],xmm19[0],xmm30[1],xmm19[1],xmm30[2],xmm19[2],xmm30[3],xmm19[3]
18036 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
18037 ; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm19, %ymm0, %ymm3
18038 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5],ymm3[6,7]
18039 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
18040 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm24, %xmm3
18041 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm7, %xmm5
18042 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
18043 ; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm27 # 16-byte Reload
18044 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm27, %xmm5
18045 ; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm30 # 16-byte Reload
18046 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm30, %xmm19
18047 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
18048 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm3 = xmm5[0,1,2],xmm3[3]
18049 ; AVX512DQ-BW-NEXT: vpsrlq $48, %zmm10, %zmm5
18050 ; AVX512DQ-BW-NEXT: vpmovqb %zmm5, %xmm5
18051 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm3 = xmm5[0,1],xmm3[2,3]
18052 ; AVX512DQ-BW-NEXT: vinserti32x4 $2, %xmm3, %zmm0, %zmm3
18053 ; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm3 {%k1}
18054 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm14, %xmm1
18055 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm17, %xmm5
18056 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
18057 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm21, %xmm5
18058 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm22, %xmm19
18059 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
18060 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
18061 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
18062 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
18063 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm25, %xmm5
18064 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm9, %xmm4
18065 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm9, %xmm18
18066 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
18067 ; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm31, %xmm5
18068 ; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm20 # 16-byte Reload
18069 ; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm20, %xmm13
18070 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]
18071 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
18072 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
18073 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
18074 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
18075 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm11, %xmm4
18076 ; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm23, %xmm0
18077 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
18078 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm6, %xmm4
18079 ; AVX512DQ-BW-NEXT: vmovdqa64 %xmm6, %xmm26
18080 ; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm29, %xmm2
18081 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
18082 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
18083 ; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
18084 ; AVX512DQ-BW-NEXT: vpsrlq $48, %zmm19, %zmm2
18085 ; AVX512DQ-BW-NEXT: vpmovqb %zmm2, %xmm2
18086 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
18087 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
18088 ; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm3, %zmm0
18089 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm1 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
18090 ; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm28, %xmm2
18091 ; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm8, %xmm3
18092 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
18093 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
18094 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm12, %xmm4
18095 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm16, %xmm5
18096 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
18097 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
18098 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
18099 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
18100 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
18101 ; AVX512DQ-BW-NEXT: vmovdqa (%rsp), %xmm5 # 16-byte Reload
18102 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm5, %xmm5
18103 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm15, %xmm8
18104 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3]
18105 ; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm8 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
18106 ; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
18107 ; AVX512DQ-BW-NEXT: vpshufb %xmm8, %xmm6, %xmm9
18108 ; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
18109 ; AVX512DQ-BW-NEXT: vpshufb %xmm8, %xmm6, %xmm13
18110 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm13[0],xmm9[0],xmm13[1],xmm9[1],xmm13[2],xmm9[2],xmm13[3],xmm9[3]
18111 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
18112 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
18113 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4],ymm5[5],ymm9[6,7]
18114 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3,4,5],ymm2[6,7]
18115 ; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm24, %xmm5
18116 ; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm7, %xmm9
18117 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
18118 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm27, %xmm9
18119 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm30, %xmm6
18120 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm9[0],xmm6[1],xmm9[1],xmm6[2],xmm9[2],xmm6[3],xmm9[3]
18121 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3]
18122 ; AVX512DQ-BW-NEXT: vpsrlq $56, %zmm10, %zmm6
18123 ; AVX512DQ-BW-NEXT: vpmovqb %zmm6, %xmm6
18124 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]
18125 ; AVX512DQ-BW-NEXT: vinserti32x4 $2, %xmm5, %zmm0, %zmm5
18126 ; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm5 {%k1}
18127 ; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm14, %xmm2
18128 ; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm17, %xmm6
18129 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
18130 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm21, %xmm6
18131 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm22, %xmm9
18132 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm9[0],xmm6[0],xmm9[1],xmm6[1],xmm9[2],xmm6[2],xmm9[3],xmm6[3]
18133 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
18134 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
18135 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3,4,5,6],ymm2[7]
18136 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm25, %xmm6
18137 ; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm18, %xmm4
18138 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
18139 ; AVX512DQ-BW-NEXT: vpshufb %xmm8, %xmm31, %xmm6
18140 ; AVX512DQ-BW-NEXT: vpshufb %xmm8, %xmm20, %xmm8
18141 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]
18142 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
18143 ; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
18144 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm6[0,1,2,3,4],ymm4[5],ymm6[6,7]
18145 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
18146 ; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm11, %xmm4
18147 ; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm23, %xmm1
18148 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
18149 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm26, %xmm4
18150 ; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm29, %xmm3
18151 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
18152 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
18153 ; AVX512DQ-BW-NEXT: vpsrlq $56, %zmm19, %zmm3
18154 ; AVX512DQ-BW-NEXT: vpmovqb %zmm3, %xmm3
18155 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
18156 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
18157 ; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm1, %zmm5, %zmm1
18158 ; AVX512DQ-BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
18159 ; AVX512DQ-BW-NEXT: vmovaps %zmm2, (%rsi)
18160 ; AVX512DQ-BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
18161 ; AVX512DQ-BW-NEXT: vmovaps %zmm2, (%rdx)
18162 ; AVX512DQ-BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
18163 ; AVX512DQ-BW-NEXT: vmovaps %zmm2, (%rcx)
18164 ; AVX512DQ-BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
18165 ; AVX512DQ-BW-NEXT: vmovaps %zmm2, (%r8)
18166 ; AVX512DQ-BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
18167 ; AVX512DQ-BW-NEXT: vmovaps %zmm2, (%r9)
18168 ; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
18169 ; AVX512DQ-BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
18170 ; AVX512DQ-BW-NEXT: vmovaps %zmm2, (%rax)
18171 ; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
18172 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, (%rax)
18173 ; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
18174 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, (%rax)
18175 ; AVX512DQ-BW-NEXT: addq $744, %rsp # imm = 0x2E8
18176 ; AVX512DQ-BW-NEXT: vzeroupper
18177 ; AVX512DQ-BW-NEXT: retq
18178 ;
18179 ; AVX512DQ-BW-FCP-LABEL: load_i8_stride8_vf64:
18180 ; AVX512DQ-BW-FCP: # %bb.0:
18181 ; AVX512DQ-BW-FCP-NEXT: subq $328, %rsp # imm = 0x148
18182 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 256(%rdi), %zmm18
18183 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm11 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
18184 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,2,2,3,0,2,4,6]
18185 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 480(%rdi), %ymm1
18186 ; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
18187 ; AVX512DQ-BW-FCP-NEXT: vpermd %ymm1, %ymm0, %ymm30
18188 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm11, %ymm30, %ymm1
18189 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm8 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
18190 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 448(%rdi), %ymm2
18191 ; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm2, (%rsp) # 32-byte Spill
18192 ; AVX512DQ-BW-FCP-NEXT: vpermd %ymm2, %ymm0, %ymm31
18193 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm8, %ymm31, %ymm2
18194 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
18195 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm9 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
18196 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 416(%rdi), %ymm2
18197 ; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
18198 ; AVX512DQ-BW-FCP-NEXT: vpermd %ymm2, %ymm0, %ymm19
18199 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm9, %ymm19, %ymm2
18200 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm10 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
18201 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 384(%rdi), %ymm29
18202 ; AVX512DQ-BW-FCP-NEXT: vpermd %ymm29, %ymm0, %ymm14
18203 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm10, %ymm14, %ymm3
18204 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
18205 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
18206 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm7 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
18207 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 368(%rdi), %xmm21
18208 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm7, %xmm21, %xmm2
18209 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 352(%rdi), %xmm4
18210 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm7, %xmm4, %xmm3
18211 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
18212 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm23 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
18213 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 336(%rdi), %xmm12
18214 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm23, %xmm12, %xmm5
18215 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 320(%rdi), %xmm28
18216 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm23, %xmm28, %xmm6
18217 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
18218 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1,2],xmm2[3]
18219 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm18, %xmm5
18220 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
18221 ; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm20
18222 ; AVX512DQ-BW-FCP-NEXT: movb $-64, %al
18223 ; AVX512DQ-BW-FCP-NEXT: kmovd %eax, %k1
18224 ; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm20 {%k1}
18225 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 224(%rdi), %ymm1
18226 ; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
18227 ; AVX512DQ-BW-FCP-NEXT: vpermd %ymm1, %ymm0, %ymm17
18228 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm11, %ymm17, %ymm1
18229 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm2
18230 ; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
18231 ; AVX512DQ-BW-FCP-NEXT: vpermd %ymm2, %ymm0, %ymm3
18232 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm8, %ymm3, %ymm5
18233 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm1[7]
18234 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 160(%rdi), %ymm1
18235 ; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
18236 ; AVX512DQ-BW-FCP-NEXT: vpermd %ymm1, %ymm0, %ymm2
18237 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm9, %ymm2, %ymm6
18238 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm27
18239 ; AVX512DQ-BW-FCP-NEXT: vpermd %ymm27, %ymm0, %ymm16
18240 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm10, %ymm16, %ymm8
18241 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm8[0,1,2,3,4],ymm6[5],ymm8[6,7]
18242 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1,2,3,4,5],ymm5[6,7]
18243 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 112(%rdi), %xmm26
18244 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm7, %xmm26, %xmm8
18245 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 96(%rdi), %xmm24
18246 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm7, %xmm24, %xmm7
18247 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
18248 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 80(%rdi), %xmm22
18249 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm23, %xmm22, %xmm25
18250 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 64(%rdi), %xmm8
18251 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm23, %xmm8, %xmm23
18252 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm23[0],xmm25[0],xmm23[1],xmm25[1],xmm23[2],xmm25[2],xmm23[3],xmm25[3]
18253 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm15 = xmm15[0,1,2],xmm10[3]
18254 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm10
18255 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm10, %xmm13
18256 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm13 = xmm13[0,1],xmm15[2,3]
18257 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3],ymm0[4,5,6,7]
18258 ; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm20, %zmm0
18259 ; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
18260 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm11 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
18261 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm11, %ymm30, %ymm0
18262 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm6 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
18263 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm6, %ymm31, %ymm13
18264 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3,4,5,6],ymm0[7]
18265 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm9 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
18266 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm9, %ymm19, %ymm13
18267 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm1 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
18268 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm1, %ymm14, %ymm15
18269 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm13 = ymm15[0,1,2,3,4],ymm13[5],ymm15[6,7]
18270 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3,4,5],ymm0[6,7]
18271 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm13 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
18272 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %xmm21, %xmm5
18273 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm13, %xmm21, %xmm15
18274 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm13, %xmm4, %xmm23
18275 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm23[0],xmm15[0],xmm23[1],xmm15[1],xmm23[2],xmm15[2],xmm23[3],xmm15[3]
18276 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm23 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
18277 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm12, %xmm7
18278 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm23, %xmm12, %xmm25
18279 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm23, %xmm28, %xmm20
18280 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm20[0],xmm25[0],xmm20[1],xmm25[1],xmm20[2],xmm25[2],xmm20[3],xmm25[3]
18281 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm12[0,1,2],xmm15[3]
18282 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $8, %zmm18, %zmm15
18283 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm15, %xmm15
18284 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm15[0,1],xmm12[2,3]
18285 ; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $2, %xmm12, %zmm0, %zmm12
18286 ; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm12 {%k1}
18287 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm11, %ymm17, %ymm0
18288 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm15
18289 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm15[0,1,2,3,4,5,6],ymm0[7]
18290 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm9, %ymm2, %ymm15
18291 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm1, %ymm16, %ymm11
18292 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3,4],ymm15[5],ymm11[6,7]
18293 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3,4,5],ymm0[6,7]
18294 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm13, %xmm26, %xmm11
18295 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm13, %xmm24, %xmm13
18296 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1],xmm13[2],xmm11[2],xmm13[3],xmm11[3]
18297 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm23, %xmm22, %xmm13
18298 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm23, %xmm8, %xmm15
18299 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm15[0],xmm13[0],xmm15[1],xmm13[1],xmm15[2],xmm13[2],xmm15[3],xmm13[3]
18300 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm13[0,1,2],xmm11[3]
18301 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $8, %zmm10, %zmm13
18302 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm13, %xmm13
18303 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm13[0,1],xmm11[2,3]
18304 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3],ymm0[4,5,6,7]
18305 ; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm12, %zmm0
18306 ; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
18307 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm21 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
18308 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm21, %ymm30, %ymm0
18309 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm25 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
18310 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm25, %ymm31, %ymm11
18311 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3,4,5,6],ymm0[7]
18312 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm9 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
18313 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm9, %ymm19, %ymm11
18314 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm6 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
18315 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm6, %ymm14, %ymm12
18316 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2,3,4],ymm11[5],ymm12[6,7]
18317 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3,4,5],ymm0[6,7]
18318 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
18319 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm11, %xmm5, %xmm12
18320 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm11, %xmm4, %xmm13
18321 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm4, %xmm1
18322 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
18323 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm13 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
18324 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm13, %xmm7, %xmm15
18325 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %xmm7, %xmm23
18326 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm13, %xmm28, %xmm20
18327 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm20[0],xmm15[0],xmm20[1],xmm15[1],xmm20[2],xmm15[2],xmm20[3],xmm15[3]
18328 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm15[0,1,2],xmm12[3]
18329 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $16, %zmm18, %zmm15
18330 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm15, %xmm15
18331 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm15[0,1],xmm12[2,3]
18332 ; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $2, %xmm12, %zmm0, %zmm12
18333 ; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm12 {%k1}
18334 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm21, %ymm17, %ymm0
18335 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm25, %ymm3, %ymm15
18336 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm15[0,1,2,3,4,5,6],ymm0[7]
18337 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm9, %ymm2, %ymm15
18338 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm6, %ymm16, %ymm9
18339 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4],ymm15[5],ymm9[6,7]
18340 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3,4,5],ymm0[6,7]
18341 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm11, %xmm26, %xmm9
18342 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm11, %xmm24, %xmm11
18343 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
18344 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm13, %xmm22, %xmm11
18345 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm13, %xmm8, %xmm13
18346 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1],xmm13[2],xmm11[2],xmm13[3],xmm11[3]
18347 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm11[0,1,2],xmm9[3]
18348 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $16, %zmm10, %zmm11
18349 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm11, %xmm11
18350 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm11[0,1],xmm9[2,3]
18351 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3],ymm0[4,5,6,7]
18352 ; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm12, %zmm0
18353 ; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
18354 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm6 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
18355 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm6, %ymm30, %ymm0
18356 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm4 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
18357 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm4, %ymm31, %ymm9
18358 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3,4,5,6],ymm0[7]
18359 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm31 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
18360 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm31, %ymm19, %ymm9
18361 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm30 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
18362 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm30, %ymm14, %ymm11
18363 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3,4],ymm9[5],ymm11[6,7]
18364 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3,4,5],ymm0[6,7]
18365 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm9 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
18366 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %xmm5, %xmm19
18367 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm9, %xmm5, %xmm11
18368 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %xmm1, %xmm20
18369 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm9, %xmm1, %xmm12
18370 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
18371 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm12 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
18372 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm12, %xmm7, %xmm14
18373 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm12, %xmm28, %xmm15
18374 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
18375 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm14[0,1,2],xmm11[3]
18376 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $24, %zmm18, %zmm14
18377 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm18, %zmm25
18378 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm14, %xmm14
18379 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm14[0,1],xmm11[2,3]
18380 ; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $2, %xmm11, %zmm0, %zmm11
18381 ; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm11 {%k1}
18382 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm6, %ymm17, %ymm0
18383 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm4, %ymm3, %ymm3
18384 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5,6],ymm0[7]
18385 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm31, %ymm2, %ymm2
18386 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm30, %ymm16, %ymm1
18387 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5],ymm1[6,7]
18388 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
18389 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm9, %xmm26, %xmm1
18390 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm9, %xmm24, %xmm2
18391 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
18392 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm12, %xmm22, %xmm2
18393 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm12, %xmm8, %xmm3
18394 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
18395 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
18396 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $24, %zmm10, %zmm2
18397 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm2, %xmm2
18398 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
18399 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
18400 ; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm11, %zmm0
18401 ; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
18402 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [1,3,2,3,1,3,5,7]
18403 ; AVX512DQ-BW-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm6 # 32-byte Folded Reload
18404 ; AVX512DQ-BW-FCP-NEXT: vpermd (%rsp), %ymm3, %ymm7 # 32-byte Folded Reload
18405 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm13 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
18406 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm13, %ymm6, %ymm2
18407 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm1 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
18408 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm1, %ymm7, %ymm9
18409 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5,6],ymm2[7]
18410 ; AVX512DQ-BW-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm5 # 32-byte Folded Reload
18411 ; AVX512DQ-BW-FCP-NEXT: vpermd %ymm29, %ymm3, %ymm14
18412 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm0 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
18413 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm0, %ymm5, %ymm11
18414 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm4 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
18415 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm4, %ymm14, %ymm12
18416 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2,3,4],ymm11[5],ymm12[6,7]
18417 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3,4,5],ymm9[6,7]
18418 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
18419 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm11, %xmm19, %xmm12
18420 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %xmm19, %xmm16
18421 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm11, %xmm20, %xmm15
18422 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %xmm20, %xmm17
18423 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm15[0],xmm12[0],xmm15[1],xmm12[1],xmm15[2],xmm12[2],xmm15[3],xmm12[3]
18424 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm20 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
18425 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm20, %xmm23, %xmm15
18426 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm20, %xmm28, %xmm29
18427 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm29[0],xmm15[0],xmm29[1],xmm15[1],xmm29[2],xmm15[2],xmm29[3],xmm15[3]
18428 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm15[0,1,2],xmm12[3]
18429 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $32, %zmm18, %zmm15
18430 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm15, %xmm15
18431 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm15[0,1],xmm12[2,3]
18432 ; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $2, %xmm12, %zmm0, %zmm12
18433 ; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm9, %zmm0, %zmm12 {%k1}
18434 ; AVX512DQ-BW-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm19 # 32-byte Folded Reload
18435 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm13, %ymm19, %ymm9
18436 ; AVX512DQ-BW-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm18 # 32-byte Folded Reload
18437 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm1, %ymm18, %ymm15
18438 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm15[0,1,2,3,4,5,6],ymm9[7]
18439 ; AVX512DQ-BW-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm15 # 32-byte Folded Reload
18440 ; AVX512DQ-BW-FCP-NEXT: vpermd %ymm27, %ymm3, %ymm21
18441 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm0, %ymm15, %ymm3
18442 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm4, %ymm21, %ymm13
18443 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm13[0,1,2,3,4],ymm3[5],ymm13[6,7]
18444 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm9[6,7]
18445 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm11, %xmm26, %xmm9
18446 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm11, %xmm24, %xmm11
18447 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
18448 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm20, %xmm22, %xmm11
18449 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm20, %xmm8, %xmm13
18450 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1],xmm13[2],xmm11[2],xmm13[3],xmm11[3]
18451 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm11[0,1,2],xmm9[3]
18452 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $32, %zmm10, %zmm11
18453 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm11, %xmm11
18454 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm11[0,1],xmm9[2,3]
18455 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm9[0,1,2,3],ymm3[4,5,6,7]
18456 ; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm3, %zmm12, %zmm29
18457 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm4 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
18458 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm4, %ymm6, %ymm9
18459 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm0 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
18460 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm0, %ymm7, %ymm11
18461 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3,4,5,6],ymm9[7]
18462 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm2 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
18463 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm2, %ymm5, %ymm11
18464 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm1 = [1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13]
18465 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm1, %ymm14, %ymm12
18466 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2,3,4],ymm11[5],ymm12[6,7]
18467 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3,4,5],ymm9[6,7]
18468 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
18469 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm11, %xmm16, %xmm12
18470 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm11, %xmm17, %xmm13
18471 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
18472 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm13 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
18473 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm13, %xmm23, %xmm20
18474 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm13, %xmm28, %xmm27
18475 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm27[0],xmm20[0],xmm27[1],xmm20[1],xmm27[2],xmm20[2],xmm27[3],xmm20[3]
18476 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1,2],xmm12[3]
18477 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm25, %zmm27
18478 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $40, %zmm25, %zmm12
18479 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm12, %xmm12
18480 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm3 = xmm12[0,1],xmm3[2,3]
18481 ; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $2, %xmm3, %zmm0, %zmm3
18482 ; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm9, %zmm0, %zmm3 {%k1}
18483 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm4, %ymm19, %ymm9
18484 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm0, %ymm18, %ymm12
18485 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm12[0,1,2,3,4,5,6],ymm9[7]
18486 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm2, %ymm15, %ymm12
18487 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm1, %ymm21, %ymm4
18488 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4],ymm12[5],ymm4[6,7]
18489 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],ymm9[6,7]
18490 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm11, %xmm26, %xmm9
18491 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm11, %xmm24, %xmm11
18492 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
18493 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm13, %xmm22, %xmm11
18494 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm13, %xmm8, %xmm12
18495 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
18496 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm11[0,1,2],xmm9[3]
18497 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $40, %zmm10, %zmm11
18498 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm11, %xmm11
18499 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm11[0,1],xmm9[2,3]
18500 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3],ymm4[4,5,6,7]
18501 ; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm3, %zmm20
18502 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm0 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
18503 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm0, %ymm6, %ymm3
18504 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm1 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
18505 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm1, %ymm7, %ymm4
18506 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5,6],ymm3[7]
18507 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm2 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
18508 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm2, %ymm5, %ymm4
18509 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm25 = [2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14]
18510 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm25, %ymm14, %ymm9
18511 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4],ymm4[5],ymm9[6,7]
18512 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
18513 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm4 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
18514 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm4, %xmm16, %xmm9
18515 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm4, %xmm17, %xmm11
18516 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
18517 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm11 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
18518 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm11, %xmm23, %xmm12
18519 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm11, %xmm28, %xmm13
18520 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
18521 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm12[0,1,2],xmm9[3]
18522 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $48, %zmm27, %zmm12
18523 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm12, %xmm12
18524 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm12[0,1],xmm9[2,3]
18525 ; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $2, %xmm9, %zmm0, %zmm9
18526 ; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm9 {%k1}
18527 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm0, %ymm19, %ymm3
18528 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm1, %ymm18, %ymm12
18529 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm12[0,1,2,3,4,5,6],ymm3[7]
18530 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm2, %ymm15, %ymm12
18531 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm25, %ymm21, %ymm13
18532 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3,4],ymm12[5],ymm13[6,7]
18533 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm12[0,1,2,3,4,5],ymm3[6,7]
18534 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm4, %xmm26, %xmm12
18535 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm4, %xmm24, %xmm4
18536 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm12[0],xmm4[1],xmm12[1],xmm4[2],xmm12[2],xmm4[3],xmm12[3]
18537 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm11, %xmm22, %xmm12
18538 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm11, %xmm8, %xmm11
18539 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]
18540 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm11[0,1,2],xmm4[3]
18541 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $48, %zmm10, %zmm11
18542 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm11, %xmm11
18543 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm11[0,1],xmm4[2,3]
18544 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
18545 ; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm3, %zmm9, %zmm3
18546 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm12 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
18547 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm12, %ymm6, %ymm0
18548 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm6 = [3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15]
18549 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm6, %ymm7, %ymm1
18550 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6],ymm0[7]
18551 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm31, %ymm5, %ymm1
18552 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm30, %ymm14, %ymm2
18553 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4],ymm1[5],ymm2[6,7]
18554 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
18555 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm1 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
18556 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm1, %xmm16, %xmm2
18557 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm1, %xmm17, %xmm4
18558 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
18559 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw {{.*#+}} xmm4 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
18560 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm4, %xmm23, %xmm9
18561 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm4, %xmm28, %xmm11
18562 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
18563 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm9[0,1,2],xmm2[3]
18564 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $56, %zmm27, %zmm9
18565 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm9, %xmm9
18566 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm9[0,1],xmm2[2,3]
18567 ; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm2
18568 ; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm2 {%k1}
18569 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm12, %ymm19, %ymm0
18570 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm6, %ymm18, %ymm9
18571 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3,4,5,6],ymm0[7]
18572 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm31, %ymm15, %ymm9
18573 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm30, %ymm21, %ymm11
18574 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3,4],ymm9[5],ymm11[6,7]
18575 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3,4,5],ymm0[6,7]
18576 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm1, %xmm26, %xmm5
18577 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm1, %xmm24, %xmm1
18578 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
18579 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm4, %xmm22, %xmm5
18580 ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm4, %xmm8, %xmm4
18581 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
18582 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm4[0,1,2],xmm1[3]
18583 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $56, %zmm10, %zmm4
18584 ; AVX512DQ-BW-FCP-NEXT: vpmovqb %zmm4, %xmm4
18585 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm4[0,1],xmm1[2,3]
18586 ; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
18587 ; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm0
18588 ; AVX512DQ-BW-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
18589 ; AVX512DQ-BW-FCP-NEXT: vmovaps %zmm1, (%rsi)
18590 ; AVX512DQ-BW-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
18591 ; AVX512DQ-BW-FCP-NEXT: vmovaps %zmm1, (%rdx)
18592 ; AVX512DQ-BW-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
18593 ; AVX512DQ-BW-FCP-NEXT: vmovaps %zmm1, (%rcx)
18594 ; AVX512DQ-BW-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
18595 ; AVX512DQ-BW-FCP-NEXT: vmovaps %zmm1, (%r8)
18596 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm29, (%r9)
18597 ; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
18598 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm20, (%rax)
18599 ; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
18600 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm3, (%rax)
18601 ; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
18602 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, (%rax)
18603 ; AVX512DQ-BW-FCP-NEXT: addq $328, %rsp # imm = 0x148
18604 ; AVX512DQ-BW-FCP-NEXT: vzeroupper
18605 ; AVX512DQ-BW-FCP-NEXT: retq
18606 %wide.vec = load <512 x i8>, ptr %in.vec, align 64
18607 %strided.vec0 = shufflevector <512 x i8> %wide.vec, <512 x i8> poison, <64 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 40, i32 48, i32 56, i32 64, i32 72, i32 80, i32 88, i32 96, i32 104, i32 112, i32 120, i32 128, i32 136, i32 144, i32 152, i32 160, i32 168, i32 176, i32 184, i32 192, i32 200, i32 208, i32 216, i32 224, i32 232, i32 240, i32 248, i32 256, i32 264, i32 272, i32 280, i32 288, i32 296, i32 304, i32 312, i32 320, i32 328, i32 336, i32 344, i32 352, i32 360, i32 368, i32 376, i32 384, i32 392, i32 400, i32 408, i32 416, i32 424, i32 432, i32 440, i32 448, i32 456, i32 464, i32 472, i32 480, i32 488, i32 496, i32 504>
18608 %strided.vec1 = shufflevector <512 x i8> %wide.vec, <512 x i8> poison, <64 x i32> <i32 1, i32 9, i32 17, i32 25, i32 33, i32 41, i32 49, i32 57, i32 65, i32 73, i32 81, i32 89, i32 97, i32 105, i32 113, i32 121, i32 129, i32 137, i32 145, i32 153, i32 161, i32 169, i32 177, i32 185, i32 193, i32 201, i32 209, i32 217, i32 225, i32 233, i32 241, i32 249, i32 257, i32 265, i32 273, i32 281, i32 289, i32 297, i32 305, i32 313, i32 321, i32 329, i32 337, i32 345, i32 353, i32 361, i32 369, i32 377, i32 385, i32 393, i32 401, i32 409, i32 417, i32 425, i32 433, i32 441, i32 449, i32 457, i32 465, i32 473, i32 481, i32 489, i32 497, i32 505>
18609 %strided.vec2 = shufflevector <512 x i8> %wide.vec, <512 x i8> poison, <64 x i32> <i32 2, i32 10, i32 18, i32 26, i32 34, i32 42, i32 50, i32 58, i32 66, i32 74, i32 82, i32 90, i32 98, i32 106, i32 114, i32 122, i32 130, i32 138, i32 146, i32 154, i32 162, i32 170, i32 178, i32 186, i32 194, i32 202, i32 210, i32 218, i32 226, i32 234, i32 242, i32 250, i32 258, i32 266, i32 274, i32 282, i32 290, i32 298, i32 306, i32 314, i32 322, i32 330, i32 338, i32 346, i32 354, i32 362, i32 370, i32 378, i32 386, i32 394, i32 402, i32 410, i32 418, i32 426, i32 434, i32 442, i32 450, i32 458, i32 466, i32 474, i32 482, i32 490, i32 498, i32 506>
18610 %strided.vec3 = shufflevector <512 x i8> %wide.vec, <512 x i8> poison, <64 x i32> <i32 3, i32 11, i32 19, i32 27, i32 35, i32 43, i32 51, i32 59, i32 67, i32 75, i32 83, i32 91, i32 99, i32 107, i32 115, i32 123, i32 131, i32 139, i32 147, i32 155, i32 163, i32 171, i32 179, i32 187, i32 195, i32 203, i32 211, i32 219, i32 227, i32 235, i32 243, i32 251, i32 259, i32 267, i32 275, i32 283, i32 291, i32 299, i32 307, i32 315, i32 323, i32 331, i32 339, i32 347, i32 355, i32 363, i32 371, i32 379, i32 387, i32 395, i32 403, i32 411, i32 419, i32 427, i32 435, i32 443, i32 451, i32 459, i32 467, i32 475, i32 483, i32 491, i32 499, i32 507>
18611 %strided.vec4 = shufflevector <512 x i8> %wide.vec, <512 x i8> poison, <64 x i32> <i32 4, i32 12, i32 20, i32 28, i32 36, i32 44, i32 52, i32 60, i32 68, i32 76, i32 84, i32 92, i32 100, i32 108, i32 116, i32 124, i32 132, i32 140, i32 148, i32 156, i32 164, i32 172, i32 180, i32 188, i32 196, i32 204, i32 212, i32 220, i32 228, i32 236, i32 244, i32 252, i32 260, i32 268, i32 276, i32 284, i32 292, i32 300, i32 308, i32 316, i32 324, i32 332, i32 340, i32 348, i32 356, i32 364, i32 372, i32 380, i32 388, i32 396, i32 404, i32 412, i32 420, i32 428, i32 436, i32 444, i32 452, i32 460, i32 468, i32 476, i32 484, i32 492, i32 500, i32 508>
18612 %strided.vec5 = shufflevector <512 x i8> %wide.vec, <512 x i8> poison, <64 x i32> <i32 5, i32 13, i32 21, i32 29, i32 37, i32 45, i32 53, i32 61, i32 69, i32 77, i32 85, i32 93, i32 101, i32 109, i32 117, i32 125, i32 133, i32 141, i32 149, i32 157, i32 165, i32 173, i32 181, i32 189, i32 197, i32 205, i32 213, i32 221, i32 229, i32 237, i32 245, i32 253, i32 261, i32 269, i32 277, i32 285, i32 293, i32 301, i32 309, i32 317, i32 325, i32 333, i32 341, i32 349, i32 357, i32 365, i32 373, i32 381, i32 389, i32 397, i32 405, i32 413, i32 421, i32 429, i32 437, i32 445, i32 453, i32 461, i32 469, i32 477, i32 485, i32 493, i32 501, i32 509>
18613 %strided.vec6 = shufflevector <512 x i8> %wide.vec, <512 x i8> poison, <64 x i32> <i32 6, i32 14, i32 22, i32 30, i32 38, i32 46, i32 54, i32 62, i32 70, i32 78, i32 86, i32 94, i32 102, i32 110, i32 118, i32 126, i32 134, i32 142, i32 150, i32 158, i32 166, i32 174, i32 182, i32 190, i32 198, i32 206, i32 214, i32 222, i32 230, i32 238, i32 246, i32 254, i32 262, i32 270, i32 278, i32 286, i32 294, i32 302, i32 310, i32 318, i32 326, i32 334, i32 342, i32 350, i32 358, i32 366, i32 374, i32 382, i32 390, i32 398, i32 406, i32 414, i32 422, i32 430, i32 438, i32 446, i32 454, i32 462, i32 470, i32 478, i32 486, i32 494, i32 502, i32 510>
18614 %strided.vec7 = shufflevector <512 x i8> %wide.vec, <512 x i8> poison, <64 x i32> <i32 7, i32 15, i32 23, i32 31, i32 39, i32 47, i32 55, i32 63, i32 71, i32 79, i32 87, i32 95, i32 103, i32 111, i32 119, i32 127, i32 135, i32 143, i32 151, i32 159, i32 167, i32 175, i32 183, i32 191, i32 199, i32 207, i32 215, i32 223, i32 231, i32 239, i32 247, i32 255, i32 263, i32 271, i32 279, i32 287, i32 295, i32 303, i32 311, i32 319, i32 327, i32 335, i32 343, i32 351, i32 359, i32 367, i32 375, i32 383, i32 391, i32 399, i32 407, i32 415, i32 423, i32 431, i32 439, i32 447, i32 455, i32 463, i32 471, i32 479, i32 487, i32 495, i32 503, i32 511>
18615 store <64 x i8> %strided.vec0, ptr %out.vec0, align 64
18616 store <64 x i8> %strided.vec1, ptr %out.vec1, align 64
18617 store <64 x i8> %strided.vec2, ptr %out.vec2, align 64
18618 store <64 x i8> %strided.vec3, ptr %out.vec3, align 64
18619 store <64 x i8> %strided.vec4, ptr %out.vec4, align 64
18620 store <64 x i8> %strided.vec5, ptr %out.vec5, align 64
18621 store <64 x i8> %strided.vec6, ptr %out.vec6, align 64
18622 store <64 x i8> %strided.vec7, ptr %out.vec7, align 64
18623 ret void
18624 }