| blob | 993029374b700f63bedbfdd9ccda4dfbbb2b5bb8 |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,FALLBACK0
3 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1,AVX1-ONLY,FALLBACK1
4 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX1,AVX2,AVX2-ONLY,AVX2-SLOW,FALLBACK2
5 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX1,AVX2,AVX2-ONLY,AVX2-FAST,FALLBACK3
6 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX1,AVX2,AVX2-ONLY,AVX2-FAST-PERLANE,FALLBACK4
7 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX,AVX2,AVX512,AVX512F,AVX512F-SLOW,AVX512F-ONLY-SLOW,FALLBACK5
8 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2,AVX512,AVX512F,AVX512F-FAST,AVX512F-ONLY-FAST,FALLBACK6
9 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX,AVX2,AVX512,AVX512F,AVX512F-SLOW,AVX512DQ-SLOW,FALLBACK7
10 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2,AVX512,AVX512F,AVX512F-FAST,AVX512DQ-FAST,FALLBACK8
11 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX,AVX2,AVX512,AVX512BW,AVX512BW-SLOW,AVX512BW-ONLY-SLOW,FALLBACK9
12 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2,AVX512,AVX512BW,AVX512BW-FAST,AVX512BW-ONLY-FAST,FALLBACK10
13 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX,AVX2,AVX512,AVX512BW,AVX512BW-SLOW,AVX512DQBW-SLOW,FALLBACK11
14 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2,AVX512,AVX512BW,AVX512BW-FAST,AVX512DQBW-FAST,FALLBACK12
16 ; These patterns are produced by LoopVectorizer for interleaved loads.
18 define void @load_i16_stride6_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4, ptr %out.vec5) nounwind {
19 ; SSE-LABEL: load_i16_stride6_vf2:
20 ; SSE: # %bb.0:
21 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
22 ; SSE-NEXT: movdqa (%rdi), %xmm0
23 ; SSE-NEXT: movdqa 16(%rdi), %xmm1
24 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,3,2,3]
25 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm2[0,2,2,3,4,5,6,7]
26 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]
27 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
28 ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
29 ; SSE-NEXT: movdqa %xmm0, %xmm5
30 ; SSE-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
31 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]
32 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[0,3,2,3,4,5,6,7]
33 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm1[1,1,1,1]
34 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm0[2,3,2,3]
35 ; SSE-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
36 ; SSE-NEXT: psrlq $48, %xmm1
37 ; SSE-NEXT: psrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
38 ; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
39 ; SSE-NEXT: movd %xmm3, (%rsi)
40 ; SSE-NEXT: movd %xmm2, (%rdx)
41 ; SSE-NEXT: movd %xmm4, (%rcx)
42 ; SSE-NEXT: movd %xmm5, (%r8)
43 ; SSE-NEXT: movd %xmm7, (%r9)
44 ; SSE-NEXT: movd %xmm0, (%rax)
45 ; SSE-NEXT: retq
46 ;
47 ; AVX1-ONLY-LABEL: load_i16_stride6_vf2:
48 ; AVX1-ONLY: # %bb.0:
49 ; AVX1-ONLY-NEXT: movq {{[0-9]+}}(%rsp), %rax
50 ; AVX1-ONLY-NEXT: vmovdqa (%rdi), %xmm0
51 ; AVX1-ONLY-NEXT: vmovdqa 16(%rdi), %xmm1
52 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,3,2,3]
53 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[0,2,2,3,4,5,6,7]
54 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]
55 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
56 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
57 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
58 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]
59 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,3,2,3,4,5,6,7]
60 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm6 = xmm1[1,1,1,1]
61 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm7 = xmm0[2,3,2,3]
62 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
63 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm1, %xmm1
64 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
65 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
66 ; AVX1-ONLY-NEXT: vmovd %xmm3, (%rsi)
67 ; AVX1-ONLY-NEXT: vmovd %xmm2, (%rdx)
68 ; AVX1-ONLY-NEXT: vmovd %xmm4, (%rcx)
69 ; AVX1-ONLY-NEXT: vmovd %xmm5, (%r8)
70 ; AVX1-ONLY-NEXT: vmovd %xmm6, (%r9)
71 ; AVX1-ONLY-NEXT: vmovd %xmm0, (%rax)
72 ; AVX1-ONLY-NEXT: retq
73 ;
74 ; AVX2-SLOW-LABEL: load_i16_stride6_vf2:
75 ; AVX2-SLOW: # %bb.0:
76 ; AVX2-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
77 ; AVX2-SLOW-NEXT: vmovdqa (%rdi), %xmm0
78 ; AVX2-SLOW-NEXT: vmovdqa 16(%rdi), %xmm1
79 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,3,2,3]
80 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[0,2,2,3,4,5,6,7]
81 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]
82 ; AVX2-SLOW-NEXT: vpbroadcastw 4(%rdi), %xmm4
83 ; AVX2-SLOW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
84 ; AVX2-SLOW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
85 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]
86 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,3,2,3,4,5,6,7]
87 ; AVX2-SLOW-NEXT: vpbroadcastw 20(%rdi), %xmm6
88 ; AVX2-SLOW-NEXT: vpbroadcastw 8(%rdi), %xmm7
89 ; AVX2-SLOW-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
90 ; AVX2-SLOW-NEXT: vpsrlq $48, %xmm1, %xmm1
91 ; AVX2-SLOW-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
92 ; AVX2-SLOW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
93 ; AVX2-SLOW-NEXT: vmovd %xmm3, (%rsi)
94 ; AVX2-SLOW-NEXT: vmovd %xmm2, (%rdx)
95 ; AVX2-SLOW-NEXT: vmovd %xmm4, (%rcx)
96 ; AVX2-SLOW-NEXT: vmovd %xmm5, (%r8)
97 ; AVX2-SLOW-NEXT: vmovd %xmm6, (%r9)
98 ; AVX2-SLOW-NEXT: vmovd %xmm0, (%rax)
99 ; AVX2-SLOW-NEXT: retq
100 ;
101 ; AVX2-FAST-LABEL: load_i16_stride6_vf2:
102 ; AVX2-FAST: # %bb.0:
103 ; AVX2-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
104 ; AVX2-FAST-NEXT: vmovdqa (%rdi), %xmm0
105 ; AVX2-FAST-NEXT: vmovdqa 16(%rdi), %xmm1
106 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,3,2,3]
107 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[0,2,2,3,4,5,6,7]
108 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]
109 ; AVX2-FAST-NEXT: vpbroadcastw 4(%rdi), %xmm4
110 ; AVX2-FAST-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
111 ; AVX2-FAST-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
112 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[12,13,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
113 ; AVX2-FAST-NEXT: vpbroadcastw 20(%rdi), %xmm6
114 ; AVX2-FAST-NEXT: vpbroadcastw 8(%rdi), %xmm7
115 ; AVX2-FAST-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
116 ; AVX2-FAST-NEXT: vpsrlq $48, %xmm1, %xmm1
117 ; AVX2-FAST-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
118 ; AVX2-FAST-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
119 ; AVX2-FAST-NEXT: vmovd %xmm3, (%rsi)
120 ; AVX2-FAST-NEXT: vmovd %xmm2, (%rdx)
121 ; AVX2-FAST-NEXT: vmovd %xmm4, (%rcx)
122 ; AVX2-FAST-NEXT: vmovd %xmm5, (%r8)
123 ; AVX2-FAST-NEXT: vmovd %xmm6, (%r9)
124 ; AVX2-FAST-NEXT: vmovd %xmm0, (%rax)
125 ; AVX2-FAST-NEXT: retq
126 ;
127 ; AVX2-FAST-PERLANE-LABEL: load_i16_stride6_vf2:
128 ; AVX2-FAST-PERLANE: # %bb.0:
129 ; AVX2-FAST-PERLANE-NEXT: movq {{[0-9]+}}(%rsp), %rax
130 ; AVX2-FAST-PERLANE-NEXT: vmovdqa (%rdi), %xmm0
131 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 16(%rdi), %xmm1
132 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,3,2,3]
133 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[0,2,2,3,4,5,6,7]
134 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]
135 ; AVX2-FAST-PERLANE-NEXT: vpbroadcastw 4(%rdi), %xmm4
136 ; AVX2-FAST-PERLANE-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
137 ; AVX2-FAST-PERLANE-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
138 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[12,13,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
139 ; AVX2-FAST-PERLANE-NEXT: vpbroadcastw 20(%rdi), %xmm6
140 ; AVX2-FAST-PERLANE-NEXT: vpbroadcastw 8(%rdi), %xmm7
141 ; AVX2-FAST-PERLANE-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
142 ; AVX2-FAST-PERLANE-NEXT: vpsrlq $48, %xmm1, %xmm1
143 ; AVX2-FAST-PERLANE-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
144 ; AVX2-FAST-PERLANE-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
145 ; AVX2-FAST-PERLANE-NEXT: vmovd %xmm3, (%rsi)
146 ; AVX2-FAST-PERLANE-NEXT: vmovd %xmm2, (%rdx)
147 ; AVX2-FAST-PERLANE-NEXT: vmovd %xmm4, (%rcx)
148 ; AVX2-FAST-PERLANE-NEXT: vmovd %xmm5, (%r8)
149 ; AVX2-FAST-PERLANE-NEXT: vmovd %xmm6, (%r9)
150 ; AVX2-FAST-PERLANE-NEXT: vmovd %xmm0, (%rax)
151 ; AVX2-FAST-PERLANE-NEXT: retq
152 ;
153 ; AVX512F-SLOW-LABEL: load_i16_stride6_vf2:
154 ; AVX512F-SLOW: # %bb.0:
155 ; AVX512F-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
156 ; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %xmm0
157 ; AVX512F-SLOW-NEXT: vmovdqa 16(%rdi), %xmm1
158 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,3,2,3]
159 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[0,2,2,3,4,5,6,7]
160 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]
161 ; AVX512F-SLOW-NEXT: vpbroadcastw 4(%rdi), %xmm4
162 ; AVX512F-SLOW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
163 ; AVX512F-SLOW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
164 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]
165 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,3,2,3,4,5,6,7]
166 ; AVX512F-SLOW-NEXT: vpbroadcastw 20(%rdi), %xmm6
167 ; AVX512F-SLOW-NEXT: vpbroadcastw 8(%rdi), %xmm7
168 ; AVX512F-SLOW-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
169 ; AVX512F-SLOW-NEXT: vpsrlq $48, %xmm1, %xmm1
170 ; AVX512F-SLOW-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
171 ; AVX512F-SLOW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
172 ; AVX512F-SLOW-NEXT: vmovd %xmm3, (%rsi)
173 ; AVX512F-SLOW-NEXT: vmovd %xmm2, (%rdx)
174 ; AVX512F-SLOW-NEXT: vmovd %xmm4, (%rcx)
175 ; AVX512F-SLOW-NEXT: vmovd %xmm5, (%r8)
176 ; AVX512F-SLOW-NEXT: vmovd %xmm6, (%r9)
177 ; AVX512F-SLOW-NEXT: vmovd %xmm0, (%rax)
178 ; AVX512F-SLOW-NEXT: retq
179 ;
180 ; AVX512F-FAST-LABEL: load_i16_stride6_vf2:
181 ; AVX512F-FAST: # %bb.0:
182 ; AVX512F-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
183 ; AVX512F-FAST-NEXT: vmovdqa (%rdi), %xmm0
184 ; AVX512F-FAST-NEXT: vmovdqa 16(%rdi), %xmm1
185 ; AVX512F-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,3,2,3]
186 ; AVX512F-FAST-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[0,2,2,3,4,5,6,7]
187 ; AVX512F-FAST-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]
188 ; AVX512F-FAST-NEXT: vpbroadcastw 4(%rdi), %xmm4
189 ; AVX512F-FAST-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
190 ; AVX512F-FAST-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
191 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[12,13,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
192 ; AVX512F-FAST-NEXT: vpbroadcastw 20(%rdi), %xmm6
193 ; AVX512F-FAST-NEXT: vpbroadcastw 8(%rdi), %xmm7
194 ; AVX512F-FAST-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
195 ; AVX512F-FAST-NEXT: vpsrlq $48, %xmm1, %xmm1
196 ; AVX512F-FAST-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
197 ; AVX512F-FAST-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
198 ; AVX512F-FAST-NEXT: vmovd %xmm3, (%rsi)
199 ; AVX512F-FAST-NEXT: vmovd %xmm2, (%rdx)
200 ; AVX512F-FAST-NEXT: vmovd %xmm4, (%rcx)
201 ; AVX512F-FAST-NEXT: vmovd %xmm5, (%r8)
202 ; AVX512F-FAST-NEXT: vmovd %xmm6, (%r9)
203 ; AVX512F-FAST-NEXT: vmovd %xmm0, (%rax)
204 ; AVX512F-FAST-NEXT: retq
205 ;
206 ; AVX512BW-SLOW-LABEL: load_i16_stride6_vf2:
207 ; AVX512BW-SLOW: # %bb.0:
208 ; AVX512BW-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
209 ; AVX512BW-SLOW-NEXT: vmovdqa (%rdi), %xmm0
210 ; AVX512BW-SLOW-NEXT: vmovdqa 16(%rdi), %xmm1
211 ; AVX512BW-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,3,2,3]
212 ; AVX512BW-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[0,2,2,3,4,5,6,7]
213 ; AVX512BW-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]
214 ; AVX512BW-SLOW-NEXT: vpbroadcastw 4(%rdi), %xmm4
215 ; AVX512BW-SLOW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
216 ; AVX512BW-SLOW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
217 ; AVX512BW-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]
218 ; AVX512BW-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,3,2,3,4,5,6,7]
219 ; AVX512BW-SLOW-NEXT: vpbroadcastw 20(%rdi), %xmm6
220 ; AVX512BW-SLOW-NEXT: vpbroadcastw 8(%rdi), %xmm7
221 ; AVX512BW-SLOW-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
222 ; AVX512BW-SLOW-NEXT: vpsrlq $48, %xmm1, %xmm1
223 ; AVX512BW-SLOW-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
224 ; AVX512BW-SLOW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
225 ; AVX512BW-SLOW-NEXT: vmovd %xmm3, (%rsi)
226 ; AVX512BW-SLOW-NEXT: vmovd %xmm2, (%rdx)
227 ; AVX512BW-SLOW-NEXT: vmovd %xmm4, (%rcx)
228 ; AVX512BW-SLOW-NEXT: vmovd %xmm5, (%r8)
229 ; AVX512BW-SLOW-NEXT: vmovd %xmm6, (%r9)
230 ; AVX512BW-SLOW-NEXT: vmovd %xmm0, (%rax)
231 ; AVX512BW-SLOW-NEXT: retq
232 ;
233 ; AVX512BW-FAST-LABEL: load_i16_stride6_vf2:
234 ; AVX512BW-FAST: # %bb.0:
235 ; AVX512BW-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
236 ; AVX512BW-FAST-NEXT: vmovdqa (%rdi), %xmm0
237 ; AVX512BW-FAST-NEXT: vmovdqa 16(%rdi), %xmm1
238 ; AVX512BW-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,3,2,3]
239 ; AVX512BW-FAST-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[0,2,2,3,4,5,6,7]
240 ; AVX512BW-FAST-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]
241 ; AVX512BW-FAST-NEXT: vpbroadcastw 4(%rdi), %xmm4
242 ; AVX512BW-FAST-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
243 ; AVX512BW-FAST-NEXT: vpbroadcastd {{.*#+}} xmm5 = [3,9,3,9,3,9,3,9]
244 ; AVX512BW-FAST-NEXT: vpermi2w %xmm1, %xmm0, %xmm5
245 ; AVX512BW-FAST-NEXT: vpbroadcastw 20(%rdi), %xmm6
246 ; AVX512BW-FAST-NEXT: vpbroadcastw 8(%rdi), %xmm7
247 ; AVX512BW-FAST-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
248 ; AVX512BW-FAST-NEXT: vpsrlq $48, %xmm1, %xmm1
249 ; AVX512BW-FAST-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
250 ; AVX512BW-FAST-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
251 ; AVX512BW-FAST-NEXT: vmovd %xmm3, (%rsi)
252 ; AVX512BW-FAST-NEXT: vmovd %xmm2, (%rdx)
253 ; AVX512BW-FAST-NEXT: vmovd %xmm4, (%rcx)
254 ; AVX512BW-FAST-NEXT: vmovd %xmm5, (%r8)
255 ; AVX512BW-FAST-NEXT: vmovd %xmm6, (%r9)
256 ; AVX512BW-FAST-NEXT: vmovd %xmm0, (%rax)
257 ; AVX512BW-FAST-NEXT: retq
258 %wide.vec = load <12 x i16>, ptr %in.vec, align 64
259 %strided.vec0 = shufflevector <12 x i16> %wide.vec, <12 x i16> poison, <2 x i32> <i32 0, i32 6>
260 %strided.vec1 = shufflevector <12 x i16> %wide.vec, <12 x i16> poison, <2 x i32> <i32 1, i32 7>
261 %strided.vec2 = shufflevector <12 x i16> %wide.vec, <12 x i16> poison, <2 x i32> <i32 2, i32 8>
262 %strided.vec3 = shufflevector <12 x i16> %wide.vec, <12 x i16> poison, <2 x i32> <i32 3, i32 9>
263 %strided.vec4 = shufflevector <12 x i16> %wide.vec, <12 x i16> poison, <2 x i32> <i32 4, i32 10>
264 %strided.vec5 = shufflevector <12 x i16> %wide.vec, <12 x i16> poison, <2 x i32> <i32 5, i32 11>
265 store <2 x i16> %strided.vec0, ptr %out.vec0, align 64
266 store <2 x i16> %strided.vec1, ptr %out.vec1, align 64
267 store <2 x i16> %strided.vec2, ptr %out.vec2, align 64
268 store <2 x i16> %strided.vec3, ptr %out.vec3, align 64
269 store <2 x i16> %strided.vec4, ptr %out.vec4, align 64
270 store <2 x i16> %strided.vec5, ptr %out.vec5, align 64
271 ret void
272 }
274 define void @load_i16_stride6_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4, ptr %out.vec5) nounwind {
275 ; SSE-LABEL: load_i16_stride6_vf4:
276 ; SSE: # %bb.0:
277 ; SSE-NEXT: movdqa (%rdi), %xmm0
278 ; SSE-NEXT: movdqa 16(%rdi), %xmm1
279 ; SSE-NEXT: movdqa 32(%rdi), %xmm5
280 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,1,0,3]
281 ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm4[0,1,2,3,4,6,6,7]
282 ; SSE-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm1[2],xmm3[3],xmm1[3]
283 ; SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
284 ; SSE-NEXT: movdqa {{.*#+}} xmm2 = [65535,65535,65535,0,65535,65535,65535,65535]
285 ; SSE-NEXT: movdqa %xmm2, %xmm6
286 ; SSE-NEXT: pandn %xmm5, %xmm6
287 ; SSE-NEXT: movdqa %xmm1, %xmm7
288 ; SSE-NEXT: psrld $16, %xmm7
289 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,7,6,7]
290 ; SSE-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm7[2],xmm4[3],xmm7[3]
291 ; SSE-NEXT: pand %xmm2, %xmm4
292 ; SSE-NEXT: por %xmm6, %xmm4
293 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm5[2,2,3,3]
294 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm5[0,3,2,3]
295 ; SSE-NEXT: psrldq {{.*#+}} xmm5 = xmm5[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
296 ; SSE-NEXT: movdqa %xmm2, %xmm8
297 ; SSE-NEXT: pandn %xmm5, %xmm8
298 ; SSE-NEXT: movdqa %xmm0, %xmm5
299 ; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[1,0],xmm1[0,0]
300 ; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[2,0],xmm1[2,3]
301 ; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm5[0,2,2,3,4,5,6,7]
302 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,3,2,3]
303 ; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[1,0,2,3,4,5,6,7]
304 ; SSE-NEXT: pand %xmm2, %xmm9
305 ; SSE-NEXT: por %xmm8, %xmm9
306 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
307 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
308 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
309 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[3,1,2,3,4,5,6,7]
310 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,3,2,3]
311 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[0,1,3,3,4,5,6,7]
312 ; SSE-NEXT: pand %xmm2, %xmm5
313 ; SSE-NEXT: pandn %xmm6, %xmm2
314 ; SSE-NEXT: por %xmm5, %xmm2
315 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[1,1,1,1]
316 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm0[2,3,2,3]
317 ; SSE-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
318 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm7[0,2,2,3,4,5,6,7]
319 ; SSE-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
320 ; SSE-NEXT: psrlq $48, %xmm1
321 ; SSE-NEXT: psrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
322 ; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
323 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm7[1,3,2,3,4,5,6,7]
324 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
325 ; SSE-NEXT: movq %xmm3, (%rsi)
326 ; SSE-NEXT: movq %xmm4, (%rdx)
327 ; SSE-NEXT: movq %xmm9, (%rcx)
328 ; SSE-NEXT: movq %xmm2, (%r8)
329 ; SSE-NEXT: movq %xmm6, (%r9)
330 ; SSE-NEXT: movq %xmm0, (%rax)
331 ; SSE-NEXT: retq
332 ;
333 ; AVX1-ONLY-LABEL: load_i16_stride6_vf4:
334 ; AVX1-ONLY: # %bb.0:
335 ; AVX1-ONLY-NEXT: movq {{[0-9]+}}(%rsp), %rax
336 ; AVX1-ONLY-NEXT: vmovdqa (%rdi), %xmm0
337 ; AVX1-ONLY-NEXT: vmovdqa 16(%rdi), %xmm1
338 ; AVX1-ONLY-NEXT: vmovdqa 32(%rdi), %xmm2
339 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[0,1,0,3]
340 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm4 = xmm3[0,1,2,3,4,6,6,7]
341 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm4 = xmm4[2],xmm1[2],xmm4[3],xmm1[3]
342 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
343 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,2,2,3,4,5,6,7]
344 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
345 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm1, %xmm5
346 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,7,6,7]
347 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm3 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
348 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm2[3],xmm3[4,5,6,7]
349 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm5 = xmm2[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
350 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm6 = xmm1[0,1],xmm0[2,3],xmm1[4,5,6,7]
351 ; AVX1-ONLY-NEXT: vpshufb {{.*#+}} xmm7 = xmm6[4,5,0,1,12,13,u,u,u,u,u,u,u,u,u,u]
352 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm5 = xmm7[0,1,2],xmm5[3],xmm7[4,5,6,7]
353 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm7 = xmm2[2,2,3,3]
354 ; AVX1-ONLY-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[6,7,2,3,14,15,u,u,u,u,u,u,u,u,u,u]
355 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2],xmm7[3],xmm6[4,5,6,7]
356 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm7 = xmm1[1,1,1,1]
357 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm8 = xmm0[2,3,2,3]
358 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
359 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]
360 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm8 = xmm2[0,2,2,3,4,5,6,7]
361 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
362 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm1, %xmm1
363 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
364 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
365 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm2[1,3,2,3,4,5,6,7]
366 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
367 ; AVX1-ONLY-NEXT: vmovq %xmm4, (%rsi)
368 ; AVX1-ONLY-NEXT: vmovq %xmm3, (%rdx)
369 ; AVX1-ONLY-NEXT: vmovq %xmm5, (%rcx)
370 ; AVX1-ONLY-NEXT: vmovq %xmm6, (%r8)
371 ; AVX1-ONLY-NEXT: vmovq %xmm7, (%r9)
372 ; AVX1-ONLY-NEXT: vmovq %xmm0, (%rax)
373 ; AVX1-ONLY-NEXT: retq
374 ;
375 ; AVX2-SLOW-LABEL: load_i16_stride6_vf4:
376 ; AVX2-SLOW: # %bb.0:
377 ; AVX2-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
378 ; AVX2-SLOW-NEXT: vmovdqa (%rdi), %xmm0
379 ; AVX2-SLOW-NEXT: vmovdqa 16(%rdi), %xmm1
380 ; AVX2-SLOW-NEXT: vmovdqa 32(%rdi), %xmm2
381 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm3 = xmm0[0],xmm2[1],xmm0[2,3]
382 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1],xmm1[2],xmm3[3]
383 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[0,1,12,13,8,9,4,5,u,u,u,u,u,u,u,u]
384 ; AVX2-SLOW-NEXT: vpsrld $16, %xmm1, %xmm4
385 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[0,1,0,3]
386 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,7,6,7]
387 ; AVX2-SLOW-NEXT: vpunpckhdq {{.*#+}} xmm4 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]
388 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2],xmm2[3],xmm4[4,5,6,7]
389 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm5 = xmm0[0,1],xmm2[2,3]
390 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm5 = xmm1[0],xmm5[1,2],xmm1[3]
391 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm6 = xmm5[4,5,0,1,12,13,8,9,u,u,u,u,u,u,u,u]
392 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[6,7,2,3,14,15,10,11,u,u,u,u,u,u,u,u]
393 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3]
394 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
395 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[8,9,4,5,0,1,12,13,u,u,u,u,u,u,u,u]
396 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[10,11,6,7,2,3,14,15,u,u,u,u,u,u,u,u]
397 ; AVX2-SLOW-NEXT: vmovq %xmm3, (%rsi)
398 ; AVX2-SLOW-NEXT: vmovq %xmm4, (%rdx)
399 ; AVX2-SLOW-NEXT: vmovq %xmm6, (%rcx)
400 ; AVX2-SLOW-NEXT: vmovq %xmm5, (%r8)
401 ; AVX2-SLOW-NEXT: vmovq %xmm1, (%r9)
402 ; AVX2-SLOW-NEXT: vmovq %xmm0, (%rax)
403 ; AVX2-SLOW-NEXT: retq
404 ;
405 ; AVX2-FAST-LABEL: load_i16_stride6_vf4:
406 ; AVX2-FAST: # %bb.0:
407 ; AVX2-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
408 ; AVX2-FAST-NEXT: vmovdqa (%rdi), %xmm0
409 ; AVX2-FAST-NEXT: vmovdqa 16(%rdi), %xmm1
410 ; AVX2-FAST-NEXT: vmovdqa 32(%rdi), %xmm2
411 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm3 = xmm0[0],xmm2[1],xmm0[2,3]
412 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1],xmm1[2],xmm3[3]
413 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[0,1,12,13,8,9,4,5,u,u,u,u,u,u,u,u]
414 ; AVX2-FAST-NEXT: vpsrld $16, %xmm1, %xmm4
415 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[u,u,u,u,u,u,u,u,2,3,14,15,12,13,14,15]
416 ; AVX2-FAST-NEXT: vpunpckhdq {{.*#+}} xmm4 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]
417 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2],xmm2[3],xmm4[4,5,6,7]
418 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm5 = xmm0[0,1],xmm2[2,3]
419 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm5 = xmm1[0],xmm5[1,2],xmm1[3]
420 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm6 = xmm5[4,5,0,1,12,13,8,9,u,u,u,u,u,u,u,u]
421 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[6,7,2,3,14,15,10,11,u,u,u,u,u,u,u,u]
422 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3]
423 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
424 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[8,9,4,5,0,1,12,13,u,u,u,u,u,u,u,u]
425 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[10,11,6,7,2,3,14,15,u,u,u,u,u,u,u,u]
426 ; AVX2-FAST-NEXT: vmovq %xmm3, (%rsi)
427 ; AVX2-FAST-NEXT: vmovq %xmm4, (%rdx)
428 ; AVX2-FAST-NEXT: vmovq %xmm6, (%rcx)
429 ; AVX2-FAST-NEXT: vmovq %xmm5, (%r8)
430 ; AVX2-FAST-NEXT: vmovq %xmm1, (%r9)
431 ; AVX2-FAST-NEXT: vmovq %xmm0, (%rax)
432 ; AVX2-FAST-NEXT: retq
433 ;
434 ; AVX2-FAST-PERLANE-LABEL: load_i16_stride6_vf4:
435 ; AVX2-FAST-PERLANE: # %bb.0:
436 ; AVX2-FAST-PERLANE-NEXT: movq {{[0-9]+}}(%rsp), %rax
437 ; AVX2-FAST-PERLANE-NEXT: vmovdqa (%rdi), %xmm0
438 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 16(%rdi), %xmm1
439 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %xmm2
440 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm3 = xmm0[0],xmm2[1],xmm0[2,3]
441 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1],xmm1[2],xmm3[3]
442 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[0,1,12,13,8,9,4,5,u,u,u,u,u,u,u,u]
443 ; AVX2-FAST-PERLANE-NEXT: vpsrld $16, %xmm1, %xmm4
444 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[u,u,u,u,u,u,u,u,2,3,14,15,12,13,14,15]
445 ; AVX2-FAST-PERLANE-NEXT: vpunpckhdq {{.*#+}} xmm4 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]
446 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2],xmm2[3],xmm4[4,5,6,7]
447 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm5 = xmm0[0,1],xmm2[2,3]
448 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm5 = xmm1[0],xmm5[1,2],xmm1[3]
449 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm6 = xmm5[4,5,0,1,12,13,8,9,u,u,u,u,u,u,u,u]
450 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[6,7,2,3,14,15,10,11,u,u,u,u,u,u,u,u]
451 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3]
452 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
453 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[8,9,4,5,0,1,12,13,u,u,u,u,u,u,u,u]
454 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[10,11,6,7,2,3,14,15,u,u,u,u,u,u,u,u]
455 ; AVX2-FAST-PERLANE-NEXT: vmovq %xmm3, (%rsi)
456 ; AVX2-FAST-PERLANE-NEXT: vmovq %xmm4, (%rdx)
457 ; AVX2-FAST-PERLANE-NEXT: vmovq %xmm6, (%rcx)
458 ; AVX2-FAST-PERLANE-NEXT: vmovq %xmm5, (%r8)
459 ; AVX2-FAST-PERLANE-NEXT: vmovq %xmm1, (%r9)
460 ; AVX2-FAST-PERLANE-NEXT: vmovq %xmm0, (%rax)
461 ; AVX2-FAST-PERLANE-NEXT: retq
462 ;
463 ; AVX512F-SLOW-LABEL: load_i16_stride6_vf4:
464 ; AVX512F-SLOW: # %bb.0:
465 ; AVX512F-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
466 ; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %xmm0
467 ; AVX512F-SLOW-NEXT: vmovdqa 16(%rdi), %xmm1
468 ; AVX512F-SLOW-NEXT: vmovdqa 32(%rdi), %xmm2
469 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} xmm3 = xmm0[0],xmm2[1],xmm0[2,3]
470 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1],xmm1[2],xmm3[3]
471 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[0,1,12,13,8,9,4,5,u,u,u,u,u,u,u,u]
472 ; AVX512F-SLOW-NEXT: vpsrld $16, %xmm1, %xmm1
473 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
474 ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,7,6,7]
475 ; AVX512F-SLOW-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
476 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm2[3],xmm0[4,5,6,7]
477 ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm1 = [4,1,10,7]
478 ; AVX512F-SLOW-NEXT: vmovdqa 32(%rdi), %ymm2
479 ; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %ymm4
480 ; AVX512F-SLOW-NEXT: vpermi2d %ymm2, %ymm4, %ymm1
481 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm5 = xmm1[4,5,0,1,12,13,8,9,u,u,u,u,u,u,u,u]
482 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[6,7,2,3,14,15,10,11,u,u,u,u,u,u,u,u]
483 ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm6 = [0,13,10,3]
484 ; AVX512F-SLOW-NEXT: vpermi2d %ymm4, %ymm2, %ymm6
485 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm2 = xmm6[8,9,4,5,0,1,12,13,u,u,u,u,u,u,u,u]
486 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm4 = xmm6[10,11,6,7,2,3,14,15,u,u,u,u,u,u,u,u]
487 ; AVX512F-SLOW-NEXT: vmovq %xmm3, (%rsi)
488 ; AVX512F-SLOW-NEXT: vmovq %xmm0, (%rdx)
489 ; AVX512F-SLOW-NEXT: vmovq %xmm5, (%rcx)
490 ; AVX512F-SLOW-NEXT: vmovq %xmm1, (%r8)
491 ; AVX512F-SLOW-NEXT: vmovq %xmm2, (%r9)
492 ; AVX512F-SLOW-NEXT: vmovq %xmm4, (%rax)
493 ; AVX512F-SLOW-NEXT: vzeroupper
494 ; AVX512F-SLOW-NEXT: retq
495 ;
496 ; AVX512F-FAST-LABEL: load_i16_stride6_vf4:
497 ; AVX512F-FAST: # %bb.0:
498 ; AVX512F-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
499 ; AVX512F-FAST-NEXT: vmovdqa (%rdi), %xmm0
500 ; AVX512F-FAST-NEXT: vmovdqa 16(%rdi), %xmm1
501 ; AVX512F-FAST-NEXT: vmovdqa 32(%rdi), %xmm2
502 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} xmm3 = xmm0[0],xmm2[1],xmm0[2,3]
503 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1],xmm1[2],xmm3[3]
504 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[0,1,12,13,8,9,4,5,u,u,u,u,u,u,u,u]
505 ; AVX512F-FAST-NEXT: vpsrld $16, %xmm1, %xmm1
506 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,u,2,3,14,15,12,13,14,15]
507 ; AVX512F-FAST-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
508 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm2[3],xmm0[4,5,6,7]
509 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm1 = [4,1,10,7]
510 ; AVX512F-FAST-NEXT: vmovdqa 32(%rdi), %ymm2
511 ; AVX512F-FAST-NEXT: vmovdqa (%rdi), %ymm4
512 ; AVX512F-FAST-NEXT: vpermi2d %ymm2, %ymm4, %ymm1
513 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm5 = xmm1[4,5,0,1,12,13,8,9,u,u,u,u,u,u,u,u]
514 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[6,7,2,3,14,15,10,11,u,u,u,u,u,u,u,u]
515 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm6 = [0,13,10,3]
516 ; AVX512F-FAST-NEXT: vpermi2d %ymm4, %ymm2, %ymm6
517 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm2 = xmm6[8,9,4,5,0,1,12,13,u,u,u,u,u,u,u,u]
518 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm4 = xmm6[10,11,6,7,2,3,14,15,u,u,u,u,u,u,u,u]
519 ; AVX512F-FAST-NEXT: vmovq %xmm3, (%rsi)
520 ; AVX512F-FAST-NEXT: vmovq %xmm0, (%rdx)
521 ; AVX512F-FAST-NEXT: vmovq %xmm5, (%rcx)
522 ; AVX512F-FAST-NEXT: vmovq %xmm1, (%r8)
523 ; AVX512F-FAST-NEXT: vmovq %xmm2, (%r9)
524 ; AVX512F-FAST-NEXT: vmovq %xmm4, (%rax)
525 ; AVX512F-FAST-NEXT: vzeroupper
526 ; AVX512F-FAST-NEXT: retq
527 ;
528 ; AVX512BW-LABEL: load_i16_stride6_vf4:
529 ; AVX512BW: # %bb.0:
530 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
531 ; AVX512BW-NEXT: vpbroadcastq {{.*#+}} xmm0 = [0,6,12,18,0,6,12,18]
532 ; AVX512BW-NEXT: vmovdqa (%rdi), %ymm1
533 ; AVX512BW-NEXT: vmovdqa 32(%rdi), %ymm2
534 ; AVX512BW-NEXT: vpermi2w %ymm2, %ymm1, %ymm0
535 ; AVX512BW-NEXT: vpbroadcastq {{.*#+}} xmm3 = [1,7,13,19,1,7,13,19]
536 ; AVX512BW-NEXT: vpermi2w %ymm2, %ymm1, %ymm3
537 ; AVX512BW-NEXT: vpbroadcastq {{.*#+}} xmm4 = [2,8,14,20,2,8,14,20]
538 ; AVX512BW-NEXT: vpermi2w %ymm2, %ymm1, %ymm4
539 ; AVX512BW-NEXT: vpbroadcastq {{.*#+}} xmm5 = [3,9,15,21,3,9,15,21]
540 ; AVX512BW-NEXT: vpermi2w %ymm2, %ymm1, %ymm5
541 ; AVX512BW-NEXT: vpbroadcastq {{.*#+}} xmm6 = [4,10,16,22,4,10,16,22]
542 ; AVX512BW-NEXT: vpermi2w %ymm2, %ymm1, %ymm6
543 ; AVX512BW-NEXT: vpbroadcastq {{.*#+}} xmm7 = [5,11,17,23,5,11,17,23]
544 ; AVX512BW-NEXT: vpermi2w %ymm2, %ymm1, %ymm7
545 ; AVX512BW-NEXT: vmovq %xmm0, (%rsi)
546 ; AVX512BW-NEXT: vmovq %xmm3, (%rdx)
547 ; AVX512BW-NEXT: vmovq %xmm4, (%rcx)
548 ; AVX512BW-NEXT: vmovq %xmm5, (%r8)
549 ; AVX512BW-NEXT: vmovq %xmm6, (%r9)
550 ; AVX512BW-NEXT: vmovq %xmm7, (%rax)
551 ; AVX512BW-NEXT: vzeroupper
552 ; AVX512BW-NEXT: retq
553 %wide.vec = load <24 x i16>, ptr %in.vec, align 64
554 %strided.vec0 = shufflevector <24 x i16> %wide.vec, <24 x i16> poison, <4 x i32> <i32 0, i32 6, i32 12, i32 18>
555 %strided.vec1 = shufflevector <24 x i16> %wide.vec, <24 x i16> poison, <4 x i32> <i32 1, i32 7, i32 13, i32 19>
556 %strided.vec2 = shufflevector <24 x i16> %wide.vec, <24 x i16> poison, <4 x i32> <i32 2, i32 8, i32 14, i32 20>
557 %strided.vec3 = shufflevector <24 x i16> %wide.vec, <24 x i16> poison, <4 x i32> <i32 3, i32 9, i32 15, i32 21>
558 %strided.vec4 = shufflevector <24 x i16> %wide.vec, <24 x i16> poison, <4 x i32> <i32 4, i32 10, i32 16, i32 22>
559 %strided.vec5 = shufflevector <24 x i16> %wide.vec, <24 x i16> poison, <4 x i32> <i32 5, i32 11, i32 17, i32 23>
560 store <4 x i16> %strided.vec0, ptr %out.vec0, align 64
561 store <4 x i16> %strided.vec1, ptr %out.vec1, align 64
562 store <4 x i16> %strided.vec2, ptr %out.vec2, align 64
563 store <4 x i16> %strided.vec3, ptr %out.vec3, align 64
564 store <4 x i16> %strided.vec4, ptr %out.vec4, align 64
565 store <4 x i16> %strided.vec5, ptr %out.vec5, align 64
566 ret void
567 }
569 define void @load_i16_stride6_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4, ptr %out.vec5) nounwind {
570 ; SSE-LABEL: load_i16_stride6_vf8:
571 ; SSE: # %bb.0:
572 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
573 ; SSE-NEXT: movdqa 64(%rdi), %xmm0
574 ; SSE-NEXT: movdqa 80(%rdi), %xmm7
575 ; SSE-NEXT: movdqa (%rdi), %xmm2
576 ; SSE-NEXT: movdqa 16(%rdi), %xmm5
577 ; SSE-NEXT: movdqa 32(%rdi), %xmm6
578 ; SSE-NEXT: movdqa 48(%rdi), %xmm4
579 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm6[0,1,1,2,4,5,6,7]
580 ; SSE-NEXT: movdqa {{.*#+}} xmm1 = [65535,65535,65535,0,0,0,65535,65535]
581 ; SSE-NEXT: movdqa %xmm1, %xmm9
582 ; SSE-NEXT: pandn %xmm3, %xmm9
583 ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm2[0,1,0,3]
584 ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm8[0,1,2,3,4,6,6,7]
585 ; SSE-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
586 ; SSE-NEXT: pand %xmm1, %xmm3
587 ; SSE-NEXT: por %xmm9, %xmm3
588 ; SSE-NEXT: movdqa %xmm0, %xmm9
589 ; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm0[2,2,3,3]
590 ; SSE-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm7[0],xmm10[1],xmm7[1],xmm10[2],xmm7[2],xmm10[3],xmm7[3]
591 ; SSE-NEXT: movdqa %xmm7, %xmm12
592 ; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[2,0],xmm0[3,0]
593 ; SSE-NEXT: movaps %xmm0, %xmm11
594 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0],xmm7[0,0]
595 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm7[2,3]
596 ; SSE-NEXT: pslld $16, %xmm7
597 ; SSE-NEXT: psrldq {{.*#+}} xmm9 = xmm9[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
598 ; SSE-NEXT: punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]
599 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm4[0,3,2,3]
600 ; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm7[0,1,0,2,4,5,6,7]
601 ; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[3,1],xmm13[1,3]
602 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm9[2,0]
603 ; SSE-NEXT: movdqa %xmm5, %xmm9
604 ; SSE-NEXT: psrld $16, %xmm9
605 ; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,5,7,6,7]
606 ; SSE-NEXT: punpckhdq {{.*#+}} xmm8 = xmm8[2],xmm9[2],xmm8[3],xmm9[3]
607 ; SSE-NEXT: pand %xmm1, %xmm8
608 ; SSE-NEXT: pandn %xmm6, %xmm1
609 ; SSE-NEXT: por %xmm8, %xmm1
610 ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,1,3,4,5,6,7]
611 ; SSE-NEXT: shufps {{.*#+}} xmm10 = xmm10[3,1],xmm7[1,3]
612 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm10[2,0]
613 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm4[1,1,1,1]
614 ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm6[2,2,3,3]
615 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm6[0,1,0,3]
616 ; SSE-NEXT: psrldq {{.*#+}} xmm6 = xmm6[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
617 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm7[0]
618 ; SSE-NEXT: movdqa {{.*#+}} xmm7 = [65535,65535,65535,0,0,65535,65535,65535]
619 ; SSE-NEXT: movdqa %xmm7, %xmm10
620 ; SSE-NEXT: pandn %xmm6, %xmm10
621 ; SSE-NEXT: movdqa %xmm2, %xmm6
622 ; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[1,0],xmm5[0,0]
623 ; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[2,0],xmm5[2,3]
624 ; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm6[0,2,2,3,4,5,6,7]
625 ; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm13[0,3,2,3]
626 ; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm13[1,0,2,3,4,5,6,7]
627 ; SSE-NEXT: pand %xmm7, %xmm13
628 ; SSE-NEXT: por %xmm10, %xmm13
629 ; SSE-NEXT: movdqa {{.*#+}} xmm10 = [65535,65535,65535,65535,65535,0,0,0]
630 ; SSE-NEXT: pand %xmm10, %xmm13
631 ; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[0,1],xmm12[0,2]
632 ; SSE-NEXT: pshufhw {{.*#+}} xmm12 = xmm11[0,1,2,3,4,6,6,7]
633 ; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm12[0,1,2,0]
634 ; SSE-NEXT: pshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,6,5,4]
635 ; SSE-NEXT: movdqa %xmm10, %xmm14
636 ; SSE-NEXT: pandn %xmm12, %xmm14
637 ; SSE-NEXT: por %xmm13, %xmm14
638 ; SSE-NEXT: movdqa %xmm4, %xmm12
639 ; SSE-NEXT: psrlq $48, %xmm12
640 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm8 = xmm8[0],xmm12[0]
641 ; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[3,1,2,3,4,5,6,7]
642 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,3,2,3]
643 ; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,1,3,3,4,5,6,7]
644 ; SSE-NEXT: pand %xmm7, %xmm6
645 ; SSE-NEXT: pandn %xmm8, %xmm7
646 ; SSE-NEXT: por %xmm6, %xmm7
647 ; SSE-NEXT: pand %xmm10, %xmm7
648 ; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm11[0,1,2,3,7,5,6,7]
649 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,1,0,2]
650 ; SSE-NEXT: movdqa %xmm10, %xmm8
651 ; SSE-NEXT: pandn %xmm6, %xmm8
652 ; SSE-NEXT: por %xmm7, %xmm8
653 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm5[1,1,1,1]
654 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm2[2,3,2,3]
655 ; SSE-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
656 ; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm9[0,1,2,3,4,5,4,6]
657 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm6 = xmm6[1],xmm4[1]
658 ; SSE-NEXT: movss {{.*#+}} xmm6 = xmm7[0],xmm6[1,2,3]
659 ; SSE-NEXT: andps %xmm10, %xmm6
660 ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm0[0,2,2,3,4,5,6,7]
661 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,1,0,3]
662 ; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5,4,6]
663 ; SSE-NEXT: movdqa %xmm10, %xmm11
664 ; SSE-NEXT: pandn %xmm7, %xmm11
665 ; SSE-NEXT: por %xmm6, %xmm11
666 ; SSE-NEXT: psrlq $48, %xmm5
667 ; SSE-NEXT: psrldq {{.*#+}} xmm2 = xmm2[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
668 ; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
669 ; SSE-NEXT: psrld $16, %xmm4
670 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm9[0,1,2,3,4,5,5,7]
671 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm5 = xmm5[1],xmm4[1]
672 ; SSE-NEXT: movss {{.*#+}} xmm5 = xmm2[0],xmm5[1,2,3]
673 ; SSE-NEXT: andps %xmm10, %xmm5
674 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7]
675 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
676 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,5,7]
677 ; SSE-NEXT: pandn %xmm0, %xmm10
678 ; SSE-NEXT: por %xmm5, %xmm10
679 ; SSE-NEXT: movaps %xmm3, (%rsi)
680 ; SSE-NEXT: movaps %xmm1, (%rdx)
681 ; SSE-NEXT: movdqa %xmm14, (%rcx)
682 ; SSE-NEXT: movdqa %xmm8, (%r8)
683 ; SSE-NEXT: movdqa %xmm11, (%r9)
684 ; SSE-NEXT: movdqa %xmm10, (%rax)
685 ; SSE-NEXT: retq
686 ;
687 ; AVX1-ONLY-LABEL: load_i16_stride6_vf8:
688 ; AVX1-ONLY: # %bb.0:
689 ; AVX1-ONLY-NEXT: movq {{[0-9]+}}(%rsp), %rax
690 ; AVX1-ONLY-NEXT: vmovdqa (%rdi), %xmm0
691 ; AVX1-ONLY-NEXT: vmovdqa 16(%rdi), %xmm2
692 ; AVX1-ONLY-NEXT: vmovdqa 32(%rdi), %xmm4
693 ; AVX1-ONLY-NEXT: vmovdqa 48(%rdi), %xmm1
694 ; AVX1-ONLY-NEXT: vpsrlq $16, %xmm4, %xmm3
695 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm7 = xmm1[0,3,2,3]
696 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm5 = xmm7[0,1,0,2,4,5,6,7]
697 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
698 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm8 = xmm0[0,1,0,3]
699 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm5 = xmm8[0,1,2,3,4,6,6,7]
700 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm5[2],xmm2[2],xmm5[3],xmm2[3]
701 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm3 = xmm5[0,1,2],xmm3[3,4,5],xmm5[6,7]
702 ; AVX1-ONLY-NEXT: vmovdqa 80(%rdi), %xmm5
703 ; AVX1-ONLY-NEXT: vpslld $16, %xmm5, %xmm9
704 ; AVX1-ONLY-NEXT: vmovdqa 64(%rdi), %xmm6
705 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm10 = xmm6[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
706 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
707 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5],xmm9[6,7]
708 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm9 = xmm4[1,1,1,1]
709 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,1,3,4,5,6,7]
710 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
711 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm2, %xmm9
712 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,5,7,6,7]
713 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm8 = xmm8[2],xmm9[2],xmm8[3],xmm9[3]
714 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3,4,5],xmm8[6,7]
715 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[2,2,3,3]
716 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3]
717 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5],xmm8[6,7]
718 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm8 = xmm1[1,1,1,1]
719 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm9 = xmm4[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
720 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm8 = xmm9[0],xmm8[0]
721 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm9 = xmm2[0,1],xmm0[2,3],xmm2[4,5,6,7]
722 ; AVX1-ONLY-NEXT: vpshufb {{.*#+}} xmm10 = xmm9[4,5,0,1,12,13,u,u,u,u,u,u,u,u,u,u]
723 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm8 = xmm10[0,1,2],xmm8[3,4],xmm10[5,6,7]
724 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm10 = xmm6[0,1,2,3],xmm5[4,5],xmm6[6,7]
725 ; AVX1-ONLY-NEXT: vpshufb {{.*#+}} xmm11 = xmm10[u,u,u,u,u,u,u,u,u,u,0,1,12,13,8,9]
726 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2,3,4],xmm11[5,6,7]
727 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm1, %xmm11
728 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm12 = xmm4[2,2,3,3]
729 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm11 = xmm12[0],xmm11[0]
730 ; AVX1-ONLY-NEXT: vpshufb {{.*#+}} xmm9 = xmm9[6,7,2,3,14,15,u,u,u,u,u,u,u,u,u,u]
731 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1,2],xmm11[3,4],xmm9[5,6,7]
732 ; AVX1-ONLY-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[u,u,u,u,u,u,u,u,u,u,2,3,14,15,10,11]
733 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1,2,3,4],xmm10[5,6,7]
734 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm10 = xmm2[1,1,1,1]
735 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm11 = xmm0[2,3,2,3]
736 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]
737 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,1,0,3]
738 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm11 = xmm4[0,1,2,3,4,5,4,6]
739 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm11 = xmm11[1],xmm1[1]
740 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1],xmm11[2,3,4,5,6,7]
741 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1],xmm6[2,3],xmm5[4,5,6,7]
742 ; AVX1-ONLY-NEXT: vpshufb {{.*#+}} xmm6 = xmm5[u,u,u,u,u,u,u,u,u,u,4,5,0,1,12,13]
743 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm6 = xmm10[0,1,2,3,4],xmm6[5,6,7]
744 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm2, %xmm2
745 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
746 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
747 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm1, %xmm1
748 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm2 = xmm4[0,1,2,3,4,5,5,7]
749 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm1 = xmm2[1],xmm1[1]
750 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
751 ; AVX1-ONLY-NEXT: vpshufb {{.*#+}} xmm1 = xmm5[u,u,u,u,u,u,u,u,u,u,6,7,2,3,14,15]
752 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm1[5,6,7]
753 ; AVX1-ONLY-NEXT: vmovdqa %xmm3, (%rsi)
754 ; AVX1-ONLY-NEXT: vmovdqa %xmm7, (%rdx)
755 ; AVX1-ONLY-NEXT: vmovdqa %xmm8, (%rcx)
756 ; AVX1-ONLY-NEXT: vmovdqa %xmm9, (%r8)
757 ; AVX1-ONLY-NEXT: vmovdqa %xmm6, (%r9)
758 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, (%rax)
759 ; AVX1-ONLY-NEXT: retq
760 ;
761 ; AVX2-SLOW-LABEL: load_i16_stride6_vf8:
762 ; AVX2-SLOW: # %bb.0:
763 ; AVX2-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
764 ; AVX2-SLOW-NEXT: vmovdqa (%rdi), %ymm3
765 ; AVX2-SLOW-NEXT: vmovdqa 32(%rdi), %ymm4
766 ; AVX2-SLOW-NEXT: vmovdqa 80(%rdi), %xmm0
767 ; AVX2-SLOW-NEXT: vpslld $16, %xmm0, %xmm2
768 ; AVX2-SLOW-NEXT: vmovdqa 64(%rdi), %xmm1
769 ; AVX2-SLOW-NEXT: vpsrldq {{.*#+}} xmm5 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
770 ; AVX2-SLOW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
771 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7]
772 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm6 = xmm5[0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15]
773 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm5, %xmm7
774 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[0,2,0,3]
775 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,6,6,7]
776 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm8[2],xmm6[3],xmm8[4,5],xmm6[6,7]
777 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm2 = xmm6[0,1,2],xmm2[3]
778 ; AVX2-SLOW-NEXT: vpbroadcastw 74(%rdi), %xmm6
779 ; AVX2-SLOW-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3]
780 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,10,11,u,u,2,3,14,15,u,u,u,u]
781 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15]
782 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1],xmm7[2],xmm5[3],xmm7[4,5],xmm5[6,7]
783 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1,2],xmm6[3]
784 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm6 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]
785 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[2,1,2,3]
786 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm7[2,1,2,0,4,5,6,7]
787 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm6, %xmm6
788 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[0,3,2,1]
789 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm9 = xmm6[0,0,2,3,4,5,6,7]
790 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[0,1,3,3]
791 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0],xmm9[1,2],xmm8[3],xmm9[4,5,6,7]
792 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm9 = xmm1[0,1],xmm0[2],xmm1[3]
793 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm10 = xmm9[u,u,u,u,u,u,u,u,u,u,0,1,12,13,8,9]
794 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2,3,4],xmm10[5,6,7]
795 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[3,1,2,1,4,5,6,7]
796 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,3,3,4,5,6,7]
797 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,7,7,7,7]
798 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0],xmm6[1,2],xmm7[3],xmm6[4,5,6,7]
799 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm7 = xmm9[u,u,u,u,u,u,u,u,u,u,2,3,14,15,10,11]
800 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm7[5,6,7]
801 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1],ymm3[2],ymm4[3,4],ymm3[5],ymm4[6,7]
802 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm3, %xmm4
803 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm4[2,2,2,2,4,5,6,7]
804 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm9 = xmm3[8,9,u,u,0,1,12,13,u,u,u,u,u,u,u,u]
805 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm7 = xmm9[0],xmm7[1],xmm9[2,3],xmm7[4],xmm9[5,6,7]
806 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
807 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[u,u,u,u,u,u,u,u,u,u,4,5,0,1,12,13]
808 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm7[0,1,2,3,4],xmm1[5,6,7]
809 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[10,11,u,u,2,3,14,15,u,u,u,u,u,u,u,u]
810 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[1,1,2,3]
811 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,5,5,5]
812 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm4[1],xmm3[2,3],xmm4[4],xmm3[5,6,7]
813 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,u,u,u,6,7,2,3,14,15]
814 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3,4],xmm0[5,6,7]
815 ; AVX2-SLOW-NEXT: vmovdqa %xmm2, (%rsi)
816 ; AVX2-SLOW-NEXT: vmovdqa %xmm5, (%rdx)
817 ; AVX2-SLOW-NEXT: vmovdqa %xmm8, (%rcx)
818 ; AVX2-SLOW-NEXT: vmovdqa %xmm6, (%r8)
819 ; AVX2-SLOW-NEXT: vmovdqa %xmm1, (%r9)
820 ; AVX2-SLOW-NEXT: vmovdqa %xmm0, (%rax)
821 ; AVX2-SLOW-NEXT: vzeroupper
822 ; AVX2-SLOW-NEXT: retq
823 ;
824 ; AVX2-FAST-LABEL: load_i16_stride6_vf8:
825 ; AVX2-FAST: # %bb.0:
826 ; AVX2-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
827 ; AVX2-FAST-NEXT: vmovdqa (%rdi), %ymm2
828 ; AVX2-FAST-NEXT: vmovdqa 32(%rdi), %ymm4
829 ; AVX2-FAST-NEXT: vmovdqa 80(%rdi), %xmm0
830 ; AVX2-FAST-NEXT: vpslld $16, %xmm0, %xmm3
831 ; AVX2-FAST-NEXT: vmovdqa 64(%rdi), %xmm1
832 ; AVX2-FAST-NEXT: vpsrldq {{.*#+}} xmm5 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
833 ; AVX2-FAST-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
834 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm5 = ymm2[0],ymm4[1],ymm2[2,3],ymm4[4],ymm2[5,6],ymm4[7]
835 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm6 = xmm5[0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15]
836 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm5, %xmm7
837 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[2,1,0,3]
838 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm8 = xmm7[u,u,u,u,0,1,u,u,8,9,12,13,u,u,u,u]
839 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm8[2],xmm6[3],xmm8[4,5],xmm6[6,7]
840 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm3 = xmm6[0,1,2],xmm3[3]
841 ; AVX2-FAST-NEXT: vpbroadcastw 74(%rdi), %xmm6
842 ; AVX2-FAST-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3]
843 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15]
844 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,2,3,u,u,10,11,14,15,u,u,u,u]
845 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1],xmm7[2],xmm5[3],xmm7[4,5],xmm5[6,7]
846 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1,2],xmm6[3]
847 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm6 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7]
848 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[2,1,2,3]
849 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm8 = xmm7[2,1,2,0,4,5,6,7]
850 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm6, %xmm6
851 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[0,3,2,1]
852 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm9 = xmm6[u,u,0,1,4,5,u,u,12,13,u,u,u,u,u,u]
853 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0],xmm9[1,2],xmm8[3],xmm9[4,5,6,7]
854 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm9 = xmm1[0,1],xmm0[2],xmm1[3]
855 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm10 = xmm9[u,u,u,u,u,u,u,u,u,u,0,1,12,13,8,9]
856 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2,3,4],xmm10[5,6,7]
857 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[3,1,2,1,4,5,6,7]
858 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[u,u,2,3,6,7,u,u,14,15,u,u,u,u,u,u]
859 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0],xmm6[1,2],xmm7[3],xmm6[4,5,6,7]
860 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm7 = xmm9[u,u,u,u,u,u,u,u,u,u,2,3,14,15,10,11]
861 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm7[5,6,7]
862 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1],ymm2[2],ymm4[3,4],ymm2[5],ymm4[6,7]
863 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm2, %xmm4
864 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm7 = xmm4[2,2,2,2,4,5,6,7]
865 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm9 = xmm2[8,9,u,u,0,1,12,13,u,u,u,u,u,u,u,u]
866 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm7 = xmm9[0],xmm7[1],xmm9[2,3],xmm7[4],xmm9[5,6,7]
867 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
868 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[u,u,u,u,u,u,u,u,u,u,4,5,0,1,12,13]
869 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm7[0,1,2,3,4],xmm1[5,6,7]
870 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[u,u,6,7,u,u,u,u,10,11,u,u,u,u,u,u]
871 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[10,11,u,u,2,3,14,15,u,u,u,u,u,u,u,u]
872 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm4[1],xmm2[2,3],xmm4[4],xmm2[5,6,7]
873 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,u,u,u,6,7,2,3,14,15]
874 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3,4],xmm0[5,6,7]
875 ; AVX2-FAST-NEXT: vmovdqa %xmm3, (%rsi)
876 ; AVX2-FAST-NEXT: vmovdqa %xmm5, (%rdx)
877 ; AVX2-FAST-NEXT: vmovdqa %xmm8, (%rcx)
878 ; AVX2-FAST-NEXT: vmovdqa %xmm6, (%r8)
879 ; AVX2-FAST-NEXT: vmovdqa %xmm1, (%r9)
880 ; AVX2-FAST-NEXT: vmovdqa %xmm0, (%rax)
881 ; AVX2-FAST-NEXT: vzeroupper
882 ; AVX2-FAST-NEXT: retq
883 ;
884 ; AVX2-FAST-PERLANE-LABEL: load_i16_stride6_vf8:
885 ; AVX2-FAST-PERLANE: # %bb.0:
886 ; AVX2-FAST-PERLANE-NEXT: movq {{[0-9]+}}(%rsp), %rax
887 ; AVX2-FAST-PERLANE-NEXT: vmovdqa (%rdi), %ymm2
888 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %ymm4
889 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 80(%rdi), %xmm0
890 ; AVX2-FAST-PERLANE-NEXT: vpslld $16, %xmm0, %xmm3
891 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 64(%rdi), %xmm1
892 ; AVX2-FAST-PERLANE-NEXT: vpsrldq {{.*#+}} xmm5 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
893 ; AVX2-FAST-PERLANE-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
894 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm5 = ymm2[0],ymm4[1],ymm2[2,3],ymm4[4],ymm2[5,6],ymm4[7]
895 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm6 = xmm5[0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15]
896 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm5, %xmm7
897 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[2,1,0,3]
898 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm8 = xmm7[u,u,u,u,0,1,u,u,8,9,12,13,u,u,u,u]
899 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm8[2],xmm6[3],xmm8[4,5],xmm6[6,7]
900 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm3 = xmm6[0,1,2],xmm3[3]
901 ; AVX2-FAST-PERLANE-NEXT: vpbroadcastw 74(%rdi), %xmm6
902 ; AVX2-FAST-PERLANE-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3]
903 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15]
904 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,2,3,u,u,10,11,14,15,u,u,u,u]
905 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1],xmm7[2],xmm5[3],xmm7[4,5],xmm5[6,7]
906 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1,2],xmm6[3]
907 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm6 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7]
908 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[2,1,2,3]
909 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm8 = xmm7[2,1,2,0,4,5,6,7]
910 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm6, %xmm6
911 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[0,3,2,1]
912 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm9 = xmm6[u,u,0,1,4,5,u,u,12,13,u,u,u,u,u,u]
913 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0],xmm9[1,2],xmm8[3],xmm9[4,5,6,7]
914 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm9 = xmm1[0,1],xmm0[2],xmm1[3]
915 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm10 = xmm9[u,u,u,u,u,u,u,u,u,u,0,1,12,13,8,9]
916 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2,3,4],xmm10[5,6,7]
917 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[3,1,2,1,4,5,6,7]
918 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[u,u,2,3,6,7,u,u,14,15,u,u,u,u,u,u]
919 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0],xmm6[1,2],xmm7[3],xmm6[4,5,6,7]
920 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm7 = xmm9[u,u,u,u,u,u,u,u,u,u,2,3,14,15,10,11]
921 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm7[5,6,7]
922 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1],ymm2[2],ymm4[3,4],ymm2[5],ymm4[6,7]
923 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm2, %xmm4
924 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm7 = xmm4[2,2,2,2,4,5,6,7]
925 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm9 = xmm2[8,9,u,u,0,1,12,13,u,u,u,u,u,u,u,u]
926 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm7 = xmm9[0],xmm7[1],xmm9[2,3],xmm7[4],xmm9[5,6,7]
927 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
928 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[u,u,u,u,u,u,u,u,u,u,4,5,0,1,12,13]
929 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm1 = xmm7[0,1,2,3,4],xmm1[5,6,7]
930 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[u,u,6,7,u,u,u,u,10,11,u,u,u,u,u,u]
931 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[10,11,u,u,2,3,14,15,u,u,u,u,u,u,u,u]
932 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm4[1],xmm2[2,3],xmm4[4],xmm2[5,6,7]
933 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,u,u,u,6,7,2,3,14,15]
934 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3,4],xmm0[5,6,7]
935 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm3, (%rsi)
936 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm5, (%rdx)
937 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm8, (%rcx)
938 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm6, (%r8)
939 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, (%r9)
940 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm0, (%rax)
941 ; AVX2-FAST-PERLANE-NEXT: vzeroupper
942 ; AVX2-FAST-PERLANE-NEXT: retq
943 ;
944 ; AVX512F-SLOW-LABEL: load_i16_stride6_vf8:
945 ; AVX512F-SLOW: # %bb.0:
946 ; AVX512F-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
947 ; AVX512F-SLOW-NEXT: vmovdqa 80(%rdi), %xmm0
948 ; AVX512F-SLOW-NEXT: vpslld $16, %xmm0, %xmm2
949 ; AVX512F-SLOW-NEXT: vmovdqa 64(%rdi), %xmm1
950 ; AVX512F-SLOW-NEXT: vpsrldq {{.*#+}} xmm3 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
951 ; AVX512F-SLOW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
952 ; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %ymm4
953 ; AVX512F-SLOW-NEXT: vmovdqa 32(%rdi), %ymm5
954 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0],ymm5[1],ymm4[2,3],ymm5[4],ymm4[5,6],ymm5[7]
955 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm6 = xmm3[0,1,12,13,u,u,4,5,u,u,u,u,u,u,u,u]
956 ; AVX512F-SLOW-NEXT: vextracti128 $1, %ymm3, %xmm7
957 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[0,2,0,3]
958 ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,6,6,7]
959 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm8[2],xmm6[3],xmm8[4,5],xmm6[6,7]
960 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} xmm2 = xmm6[0,1,2],xmm2[3]
961 ; AVX512F-SLOW-NEXT: vpbroadcastw 74(%rdi), %xmm6
962 ; AVX512F-SLOW-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3]
963 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,10,11,u,u,2,3,14,15,u,u,u,u]
964 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[2,3,14,15,u,u,6,7,u,u,u,u,u,u,u,u]
965 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm7[2],xmm3[3],xmm7[4,5],xmm3[6,7]
966 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1,2],xmm6[3]
967 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm6 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7]
968 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[2,1,2,3]
969 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm7[2,1,2,0,4,5,6,7]
970 ; AVX512F-SLOW-NEXT: vextracti128 $1, %ymm6, %xmm6
971 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[0,3,2,1]
972 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm9 = xmm6[0,0,2,3,4,5,6,7]
973 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[0,1,3,3]
974 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0],xmm9[1,2],xmm8[3],xmm9[4,5,6,7]
975 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} xmm9 = xmm1[0,1],xmm0[2],xmm1[3]
976 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm10 = xmm9[u,u,u,u,u,u,u,u,u,u,0,1,12,13,8,9]
977 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2,3,4],xmm10[5,6,7]
978 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[3,1,2,1,4,5,6,7]
979 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,3,3,4,5,6,7]
980 ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,7,7,7,7]
981 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0],xmm6[1,2],xmm7[3],xmm6[4,5,6,7]
982 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm7 = xmm9[u,u,u,u,u,u,u,u,u,u,2,3,14,15,10,11]
983 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm7[5,6,7]
984 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1],ymm4[2],ymm5[3,4],ymm4[5],ymm5[6,7]
985 ; AVX512F-SLOW-NEXT: vextracti128 $1, %ymm4, %xmm5
986 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm5[2,2,2,2,4,5,6,7]
987 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm9 = xmm4[8,9,u,u,0,1,12,13,u,u,u,u,u,u,u,u]
988 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm7 = xmm9[0],xmm7[1],xmm9[2,3],xmm7[4],xmm9[5,6,7]
989 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
990 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[u,u,u,u,u,u,u,u,u,u,4,5,0,1,12,13]
991 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm7[0,1,2,3,4],xmm1[5,6,7]
992 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[10,11,u,u,2,3,14,15,u,u,u,u,u,u,u,u]
993 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,3]
994 ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,5,5]
995 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm5[1],xmm4[2,3],xmm5[4],xmm4[5,6,7]
996 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,u,u,u,6,7,2,3,14,15]
997 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm4[0,1,2,3,4],xmm0[5,6,7]
998 ; AVX512F-SLOW-NEXT: vmovdqa %xmm2, (%rsi)
999 ; AVX512F-SLOW-NEXT: vmovdqa %xmm3, (%rdx)
1000 ; AVX512F-SLOW-NEXT: vmovdqa %xmm8, (%rcx)
1001 ; AVX512F-SLOW-NEXT: vmovdqa %xmm6, (%r8)
1002 ; AVX512F-SLOW-NEXT: vmovdqa %xmm1, (%r9)
1003 ; AVX512F-SLOW-NEXT: vmovdqa %xmm0, (%rax)
1004 ; AVX512F-SLOW-NEXT: vzeroupper
1005 ; AVX512F-SLOW-NEXT: retq
1006 ;
1007 ; AVX512F-FAST-LABEL: load_i16_stride6_vf8:
1008 ; AVX512F-FAST: # %bb.0:
1009 ; AVX512F-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
1010 ; AVX512F-FAST-NEXT: vmovdqa 80(%rdi), %xmm0
1011 ; AVX512F-FAST-NEXT: vpslld $16, %xmm0, %xmm2
1012 ; AVX512F-FAST-NEXT: vmovdqa 64(%rdi), %xmm1
1013 ; AVX512F-FAST-NEXT: vpsrldq {{.*#+}} xmm3 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
1014 ; AVX512F-FAST-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
1015 ; AVX512F-FAST-NEXT: vmovdqa (%rdi), %ymm3
1016 ; AVX512F-FAST-NEXT: vmovdqa 32(%rdi), %ymm4
1017 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm5 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7]
1018 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm6 = xmm5[0,1,12,13,u,u,4,5,u,u,u,u,u,u,u,u]
1019 ; AVX512F-FAST-NEXT: vextracti128 $1, %ymm5, %xmm7
1020 ; AVX512F-FAST-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[2,1,0,3]
1021 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm8 = xmm7[u,u,u,u,0,1,u,u,8,9,12,13,u,u,u,u]
1022 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm8[2],xmm6[3],xmm8[4,5],xmm6[6,7]
1023 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} xmm2 = xmm6[0,1,2],xmm2[3]
1024 ; AVX512F-FAST-NEXT: vpbroadcastw 74(%rdi), %xmm6
1025 ; AVX512F-FAST-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3]
1026 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[2,3,14,15,u,u,6,7,u,u,u,u,u,u,u,u]
1027 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,2,3,u,u,10,11,14,15,u,u,u,u]
1028 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1],xmm7[2],xmm5[3],xmm7[4,5],xmm5[6,7]
1029 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1,2],xmm6[3]
1030 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm6 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]
1031 ; AVX512F-FAST-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[2,1,2,3]
1032 ; AVX512F-FAST-NEXT: vpshuflw {{.*#+}} xmm8 = xmm7[2,1,2,0,4,5,6,7]
1033 ; AVX512F-FAST-NEXT: vextracti128 $1, %ymm6, %xmm6
1034 ; AVX512F-FAST-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[0,3,2,1]
1035 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm9 = xmm6[u,u,0,1,4,5,u,u,12,13,u,u,u,u,u,u]
1036 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0],xmm9[1,2],xmm8[3],xmm9[4,5,6,7]
1037 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} xmm9 = xmm1[0,1],xmm0[2],xmm1[3]
1038 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm10 = xmm9[u,u,u,u,u,u,u,u,u,u,0,1,12,13,8,9]
1039 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2,3,4],xmm10[5,6,7]
1040 ; AVX512F-FAST-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[3,1,2,1,4,5,6,7]
1041 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[u,u,2,3,6,7,u,u,14,15,u,u,u,u,u,u]
1042 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0],xmm6[1,2],xmm7[3],xmm6[4,5,6,7]
1043 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm7 = xmm9[u,u,u,u,u,u,u,u,u,u,2,3,14,15,10,11]
1044 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm7[5,6,7]
1045 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1],ymm3[2],ymm4[3,4],ymm3[5],ymm4[6,7]
1046 ; AVX512F-FAST-NEXT: vextracti128 $1, %ymm3, %xmm4
1047 ; AVX512F-FAST-NEXT: vpshuflw {{.*#+}} xmm7 = xmm4[2,2,2,2,4,5,6,7]
1048 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm9 = xmm3[8,9,u,u,0,1,12,13,u,u,u,u,u,u,u,u]
1049 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm7 = xmm9[0],xmm7[1],xmm9[2,3],xmm7[4],xmm9[5,6,7]
1050 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
1051 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[u,u,u,u,u,u,u,u,u,u,4,5,0,1,12,13]
1052 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm7[0,1,2,3,4],xmm1[5,6,7]
1053 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[u,u,6,7,u,u,u,u,10,11,u,u,u,u,u,u]
1054 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[10,11,u,u,2,3,14,15,u,u,u,u,u,u,u,u]
1055 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm4[1],xmm3[2,3],xmm4[4],xmm3[5,6,7]
1056 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,u,u,u,6,7,2,3,14,15]
1057 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3,4],xmm0[5,6,7]
1058 ; AVX512F-FAST-NEXT: vmovdqa %xmm2, (%rsi)
1059 ; AVX512F-FAST-NEXT: vmovdqa %xmm5, (%rdx)
1060 ; AVX512F-FAST-NEXT: vmovdqa %xmm8, (%rcx)
1061 ; AVX512F-FAST-NEXT: vmovdqa %xmm6, (%r8)
1062 ; AVX512F-FAST-NEXT: vmovdqa %xmm1, (%r9)
1063 ; AVX512F-FAST-NEXT: vmovdqa %xmm0, (%rax)
1064 ; AVX512F-FAST-NEXT: vzeroupper
1065 ; AVX512F-FAST-NEXT: retq
1066 ;
1067 ; AVX512BW-LABEL: load_i16_stride6_vf8:
1068 ; AVX512BW: # %bb.0:
1069 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
1070 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
1071 ; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm1
1072 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm2 = [0,6,12,18,24,30,36,42]
1073 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
1074 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm3 = [1,7,13,19,25,31,37,43]
1075 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm3
1076 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm4 = [2,8,14,20,26,32,38,44]
1077 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm4
1078 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm5 = [3,9,15,21,27,33,39,45]
1079 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm5
1080 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm6 = [4,10,16,22,28,34,40,46]
1081 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm6
1082 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm7 = [5,11,17,23,29,35,41,47]
1083 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm7
1084 ; AVX512BW-NEXT: vmovdqa %xmm2, (%rsi)
1085 ; AVX512BW-NEXT: vmovdqa %xmm3, (%rdx)
1086 ; AVX512BW-NEXT: vmovdqa %xmm4, (%rcx)
1087 ; AVX512BW-NEXT: vmovdqa %xmm5, (%r8)
1088 ; AVX512BW-NEXT: vmovdqa %xmm6, (%r9)
1089 ; AVX512BW-NEXT: vmovdqa %xmm7, (%rax)
1090 ; AVX512BW-NEXT: vzeroupper
1091 ; AVX512BW-NEXT: retq
1092 %wide.vec = load <48 x i16>, ptr %in.vec, align 64
1093 %strided.vec0 = shufflevector <48 x i16> %wide.vec, <48 x i16> poison, <8 x i32> <i32 0, i32 6, i32 12, i32 18, i32 24, i32 30, i32 36, i32 42>
1094 %strided.vec1 = shufflevector <48 x i16> %wide.vec, <48 x i16> poison, <8 x i32> <i32 1, i32 7, i32 13, i32 19, i32 25, i32 31, i32 37, i32 43>
1095 %strided.vec2 = shufflevector <48 x i16> %wide.vec, <48 x i16> poison, <8 x i32> <i32 2, i32 8, i32 14, i32 20, i32 26, i32 32, i32 38, i32 44>
1096 %strided.vec3 = shufflevector <48 x i16> %wide.vec, <48 x i16> poison, <8 x i32> <i32 3, i32 9, i32 15, i32 21, i32 27, i32 33, i32 39, i32 45>
1097 %strided.vec4 = shufflevector <48 x i16> %wide.vec, <48 x i16> poison, <8 x i32> <i32 4, i32 10, i32 16, i32 22, i32 28, i32 34, i32 40, i32 46>
1098 %strided.vec5 = shufflevector <48 x i16> %wide.vec, <48 x i16> poison, <8 x i32> <i32 5, i32 11, i32 17, i32 23, i32 29, i32 35, i32 41, i32 47>
1099 store <8 x i16> %strided.vec0, ptr %out.vec0, align 64
1100 store <8 x i16> %strided.vec1, ptr %out.vec1, align 64
1101 store <8 x i16> %strided.vec2, ptr %out.vec2, align 64
1102 store <8 x i16> %strided.vec3, ptr %out.vec3, align 64
1103 store <8 x i16> %strided.vec4, ptr %out.vec4, align 64
1104 store <8 x i16> %strided.vec5, ptr %out.vec5, align 64
1105 ret void
1106 }
1108 define void @load_i16_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4, ptr %out.vec5) nounwind {
1109 ; SSE-LABEL: load_i16_stride6_vf16:
1110 ; SSE: # %bb.0:
1111 ; SSE-NEXT: subq $104, %rsp
1112 ; SSE-NEXT: movdqa 112(%rdi), %xmm8
1113 ; SSE-NEXT: movdqa 128(%rdi), %xmm12
1114 ; SSE-NEXT: movdqa 64(%rdi), %xmm2
1115 ; SSE-NEXT: movdqa 80(%rdi), %xmm10
1116 ; SSE-NEXT: movdqa (%rdi), %xmm3
1117 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1118 ; SSE-NEXT: movdqa 16(%rdi), %xmm5
1119 ; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1120 ; SSE-NEXT: movdqa 32(%rdi), %xmm0
1121 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1122 ; SSE-NEXT: movdqa 48(%rdi), %xmm11
1123 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,2,4,5,6,7]
1124 ; SSE-NEXT: movdqa {{.*#+}} xmm13 = [65535,65535,65535,0,0,0,65535,65535]
1125 ; SSE-NEXT: movdqa %xmm13, %xmm1
1126 ; SSE-NEXT: pandn %xmm0, %xmm1
1127 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,1,0,3]
1128 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1129 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
1130 ; SSE-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm5[2],xmm0[3],xmm5[3]
1131 ; SSE-NEXT: pand %xmm13, %xmm0
1132 ; SSE-NEXT: por %xmm1, %xmm0
1133 ; SSE-NEXT: movdqa %xmm2, %xmm5
1134 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]
1135 ; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm10[0],xmm1[1],xmm10[1],xmm1[2],xmm10[2],xmm1[3],xmm10[3]
1136 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1137 ; SSE-NEXT: movdqa %xmm10, %xmm1
1138 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[3,0]
1139 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1140 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1141 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,0],xmm10[0,0]
1142 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm10[2,3]
1143 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1144 ; SSE-NEXT: pslld $16, %xmm10
1145 ; SSE-NEXT: psrldq {{.*#+}} xmm5 = xmm5[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
1146 ; SSE-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm10[0],xmm5[1],xmm10[1],xmm5[2],xmm10[2],xmm5[3],xmm10[3]
1147 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm11[0,3,2,3]
1148 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm7[0,1,0,2,4,5,6,7]
1149 ; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[3,1],xmm1[1,3]
1150 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm5[2,0]
1151 ; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1152 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm12[0,1,1,2,4,5,6,7]
1153 ; SSE-NEXT: movdqa %xmm13, %xmm2
1154 ; SSE-NEXT: pandn %xmm1, %xmm2
1155 ; SSE-NEXT: movdqa 96(%rdi), %xmm1
1156 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1157 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm1[0,1,0,3]
1158 ; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm6[0,1,2,3,4,6,6,7]
1159 ; SSE-NEXT: movdqa %xmm8, %xmm3
1160 ; SSE-NEXT: punpckhdq {{.*#+}} xmm10 = xmm10[2],xmm8[2],xmm10[3],xmm8[3]
1161 ; SSE-NEXT: pand %xmm13, %xmm10
1162 ; SSE-NEXT: por %xmm2, %xmm10
1163 ; SSE-NEXT: movdqa 160(%rdi), %xmm4
1164 ; SSE-NEXT: movdqa 176(%rdi), %xmm1
1165 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm4[2,2,3,3]
1166 ; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
1167 ; SSE-NEXT: movdqa %xmm1, %xmm0
1168 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm4[3,0]
1169 ; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1170 ; SSE-NEXT: movdqa %xmm4, %xmm15
1171 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1172 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[1,0],xmm1[0,0]
1173 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm1[2,3]
1174 ; SSE-NEXT: pslld $16, %xmm1
1175 ; SSE-NEXT: psrldq {{.*#+}} xmm15 = xmm15[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
1176 ; SSE-NEXT: punpcklwd {{.*#+}} xmm15 = xmm15[0],xmm1[0],xmm15[1],xmm1[1],xmm15[2],xmm1[2],xmm15[3],xmm1[3]
1177 ; SSE-NEXT: movdqa 144(%rdi), %xmm8
1178 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm8[0,3,2,3]
1179 ; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm1[0,1,0,2,4,5,6,7]
1180 ; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[3,1],xmm9[1,3]
1181 ; SSE-NEXT: shufps {{.*#+}} xmm10 = xmm10[0,1],xmm15[2,0]
1182 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
1183 ; SSE-NEXT: movdqa %xmm5, %xmm15
1184 ; SSE-NEXT: psrld $16, %xmm15
1185 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload
1186 ; SSE-NEXT: # xmm9 = mem[0,1,2,3,5,7,6,7]
1187 ; SSE-NEXT: punpckhdq {{.*#+}} xmm9 = xmm9[2],xmm15[2],xmm9[3],xmm15[3]
1188 ; SSE-NEXT: movdqa %xmm13, %xmm15
1189 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
1190 ; SSE-NEXT: pandn %xmm14, %xmm15
1191 ; SSE-NEXT: pand %xmm13, %xmm9
1192 ; SSE-NEXT: por %xmm15, %xmm9
1193 ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,1,3,4,5,6,7]
1194 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
1195 ; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[3,1],xmm7[1,3]
1196 ; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[0,1],xmm15[2,0]
1197 ; SSE-NEXT: movdqa %xmm3, %xmm7
1198 ; SSE-NEXT: psrld $16, %xmm7
1199 ; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,5,7,6,7]
1200 ; SSE-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm7[2],xmm6[3],xmm7[3]
1201 ; SSE-NEXT: pand %xmm13, %xmm6
1202 ; SSE-NEXT: movdqa %xmm12, %xmm15
1203 ; SSE-NEXT: pandn %xmm12, %xmm13
1204 ; SSE-NEXT: por %xmm6, %xmm13
1205 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]
1206 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,1],xmm1[1,3]
1207 ; SSE-NEXT: shufps {{.*#+}} xmm13 = xmm13[0,1],xmm2[2,0]
1208 ; SSE-NEXT: movaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1209 ; SSE-NEXT: movdqa %xmm14, %xmm1
1210 ; SSE-NEXT: psrldq {{.*#+}} xmm1 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
1211 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm11[1,1,1,1]
1212 ; SSE-NEXT: movdqa %xmm11, %xmm13
1213 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
1214 ; SSE-NEXT: movdqa {{.*#+}} xmm0 = [65535,65535,65535,0,0,65535,65535,65535]
1215 ; SSE-NEXT: movdqa %xmm0, %xmm6
1216 ; SSE-NEXT: pandn %xmm1, %xmm6
1217 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
1218 ; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[1,0],xmm5[0,0]
1219 ; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[2,0],xmm5[2,3]
1220 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm7[0,2,2,3,4,5,6,7]
1221 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3]
1222 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,0,2,3,4,5,6,7]
1223 ; SSE-NEXT: pand %xmm0, %xmm1
1224 ; SSE-NEXT: por %xmm6, %xmm1
1225 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
1226 ; SSE-NEXT: shufps $132, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
1227 ; SSE-NEXT: # xmm2 = xmm2[0,1],mem[0,2]
1228 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1229 ; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm2[0,1,2,3,4,6,6,7]
1230 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,1,2,0]
1231 ; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,6,5,4]
1232 ; SSE-NEXT: movdqa {{.*#+}} xmm12 = [65535,65535,65535,65535,65535,0,0,0]
1233 ; SSE-NEXT: movdqa %xmm12, %xmm5
1234 ; SSE-NEXT: pandn %xmm6, %xmm5
1235 ; SSE-NEXT: pand %xmm12, %xmm1
1236 ; SSE-NEXT: por %xmm1, %xmm5
1237 ; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1238 ; SSE-NEXT: movdqa %xmm15, %xmm1
1239 ; SSE-NEXT: movdqa %xmm15, %xmm2
1240 ; SSE-NEXT: movdqa %xmm15, (%rsp) # 16-byte Spill
1241 ; SSE-NEXT: psrldq {{.*#+}} xmm1 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
1242 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm8[1,1,1,1]
1243 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm6[0]
1244 ; SSE-NEXT: movdqa %xmm0, %xmm6
1245 ; SSE-NEXT: pandn %xmm1, %xmm6
1246 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
1247 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1248 ; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[1,0],xmm3[0,0]
1249 ; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[2,0],xmm3[2,3]
1250 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm15[0,2,2,3,4,5,6,7]
1251 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3]
1252 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,0,2,3,4,5,6,7]
1253 ; SSE-NEXT: pand %xmm0, %xmm1
1254 ; SSE-NEXT: por %xmm6, %xmm1
1255 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
1256 ; SSE-NEXT: shufps $132, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
1257 ; SSE-NEXT: # xmm3 = xmm3[0,1],mem[0,2]
1258 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm3[0,1,2,3,4,6,6,7]
1259 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,1,2,0]
1260 ; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm5[0,1,2,3,4,6,5,4]
1261 ; SSE-NEXT: movdqa %xmm12, %xmm11
1262 ; SSE-NEXT: pandn %xmm6, %xmm11
1263 ; SSE-NEXT: pand %xmm12, %xmm1
1264 ; SSE-NEXT: por %xmm1, %xmm11
1265 ; SSE-NEXT: movdqa %xmm13, %xmm1
1266 ; SSE-NEXT: psrlq $48, %xmm1
1267 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm14[2,2,3,3]
1268 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm1[0]
1269 ; SSE-NEXT: movdqa %xmm0, %xmm1
1270 ; SSE-NEXT: pandn %xmm6, %xmm1
1271 ; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm7[3,1,2,3,4,5,6,7]
1272 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,3,2,3]
1273 ; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,1,3,3,4,5,6,7]
1274 ; SSE-NEXT: pand %xmm0, %xmm6
1275 ; SSE-NEXT: por %xmm1, %xmm6
1276 ; SSE-NEXT: pshufhw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
1277 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,7,5,6,7]
1278 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,2]
1279 ; SSE-NEXT: movdqa %xmm12, %xmm7
1280 ; SSE-NEXT: pandn %xmm1, %xmm7
1281 ; SSE-NEXT: pand %xmm12, %xmm6
1282 ; SSE-NEXT: por %xmm6, %xmm7
1283 ; SSE-NEXT: movdqa %xmm8, %xmm1
1284 ; SSE-NEXT: psrlq $48, %xmm1
1285 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm2[2,2,3,3]
1286 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm1[0]
1287 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm15[3,1,2,3,4,5,6,7]
1288 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3]
1289 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,3,3,4,5,6,7]
1290 ; SSE-NEXT: pand %xmm0, %xmm1
1291 ; SSE-NEXT: pandn %xmm6, %xmm0
1292 ; SSE-NEXT: por %xmm1, %xmm0
1293 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm3[0,1,2,3,7,5,6,7]
1294 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,2]
1295 ; SSE-NEXT: movdqa %xmm12, %xmm3
1296 ; SSE-NEXT: pandn %xmm1, %xmm3
1297 ; SSE-NEXT: pand %xmm12, %xmm0
1298 ; SSE-NEXT: por %xmm0, %xmm3
1299 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
1300 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,1,1]
1301 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
1302 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm5[2,3,2,3]
1303 ; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
1304 ; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm14[0,1,0,3]
1305 ; SSE-NEXT: pshufhw {{.*#+}} xmm15 = xmm14[0,1,2,3,4,5,4,6]
1306 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm15 = xmm15[1],xmm13[1]
1307 ; SSE-NEXT: movss {{.*#+}} xmm15 = xmm1[0],xmm15[1,2,3]
1308 ; SSE-NEXT: pshuflw $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1309 ; SSE-NEXT: # xmm0 = mem[0,2,2,3,4,5,6,7]
1310 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
1311 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,4,6]
1312 ; SSE-NEXT: movdqa %xmm12, %xmm2
1313 ; SSE-NEXT: pandn %xmm0, %xmm2
1314 ; SSE-NEXT: andps %xmm12, %xmm15
1315 ; SSE-NEXT: por %xmm15, %xmm2
1316 ; SSE-NEXT: pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1317 ; SSE-NEXT: # xmm0 = mem[1,1,1,1]
1318 ; SSE-NEXT: pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
1319 ; SSE-NEXT: # xmm15 = mem[2,3,2,3]
1320 ; SSE-NEXT: punpcklwd {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
1321 ; SSE-NEXT: pshufd $196, (%rsp), %xmm0 # 16-byte Folded Reload
1322 ; SSE-NEXT: # xmm0 = mem[0,1,0,3]
1323 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1324 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,4,6]
1325 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm8[1]
1326 ; SSE-NEXT: movss {{.*#+}} xmm0 = xmm15[0],xmm0[1,2,3]
1327 ; SSE-NEXT: pshuflw {{.*#+}} xmm15 = xmm4[0,2,2,3,4,5,6,7]
1328 ; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm15[0,1,0,3]
1329 ; SSE-NEXT: pshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,5,4,6]
1330 ; SSE-NEXT: movdqa %xmm12, %xmm1
1331 ; SSE-NEXT: pandn %xmm15, %xmm1
1332 ; SSE-NEXT: andps %xmm12, %xmm0
1333 ; SSE-NEXT: por %xmm0, %xmm1
1334 ; SSE-NEXT: psrlq $48, %xmm6
1335 ; SSE-NEXT: psrldq {{.*#+}} xmm5 = xmm5[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1336 ; SSE-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
1337 ; SSE-NEXT: psrld $16, %xmm13
1338 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm14[0,1,2,3,4,5,5,7]
1339 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm13[1]
1340 ; SSE-NEXT: movss {{.*#+}} xmm0 = xmm5[0],xmm0[1,2,3]
1341 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
1342 ; SSE-NEXT: # xmm6 = mem[3,1,2,3,4,5,6,7]
1343 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,1,0,3]
1344 ; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,4,5,7]
1345 ; SSE-NEXT: movdqa %xmm12, %xmm15
1346 ; SSE-NEXT: pandn %xmm6, %xmm15
1347 ; SSE-NEXT: andps %xmm12, %xmm0
1348 ; SSE-NEXT: por %xmm0, %xmm15
1349 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1350 ; SSE-NEXT: psrlq $48, %xmm0
1351 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
1352 ; SSE-NEXT: psrldq {{.*#+}} xmm5 = xmm5[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1353 ; SSE-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
1354 ; SSE-NEXT: psrld $16, %xmm8
1355 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1356 ; SSE-NEXT: # xmm0 = mem[0,1,2,3,4,5,5,7]
1357 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm8[1]
1358 ; SSE-NEXT: movss {{.*#+}} xmm0 = xmm5[0],xmm0[1,2,3]
1359 ; SSE-NEXT: andps %xmm12, %xmm0
1360 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7]
1361 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,3]
1362 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,4,5,7]
1363 ; SSE-NEXT: pandn %xmm4, %xmm12
1364 ; SSE-NEXT: por %xmm0, %xmm12
1365 ; SSE-NEXT: movaps %xmm10, 16(%rsi)
1366 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1367 ; SSE-NEXT: movaps %xmm0, (%rsi)
1368 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1369 ; SSE-NEXT: movaps %xmm0, 16(%rdx)
1370 ; SSE-NEXT: movaps %xmm9, (%rdx)
1371 ; SSE-NEXT: movdqa %xmm11, 16(%rcx)
1372 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1373 ; SSE-NEXT: movaps %xmm0, (%rcx)
1374 ; SSE-NEXT: movdqa %xmm3, 16(%r8)
1375 ; SSE-NEXT: movdqa %xmm7, (%r8)
1376 ; SSE-NEXT: movdqa %xmm1, 16(%r9)
1377 ; SSE-NEXT: movdqa %xmm2, (%r9)
1378 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
1379 ; SSE-NEXT: movdqa %xmm12, 16(%rax)
1380 ; SSE-NEXT: movdqa %xmm15, (%rax)
1381 ; SSE-NEXT: addq $104, %rsp
1382 ; SSE-NEXT: retq
1383 ;
1384 ; AVX1-ONLY-LABEL: load_i16_stride6_vf16:
1385 ; AVX1-ONLY: # %bb.0:
1386 ; AVX1-ONLY-NEXT: subq $104, %rsp
1387 ; AVX1-ONLY-NEXT: vmovdqa 96(%rdi), %xmm0
1388 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1389 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm13 = xmm0[0,1,0,3]
1390 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm0 = xmm13[0,1,2,3,4,6,6,7]
1391 ; AVX1-ONLY-NEXT: vmovdqa 112(%rdi), %xmm1
1392 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1393 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, %xmm7
1394 ; AVX1-ONLY-NEXT: vmovdqa 80(%rdi), %xmm1
1395 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1396 ; AVX1-ONLY-NEXT: vpslld $16, %xmm1, %xmm2
1397 ; AVX1-ONLY-NEXT: vmovdqa 64(%rdi), %xmm4
1398 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm3 = xmm4[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
1399 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
1400 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm5
1401 ; AVX1-ONLY-NEXT: vmovdqa (%rdi), %xmm1
1402 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1403 ; AVX1-ONLY-NEXT: vmovdqa 16(%rdi), %xmm6
1404 ; AVX1-ONLY-NEXT: vmovdqa 32(%rdi), %xmm8
1405 ; AVX1-ONLY-NEXT: vmovdqa 48(%rdi), %xmm2
1406 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1407 ; AVX1-ONLY-NEXT: vpsrlq $16, %xmm8, %xmm0
1408 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm15 = xmm2[0,3,2,3]
1409 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm10 = xmm15[0,1,0,2,4,5,6,7]
1410 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm0[0],xmm10[1],xmm0[1]
1411 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[0,1,0,3]
1412 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm11 = xmm2[0,1,2,3,4,6,6,7]
1413 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm11 = xmm11[2],xmm6[2],xmm11[3],xmm6[3]
1414 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0,1,2],xmm10[3,4,5],xmm11[6,7]
1415 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm3 = ymm10[0,1,2],ymm5[3,4,5,6,7]
1416 ; AVX1-ONLY-NEXT: vmovdqa 176(%rdi), %xmm10
1417 ; AVX1-ONLY-NEXT: vpslld $16, %xmm10, %xmm5
1418 ; AVX1-ONLY-NEXT: vmovdqa 160(%rdi), %xmm11
1419 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm12 = xmm11[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
1420 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm12[0],xmm5[0],xmm12[1],xmm5[1],xmm12[2],xmm5[2],xmm12[3],xmm5[3]
1421 ; AVX1-ONLY-NEXT: vmovdqa 128(%rdi), %xmm12
1422 ; AVX1-ONLY-NEXT: vpsrlq $16, %xmm12, %xmm0
1423 ; AVX1-ONLY-NEXT: vmovdqa 144(%rdi), %xmm1
1424 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1425 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,3,2,3]
1426 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm9 = xmm1[0,1,0,2,4,5,6,7]
1427 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm9[0],xmm0[0],xmm9[1],xmm0[1]
1428 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm14[6,7]
1429 ; AVX1-ONLY-NEXT: vmovaps {{.*#+}} ymm14 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0]
1430 ; AVX1-ONLY-NEXT: vandps %ymm3, %ymm14, %ymm3
1431 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
1432 ; AVX1-ONLY-NEXT: vandnps %ymm0, %ymm14, %ymm0
1433 ; AVX1-ONLY-NEXT: vorps %ymm0, %ymm3, %ymm0
1434 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
1435 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm0 = xmm13[0,1,2,3,5,7,6,7]
1436 ; AVX1-ONLY-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1437 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm7, %xmm3
1438 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm3[2],xmm0[3],xmm3[3]
1439 ; AVX1-ONLY-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1440 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm3 = xmm4[2,2,3,3]
1441 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
1442 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm13[0],xmm3[1],xmm13[1],xmm3[2],xmm13[2],xmm3[3],xmm13[3]
1443 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm3, %ymm0
1444 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm3 = xmm15[0,1,1,3,4,5,6,7]
1445 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm9 = xmm8[1,1,1,1]
1446 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1]
1447 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,7,6,7]
1448 ; AVX1-ONLY-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1449 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm6, %xmm9
1450 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm2[2],xmm9[2],xmm2[3],xmm9[3]
1451 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[3,4,5],xmm2[6,7]
1452 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
1453 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]
1454 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm12[1,1,1,1]
1455 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
1456 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm11[2,2,3,3]
1457 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3]
1458 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm2[6,7]
1459 ; AVX1-ONLY-NEXT: vandps %ymm0, %ymm14, %ymm0
1460 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
1461 ; AVX1-ONLY-NEXT: vandnps %ymm1, %ymm14, %ymm1
1462 ; AVX1-ONLY-NEXT: vorps %ymm1, %ymm0, %ymm0
1463 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
1464 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
1465 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
1466 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm1 = xmm8[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
1467 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
1468 ; AVX1-ONLY-NEXT: vmovdqa {{.*#+}} xmm1 = [4,5,0,1,12,13,14,15,8,9,10,11,12,13,14,15]
1469 ; AVX1-ONLY-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm5 # 16-byte Folded Reload
1470 ; AVX1-ONLY-NEXT: # xmm5 = xmm6[0,1],mem[2,3],xmm6[4,5,6,7]
1471 ; AVX1-ONLY-NEXT: vpshufb %xmm1, %xmm5, %xmm3
1472 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[3,4],xmm3[5,6,7]
1473 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
1474 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm3 = xmm7[0,1],xmm6[2,3],xmm7[4,5,6,7]
1475 ; AVX1-ONLY-NEXT: vpshufb %xmm1, %xmm3, %xmm1
1476 ; AVX1-ONLY-NEXT: vmovdqa {{.*#+}} xmm9 = [0,1,2,3,4,5,6,7,8,9,0,1,12,13,8,9]
1477 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm15 = xmm4[0,1,2,3],xmm13[4,5],xmm4[6,7]
1478 ; AVX1-ONLY-NEXT: vpshufb %xmm9, %xmm15, %xmm13
1479 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm1, %ymm13, %ymm1
1480 ; AVX1-ONLY-NEXT: vmovaps {{.*#+}} ymm13 = [0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
1481 ; AVX1-ONLY-NEXT: vandnps %ymm0, %ymm13, %ymm0
1482 ; AVX1-ONLY-NEXT: vandps %ymm1, %ymm13, %ymm1
1483 ; AVX1-ONLY-NEXT: vorps %ymm0, %ymm1, %ymm0
1484 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
1485 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm1 = xmm4[1,1,1,1]
1486 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm7 = xmm12[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
1487 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm7[0],xmm1[0]
1488 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm7 = xmm11[0,1,2,3],xmm10[4,5],xmm11[6,7]
1489 ; AVX1-ONLY-NEXT: vpshufb %xmm9, %xmm7, %xmm9
1490 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm9[0,1,2],xmm1[3,4],xmm9[5,6,7]
1491 ; AVX1-ONLY-NEXT: vandps %ymm0, %ymm14, %ymm0
1492 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
1493 ; AVX1-ONLY-NEXT: vandnps %ymm1, %ymm14, %ymm1
1494 ; AVX1-ONLY-NEXT: vorps %ymm1, %ymm0, %ymm0
1495 ; AVX1-ONLY-NEXT: vmovups %ymm0, (%rsp) # 32-byte Spill
1496 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, %xmm0
1497 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm2, %xmm1
1498 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm9 = xmm8[2,2,3,3]
1499 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm9[0],xmm1[0]
1500 ; AVX1-ONLY-NEXT: vmovdqa {{.*#+}} xmm9 = [6,7,2,3,14,15,14,15,8,9,10,11,12,13,14,15]
1501 ; AVX1-ONLY-NEXT: vpshufb %xmm9, %xmm5, %xmm2
1502 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3,4],xmm2[5,6,7]
1503 ; AVX1-ONLY-NEXT: vpshufb %xmm9, %xmm3, %xmm2
1504 ; AVX1-ONLY-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,2,3,4,5,6,7,14,15,2,3,14,15,10,11]
1505 ; AVX1-ONLY-NEXT: vpshufb %xmm3, %xmm15, %xmm9
1506 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm2, %ymm9, %ymm2
1507 ; AVX1-ONLY-NEXT: vandnps %ymm1, %ymm13, %ymm1
1508 ; AVX1-ONLY-NEXT: vandps %ymm2, %ymm13, %ymm2
1509 ; AVX1-ONLY-NEXT: vorps %ymm1, %ymm2, %ymm1
1510 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm4, %xmm2
1511 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm9 = xmm12[2,2,3,3]
1512 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm9[0],xmm2[0]
1513 ; AVX1-ONLY-NEXT: vpshufb %xmm3, %xmm7, %xmm3
1514 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3,4],xmm3[5,6,7]
1515 ; AVX1-ONLY-NEXT: vandps %ymm1, %ymm14, %ymm1
1516 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2
1517 ; AVX1-ONLY-NEXT: vandnps %ymm2, %ymm14, %ymm2
1518 ; AVX1-ONLY-NEXT: vorps %ymm2, %ymm1, %ymm14
1519 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
1520 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm1 = xmm5[1,1,1,1]
1521 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm6[2,3,2,3]
1522 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
1523 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
1524 ; AVX1-ONLY-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload
1525 ; AVX1-ONLY-NEXT: # xmm2 = xmm2[0,1],mem[2,3],xmm2[4,5,6,7]
1526 ; AVX1-ONLY-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,4,5,4,5,6,7,0,1,4,5,0,1,12,13]
1527 ; AVX1-ONLY-NEXT: vpshufb %xmm3, %xmm2, %xmm7
1528 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm1, %ymm7, %ymm1
1529 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
1530 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,1,1]
1531 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
1532 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm9 = xmm15[2,3,2,3]
1533 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]
1534 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[0,1,0,3]
1535 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm9 = xmm8[0,1,2,3,4,5,4,6]
1536 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm9 = xmm9[1],xmm0[1]
1537 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1],xmm9[2,3,4,5,6,7]
1538 ; AVX1-ONLY-NEXT: vmovaps {{.*#+}} ymm9 = [65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
1539 ; AVX1-ONLY-NEXT: vandnps %ymm1, %ymm9, %ymm1
1540 ; AVX1-ONLY-NEXT: vandps %ymm7, %ymm9, %ymm7
1541 ; AVX1-ONLY-NEXT: vorps %ymm1, %ymm7, %ymm1
1542 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm7 = xmm10[0,1],xmm11[2,3],xmm10[4,5,6,7]
1543 ; AVX1-ONLY-NEXT: vpshufb %xmm3, %xmm7, %xmm3
1544 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm10 = xmm12[0,1,0,3]
1545 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm11 = xmm10[0,1,2,3,4,5,4,6]
1546 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
1547 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm11 = xmm11[1],xmm12[1]
1548 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm3 = xmm11[0,1,2,3,4],xmm3[5,6,7]
1549 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3
1550 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
1551 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm5, %xmm3
1552 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm11 = xmm6[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1553 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm11[0],xmm3[0],xmm11[1],xmm3[1],xmm11[2],xmm3[2],xmm11[3],xmm3[3]
1554 ; AVX1-ONLY-NEXT: vmovdqa {{.*#+}} xmm11 = [6,7,2,3,4,5,6,7,6,7,6,7,2,3,14,15]
1555 ; AVX1-ONLY-NEXT: vpshufb %xmm11, %xmm2, %xmm2
1556 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2
1557 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm4, %xmm3
1558 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm4 = xmm15[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1559 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
1560 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm0, %xmm4
1561 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm6 = xmm8[0,1,2,3,4,5,5,7]
1562 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm4 = xmm6[1],xmm4[1]
1563 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3,4,5,6,7]
1564 ; AVX1-ONLY-NEXT: vandnps %ymm2, %ymm9, %ymm2
1565 ; AVX1-ONLY-NEXT: vandps %ymm3, %ymm9, %ymm3
1566 ; AVX1-ONLY-NEXT: vorps %ymm2, %ymm3, %ymm2
1567 ; AVX1-ONLY-NEXT: vpshufb %xmm11, %xmm7, %xmm3
1568 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm12, %xmm4
1569 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm5 = xmm10[0,1,2,3,4,5,5,7]
1570 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm4 = xmm5[1],xmm4[1]
1571 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3,4],xmm3[5,6,7]
1572 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3
1573 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5,6,7]
1574 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
1575 ; AVX1-ONLY-NEXT: vmovaps %ymm3, (%rsi)
1576 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
1577 ; AVX1-ONLY-NEXT: vmovaps %ymm0, (%rdx)
1578 ; AVX1-ONLY-NEXT: vmovups (%rsp), %ymm0 # 32-byte Reload
1579 ; AVX1-ONLY-NEXT: vmovaps %ymm0, (%rcx)
1580 ; AVX1-ONLY-NEXT: vmovaps %ymm14, (%r8)
1581 ; AVX1-ONLY-NEXT: vmovaps %ymm1, (%r9)
1582 ; AVX1-ONLY-NEXT: movq {{[0-9]+}}(%rsp), %rax
1583 ; AVX1-ONLY-NEXT: vmovaps %ymm2, (%rax)
1584 ; AVX1-ONLY-NEXT: addq $104, %rsp
1585 ; AVX1-ONLY-NEXT: vzeroupper
1586 ; AVX1-ONLY-NEXT: retq
1587 ;
1588 ; AVX2-SLOW-LABEL: load_i16_stride6_vf16:
1589 ; AVX2-SLOW: # %bb.0:
1590 ; AVX2-SLOW-NEXT: vmovdqa (%rdi), %ymm3
1591 ; AVX2-SLOW-NEXT: vmovdqa 32(%rdi), %ymm4
1592 ; AVX2-SLOW-NEXT: vmovdqa 64(%rdi), %ymm2
1593 ; AVX2-SLOW-NEXT: vmovdqa 96(%rdi), %ymm5
1594 ; AVX2-SLOW-NEXT: vmovdqa 160(%rdi), %ymm0
1595 ; AVX2-SLOW-NEXT: vmovdqa 128(%rdi), %ymm1
1596 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm8 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
1597 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm8, %xmm9
1598 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm6 = xmm9[0,1,4,5,4,5,u,u,0,1,12,13,u,u,4,5]
1599 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm8[2,2,2,2,4,5,6,7]
1600 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[0,1,2,2]
1601 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2],xmm7[3],xmm6[4,5],xmm7[6],xmm6[7]
1602 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm10
1603 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm11 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7]
1604 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm6 = xmm11[0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15]
1605 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm11, %xmm12
1606 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm12[0,2,0,3]
1607 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,6,6,7]
1608 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm13 = xmm6[0,1],xmm7[2],xmm6[3],xmm7[4,5],xmm6[6,7]
1609 ; AVX2-SLOW-NEXT: vperm2i128 {{.*#+}} ymm6 = ymm2[2,3],ymm5[2,3]
1610 ; AVX2-SLOW-NEXT: vperm2i128 {{.*#+}} ymm7 = ymm2[0,1],ymm5[0,1]
1611 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm7[0],ymm6[1],ymm7[2,3,4,5],ymm6[6],ymm7[7]
1612 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm2 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
1613 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm13[0,1,2],ymm2[3,4,5,6,7]
1614 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm10 = ymm2[0,1,2],ymm10[3,4,5,6,7],ymm2[8,9,10],ymm10[11,12,13,14,15]
1615 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm10[4,5,6,7]
1616 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,5,5,5,5]
1617 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm9 = xmm9[6,7,2,3,4,5,u,u,2,3,14,15,u,u,6,7]
1618 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3],xmm9[4,5],xmm8[6],xmm9[7]
1619 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
1620 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm9 = xmm12[u,u,u,u,10,11,u,u,2,3,14,15,u,u,u,u]
1621 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm10 = xmm11[2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15]
1622 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm9 = xmm10[0,1],xmm9[2],xmm10[3],xmm9[4,5],xmm10[6,7]
1623 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,10,11,6,7,18,19,30,31,26,27,26,27,30,31,26,27,26,27,22,23]
1624 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0,1,2],ymm5[3,4,5,6,7]
1625 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm8 = ymm5[0,1,2],ymm8[3,4,5,6,7],ymm5[8,9,10],ymm8[11,12,13,14,15]
1626 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm8[4,5,6,7]
1627 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm8 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
1628 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm8, %xmm9
1629 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[0,1,2,1]
1630 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm10 = xmm9[0,1,2,3,6,5,6,4]
1631 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm8[2,1,0,3]
1632 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm11[0,0,0,0,4,5,6,7]
1633 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,4,6,7]
1634 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2,3],xmm10[4],xmm8[5,6],xmm10[7]
1635 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
1636 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm10 = ymm7[0,1],ymm6[2],ymm7[3],ymm6[4],ymm7[5,6],ymm6[7]
1637 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm12 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]
1638 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm12[2,1,2,3]
1639 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm14 = xmm13[2,1,2,0,4,5,6,7]
1640 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm12, %xmm12
1641 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[0,3,2,1]
1642 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm15 = xmm12[0,0,2,3,4,5,6,7]
1643 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm15 = xmm15[0,1,3,3]
1644 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm14 = xmm14[0],xmm15[1,2],xmm14[3],xmm15[4,5,6,7]
1645 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm15 = ymm10[4,5,0,1,12,13,14,15,8,9,12,13,0,1,u,u,20,21,16,17,28,29,30,31,24,25,28,29,16,17,u,u]
1646 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm8 = ymm15[0,1,2],ymm8[3,4,5,6,7],ymm15[8,9,10],ymm8[11,12,13,14,15]
1647 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,6,5,4]
1648 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm14 = xmm14[0,1,2,3,4],xmm15[5,6,7]
1649 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3],ymm8[4,5,6,7]
1650 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,7,5,6,5]
1651 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,1,1,1,4,5,6,7]
1652 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,4,5,7,7]
1653 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm9 = xmm11[0,1,2,3],xmm9[4],xmm11[5,6],xmm9[7]
1654 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm11 = xmm13[3,1,2,1,4,5,6,7]
1655 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[0,1,3,3,4,5,6,7]
1656 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,7,7,7,7]
1657 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm11 = xmm11[0],xmm12[1,2],xmm11[3],xmm12[4,5,6,7]
1658 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
1659 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm10 = ymm10[6,7,2,3,14,15,14,15,14,15,10,11,u,u,2,3,22,23,18,19,30,31,30,31,30,31,26,27,u,u,18,19]
1660 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm9 = ymm10[0,1,2],ymm9[3,4,5,6,7],ymm10[8,9,10],ymm9[11,12,13,14,15]
1661 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[0,1,3,2]
1662 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0,1,2,3,4],xmm10[5,6,7]
1663 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3],ymm9[4,5,6,7]
1664 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1],ymm3[2],ymm4[3,4],ymm3[5],ymm4[6,7]
1665 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm3, %xmm4
1666 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm10 = xmm4[2,2,2,2,4,5,6,7]
1667 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm11 = xmm3[8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15]
1668 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0],xmm10[1],xmm11[2,3],xmm10[4],xmm11[5,6,7]
1669 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0],ymm7[1],ymm6[2,3,4,5],ymm7[6],ymm6[7]
1670 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm7 = ymm6[8,9,4,5,4,5,6,7,0,1,4,5,0,1,12,13,24,25,20,21,u,u,u,u,u,u,u,u,u,u,u,u]
1671 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} xmm11 = [255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0]
1672 ; AVX2-SLOW-NEXT: vpblendvb %ymm11, %ymm10, %ymm7, %ymm7
1673 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]
1674 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm1 = xmm3[10,11,u,u,2,3,14,15,u,u,10,11,12,13,14,15]
1675 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm4[1,1,2,3]
1676 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,5,5]
1677 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm3[1],xmm1[2,3],xmm3[4],xmm1[5,6,7]
1678 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm3
1679 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm4 = ymm6[10,11,6,7,4,5,6,7,6,7,6,7,2,3,14,15,26,27,22,23,u,u,u,u,u,u,u,u,u,u,u,u]
1680 ; AVX2-SLOW-NEXT: vpblendvb %ymm11, %ymm1, %ymm4, %ymm1
1681 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm4 = xmm3[u,u,u,u,u,u,u,u,8,9,u,u,0,1,12,13]
1682 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]
1683 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm0[0,1,0,2,4,5,6,7]
1684 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,6,6,6,6]
1685 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm4 = xmm6[0,1,2,3],xmm4[4],xmm6[5],xmm4[6,7]
1686 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
1687 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm7[0,1,2,3,4],ymm4[5,6,7]
1688 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u,u,10,11,u,u,2,3,14,15]
1689 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]
1690 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,3,3]
1691 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4],xmm0[5],xmm3[6,7]
1692 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
1693 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5,6,7]
1694 ; AVX2-SLOW-NEXT: vmovdqa %ymm2, (%rsi)
1695 ; AVX2-SLOW-NEXT: vmovdqa %ymm5, (%rdx)
1696 ; AVX2-SLOW-NEXT: vmovdqa %ymm8, (%rcx)
1697 ; AVX2-SLOW-NEXT: vmovdqa %ymm9, (%r8)
1698 ; AVX2-SLOW-NEXT: vmovdqa %ymm4, (%r9)
1699 ; AVX2-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
1700 ; AVX2-SLOW-NEXT: vmovdqa %ymm0, (%rax)
1701 ; AVX2-SLOW-NEXT: vzeroupper
1702 ; AVX2-SLOW-NEXT: retq
1703 ;
1704 ; AVX2-FAST-LABEL: load_i16_stride6_vf16:
1705 ; AVX2-FAST: # %bb.0:
1706 ; AVX2-FAST-NEXT: vmovdqa (%rdi), %ymm3
1707 ; AVX2-FAST-NEXT: vmovdqa 32(%rdi), %ymm4
1708 ; AVX2-FAST-NEXT: vmovdqa 64(%rdi), %ymm2
1709 ; AVX2-FAST-NEXT: vmovdqa 96(%rdi), %ymm5
1710 ; AVX2-FAST-NEXT: vmovdqa 160(%rdi), %ymm0
1711 ; AVX2-FAST-NEXT: vmovdqa 128(%rdi), %ymm1
1712 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm8 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
1713 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm6 = xmm8[u,u,u,u,u,u,4,5,u,u,u,u,8,9,u,u]
1714 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm8, %xmm9
1715 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm7 = xmm9[0,1,4,5,4,5,u,u,0,1,12,13,u,u,4,5]
1716 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0,1,2],xmm6[3],xmm7[4,5],xmm6[6],xmm7[7]
1717 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm10
1718 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm11 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7]
1719 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm6 = xmm11[0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15]
1720 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm11, %xmm7
1721 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm12 = xmm7[2,1,0,3]
1722 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm7 = xmm12[u,u,u,u,0,1,u,u,8,9,12,13,u,u,u,u]
1723 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm13 = xmm6[0,1],xmm7[2],xmm6[3],xmm7[4,5],xmm6[6,7]
1724 ; AVX2-FAST-NEXT: vperm2i128 {{.*#+}} ymm6 = ymm2[2,3],ymm5[2,3]
1725 ; AVX2-FAST-NEXT: vperm2i128 {{.*#+}} ymm7 = ymm2[0,1],ymm5[0,1]
1726 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm5 = ymm7[0],ymm6[1],ymm7[2,3,4,5],ymm6[6],ymm7[7]
1727 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm2 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
1728 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm13[0,1,2],ymm2[3,4,5,6,7]
1729 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm10 = ymm2[0,1,2],ymm10[3,4,5,6,7],ymm2[8,9,10],ymm10[11,12,13,14,15]
1730 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm10[4,5,6,7]
1731 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,5,5,5,5]
1732 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm9 = xmm9[6,7,2,3,4,5,u,u,2,3,14,15,u,u,6,7]
1733 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3],xmm9[4,5],xmm8[6],xmm9[7]
1734 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
1735 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm9 = xmm11[2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15]
1736 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm10 = xmm12[u,u,u,u,2,3,u,u,10,11,14,15,u,u,u,u]
1737 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1],xmm10[2],xmm9[3],xmm10[4,5],xmm9[6,7]
1738 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,10,11,6,7,18,19,30,31,26,27,26,27,30,31,26,27,26,27,22,23]
1739 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0,1,2],ymm5[3,4,5,6,7]
1740 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm8 = ymm5[0,1,2],ymm8[3,4,5,6,7],ymm5[8,9,10],ymm8[11,12,13,14,15]
1741 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm8[4,5,6,7]
1742 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm8 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
1743 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm8, %xmm9
1744 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[0,1,2,1]
1745 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm10 = xmm9[0,1,2,3,6,5,6,4]
1746 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm11 = xmm8[2,1,0,3]
1747 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm8 = xmm11[0,1,0,1,0,1,0,1,u,u,8,9,12,13,u,u]
1748 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2,3],xmm10[4],xmm8[5,6],xmm10[7]
1749 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
1750 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm10 = ymm7[0,1],ymm6[2],ymm7[3],ymm6[4],ymm7[5,6],ymm6[7]
1751 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm12 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]
1752 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm13 = xmm12[2,1,2,3]
1753 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm14 = xmm13[2,1,2,0,4,5,6,7]
1754 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm12, %xmm12
1755 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[0,3,2,1]
1756 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm15 = xmm12[u,u,0,1,4,5,u,u,12,13,u,u,u,u,u,u]
1757 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm14 = xmm14[0],xmm15[1,2],xmm14[3],xmm15[4,5,6,7]
1758 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm15 = ymm10[4,5,0,1,12,13,14,15,8,9,12,13,0,1,u,u,20,21,16,17,28,29,30,31,24,25,28,29,16,17,u,u]
1759 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm8 = ymm15[0,1,2],ymm8[3,4,5,6,7],ymm15[8,9,10],ymm8[11,12,13,14,15]
1760 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,6,5,4]
1761 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm14 = xmm14[0,1,2,3,4],xmm15[5,6,7]
1762 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3],ymm8[4,5,6,7]
1763 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,7,5,6,5]
1764 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm11 = xmm11[2,3,2,3,2,3,2,3,u,u,10,11,14,15,u,u]
1765 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm9 = xmm11[0,1,2,3],xmm9[4],xmm11[5,6],xmm9[7]
1766 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm11 = xmm13[3,1,2,1,4,5,6,7]
1767 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm12 = xmm12[u,u,2,3,6,7,u,u,14,15,u,u,u,u,u,u]
1768 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm11 = xmm11[0],xmm12[1,2],xmm11[3],xmm12[4,5,6,7]
1769 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
1770 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm10 = ymm10[6,7,2,3,14,15,14,15,14,15,10,11,u,u,2,3,22,23,18,19,30,31,30,31,30,31,26,27,u,u,18,19]
1771 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm9 = ymm10[0,1,2],ymm9[3,4,5,6,7],ymm10[8,9,10],ymm9[11,12,13,14,15]
1772 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[0,1,3,2]
1773 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0,1,2,3,4],xmm10[5,6,7]
1774 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3],ymm9[4,5,6,7]
1775 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1],ymm3[2],ymm4[3,4],ymm3[5],ymm4[6,7]
1776 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm3, %xmm4
1777 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm10 = xmm4[2,2,2,2,4,5,6,7]
1778 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm11 = xmm3[8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15]
1779 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0],xmm10[1],xmm11[2,3],xmm10[4],xmm11[5,6,7]
1780 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0],ymm7[1],ymm6[2,3,4,5],ymm7[6],ymm6[7]
1781 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm7 = ymm6[8,9,4,5,4,5,6,7,0,1,4,5,0,1,12,13,24,25,20,21,u,u,u,u,u,u,u,u,u,u,u,u]
1782 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm11 = [255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0]
1783 ; AVX2-FAST-NEXT: vpblendvb %ymm11, %ymm10, %ymm7, %ymm7
1784 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]
1785 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm1 = xmm4[u,u,6,7,u,u,u,u,10,11,u,u,u,u,u,u]
1786 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[10,11,u,u,2,3,14,15,u,u,10,11,12,13,14,15]
1787 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0],xmm1[1],xmm3[2,3],xmm1[4],xmm3[5,6,7]
1788 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
1789 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm4 = ymm6[10,11,6,7,4,5,6,7,6,7,6,7,2,3,14,15,26,27,22,23,u,u,u,u,u,u,u,u,u,u,u,u]
1790 ; AVX2-FAST-NEXT: vpblendvb %ymm11, %ymm1, %ymm4, %ymm1
1791 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm4 = xmm3[u,u,u,u,u,u,u,u,8,9,u,u,0,1,12,13]
1792 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]
1793 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm6 = xmm0[0,1,2,3,0,1,4,5,u,u,12,13,u,u,u,u]
1794 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm4 = xmm6[0,1,2,3],xmm4[4],xmm6[5],xmm4[6,7]
1795 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
1796 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm4 = ymm7[0,1,2,3,4],ymm4[5,6,7]
1797 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u,u,10,11,u,u,2,3,14,15]
1798 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,2,3,6,7,u,u,14,15,u,u,u,u]
1799 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4],xmm0[5],xmm3[6,7]
1800 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
1801 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5,6,7]
1802 ; AVX2-FAST-NEXT: vmovdqa %ymm2, (%rsi)
1803 ; AVX2-FAST-NEXT: vmovdqa %ymm5, (%rdx)
1804 ; AVX2-FAST-NEXT: vmovdqa %ymm8, (%rcx)
1805 ; AVX2-FAST-NEXT: vmovdqa %ymm9, (%r8)
1806 ; AVX2-FAST-NEXT: vmovdqa %ymm4, (%r9)
1807 ; AVX2-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
1808 ; AVX2-FAST-NEXT: vmovdqa %ymm0, (%rax)
1809 ; AVX2-FAST-NEXT: vzeroupper
1810 ; AVX2-FAST-NEXT: retq
1811 ;
1812 ; AVX2-FAST-PERLANE-LABEL: load_i16_stride6_vf16:
1813 ; AVX2-FAST-PERLANE: # %bb.0:
1814 ; AVX2-FAST-PERLANE-NEXT: vmovdqa (%rdi), %ymm3
1815 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %ymm4
1816 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 64(%rdi), %ymm2
1817 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 96(%rdi), %ymm5
1818 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 160(%rdi), %ymm0
1819 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 128(%rdi), %ymm1
1820 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm8 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
1821 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm6 = xmm8[u,u,u,u,u,u,4,5,u,u,u,u,8,9,u,u]
1822 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm8, %xmm9
1823 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm7 = xmm9[0,1,4,5,4,5,u,u,0,1,12,13,u,u,4,5]
1824 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0,1,2],xmm6[3],xmm7[4,5],xmm6[6],xmm7[7]
1825 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm10
1826 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm11 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7]
1827 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm6 = xmm11[0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15]
1828 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm11, %xmm7
1829 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm12 = xmm7[2,1,0,3]
1830 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm7 = xmm12[u,u,u,u,0,1,u,u,8,9,12,13,u,u,u,u]
1831 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm13 = xmm6[0,1],xmm7[2],xmm6[3],xmm7[4,5],xmm6[6,7]
1832 ; AVX2-FAST-PERLANE-NEXT: vperm2i128 {{.*#+}} ymm6 = ymm2[2,3],ymm5[2,3]
1833 ; AVX2-FAST-PERLANE-NEXT: vperm2i128 {{.*#+}} ymm7 = ymm2[0,1],ymm5[0,1]
1834 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm5 = ymm7[0],ymm6[1],ymm7[2,3,4,5],ymm6[6],ymm7[7]
1835 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} ymm2 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
1836 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm2 = ymm13[0,1,2],ymm2[3,4,5,6,7]
1837 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm10 = ymm2[0,1,2],ymm10[3,4,5,6,7],ymm2[8,9,10],ymm10[11,12,13,14,15]
1838 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm10[4,5,6,7]
1839 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,5,5,5,5]
1840 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm9 = xmm9[6,7,2,3,4,5,u,u,2,3,14,15,u,u,6,7]
1841 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3],xmm9[4,5],xmm8[6],xmm9[7]
1842 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
1843 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm9 = xmm11[2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15]
1844 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm10 = xmm12[u,u,u,u,2,3,u,u,10,11,14,15,u,u,u,u]
1845 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1],xmm10[2],xmm9[3],xmm10[4,5],xmm9[6,7]
1846 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,10,11,6,7,18,19,30,31,26,27,26,27,30,31,26,27,26,27,22,23]
1847 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0,1,2],ymm5[3,4,5,6,7]
1848 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm8 = ymm5[0,1,2],ymm8[3,4,5,6,7],ymm5[8,9,10],ymm8[11,12,13,14,15]
1849 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm8[4,5,6,7]
1850 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm8 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
1851 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm8, %xmm9
1852 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[0,1,2,1]
1853 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm10 = xmm9[0,1,2,3,6,5,6,4]
1854 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm11 = xmm8[2,1,0,3]
1855 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm8 = xmm11[0,1,0,1,0,1,0,1,u,u,8,9,12,13,u,u]
1856 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2,3],xmm10[4],xmm8[5,6],xmm10[7]
1857 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
1858 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm10 = ymm7[0,1],ymm6[2],ymm7[3],ymm6[4],ymm7[5,6],ymm6[7]
1859 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm12 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]
1860 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm13 = xmm12[2,1,2,3]
1861 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm14 = xmm13[2,1,2,0,4,5,6,7]
1862 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm12, %xmm12
1863 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[0,3,2,1]
1864 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm15 = xmm12[u,u,0,1,4,5,u,u,12,13,u,u,u,u,u,u]
1865 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm14 = xmm14[0],xmm15[1,2],xmm14[3],xmm15[4,5,6,7]
1866 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} ymm15 = ymm10[4,5,0,1,12,13,14,15,8,9,12,13,0,1,u,u,20,21,16,17,28,29,30,31,24,25,28,29,16,17,u,u]
1867 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm8 = ymm15[0,1,2],ymm8[3,4,5,6,7],ymm15[8,9,10],ymm8[11,12,13,14,15]
1868 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,6,5,4]
1869 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm14 = xmm14[0,1,2,3,4],xmm15[5,6,7]
1870 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3],ymm8[4,5,6,7]
1871 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,7,5,6,5]
1872 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm11 = xmm11[2,3,2,3,2,3,2,3,u,u,10,11,14,15,u,u]
1873 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm9 = xmm11[0,1,2,3],xmm9[4],xmm11[5,6],xmm9[7]
1874 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm11 = xmm13[3,1,2,1,4,5,6,7]
1875 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm12 = xmm12[u,u,2,3,6,7,u,u,14,15,u,u,u,u,u,u]
1876 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm11 = xmm11[0],xmm12[1,2],xmm11[3],xmm12[4,5,6,7]
1877 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
1878 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} ymm10 = ymm10[6,7,2,3,14,15,14,15,14,15,10,11,u,u,2,3,22,23,18,19,30,31,30,31,30,31,26,27,u,u,18,19]
1879 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm9 = ymm10[0,1,2],ymm9[3,4,5,6,7],ymm10[8,9,10],ymm9[11,12,13,14,15]
1880 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[0,1,3,2]
1881 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0,1,2,3,4],xmm10[5,6,7]
1882 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3],ymm9[4,5,6,7]
1883 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1],ymm3[2],ymm4[3,4],ymm3[5],ymm4[6,7]
1884 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm3, %xmm4
1885 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm10 = xmm4[2,2,2,2,4,5,6,7]
1886 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm11 = xmm3[8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15]
1887 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0],xmm10[1],xmm11[2,3],xmm10[4],xmm11[5,6,7]
1888 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0],ymm7[1],ymm6[2,3,4,5],ymm7[6],ymm6[7]
1889 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} ymm7 = ymm6[8,9,4,5,4,5,6,7,0,1,4,5,0,1,12,13,24,25,20,21,u,u,u,u,u,u,u,u,u,u,u,u]
1890 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm11 = [255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0]
1891 ; AVX2-FAST-PERLANE-NEXT: vpblendvb %ymm11, %ymm10, %ymm7, %ymm7
1892 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]
1893 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm1 = xmm4[u,u,6,7,u,u,u,u,10,11,u,u,u,u,u,u]
1894 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[10,11,u,u,2,3,14,15,u,u,10,11,12,13,14,15]
1895 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0],xmm1[1],xmm3[2,3],xmm1[4],xmm3[5,6,7]
1896 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm0, %xmm3
1897 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} ymm4 = ymm6[10,11,6,7,4,5,6,7,6,7,6,7,2,3,14,15,26,27,22,23,u,u,u,u,u,u,u,u,u,u,u,u]
1898 ; AVX2-FAST-PERLANE-NEXT: vpblendvb %ymm11, %ymm1, %ymm4, %ymm1
1899 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm4 = xmm3[u,u,u,u,u,u,u,u,8,9,u,u,0,1,12,13]
1900 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]
1901 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm6 = xmm0[0,1,2,3,0,1,4,5,u,u,12,13,u,u,u,u]
1902 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm4 = xmm6[0,1,2,3],xmm4[4],xmm6[5],xmm4[6,7]
1903 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
1904 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm4 = ymm7[0,1,2,3,4],ymm4[5,6,7]
1905 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u,u,10,11,u,u,2,3,14,15]
1906 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,2,3,6,7,u,u,14,15,u,u,u,u]
1907 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4],xmm0[5],xmm3[6,7]
1908 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
1909 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5,6,7]
1910 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm2, (%rsi)
1911 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm5, (%rdx)
1912 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm8, (%rcx)
1913 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm9, (%r8)
1914 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm4, (%r9)
1915 ; AVX2-FAST-PERLANE-NEXT: movq {{[0-9]+}}(%rsp), %rax
1916 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm0, (%rax)
1917 ; AVX2-FAST-PERLANE-NEXT: vzeroupper
1918 ; AVX2-FAST-PERLANE-NEXT: retq
1919 ;
1920 ; AVX512F-SLOW-LABEL: load_i16_stride6_vf16:
1921 ; AVX512F-SLOW: # %bb.0:
1922 ; AVX512F-SLOW-NEXT: vmovdqa 160(%rdi), %ymm0
1923 ; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %ymm3
1924 ; AVX512F-SLOW-NEXT: vmovdqa 32(%rdi), %ymm4
1925 ; AVX512F-SLOW-NEXT: vmovdqa 64(%rdi), %ymm1
1926 ; AVX512F-SLOW-NEXT: vmovdqa 128(%rdi), %ymm2
1927 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm2[0,1],ymm0[2],ymm2[3,4],ymm0[5],ymm2[6,7]
1928 ; AVX512F-SLOW-NEXT: vextracti128 $1, %ymm5, %xmm8
1929 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm6 = xmm8[0,1,4,5,4,5,u,u,0,1,12,13,u,u,4,5]
1930 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm5[2,2,2,2,4,5,6,7]
1931 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[0,1,2,2]
1932 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2],xmm7[3],xmm6[4,5],xmm7[6],xmm6[7]
1933 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm9
1934 ; AVX512F-SLOW-NEXT: vperm2i128 {{.*#+}} ymm6 = ymm1[2,3],mem[2,3]
1935 ; AVX512F-SLOW-NEXT: vinserti128 $1, 96(%rdi), %ymm1, %ymm7
1936 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm10 = ymm7[0],ymm6[1],ymm7[2,3,4,5],ymm6[6],ymm7[7]
1937 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm1 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
1938 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm11 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7]
1939 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm12 = xmm11[0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15]
1940 ; AVX512F-SLOW-NEXT: vextracti128 $1, %ymm11, %xmm13
1941 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm14 = xmm13[0,2,0,3]
1942 ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,4,6,6,7]
1943 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1],xmm14[2],xmm12[3],xmm14[4,5],xmm12[6,7]
1944 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm12[0,1,2],ymm1[3,4,5,6,7]
1945 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm9 = ymm1[0,1,2],ymm9[3,4,5,6,7],ymm1[8,9,10],ymm9[11,12,13,14,15]
1946 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm9[4,5,6,7]
1947 ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,5,5]
1948 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[6,7,2,3,4,5,u,u,2,3,14,15,u,u,6,7]
1949 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm5 = xmm8[0,1,2],xmm5[3],xmm8[4,5],xmm5[6],xmm8[7]
1950 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
1951 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm8 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,10,11,6,7,18,19,30,31,26,27,26,27,30,31,26,27,26,27,22,23]
1952 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm9 = xmm13[u,u,u,u,10,11,u,u,2,3,14,15,u,u,u,u]
1953 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm10 = xmm11[2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15]
1954 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm9 = xmm10[0,1],xmm9[2],xmm10[3],xmm9[4,5],xmm10[6,7]
1955 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2],ymm8[3,4,5,6,7]
1956 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm5 = ymm8[0,1,2],ymm5[3,4,5,6,7],ymm8[8,9,10],ymm5[11,12,13,14,15]
1957 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3],ymm5[4,5,6,7]
1958 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm8 = ymm0[0,1],ymm2[2],ymm0[3,4],ymm2[5],ymm0[6,7]
1959 ; AVX512F-SLOW-NEXT: vextracti128 $1, %ymm8, %xmm9
1960 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[0,1,2,1]
1961 ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} xmm10 = xmm9[0,1,2,3,6,5,6,4]
1962 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm8[2,1,0,3]
1963 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm11[0,0,0,0,4,5,6,7]
1964 ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,4,6,7]
1965 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2,3],xmm10[4],xmm8[5,6],xmm10[7]
1966 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
1967 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm10 = ymm7[0,1],ymm6[2],ymm7[3],ymm6[4],ymm7[5,6],ymm6[7]
1968 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm12 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]
1969 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm12[2,1,2,3]
1970 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm14 = xmm13[2,1,2,0,4,5,6,7]
1971 ; AVX512F-SLOW-NEXT: vextracti128 $1, %ymm12, %xmm12
1972 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[0,3,2,1]
1973 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm15 = xmm12[0,0,2,3,4,5,6,7]
1974 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm15 = xmm15[0,1,3,3]
1975 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm14 = xmm14[0],xmm15[1,2],xmm14[3],xmm15[4,5,6,7]
1976 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm15 = ymm10[4,5,0,1,12,13,14,15,8,9,0,1,12,13,8,9,20,21,16,17,28,29,30,31,24,25,16,17,28,29,24,25]
1977 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm8 = ymm15[0,1,2],ymm8[3,4,5,6,7],ymm15[8,9,10],ymm8[11,12,13,14,15]
1978 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm14 = xmm14[0,1,2,3,4],xmm15[5,6,7]
1979 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3],ymm8[4,5,6,7]
1980 ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,7,5,6,5]
1981 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,1,1,1,4,5,6,7]
1982 ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,4,5,7,7]
1983 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm9 = xmm11[0,1,2,3],xmm9[4],xmm11[5,6],xmm9[7]
1984 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm11 = xmm13[3,1,2,1,4,5,6,7]
1985 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[0,1,3,3,4,5,6,7]
1986 ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,7,7,7,7]
1987 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm11 = xmm11[0],xmm12[1,2],xmm11[3],xmm12[4,5,6,7]
1988 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
1989 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm10 = ymm10[6,7,2,3,14,15,14,15,14,15,2,3,14,15,10,11,22,23,18,19,30,31,30,31,30,31,18,19,30,31,26,27]
1990 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm9 = ymm10[0,1,2],ymm9[3,4,5,6,7],ymm10[8,9,10],ymm9[11,12,13,14,15]
1991 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0,1,2,3,4],xmm10[5,6,7]
1992 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3],ymm9[4,5,6,7]
1993 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0],ymm7[1],ymm6[2,3,4,5],ymm7[6],ymm6[7]
1994 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[4,5,0,1,12,13,24,25,20,21],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1995 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1],ymm3[2],ymm4[3,4],ymm3[5],ymm4[6,7]
1996 ; AVX512F-SLOW-NEXT: vextracti128 $1, %ymm3, %xmm4
1997 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm10 = xmm4[2,2,2,2,4,5,6,7]
1998 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm11 = xmm3[8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15]
1999 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0],xmm10[1],xmm11[2,3],xmm10[4],xmm11[5,6,7]
2000 ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} ymm11 = [65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
2001 ; AVX512F-SLOW-NEXT: vpternlogq $236, %ymm11, %ymm7, %ymm10
2002 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7]
2003 ; AVX512F-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm2
2004 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm7 = xmm2[u,u,u,u,u,u,u,u,8,9,u,u,0,1,12,13]
2005 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]
2006 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm12 = xmm0[0,1,0,2,4,5,6,7]
2007 ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,6,6,6,6]
2008 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm7 = xmm12[0,1,2,3],xmm7[4],xmm12[5],xmm7[6,7]
2009 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
2010 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm7 = ymm10[0,1,2,3,4],ymm7[5,6,7]
2011 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[10,11,u,u,2,3,14,15,u,u,10,11,12,13,14,15]
2012 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[1,1,2,3]
2013 ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,5,5,5]
2014 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm4[1],xmm3[2,3],xmm4[4],xmm3[5,6,7]
2015 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[6,7,2,3,14,15,26,27,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
2016 ; AVX512F-SLOW-NEXT: vpternlogq $236, %ymm11, %ymm4, %ymm3
2017 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,u,u,10,11,u,u,2,3,14,15]
2018 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]
2019 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,3,3]
2020 ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4],xmm0[5],xmm2[6,7]
2021 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
2022 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4],ymm0[5,6,7]
2023 ; AVX512F-SLOW-NEXT: vmovdqa %ymm1, (%rsi)
2024 ; AVX512F-SLOW-NEXT: vmovdqa %ymm5, (%rdx)
2025 ; AVX512F-SLOW-NEXT: vmovdqa %ymm8, (%rcx)
2026 ; AVX512F-SLOW-NEXT: vmovdqa %ymm9, (%r8)
2027 ; AVX512F-SLOW-NEXT: vmovdqa %ymm7, (%r9)
2028 ; AVX512F-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
2029 ; AVX512F-SLOW-NEXT: vmovdqa %ymm0, (%rax)
2030 ; AVX512F-SLOW-NEXT: vzeroupper
2031 ; AVX512F-SLOW-NEXT: retq
2032 ;
2033 ; AVX512F-FAST-LABEL: load_i16_stride6_vf16:
2034 ; AVX512F-FAST: # %bb.0:
2035 ; AVX512F-FAST-NEXT: vmovdqa 160(%rdi), %ymm0
2036 ; AVX512F-FAST-NEXT: vmovdqa (%rdi), %ymm3
2037 ; AVX512F-FAST-NEXT: vmovdqa 32(%rdi), %ymm4
2038 ; AVX512F-FAST-NEXT: vmovdqa 64(%rdi), %ymm2
2039 ; AVX512F-FAST-NEXT: vmovdqa 128(%rdi), %ymm1
2040 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm5 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
2041 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm6 = xmm5[u,u,u,u,u,u,4,5,u,u,u,u,8,9,u,u]
2042 ; AVX512F-FAST-NEXT: vextracti128 $1, %ymm5, %xmm8
2043 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm7 = xmm8[0,1,4,5,4,5,u,u,0,1,12,13,u,u,4,5]
2044 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0,1,2],xmm6[3],xmm7[4,5],xmm6[6],xmm7[7]
2045 ; AVX512F-FAST-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm9
2046 ; AVX512F-FAST-NEXT: vperm2i128 {{.*#+}} ymm6 = ymm2[2,3],mem[2,3]
2047 ; AVX512F-FAST-NEXT: vinserti128 $1, 96(%rdi), %ymm2, %ymm7
2048 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm10 = ymm7[0],ymm6[1],ymm7[2,3,4,5],ymm6[6],ymm7[7]
2049 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm2 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
2050 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm11 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7]
2051 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm12 = xmm11[0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15]
2052 ; AVX512F-FAST-NEXT: vextracti128 $1, %ymm11, %xmm13
2053 ; AVX512F-FAST-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[2,1,0,3]
2054 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm14 = xmm13[u,u,u,u,0,1,u,u,8,9,12,13,u,u,u,u]
2055 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1],xmm14[2],xmm12[3],xmm14[4,5],xmm12[6,7]
2056 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm12[0,1,2],ymm2[3,4,5,6,7]
2057 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm9 = ymm2[0,1,2],ymm9[3,4,5,6,7],ymm2[8,9,10],ymm9[11,12,13,14,15]
2058 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm9[4,5,6,7]
2059 ; AVX512F-FAST-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,5,5]
2060 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[6,7,2,3,4,5,u,u,2,3,14,15,u,u,6,7]
2061 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm5 = xmm8[0,1,2],xmm5[3],xmm8[4,5],xmm5[6],xmm8[7]
2062 ; AVX512F-FAST-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
2063 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm8 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,10,11,6,7,18,19,30,31,26,27,26,27,30,31,26,27,26,27,22,23]
2064 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm9 = xmm11[2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15]
2065 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm10 = xmm13[u,u,u,u,2,3,u,u,10,11,14,15,u,u,u,u]
2066 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1],xmm10[2],xmm9[3],xmm10[4,5],xmm9[6,7]
2067 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2],ymm8[3,4,5,6,7]
2068 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm5 = ymm8[0,1,2],ymm5[3,4,5,6,7],ymm8[8,9,10],ymm5[11,12,13,14,15]
2069 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3],ymm5[4,5,6,7]
2070 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm8 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
2071 ; AVX512F-FAST-NEXT: vextracti128 $1, %ymm8, %xmm9
2072 ; AVX512F-FAST-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[0,1,2,1]
2073 ; AVX512F-FAST-NEXT: vpshufhw {{.*#+}} xmm10 = xmm9[0,1,2,3,6,5,6,4]
2074 ; AVX512F-FAST-NEXT: vpshufd {{.*#+}} xmm11 = xmm8[2,1,0,3]
2075 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm8 = xmm11[0,1,0,1,0,1,0,1,u,u,8,9,12,13,u,u]
2076 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2,3],xmm10[4],xmm8[5,6],xmm10[7]
2077 ; AVX512F-FAST-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
2078 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm10 = ymm7[0,1],ymm6[2],ymm7[3],ymm6[4],ymm7[5,6],ymm6[7]
2079 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm12 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]
2080 ; AVX512F-FAST-NEXT: vpshufd {{.*#+}} xmm13 = xmm12[2,1,2,3]
2081 ; AVX512F-FAST-NEXT: vpshuflw {{.*#+}} xmm14 = xmm13[2,1,2,0,4,5,6,7]
2082 ; AVX512F-FAST-NEXT: vextracti128 $1, %ymm12, %xmm12
2083 ; AVX512F-FAST-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[0,3,2,1]
2084 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm15 = xmm12[u,u,0,1,4,5,u,u,12,13,u,u,u,u,u,u]
2085 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm14 = xmm14[0],xmm15[1,2],xmm14[3],xmm15[4,5,6,7]
2086 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm15 = ymm10[4,5,0,1,12,13,14,15,8,9,0,1,12,13,8,9,20,21,16,17,28,29,30,31,24,25,16,17,28,29,24,25]
2087 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm8 = ymm15[0,1,2],ymm8[3,4,5,6,7],ymm15[8,9,10],ymm8[11,12,13,14,15]
2088 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm14 = xmm14[0,1,2,3,4],xmm15[5,6,7]
2089 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3],ymm8[4,5,6,7]
2090 ; AVX512F-FAST-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,7,5,6,5]
2091 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm11 = xmm11[2,3,2,3,2,3,2,3,u,u,10,11,14,15,u,u]
2092 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm9 = xmm11[0,1,2,3],xmm9[4],xmm11[5,6],xmm9[7]
2093 ; AVX512F-FAST-NEXT: vpshuflw {{.*#+}} xmm11 = xmm13[3,1,2,1,4,5,6,7]
2094 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm12 = xmm12[u,u,2,3,6,7,u,u,14,15,u,u,u,u,u,u]
2095 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm11 = xmm11[0],xmm12[1,2],xmm11[3],xmm12[4,5,6,7]
2096 ; AVX512F-FAST-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
2097 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm10 = ymm10[6,7,2,3,14,15,14,15,14,15,2,3,14,15,10,11,22,23,18,19,30,31,30,31,30,31,18,19,30,31,26,27]
2098 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm9 = ymm10[0,1,2],ymm9[3,4,5,6,7],ymm10[8,9,10],ymm9[11,12,13,14,15]
2099 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0,1,2,3,4],xmm10[5,6,7]
2100 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3],ymm9[4,5,6,7]
2101 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0],ymm7[1],ymm6[2,3,4,5],ymm7[6],ymm6[7]
2102 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[4,5,0,1,12,13,24,25,20,21],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
2103 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1],ymm3[2],ymm4[3,4],ymm3[5],ymm4[6,7]
2104 ; AVX512F-FAST-NEXT: vextracti128 $1, %ymm3, %xmm4
2105 ; AVX512F-FAST-NEXT: vpshuflw {{.*#+}} xmm10 = xmm4[2,2,2,2,4,5,6,7]
2106 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm11 = xmm3[8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15]
2107 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0],xmm10[1],xmm11[2,3],xmm10[4],xmm11[5,6,7]
2108 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm11 = [65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
2109 ; AVX512F-FAST-NEXT: vpternlogq $236, %ymm11, %ymm7, %ymm10
2110 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]
2111 ; AVX512F-FAST-NEXT: vextracti128 $1, %ymm0, %xmm1
2112 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm7 = xmm1[u,u,u,u,u,u,u,u,8,9,u,u,0,1,12,13]
2113 ; AVX512F-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]
2114 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm12 = xmm0[0,1,2,3,0,1,4,5,u,u,12,13,u,u,u,u]
2115 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm7 = xmm12[0,1,2,3],xmm7[4],xmm12[5],xmm7[6,7]
2116 ; AVX512F-FAST-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
2117 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm7 = ymm10[0,1,2,3,4],ymm7[5,6,7]
2118 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[u,u,6,7,u,u,u,u,10,11,u,u,u,u,u,u]
2119 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[10,11,u,u,2,3,14,15,u,u,10,11,12,13,14,15]
2120 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm4[1],xmm3[2,3],xmm4[4],xmm3[5,6,7]
2121 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[6,7,2,3,14,15,26,27,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
2122 ; AVX512F-FAST-NEXT: vpternlogq $236, %ymm11, %ymm4, %ymm3
2123 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u,u,u,u,10,11,u,u,2,3,14,15]
2124 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,2,3,6,7,u,u,14,15,u,u,u,u]
2125 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4],xmm0[5],xmm1[6,7]
2126 ; AVX512F-FAST-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
2127 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4],ymm0[5,6,7]
2128 ; AVX512F-FAST-NEXT: vmovdqa %ymm2, (%rsi)
2129 ; AVX512F-FAST-NEXT: vmovdqa %ymm5, (%rdx)
2130 ; AVX512F-FAST-NEXT: vmovdqa %ymm8, (%rcx)
2131 ; AVX512F-FAST-NEXT: vmovdqa %ymm9, (%r8)
2132 ; AVX512F-FAST-NEXT: vmovdqa %ymm7, (%r9)
2133 ; AVX512F-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
2134 ; AVX512F-FAST-NEXT: vmovdqa %ymm0, (%rax)
2135 ; AVX512F-FAST-NEXT: vzeroupper
2136 ; AVX512F-FAST-NEXT: retq
2137 ;
2138 ; AVX512BW-LABEL: load_i16_stride6_vf16:
2139 ; AVX512BW: # %bb.0:
2140 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
2141 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm2
2142 ; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm3
2143 ; AVX512BW-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [0,0,0,2,8,14,20,26,0,0,0,2,8,14,20,26]
2144 ; AVX512BW-NEXT: # ymm0 = mem[0,1,0,1]
2145 ; AVX512BW-NEXT: vmovdqa 160(%rdi), %ymm4
2146 ; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm5
2147 ; AVX512BW-NEXT: vpermi2w %ymm4, %ymm5, %ymm0
2148 ; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm1 = <0,6,12,18,24,30,36,42,48,54,60,u,u,u,u,u>
2149 ; AVX512BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm1
2150 ; AVX512BW-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
2151 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
2152 ; AVX512BW-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [0,0,0,3,9,15,21,27,0,0,0,3,9,15,21,27]
2153 ; AVX512BW-NEXT: # ymm1 = mem[0,1,0,1]
2154 ; AVX512BW-NEXT: vpermi2w %ymm4, %ymm5, %ymm1
2155 ; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm6 = <1,7,13,19,25,31,37,43,49,55,61,u,u,u,u,u>
2156 ; AVX512BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm6
2157 ; AVX512BW-NEXT: vpblendw {{.*#+}} ymm1 = ymm6[0,1,2],ymm1[3,4,5,6,7],ymm6[8,9,10],ymm1[11,12,13,14,15]
2158 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm6[0,1,2,3],ymm1[4,5,6,7]
2159 ; AVX512BW-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [0,0,0,20,26,0,6,12,0,0,0,20,26,0,6,12]
2160 ; AVX512BW-NEXT: # ymm6 = mem[0,1,0,1]
2161 ; AVX512BW-NEXT: vpermi2w %ymm5, %ymm4, %ymm6
2162 ; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm7 = <34,40,46,52,58,0,6,12,18,24,30,u,u,u,u,u>
2163 ; AVX512BW-NEXT: vpermi2w %zmm2, %zmm3, %zmm7
2164 ; AVX512BW-NEXT: vpblendw {{.*#+}} ymm6 = ymm7[0,1,2],ymm6[3,4,5,6,7],ymm7[8,9,10],ymm6[11,12,13,14,15]
2165 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7]
2166 ; AVX512BW-NEXT: vbroadcasti128 {{.*#+}} ymm7 = [0,0,0,21,27,1,7,13,0,0,0,21,27,1,7,13]
2167 ; AVX512BW-NEXT: # ymm7 = mem[0,1,0,1]
2168 ; AVX512BW-NEXT: vpermi2w %ymm5, %ymm4, %ymm7
2169 ; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm8 = <35,41,47,53,59,1,7,13,19,25,31,u,u,u,u,u>
2170 ; AVX512BW-NEXT: vpermi2w %zmm2, %zmm3, %zmm8
2171 ; AVX512BW-NEXT: vpblendw {{.*#+}} ymm7 = ymm8[0,1,2],ymm7[3,4,5,6,7],ymm8[8,9,10],ymm7[11,12,13,14,15]
2172 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7]
2173 ; AVX512BW-NEXT: vbroadcasti128 {{.*#+}} ymm8 = [0,0,0,6,12,18,24,30,0,0,0,6,12,18,24,30]
2174 ; AVX512BW-NEXT: # ymm8 = mem[0,1,0,1]
2175 ; AVX512BW-NEXT: vpermi2w %ymm4, %ymm5, %ymm8
2176 ; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm9 = <4,10,16,22,28,34,40,46,52,58,u,u,u,u,u,u>
2177 ; AVX512BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm9
2178 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5,6,7]
2179 ; AVX512BW-NEXT: vbroadcasti128 {{.*#+}} ymm9 = [0,0,1,7,13,19,25,31,0,0,1,7,13,19,25,31]
2180 ; AVX512BW-NEXT: # ymm9 = mem[0,1,0,1]
2181 ; AVX512BW-NEXT: vpermi2w %ymm4, %ymm5, %ymm9
2182 ; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm4 = <5,11,17,23,29,35,41,47,53,59,u,u,u,u,u,u>
2183 ; AVX512BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm4
2184 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4],ymm9[5,6,7]
2185 ; AVX512BW-NEXT: vmovdqa %ymm0, (%rsi)
2186 ; AVX512BW-NEXT: vmovdqa %ymm1, (%rdx)
2187 ; AVX512BW-NEXT: vmovdqa %ymm6, (%rcx)
2188 ; AVX512BW-NEXT: vmovdqa %ymm7, (%r8)
2189 ; AVX512BW-NEXT: vmovdqa %ymm8, (%r9)
2190 ; AVX512BW-NEXT: vmovdqa %ymm2, (%rax)
2191 ; AVX512BW-NEXT: vzeroupper
2192 ; AVX512BW-NEXT: retq
2193 %wide.vec = load <96 x i16>, ptr %in.vec, align 64
2194 %strided.vec0 = shufflevector <96 x i16> %wide.vec, <96 x i16> poison, <16 x i32> <i32 0, i32 6, i32 12, i32 18, i32 24, i32 30, i32 36, i32 42, i32 48, i32 54, i32 60, i32 66, i32 72, i32 78, i32 84, i32 90>
2195 %strided.vec1 = shufflevector <96 x i16> %wide.vec, <96 x i16> poison, <16 x i32> <i32 1, i32 7, i32 13, i32 19, i32 25, i32 31, i32 37, i32 43, i32 49, i32 55, i32 61, i32 67, i32 73, i32 79, i32 85, i32 91>
2196 %strided.vec2 = shufflevector <96 x i16> %wide.vec, <96 x i16> poison, <16 x i32> <i32 2, i32 8, i32 14, i32 20, i32 26, i32 32, i32 38, i32 44, i32 50, i32 56, i32 62, i32 68, i32 74, i32 80, i32 86, i32 92>
2197 %strided.vec3 = shufflevector <96 x i16> %wide.vec, <96 x i16> poison, <16 x i32> <i32 3, i32 9, i32 15, i32 21, i32 27, i32 33, i32 39, i32 45, i32 51, i32 57, i32 63, i32 69, i32 75, i32 81, i32 87, i32 93>
2198 %strided.vec4 = shufflevector <96 x i16> %wide.vec, <96 x i16> poison, <16 x i32> <i32 4, i32 10, i32 16, i32 22, i32 28, i32 34, i32 40, i32 46, i32 52, i32 58, i32 64, i32 70, i32 76, i32 82, i32 88, i32 94>
2199 %strided.vec5 = shufflevector <96 x i16> %wide.vec, <96 x i16> poison, <16 x i32> <i32 5, i32 11, i32 17, i32 23, i32 29, i32 35, i32 41, i32 47, i32 53, i32 59, i32 65, i32 71, i32 77, i32 83, i32 89, i32 95>
2200 store <16 x i16> %strided.vec0, ptr %out.vec0, align 64
2201 store <16 x i16> %strided.vec1, ptr %out.vec1, align 64
2202 store <16 x i16> %strided.vec2, ptr %out.vec2, align 64
2203 store <16 x i16> %strided.vec3, ptr %out.vec3, align 64
2204 store <16 x i16> %strided.vec4, ptr %out.vec4, align 64
2205 store <16 x i16> %strided.vec5, ptr %out.vec5, align 64
2206 ret void
2207 }
2209 define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4, ptr %out.vec5) nounwind {
2210 ; SSE-LABEL: load_i16_stride6_vf32:
2211 ; SSE: # %bb.0:
2212 ; SSE-NEXT: subq $488, %rsp # imm = 0x1E8
2213 ; SSE-NEXT: movdqa 304(%rdi), %xmm5
2214 ; SSE-NEXT: movdqa 320(%rdi), %xmm7
2215 ; SSE-NEXT: movdqa 64(%rdi), %xmm2
2216 ; SSE-NEXT: movdqa 80(%rdi), %xmm10
2217 ; SSE-NEXT: movdqa (%rdi), %xmm3
2218 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2219 ; SSE-NEXT: movdqa 16(%rdi), %xmm4
2220 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2221 ; SSE-NEXT: movdqa 32(%rdi), %xmm0
2222 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2223 ; SSE-NEXT: movdqa 48(%rdi), %xmm6
2224 ; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2225 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,2,4,5,6,7]
2226 ; SSE-NEXT: movdqa {{.*#+}} xmm8 = [65535,65535,65535,0,0,0,65535,65535]
2227 ; SSE-NEXT: movdqa %xmm8, %xmm1
2228 ; SSE-NEXT: pandn %xmm0, %xmm1
2229 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,1,0,3]
2230 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2231 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
2232 ; SSE-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm4[2],xmm0[3],xmm4[3]
2233 ; SSE-NEXT: pand %xmm8, %xmm0
2234 ; SSE-NEXT: por %xmm1, %xmm0
2235 ; SSE-NEXT: movdqa %xmm0, %xmm1
2236 ; SSE-NEXT: movdqa %xmm2, %xmm11
2237 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,2,3,3]
2238 ; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
2239 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2240 ; SSE-NEXT: movdqa %xmm10, %xmm0
2241 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm2[3,0]
2242 ; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2243 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2244 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,0],xmm10[0,0]
2245 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm10[2,3]
2246 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2247 ; SSE-NEXT: pslld $16, %xmm10
2248 ; SSE-NEXT: psrldq {{.*#+}} xmm11 = xmm11[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
2249 ; SSE-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]
2250 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,3,2,3]
2251 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2252 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
2253 ; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[3,1],xmm0[1,3]
2254 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm11[2,0]
2255 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2256 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm7[0,1,1,2,4,5,6,7]
2257 ; SSE-NEXT: movdqa %xmm8, %xmm1
2258 ; SSE-NEXT: pandn %xmm0, %xmm1
2259 ; SSE-NEXT: movdqa 288(%rdi), %xmm0
2260 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2261 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
2262 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2263 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
2264 ; SSE-NEXT: movdqa %xmm5, %xmm6
2265 ; SSE-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm5[2],xmm0[3],xmm5[3]
2266 ; SSE-NEXT: pand %xmm8, %xmm0
2267 ; SSE-NEXT: movdqa %xmm8, %xmm10
2268 ; SSE-NEXT: por %xmm1, %xmm0
2269 ; SSE-NEXT: movdqa %xmm0, %xmm1
2270 ; SSE-NEXT: movdqa 352(%rdi), %xmm4
2271 ; SSE-NEXT: movdqa 368(%rdi), %xmm0
2272 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm4[2,2,3,3]
2273 ; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2274 ; SSE-NEXT: movdqa %xmm0, %xmm3
2275 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm4[3,0]
2276 ; SSE-NEXT: movaps %xmm3, (%rsp) # 16-byte Spill
2277 ; SSE-NEXT: movdqa %xmm4, %xmm3
2278 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2279 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[1,0],xmm0[0,0]
2280 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm0[2,3]
2281 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2282 ; SSE-NEXT: pslld $16, %xmm0
2283 ; SSE-NEXT: psrldq {{.*#+}} xmm3 = xmm3[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
2284 ; SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
2285 ; SSE-NEXT: movdqa 336(%rdi), %xmm0
2286 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2287 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3]
2288 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2289 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
2290 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,1],xmm0[1,3]
2291 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,0]
2292 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2293 ; SSE-NEXT: movdqa 224(%rdi), %xmm15
2294 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm15[0,1,1,2,4,5,6,7]
2295 ; SSE-NEXT: movdqa %xmm8, %xmm3
2296 ; SSE-NEXT: pandn %xmm0, %xmm3
2297 ; SSE-NEXT: movdqa 208(%rdi), %xmm1
2298 ; SSE-NEXT: movdqa 192(%rdi), %xmm0
2299 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2300 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
2301 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2302 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
2303 ; SSE-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
2304 ; SSE-NEXT: movdqa %xmm1, %xmm14
2305 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2306 ; SSE-NEXT: pand %xmm8, %xmm0
2307 ; SSE-NEXT: por %xmm3, %xmm0
2308 ; SSE-NEXT: movdqa %xmm0, %xmm1
2309 ; SSE-NEXT: movdqa 256(%rdi), %xmm4
2310 ; SSE-NEXT: movdqa 272(%rdi), %xmm0
2311 ; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm4[2,2,3,3]
2312 ; SSE-NEXT: punpcklwd {{.*#+}} xmm13 = xmm13[0],xmm0[0],xmm13[1],xmm0[1],xmm13[2],xmm0[2],xmm13[3],xmm0[3]
2313 ; SSE-NEXT: movdqa %xmm0, %xmm3
2314 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm4[3,0]
2315 ; SSE-NEXT: movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2316 ; SSE-NEXT: movdqa %xmm4, %xmm3
2317 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2318 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[1,0],xmm0[0,0]
2319 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm0[2,3]
2320 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2321 ; SSE-NEXT: pslld $16, %xmm0
2322 ; SSE-NEXT: psrldq {{.*#+}} xmm3 = xmm3[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
2323 ; SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
2324 ; SSE-NEXT: movdqa 240(%rdi), %xmm0
2325 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2326 ; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm0[0,3,2,3]
2327 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm12[0,1,0,2,4,5,6,7]
2328 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,1],xmm0[1,3]
2329 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,0]
2330 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2331 ; SSE-NEXT: movdqa 128(%rdi), %xmm0
2332 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2333 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,2,4,5,6,7]
2334 ; SSE-NEXT: movdqa %xmm8, %xmm3
2335 ; SSE-NEXT: pandn %xmm0, %xmm3
2336 ; SSE-NEXT: movdqa 112(%rdi), %xmm1
2337 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2338 ; SSE-NEXT: movdqa 96(%rdi), %xmm0
2339 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2340 ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm0[0,1,0,3]
2341 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm8[0,1,2,3,4,6,6,7]
2342 ; SSE-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
2343 ; SSE-NEXT: pand %xmm10, %xmm0
2344 ; SSE-NEXT: por %xmm3, %xmm0
2345 ; SSE-NEXT: movdqa 160(%rdi), %xmm1
2346 ; SSE-NEXT: movdqa 176(%rdi), %xmm11
2347 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,2,3,3]
2348 ; SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm11[0],xmm3[1],xmm11[1],xmm3[2],xmm11[2],xmm3[3],xmm11[3]
2349 ; SSE-NEXT: movdqa %xmm11, %xmm4
2350 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm1[3,0]
2351 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2352 ; SSE-NEXT: movdqa %xmm1, %xmm9
2353 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2354 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,0],xmm11[0,0]
2355 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm11[2,3]
2356 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2357 ; SSE-NEXT: pslld $16, %xmm11
2358 ; SSE-NEXT: psrldq {{.*#+}} xmm9 = xmm9[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
2359 ; SSE-NEXT: punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1],xmm9[2],xmm11[2],xmm9[3],xmm11[3]
2360 ; SSE-NEXT: movdqa 144(%rdi), %xmm1
2361 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2362 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm1[0,3,2,3]
2363 ; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm4[0,1,0,2,4,5,6,7]
2364 ; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[3,1],xmm11[1,3]
2365 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm9[2,0]
2366 ; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2367 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
2368 ; SSE-NEXT: movdqa %xmm5, %xmm9
2369 ; SSE-NEXT: psrld $16, %xmm9
2370 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
2371 ; SSE-NEXT: # xmm11 = mem[0,1,2,3,5,7,6,7]
2372 ; SSE-NEXT: punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm9[2],xmm11[3],xmm9[3]
2373 ; SSE-NEXT: movdqa %xmm10, %xmm9
2374 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
2375 ; SSE-NEXT: pandn %xmm1, %xmm9
2376 ; SSE-NEXT: pand %xmm10, %xmm11
2377 ; SSE-NEXT: por %xmm9, %xmm11
2378 ; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload
2379 ; SSE-NEXT: # xmm9 = mem[0,1,1,3,4,5,6,7]
2380 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
2381 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,1],xmm9[1,3]
2382 ; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[0,1],xmm0[2,0]
2383 ; SSE-NEXT: movaps %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2384 ; SSE-NEXT: movdqa %xmm6, %xmm0
2385 ; SSE-NEXT: psrld $16, %xmm0
2386 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload
2387 ; SSE-NEXT: # xmm9 = mem[0,1,2,3,5,7,6,7]
2388 ; SSE-NEXT: punpckhdq {{.*#+}} xmm9 = xmm9[2],xmm0[2],xmm9[3],xmm0[3]
2389 ; SSE-NEXT: movdqa %xmm10, %xmm0
2390 ; SSE-NEXT: pandn %xmm7, %xmm0
2391 ; SSE-NEXT: pand %xmm10, %xmm9
2392 ; SSE-NEXT: por %xmm0, %xmm9
2393 ; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2394 ; SSE-NEXT: # xmm0 = mem[0,1,1,3,4,5,6,7]
2395 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,1],xmm0[1,3]
2396 ; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[0,1],xmm2[2,0]
2397 ; SSE-NEXT: movaps %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2398 ; SSE-NEXT: movdqa %xmm14, %xmm0
2399 ; SSE-NEXT: psrld $16, %xmm0
2400 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload
2401 ; SSE-NEXT: # xmm14 = mem[0,1,2,3,5,7,6,7]
2402 ; SSE-NEXT: punpckhdq {{.*#+}} xmm14 = xmm14[2],xmm0[2],xmm14[3],xmm0[3]
2403 ; SSE-NEXT: movdqa %xmm10, %xmm0
2404 ; SSE-NEXT: movdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2405 ; SSE-NEXT: pandn %xmm15, %xmm0
2406 ; SSE-NEXT: pand %xmm10, %xmm14
2407 ; SSE-NEXT: por %xmm0, %xmm14
2408 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm12[0,1,1,3,4,5,6,7]
2409 ; SSE-NEXT: shufps {{.*#+}} xmm13 = xmm13[3,1],xmm0[1,3]
2410 ; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[0,1],xmm13[2,0]
2411 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
2412 ; SSE-NEXT: psrld $16, %xmm0
2413 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm8[0,1,2,3,5,7,6,7]
2414 ; SSE-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm0[2],xmm2[3],xmm0[3]
2415 ; SSE-NEXT: pand %xmm10, %xmm2
2416 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
2417 ; SSE-NEXT: pandn %xmm12, %xmm10
2418 ; SSE-NEXT: por %xmm2, %xmm10
2419 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm4[0,1,1,3,4,5,6,7]
2420 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,1],xmm0[1,3]
2421 ; SSE-NEXT: shufps {{.*#+}} xmm10 = xmm10[0,1],xmm3[2,0]
2422 ; SSE-NEXT: movaps %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2423 ; SSE-NEXT: movdqa %xmm1, %xmm0
2424 ; SSE-NEXT: psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
2425 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
2426 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm13[1,1,1,1]
2427 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
2428 ; SSE-NEXT: movdqa {{.*#+}} xmm1 = [65535,65535,65535,0,0,65535,65535,65535]
2429 ; SSE-NEXT: movdqa %xmm1, %xmm3
2430 ; SSE-NEXT: pandn %xmm0, %xmm3
2431 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
2432 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,0],xmm5[0,0]
2433 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm5[2,3]
2434 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm2[0,2,2,3,4,5,6,7]
2435 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3]
2436 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm0[1,0,2,3,4,5,6,7]
2437 ; SSE-NEXT: pand %xmm1, %xmm4
2438 ; SSE-NEXT: por %xmm3, %xmm4
2439 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
2440 ; SSE-NEXT: shufps $132, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2441 ; SSE-NEXT: # xmm0 = xmm0[0,1],mem[0,2]
2442 ; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2443 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
2444 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,0]
2445 ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm0[0,1,2,3,4,6,5,4]
2446 ; SSE-NEXT: movdqa {{.*#+}} xmm0 = [65535,65535,65535,65535,65535,0,0,0]
2447 ; SSE-NEXT: movdqa %xmm0, %xmm5
2448 ; SSE-NEXT: pandn %xmm3, %xmm5
2449 ; SSE-NEXT: pand %xmm0, %xmm4
2450 ; SSE-NEXT: por %xmm4, %xmm5
2451 ; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2452 ; SSE-NEXT: movdqa %xmm7, %xmm9
2453 ; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2454 ; SSE-NEXT: movdqa %xmm7, %xmm3
2455 ; SSE-NEXT: psrldq {{.*#+}} xmm3 = xmm3[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
2456 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
2457 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm11[1,1,1,1]
2458 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
2459 ; SSE-NEXT: movdqa %xmm1, %xmm4
2460 ; SSE-NEXT: pandn %xmm3, %xmm4
2461 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
2462 ; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2463 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,0],xmm6[0,0]
2464 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm6[2,3]
2465 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm3[0,2,2,3,4,5,6,7]
2466 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,3,2,3]
2467 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[1,0,2,3,4,5,6,7]
2468 ; SSE-NEXT: pand %xmm1, %xmm5
2469 ; SSE-NEXT: por %xmm4, %xmm5
2470 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
2471 ; SSE-NEXT: shufps $132, (%rsp), %xmm4 # 16-byte Folded Reload
2472 ; SSE-NEXT: # xmm4 = xmm4[0,1],mem[0,2]
2473 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2474 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,6,6,7]
2475 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,2,0]
2476 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,6,5,4]
2477 ; SSE-NEXT: movdqa %xmm0, %xmm6
2478 ; SSE-NEXT: pandn %xmm4, %xmm6
2479 ; SSE-NEXT: pand %xmm0, %xmm5
2480 ; SSE-NEXT: por %xmm5, %xmm6
2481 ; SSE-NEXT: movdqa %xmm6, (%rsp) # 16-byte Spill
2482 ; SSE-NEXT: psrldq {{.*#+}} xmm15 = xmm15[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
2483 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
2484 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,1,1,1]
2485 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm15 = xmm15[0],xmm5[0]
2486 ; SSE-NEXT: movdqa %xmm1, %xmm5
2487 ; SSE-NEXT: pandn %xmm15, %xmm5
2488 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
2489 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
2490 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[1,0],xmm6[0,0]
2491 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm6[2,3]
2492 ; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm4[0,2,2,3,4,5,6,7]
2493 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,3,2,3]
2494 ; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[1,0,2,3,4,5,6,7]
2495 ; SSE-NEXT: pand %xmm1, %xmm6
2496 ; SSE-NEXT: por %xmm5, %xmm6
2497 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
2498 ; SSE-NEXT: shufps $132, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
2499 ; SSE-NEXT: # xmm5 = xmm5[0,1],mem[0,2]
2500 ; SSE-NEXT: movaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2501 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,6,6,7]
2502 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,1,2,0]
2503 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,6,5,4]
2504 ; SSE-NEXT: movdqa %xmm0, %xmm8
2505 ; SSE-NEXT: pandn %xmm5, %xmm8
2506 ; SSE-NEXT: pand %xmm0, %xmm6
2507 ; SSE-NEXT: por %xmm6, %xmm8
2508 ; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2509 ; SSE-NEXT: movdqa %xmm12, %xmm15
2510 ; SSE-NEXT: movdqa %xmm12, %xmm5
2511 ; SSE-NEXT: psrldq {{.*#+}} xmm5 = xmm5[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
2512 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
2513 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm10[1,1,1,1]
2514 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm6[0]
2515 ; SSE-NEXT: movdqa %xmm1, %xmm6
2516 ; SSE-NEXT: pandn %xmm5, %xmm6
2517 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
2518 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
2519 ; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[1,0],xmm8[0,0]
2520 ; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[2,0],xmm8[2,3]
2521 ; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm5[0,2,2,3,4,5,6,7]
2522 ; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm12[0,3,2,3]
2523 ; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm12[1,0,2,3,4,5,6,7]
2524 ; SSE-NEXT: pand %xmm1, %xmm12
2525 ; SSE-NEXT: por %xmm6, %xmm12
2526 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
2527 ; SSE-NEXT: shufps $132, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
2528 ; SSE-NEXT: # xmm6 = xmm6[0,1],mem[0,2]
2529 ; SSE-NEXT: movaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2530 ; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,6,6,7]
2531 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,1,2,0]
2532 ; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,6,5,4]
2533 ; SSE-NEXT: movdqa %xmm0, %xmm8
2534 ; SSE-NEXT: pandn %xmm6, %xmm8
2535 ; SSE-NEXT: pand %xmm0, %xmm12
2536 ; SSE-NEXT: por %xmm12, %xmm8
2537 ; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2538 ; SSE-NEXT: movdqa %xmm13, %xmm12
2539 ; SSE-NEXT: movdqa %xmm13, %xmm6
2540 ; SSE-NEXT: psrlq $48, %xmm6
2541 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
2542 ; SSE-NEXT: # xmm8 = mem[2,2,3,3]
2543 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm8 = xmm8[0],xmm6[0]
2544 ; SSE-NEXT: movdqa %xmm1, %xmm6
2545 ; SSE-NEXT: pandn %xmm8, %xmm6
2546 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]
2547 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]
2548 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,3,4,5,6,7]
2549 ; SSE-NEXT: pand %xmm1, %xmm2
2550 ; SSE-NEXT: por %xmm6, %xmm2
2551 ; SSE-NEXT: pshufhw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
2552 ; SSE-NEXT: # xmm6 = mem[0,1,2,3,7,5,6,7]
2553 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,1,0,2]
2554 ; SSE-NEXT: movdqa %xmm0, %xmm8
2555 ; SSE-NEXT: pandn %xmm6, %xmm8
2556 ; SSE-NEXT: pand %xmm0, %xmm2
2557 ; SSE-NEXT: por %xmm2, %xmm8
2558 ; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2559 ; SSE-NEXT: movdqa %xmm11, %xmm2
2560 ; SSE-NEXT: psrlq $48, %xmm2
2561 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm9[2,2,3,3]
2562 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm2[0]
2563 ; SSE-NEXT: movdqa %xmm1, %xmm2
2564 ; SSE-NEXT: pandn %xmm6, %xmm2
2565 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7]
2566 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,3,2,3]
2567 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,3,4,5,6,7]
2568 ; SSE-NEXT: pand %xmm1, %xmm3
2569 ; SSE-NEXT: por %xmm2, %xmm3
2570 ; SSE-NEXT: pshufhw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
2571 ; SSE-NEXT: # xmm2 = mem[0,1,2,3,7,5,6,7]
2572 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,2]
2573 ; SSE-NEXT: movdqa %xmm0, %xmm6
2574 ; SSE-NEXT: pandn %xmm2, %xmm6
2575 ; SSE-NEXT: pand %xmm0, %xmm3
2576 ; SSE-NEXT: por %xmm3, %xmm6
2577 ; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2578 ; SSE-NEXT: psrlq $48, %xmm7
2579 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
2580 ; SSE-NEXT: # xmm3 = mem[2,2,3,3]
2581 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm7[0]
2582 ; SSE-NEXT: movdqa %xmm1, %xmm2
2583 ; SSE-NEXT: pandn %xmm3, %xmm2
2584 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm4[3,1,2,3,4,5,6,7]
2585 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,3,2,3]
2586 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,3,4,5,6,7]
2587 ; SSE-NEXT: pand %xmm1, %xmm3
2588 ; SSE-NEXT: por %xmm2, %xmm3
2589 ; SSE-NEXT: pshufhw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
2590 ; SSE-NEXT: # xmm2 = mem[0,1,2,3,7,5,6,7]
2591 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,2]
2592 ; SSE-NEXT: movdqa %xmm0, %xmm13
2593 ; SSE-NEXT: pandn %xmm2, %xmm13
2594 ; SSE-NEXT: pand %xmm0, %xmm3
2595 ; SSE-NEXT: por %xmm3, %xmm13
2596 ; SSE-NEXT: movdqa %xmm10, %xmm9
2597 ; SSE-NEXT: movdqa %xmm10, %xmm2
2598 ; SSE-NEXT: psrlq $48, %xmm2
2599 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm15[2,2,3,3]
2600 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm2[0]
2601 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm5[3,1,2,3,4,5,6,7]
2602 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]
2603 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,3,4,5,6,7]
2604 ; SSE-NEXT: pand %xmm1, %xmm2
2605 ; SSE-NEXT: pandn %xmm3, %xmm1
2606 ; SSE-NEXT: por %xmm2, %xmm1
2607 ; SSE-NEXT: pshufhw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
2608 ; SSE-NEXT: # xmm2 = mem[0,1,2,3,7,5,6,7]
2609 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,2]
2610 ; SSE-NEXT: movdqa %xmm0, %xmm7
2611 ; SSE-NEXT: pandn %xmm2, %xmm7
2612 ; SSE-NEXT: pand %xmm0, %xmm1
2613 ; SSE-NEXT: por %xmm1, %xmm7
2614 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
2615 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm10[1,1,1,1]
2616 ; SSE-NEXT: pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
2617 ; SSE-NEXT: # xmm2 = mem[2,3,2,3]
2618 ; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
2619 ; SSE-NEXT: pshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
2620 ; SSE-NEXT: # xmm8 = mem[0,1,0,3]
2621 ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm8[0,1,2,3,4,5,4,6]
2622 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm12[1]
2623 ; SSE-NEXT: movss {{.*#+}} xmm3 = xmm2[0],xmm3[1,2,3]
2624 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
2625 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm11[0,2,2,3,4,5,6,7]
2626 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,3]
2627 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,6]
2628 ; SSE-NEXT: movdqa %xmm0, %xmm6
2629 ; SSE-NEXT: pandn %xmm2, %xmm6
2630 ; SSE-NEXT: andps %xmm0, %xmm3
2631 ; SSE-NEXT: por %xmm3, %xmm6
2632 ; SSE-NEXT: pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
2633 ; SSE-NEXT: # xmm2 = mem[1,1,1,1]
2634 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
2635 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm15[2,3,2,3]
2636 ; SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
2637 ; SSE-NEXT: pshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2638 ; SSE-NEXT: # xmm1 = mem[0,1,0,3]
2639 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2640 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm1[0,1,2,3,4,5,4,6]
2641 ; SSE-NEXT: punpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
2642 ; SSE-NEXT: # xmm2 = xmm2[1],mem[1]
2643 ; SSE-NEXT: movss {{.*#+}} xmm2 = xmm3[0],xmm2[1,2,3]
2644 ; SSE-NEXT: pshuflw $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
2645 ; SSE-NEXT: # xmm3 = mem[0,2,2,3,4,5,6,7]
2646 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,3]
2647 ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,4,6]
2648 ; SSE-NEXT: movdqa %xmm0, %xmm5
2649 ; SSE-NEXT: pandn %xmm3, %xmm5
2650 ; SSE-NEXT: andps %xmm0, %xmm2
2651 ; SSE-NEXT: por %xmm2, %xmm5
2652 ; SSE-NEXT: pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
2653 ; SSE-NEXT: # xmm2 = mem[1,1,1,1]
2654 ; SSE-NEXT: pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
2655 ; SSE-NEXT: # xmm3 = mem[2,3,2,3]
2656 ; SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
2657 ; SSE-NEXT: pshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2658 ; SSE-NEXT: # xmm1 = mem[0,1,0,3]
2659 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2660 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm1[0,1,2,3,4,5,4,6]
2661 ; SSE-NEXT: punpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
2662 ; SSE-NEXT: # xmm2 = xmm2[1],mem[1]
2663 ; SSE-NEXT: movss {{.*#+}} xmm2 = xmm3[0],xmm2[1,2,3]
2664 ; SSE-NEXT: pshuflw $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
2665 ; SSE-NEXT: # xmm3 = mem[0,2,2,3,4,5,6,7]
2666 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,3]
2667 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm3[0,1,2,3,4,5,4,6]
2668 ; SSE-NEXT: movdqa %xmm0, %xmm4
2669 ; SSE-NEXT: pandn %xmm1, %xmm4
2670 ; SSE-NEXT: andps %xmm0, %xmm2
2671 ; SSE-NEXT: por %xmm2, %xmm4
2672 ; SSE-NEXT: pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2673 ; SSE-NEXT: # xmm1 = mem[1,1,1,1]
2674 ; SSE-NEXT: pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
2675 ; SSE-NEXT: # xmm2 = mem[2,3,2,3]
2676 ; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
2677 ; SSE-NEXT: pshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2678 ; SSE-NEXT: # xmm1 = mem[0,1,0,3]
2679 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2680 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,6]
2681 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm1 = xmm1[1],xmm9[1]
2682 ; SSE-NEXT: movss {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
2683 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
2684 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm9[0,2,2,3,4,5,6,7]
2685 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,3]
2686 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,6]
2687 ; SSE-NEXT: movdqa %xmm0, %xmm3
2688 ; SSE-NEXT: pandn %xmm2, %xmm3
2689 ; SSE-NEXT: andps %xmm0, %xmm1
2690 ; SSE-NEXT: por %xmm1, %xmm3
2691 ; SSE-NEXT: psrlq $48, %xmm10
2692 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
2693 ; SSE-NEXT: psrldq {{.*#+}} xmm1 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
2694 ; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm10[0],xmm1[1],xmm10[1],xmm1[2],xmm10[2],xmm1[3],xmm10[3]
2695 ; SSE-NEXT: psrld $16, %xmm12
2696 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm8[0,1,2,3,4,5,5,7]
2697 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm12[1]
2698 ; SSE-NEXT: movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
2699 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm11[3,1,2,3,4,5,6,7]
2700 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3]
2701 ; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm1[0,1,2,3,4,4,5,7]
2702 ; SSE-NEXT: movdqa %xmm0, %xmm1
2703 ; SSE-NEXT: pandn %xmm8, %xmm1
2704 ; SSE-NEXT: andps %xmm0, %xmm2
2705 ; SSE-NEXT: por %xmm2, %xmm1
2706 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
2707 ; SSE-NEXT: psrlq $48, %xmm8
2708 ; SSE-NEXT: psrldq {{.*#+}} xmm15 = xmm15[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
2709 ; SSE-NEXT: punpcklwd {{.*#+}} xmm15 = xmm15[0],xmm8[0],xmm15[1],xmm8[1],xmm15[2],xmm8[2],xmm15[3],xmm8[3]
2710 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
2711 ; SSE-NEXT: psrld $16, %xmm11
2712 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
2713 ; SSE-NEXT: # xmm2 = mem[0,1,2,3,4,5,5,7]
2714 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm11[1]
2715 ; SSE-NEXT: movss {{.*#+}} xmm2 = xmm15[0],xmm2[1,2,3]
2716 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
2717 ; SSE-NEXT: # xmm8 = mem[3,1,2,3,4,5,6,7]
2718 ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,1,0,3]
2719 ; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,4,5,7]
2720 ; SSE-NEXT: movdqa %xmm0, %xmm15
2721 ; SSE-NEXT: pandn %xmm8, %xmm15
2722 ; SSE-NEXT: andps %xmm0, %xmm2
2723 ; SSE-NEXT: por %xmm2, %xmm15
2724 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
2725 ; SSE-NEXT: psrlq $48, %xmm8
2726 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
2727 ; SSE-NEXT: psrldq {{.*#+}} xmm2 = xmm2[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
2728 ; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm8[0],xmm2[1],xmm8[1],xmm2[2],xmm8[2],xmm2[3],xmm8[3]
2729 ; SSE-NEXT: movdqa %xmm2, %xmm8
2730 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
2731 ; SSE-NEXT: psrld $16, %xmm11
2732 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
2733 ; SSE-NEXT: # xmm2 = mem[0,1,2,3,4,5,5,7]
2734 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm11[1]
2735 ; SSE-NEXT: movss {{.*#+}} xmm2 = xmm8[0],xmm2[1,2,3]
2736 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
2737 ; SSE-NEXT: # xmm8 = mem[3,1,2,3,4,5,6,7]
2738 ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,1,0,3]
2739 ; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,4,5,7]
2740 ; SSE-NEXT: movdqa %xmm0, %xmm12
2741 ; SSE-NEXT: pandn %xmm8, %xmm12
2742 ; SSE-NEXT: andps %xmm0, %xmm2
2743 ; SSE-NEXT: por %xmm2, %xmm12
2744 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
2745 ; SSE-NEXT: psrlq $48, %xmm8
2746 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
2747 ; SSE-NEXT: psrldq {{.*#+}} xmm2 = xmm2[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
2748 ; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm8[0],xmm2[1],xmm8[1],xmm2[2],xmm8[2],xmm2[3],xmm8[3]
2749 ; SSE-NEXT: movdqa %xmm2, %xmm8
2750 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
2751 ; SSE-NEXT: psrld $16, %xmm11
2752 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
2753 ; SSE-NEXT: # xmm2 = mem[0,1,2,3,4,5,5,7]
2754 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm11[1]
2755 ; SSE-NEXT: movss {{.*#+}} xmm2 = xmm8[0],xmm2[1,2,3]
2756 ; SSE-NEXT: andps %xmm0, %xmm2
2757 ; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm9[3,1,2,3,4,5,6,7]
2758 ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,1,0,3]
2759 ; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,4,5,7]
2760 ; SSE-NEXT: pandn %xmm8, %xmm0
2761 ; SSE-NEXT: por %xmm2, %xmm0
2762 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
2763 ; SSE-NEXT: movaps %xmm2, 16(%rsi)
2764 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
2765 ; SSE-NEXT: movaps %xmm2, 32(%rsi)
2766 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
2767 ; SSE-NEXT: movaps %xmm2, 48(%rsi)
2768 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
2769 ; SSE-NEXT: movaps %xmm2, (%rsi)
2770 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
2771 ; SSE-NEXT: movaps %xmm2, 16(%rdx)
2772 ; SSE-NEXT: movaps %xmm14, 32(%rdx)
2773 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
2774 ; SSE-NEXT: movaps %xmm2, 48(%rdx)
2775 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
2776 ; SSE-NEXT: movaps %xmm2, (%rdx)
2777 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
2778 ; SSE-NEXT: movaps %xmm2, 16(%rcx)
2779 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
2780 ; SSE-NEXT: movaps %xmm2, 32(%rcx)
2781 ; SSE-NEXT: movaps (%rsp), %xmm2 # 16-byte Reload
2782 ; SSE-NEXT: movaps %xmm2, 48(%rcx)
2783 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
2784 ; SSE-NEXT: movaps %xmm2, (%rcx)
2785 ; SSE-NEXT: movdqa %xmm7, 16(%r8)
2786 ; SSE-NEXT: movdqa %xmm13, 32(%r8)
2787 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
2788 ; SSE-NEXT: movaps %xmm2, 48(%r8)
2789 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
2790 ; SSE-NEXT: movaps %xmm2, (%r8)
2791 ; SSE-NEXT: movdqa %xmm3, 16(%r9)
2792 ; SSE-NEXT: movdqa %xmm4, 32(%r9)
2793 ; SSE-NEXT: movdqa %xmm5, 48(%r9)
2794 ; SSE-NEXT: movdqa %xmm6, (%r9)
2795 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
2796 ; SSE-NEXT: movdqa %xmm0, 16(%rax)
2797 ; SSE-NEXT: movdqa %xmm12, 32(%rax)
2798 ; SSE-NEXT: movdqa %xmm15, 48(%rax)
2799 ; SSE-NEXT: movdqa %xmm1, (%rax)
2800 ; SSE-NEXT: addq $488, %rsp # imm = 0x1E8
2801 ; SSE-NEXT: retq
2802 ;
2803 ; AVX1-ONLY-LABEL: load_i16_stride6_vf32:
2804 ; AVX1-ONLY: # %bb.0:
2805 ; AVX1-ONLY-NEXT: subq $536, %rsp # imm = 0x218
2806 ; AVX1-ONLY-NEXT: vmovdqa 96(%rdi), %xmm0
2807 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2808 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm14 = xmm0[0,1,0,3]
2809 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm0 = xmm14[0,1,2,3,4,6,6,7]
2810 ; AVX1-ONLY-NEXT: vmovdqa 112(%rdi), %xmm1
2811 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2812 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
2813 ; AVX1-ONLY-NEXT: vmovdqa 80(%rdi), %xmm1
2814 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2815 ; AVX1-ONLY-NEXT: vpslld $16, %xmm1, %xmm1
2816 ; AVX1-ONLY-NEXT: vmovdqa 64(%rdi), %xmm12
2817 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm2 = xmm12[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
2818 ; AVX1-ONLY-NEXT: vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2819 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
2820 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
2821 ; AVX1-ONLY-NEXT: vmovdqa (%rdi), %xmm3
2822 ; AVX1-ONLY-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2823 ; AVX1-ONLY-NEXT: vmovdqa 16(%rdi), %xmm4
2824 ; AVX1-ONLY-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2825 ; AVX1-ONLY-NEXT: vmovdqa 32(%rdi), %xmm1
2826 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2827 ; AVX1-ONLY-NEXT: vmovdqa 48(%rdi), %xmm2
2828 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2829 ; AVX1-ONLY-NEXT: vpsrlq $16, %xmm1, %xmm1
2830 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm7 = xmm2[0,3,2,3]
2831 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm2 = xmm7[0,1,0,2,4,5,6,7]
2832 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
2833 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm8 = xmm3[0,1,0,3]
2834 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm2 = xmm8[0,1,2,3,4,6,6,7]
2835 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm2[2],xmm4[2],xmm2[3],xmm4[3]
2836 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3,4,5],xmm2[6,7]
2837 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
2838 ; AVX1-ONLY-NEXT: vmovdqa 176(%rdi), %xmm1
2839 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2840 ; AVX1-ONLY-NEXT: vpslld $16, %xmm1, %xmm1
2841 ; AVX1-ONLY-NEXT: vmovdqa 160(%rdi), %xmm2
2842 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2843 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
2844 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
2845 ; AVX1-ONLY-NEXT: vmovdqa 128(%rdi), %xmm11
2846 ; AVX1-ONLY-NEXT: vpsrlq $16, %xmm11, %xmm2
2847 ; AVX1-ONLY-NEXT: vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2848 ; AVX1-ONLY-NEXT: vmovdqa 144(%rdi), %xmm3
2849 ; AVX1-ONLY-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2850 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[0,3,2,3]
2851 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm3 = xmm6[0,1,0,2,4,5,6,7]
2852 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
2853 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3,4,5],xmm1[6,7]
2854 ; AVX1-ONLY-NEXT: vmovaps {{.*#+}} ymm2 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0]
2855 ; AVX1-ONLY-NEXT: vandps %ymm2, %ymm0, %ymm0
2856 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
2857 ; AVX1-ONLY-NEXT: vandnps %ymm1, %ymm2, %ymm1
2858 ; AVX1-ONLY-NEXT: vorps %ymm1, %ymm0, %ymm0
2859 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
2860 ; AVX1-ONLY-NEXT: vmovdqa 272(%rdi), %xmm9
2861 ; AVX1-ONLY-NEXT: vpslld $16, %xmm9, %xmm0
2862 ; AVX1-ONLY-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2863 ; AVX1-ONLY-NEXT: vmovdqa 256(%rdi), %xmm1
2864 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2865 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm1 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
2866 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2867 ; AVX1-ONLY-NEXT: vmovdqa 288(%rdi), %xmm1
2868 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, (%rsp) # 16-byte Spill
2869 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[0,1,0,3]
2870 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm2 = xmm5[0,1,2,3,4,6,6,7]
2871 ; AVX1-ONLY-NEXT: vmovdqa 304(%rdi), %xmm4
2872 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm2[2],xmm4[2],xmm2[3],xmm4[3]
2873 ; AVX1-ONLY-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2874 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
2875 ; AVX1-ONLY-NEXT: vmovdqa 224(%rdi), %xmm1
2876 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2877 ; AVX1-ONLY-NEXT: vpsrlq $16, %xmm1, %xmm2
2878 ; AVX1-ONLY-NEXT: vmovdqa 240(%rdi), %xmm1
2879 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2880 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm13 = xmm1[0,3,2,3]
2881 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm3 = xmm13[0,1,0,2,4,5,6,7]
2882 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
2883 ; AVX1-ONLY-NEXT: vmovdqa 192(%rdi), %xmm1
2884 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2885 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[0,1,0,3]
2886 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm10 = xmm3[0,1,2,3,4,6,6,7]
2887 ; AVX1-ONLY-NEXT: vmovdqa 208(%rdi), %xmm1
2888 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2889 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm10 = xmm10[2],xmm1[2],xmm10[3],xmm1[3]
2890 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm10[0,1,2],xmm2[3,4,5],xmm10[6,7]
2891 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm0[3,4,5,6,7]
2892 ; AVX1-ONLY-NEXT: vmovdqa 368(%rdi), %xmm0
2893 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2894 ; AVX1-ONLY-NEXT: vpslld $16, %xmm0, %xmm0
2895 ; AVX1-ONLY-NEXT: vmovdqa 352(%rdi), %xmm1
2896 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2897 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm10 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
2898 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm0[0],xmm10[1],xmm0[1],xmm10[2],xmm0[2],xmm10[3],xmm0[3]
2899 ; AVX1-ONLY-NEXT: vmovdqa 320(%rdi), %xmm0
2900 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2901 ; AVX1-ONLY-NEXT: vpsrlq $16, %xmm0, %xmm0
2902 ; AVX1-ONLY-NEXT: vmovdqa 336(%rdi), %xmm1
2903 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2904 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,3,2,3]
2905 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm15 = xmm1[0,1,0,2,4,5,6,7]
2906 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]
2907 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm10[6,7]
2908 ; AVX1-ONLY-NEXT: vmovaps {{.*#+}} ymm10 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0]
2909 ; AVX1-ONLY-NEXT: vandps %ymm2, %ymm10, %ymm2
2910 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
2911 ; AVX1-ONLY-NEXT: vandnps %ymm0, %ymm10, %ymm0
2912 ; AVX1-ONLY-NEXT: vorps %ymm0, %ymm2, %ymm0
2913 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
2914 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm0 = xmm14[0,1,2,3,5,7,6,7]
2915 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
2916 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm14, %xmm2
2917 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
2918 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm12[2,2,3,3]
2919 ; AVX1-ONLY-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload
2920 ; AVX1-ONLY-NEXT: # xmm2 = xmm2[0],mem[0],xmm2[1],mem[1],xmm2[2],mem[2],xmm2[3],mem[3]
2921 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0
2922 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm2 = xmm7[0,1,1,3,4,5,6,7]
2923 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
2924 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm7 = xmm10[1,1,1,1]
2925 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm7[0],xmm2[1],xmm7[1]
2926 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm7 = xmm8[0,1,2,3,5,7,6,7]
2927 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
2928 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm15, %xmm8
2929 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm7[2],xmm8[2],xmm7[3],xmm8[3]
2930 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm7[0,1,2],xmm2[3,4,5],xmm7[6,7]
2931 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
2932 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm2 = xmm6[0,1,1,3,4,5,6,7]
2933 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm6 = xmm11[1,1,1,1]
2934 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1]
2935 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
2936 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm6 = xmm11[2,2,3,3]
2937 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
2938 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1],xmm6[2],xmm8[2],xmm6[3],xmm8[3]
2939 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5],xmm6[6,7]
2940 ; AVX1-ONLY-NEXT: vmovaps {{.*#+}} ymm6 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0]
2941 ; AVX1-ONLY-NEXT: vandps %ymm6, %ymm0, %ymm0
2942 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2
2943 ; AVX1-ONLY-NEXT: vandnps %ymm2, %ymm6, %ymm2
2944 ; AVX1-ONLY-NEXT: vorps %ymm2, %ymm0, %ymm0
2945 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
2946 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm0 = xmm5[0,1,2,3,5,7,6,7]
2947 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm4, %xmm2
2948 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
2949 ; AVX1-ONLY-NEXT: vpshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
2950 ; AVX1-ONLY-NEXT: # xmm2 = mem[2,2,3,3]
2951 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
2952 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0
2953 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm2 = xmm13[0,1,1,3,4,5,6,7]
2954 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
2955 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm4 = xmm13[1,1,1,1]
2956 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
2957 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,7,6,7]
2958 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
2959 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm9, %xmm4
2960 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
2961 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3,4,5],xmm3[6,7]
2962 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
2963 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]
2964 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
2965 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[1,1,1,1]
2966 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
2967 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
2968 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm4[2,2,3,3]
2969 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
2970 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm12[0],xmm2[1],xmm12[1],xmm2[2],xmm12[2],xmm2[3],xmm12[3]
2971 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm2[6,7]
2972 ; AVX1-ONLY-NEXT: vandps %ymm6, %ymm0, %ymm0
2973 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
2974 ; AVX1-ONLY-NEXT: vandnps %ymm1, %ymm6, %ymm1
2975 ; AVX1-ONLY-NEXT: vorps %ymm1, %ymm0, %ymm0
2976 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
2977 ; AVX1-ONLY-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2978 ; AVX1-ONLY-NEXT: # xmm0 = mem[1,1,1,1]
2979 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm1 = xmm10[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
2980 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
2981 ; AVX1-ONLY-NEXT: vmovdqa {{.*#+}} xmm0 = [4,5,0,1,12,13,14,15,8,9,10,11,12,13,14,15]
2982 ; AVX1-ONLY-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm10 # 16-byte Folded Reload
2983 ; AVX1-ONLY-NEXT: # xmm10 = xmm15[0,1],mem[2,3],xmm15[4,5,6,7]
2984 ; AVX1-ONLY-NEXT: vpshufb %xmm0, %xmm10, %xmm2
2985 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm5 = xmm2[0,1,2],xmm1[3,4],xmm2[5,6,7]
2986 ; AVX1-ONLY-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14, %xmm1 # 16-byte Folded Reload
2987 ; AVX1-ONLY-NEXT: # xmm1 = xmm14[0,1],mem[2,3],xmm14[4,5,6,7]
2988 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2989 ; AVX1-ONLY-NEXT: vpshufb %xmm0, %xmm1, %xmm1
2990 ; AVX1-ONLY-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7,8,9,0,1,12,13,8,9]
2991 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
2992 ; AVX1-ONLY-NEXT: vpblendw $207, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm6 # 16-byte Folded Reload
2993 ; AVX1-ONLY-NEXT: # xmm6 = mem[0,1,2,3],xmm6[4,5],mem[6,7]
2994 ; AVX1-ONLY-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2995 ; AVX1-ONLY-NEXT: vpshufb %xmm2, %xmm6, %xmm6
2996 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm1, %ymm6, %ymm6
2997 ; AVX1-ONLY-NEXT: vmovaps {{.*#+}} ymm1 = [0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
2998 ; AVX1-ONLY-NEXT: vandnps %ymm5, %ymm1, %ymm5
2999 ; AVX1-ONLY-NEXT: vandps %ymm1, %ymm6, %ymm6
3000 ; AVX1-ONLY-NEXT: vorps %ymm5, %ymm6, %ymm6
3001 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
3002 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm5 = xmm14[1,1,1,1]
3003 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3004 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm7 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
3005 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm7 = xmm7[0],xmm5[0]
3006 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm5 = xmm11[0,1,2,3],xmm8[4,5],xmm11[6,7]
3007 ; AVX1-ONLY-NEXT: vpshufb %xmm2, %xmm5, %xmm8
3008 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3,4],xmm8[5,6,7]
3009 ; AVX1-ONLY-NEXT: vmovaps {{.*#+}} ymm1 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0]
3010 ; AVX1-ONLY-NEXT: vandps %ymm1, %ymm6, %ymm6
3011 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm7, %ymm0, %ymm7
3012 ; AVX1-ONLY-NEXT: vandnps %ymm7, %ymm1, %ymm7
3013 ; AVX1-ONLY-NEXT: vorps %ymm7, %ymm6, %ymm1
3014 ; AVX1-ONLY-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3015 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3016 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm6 = xmm1[1,1,1,1]
3017 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm7 = xmm13[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
3018 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm7 = xmm7[0],xmm6[0]
3019 ; AVX1-ONLY-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9, %xmm6 # 16-byte Folded Reload
3020 ; AVX1-ONLY-NEXT: # xmm6 = xmm9[0,1],mem[2,3],xmm9[4,5,6,7]
3021 ; AVX1-ONLY-NEXT: vpshufb %xmm0, %xmm6, %xmm8
3022 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm11 = xmm8[0,1,2],xmm7[3,4],xmm8[5,6,7]
3023 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
3024 ; AVX1-ONLY-NEXT: vpblendw $12, (%rsp), %xmm7, %xmm7 # 16-byte Folded Reload
3025 ; AVX1-ONLY-NEXT: # xmm7 = xmm7[0,1],mem[2,3],xmm7[4,5,6,7]
3026 ; AVX1-ONLY-NEXT: vpshufb %xmm0, %xmm7, %xmm0
3027 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
3028 ; AVX1-ONLY-NEXT: vpblendw $207, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8, %xmm8 # 16-byte Folded Reload
3029 ; AVX1-ONLY-NEXT: # xmm8 = mem[0,1,2,3],xmm8[4,5],mem[6,7]
3030 ; AVX1-ONLY-NEXT: vpshufb %xmm2, %xmm8, %xmm15
3031 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm15, %ymm0
3032 ; AVX1-ONLY-NEXT: vmovaps {{.*#+}} ymm9 = [0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
3033 ; AVX1-ONLY-NEXT: vandnps %ymm11, %ymm9, %ymm11
3034 ; AVX1-ONLY-NEXT: vandps %ymm0, %ymm9, %ymm0
3035 ; AVX1-ONLY-NEXT: vorps %ymm0, %ymm11, %ymm11
3036 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
3037 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm0 = xmm13[1,1,1,1]
3038 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm15 = xmm3[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
3039 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm15 = xmm15[0],xmm0[0]
3040 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm4[0,1,2,3],xmm12[4,5],xmm4[6,7]
3041 ; AVX1-ONLY-NEXT: vpshufb %xmm2, %xmm0, %xmm2
3042 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm15[3,4],xmm2[5,6,7]
3043 ; AVX1-ONLY-NEXT: vmovaps {{.*#+}} ymm3 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0]
3044 ; AVX1-ONLY-NEXT: vandps %ymm3, %ymm11, %ymm11
3045 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2
3046 ; AVX1-ONLY-NEXT: vandnps %ymm2, %ymm3, %ymm2
3047 ; AVX1-ONLY-NEXT: vorps %ymm2, %ymm11, %ymm2
3048 ; AVX1-ONLY-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3049 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
3050 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm15, %xmm2
3051 ; AVX1-ONLY-NEXT: vpshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
3052 ; AVX1-ONLY-NEXT: # xmm11 = mem[2,2,3,3]
3053 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm11[0],xmm2[0]
3054 ; AVX1-ONLY-NEXT: vmovdqa {{.*#+}} xmm11 = [6,7,2,3,14,15,14,15,8,9,10,11,12,13,14,15]
3055 ; AVX1-ONLY-NEXT: vpshufb %xmm11, %xmm10, %xmm10
3056 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm10[0,1,2],xmm2[3,4],xmm10[5,6,7]
3057 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
3058 ; AVX1-ONLY-NEXT: vpshufb %xmm11, %xmm3, %xmm3
3059 ; AVX1-ONLY-NEXT: vmovdqa {{.*#+}} xmm10 = [0,1,2,3,4,5,6,7,14,15,2,3,14,15,10,11]
3060 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
3061 ; AVX1-ONLY-NEXT: vpshufb %xmm10, %xmm4, %xmm4
3062 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm3, %ymm4, %ymm3
3063 ; AVX1-ONLY-NEXT: vandnps %ymm2, %ymm9, %ymm2
3064 ; AVX1-ONLY-NEXT: vandps %ymm3, %ymm9, %ymm3
3065 ; AVX1-ONLY-NEXT: vmovaps %ymm9, %ymm12
3066 ; AVX1-ONLY-NEXT: vorps %ymm2, %ymm3, %ymm2
3067 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm14, %xmm3
3068 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
3069 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm4 = xmm9[2,2,3,3]
3070 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
3071 ; AVX1-ONLY-NEXT: vpshufb %xmm10, %xmm5, %xmm4
3072 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0,1,2],xmm3[3,4],xmm4[5,6,7]
3073 ; AVX1-ONLY-NEXT: vmovaps {{.*#+}} ymm4 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0]
3074 ; AVX1-ONLY-NEXT: vandps %ymm4, %ymm2, %ymm2
3075 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3
3076 ; AVX1-ONLY-NEXT: vandnps %ymm3, %ymm4, %ymm3
3077 ; AVX1-ONLY-NEXT: vmovaps %ymm4, %ymm5
3078 ; AVX1-ONLY-NEXT: vorps %ymm3, %ymm2, %ymm2
3079 ; AVX1-ONLY-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3080 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm1, %xmm2
3081 ; AVX1-ONLY-NEXT: vpshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
3082 ; AVX1-ONLY-NEXT: # xmm3 = mem[2,2,3,3]
3083 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
3084 ; AVX1-ONLY-NEXT: vpshufb %xmm11, %xmm6, %xmm3
3085 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3,4],xmm3[5,6,7]
3086 ; AVX1-ONLY-NEXT: vpshufb %xmm11, %xmm7, %xmm3
3087 ; AVX1-ONLY-NEXT: vpshufb %xmm10, %xmm8, %xmm4
3088 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm3, %ymm4, %ymm3
3089 ; AVX1-ONLY-NEXT: vandnps %ymm2, %ymm12, %ymm2
3090 ; AVX1-ONLY-NEXT: vandps %ymm3, %ymm12, %ymm1
3091 ; AVX1-ONLY-NEXT: vorps %ymm2, %ymm1, %ymm1
3092 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm13, %xmm2
3093 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
3094 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm3 = xmm12[2,2,3,3]
3095 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
3096 ; AVX1-ONLY-NEXT: vpshufb %xmm10, %xmm0, %xmm0
3097 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm2[3,4],xmm0[5,6,7]
3098 ; AVX1-ONLY-NEXT: vandps %ymm5, %ymm1, %ymm1
3099 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
3100 ; AVX1-ONLY-NEXT: vandnps %ymm0, %ymm5, %ymm0
3101 ; AVX1-ONLY-NEXT: vorps %ymm0, %ymm1, %ymm0
3102 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3103 ; AVX1-ONLY-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
3104 ; AVX1-ONLY-NEXT: # xmm1 = mem[1,1,1,1]
3105 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
3106 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]
3107 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
3108 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
3109 ; AVX1-ONLY-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm13 # 16-byte Folded Reload
3110 ; AVX1-ONLY-NEXT: # xmm13 = xmm0[0,1],mem[2,3],xmm0[4,5,6,7]
3111 ; AVX1-ONLY-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,4,5,4,5,6,7,0,1,4,5,0,1,12,13]
3112 ; AVX1-ONLY-NEXT: vpshufb %xmm2, %xmm13, %xmm3
3113 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm1
3114 ; AVX1-ONLY-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
3115 ; AVX1-ONLY-NEXT: # xmm3 = mem[1,1,1,1]
3116 ; AVX1-ONLY-NEXT: vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
3117 ; AVX1-ONLY-NEXT: # xmm5 = mem[2,3,2,3]
3118 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
3119 ; AVX1-ONLY-NEXT: vpshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
3120 ; AVX1-ONLY-NEXT: # xmm5 = mem[0,1,0,3]
3121 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm6 = xmm5[0,1,2,3,4,5,4,6]
3122 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm6 = xmm6[1],xmm15[1]
3123 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm6[2,3,4,5,6,7]
3124 ; AVX1-ONLY-NEXT: vmovaps {{.*#+}} ymm15 = [65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
3125 ; AVX1-ONLY-NEXT: vandnps %ymm1, %ymm15, %ymm1
3126 ; AVX1-ONLY-NEXT: vandps %ymm3, %ymm15, %ymm3
3127 ; AVX1-ONLY-NEXT: vorps %ymm1, %ymm3, %ymm1
3128 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
3129 ; AVX1-ONLY-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm3 # 16-byte Folded Reload
3130 ; AVX1-ONLY-NEXT: # xmm3 = xmm0[0,1],mem[2,3],xmm0[4,5,6,7]
3131 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm6 = xmm9[0,1,0,3]
3132 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm7 = xmm6[0,1,2,3,4,5,4,6]
3133 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm7 = xmm7[1],xmm14[1]
3134 ; AVX1-ONLY-NEXT: vpshufb %xmm2, %xmm3, %xmm8
3135 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4],xmm8[5,6,7]
3136 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm7, %ymm0, %ymm7
3137 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm7[5,6,7]
3138 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3139 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
3140 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm7 = xmm14[1,1,1,1]
3141 ; AVX1-ONLY-NEXT: vpshufd $238, (%rsp), %xmm8 # 16-byte Folded Reload
3142 ; AVX1-ONLY-NEXT: # xmm8 = mem[2,3,2,3]
3143 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
3144 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
3145 ; AVX1-ONLY-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
3146 ; AVX1-ONLY-NEXT: # xmm0 = xmm0[0,1],mem[2,3],xmm0[4,5,6,7]
3147 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3148 ; AVX1-ONLY-NEXT: vpshufb %xmm2, %xmm0, %xmm9
3149 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm8, %ymm9, %ymm9
3150 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
3151 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm8 = xmm0[1,1,1,1]
3152 ; AVX1-ONLY-NEXT: vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
3153 ; AVX1-ONLY-NEXT: # xmm10 = mem[2,3,2,3]
3154 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3]
3155 ; AVX1-ONLY-NEXT: vpshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
3156 ; AVX1-ONLY-NEXT: # xmm8 = mem[0,1,0,3]
3157 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm11 = xmm8[0,1,2,3,4,5,4,6]
3158 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
3159 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm11 = xmm11[1],xmm7[1]
3160 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1],xmm11[2,3,4,5,6,7]
3161 ; AVX1-ONLY-NEXT: vandnps %ymm9, %ymm15, %ymm9
3162 ; AVX1-ONLY-NEXT: vandps %ymm15, %ymm10, %ymm10
3163 ; AVX1-ONLY-NEXT: vorps %ymm9, %ymm10, %ymm11
3164 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3165 ; AVX1-ONLY-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm10 # 16-byte Folded Reload
3166 ; AVX1-ONLY-NEXT: # xmm10 = xmm1[0,1],mem[2,3],xmm1[4,5,6,7]
3167 ; AVX1-ONLY-NEXT: vpshufb %xmm2, %xmm10, %xmm2
3168 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm9 = xmm12[0,1,0,3]
3169 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm12 = xmm9[0,1,2,3,4,5,4,6]
3170 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3171 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm12 = xmm12[1],xmm1[1]
3172 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm12[0,1,2,3,4],xmm2[5,6,7]
3173 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2
3174 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm2 = ymm11[0,1,2,3,4],ymm2[5,6,7]
3175 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
3176 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm11, %xmm11
3177 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm12 = xmm4[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
3178 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
3179 ; AVX1-ONLY-NEXT: vmovdqa {{.*#+}} xmm11 = [6,7,2,3,4,5,6,7,6,7,6,7,2,3,14,15]
3180 ; AVX1-ONLY-NEXT: vpshufb %xmm11, %xmm13, %xmm4
3181 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm12, %ymm4, %ymm4
3182 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
3183 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm12, %xmm12
3184 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
3185 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm13 = xmm13[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
3186 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
3187 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
3188 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm13, %xmm13
3189 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,5,7]
3190 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm5 = xmm5[1],xmm13[1]
3191 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm5 = xmm12[0,1],xmm5[2,3,4,5,6,7]
3192 ; AVX1-ONLY-NEXT: vandnps %ymm4, %ymm15, %ymm4
3193 ; AVX1-ONLY-NEXT: vandps %ymm5, %ymm15, %ymm5
3194 ; AVX1-ONLY-NEXT: vorps %ymm4, %ymm5, %ymm4
3195 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
3196 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm5, %xmm5
3197 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,5,5,7]
3198 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm5 = xmm6[1],xmm5[1]
3199 ; AVX1-ONLY-NEXT: vpshufb %xmm11, %xmm3, %xmm3
3200 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm3 = xmm5[0,1,2,3,4],xmm3[5,6,7]
3201 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3
3202 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3,4],ymm3[5,6,7]
3203 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm14, %xmm4
3204 ; AVX1-ONLY-NEXT: vmovdqa (%rsp), %xmm5 # 16-byte Reload
3205 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm5 = xmm5[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
3206 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
3207 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
3208 ; AVX1-ONLY-NEXT: vpshufb %xmm11, %xmm5, %xmm5
3209 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm4, %ymm5, %ymm4
3210 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm0, %xmm5
3211 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
3212 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm6 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
3213 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
3214 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm7, %xmm6
3215 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm7 = xmm8[0,1,2,3,4,5,5,7]
3216 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm6 = xmm7[1],xmm6[1]
3217 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1],xmm6[2,3,4,5,6,7]
3218 ; AVX1-ONLY-NEXT: vandnps %ymm4, %ymm15, %ymm4
3219 ; AVX1-ONLY-NEXT: vandps %ymm5, %ymm15, %ymm5
3220 ; AVX1-ONLY-NEXT: vorps %ymm4, %ymm5, %ymm4
3221 ; AVX1-ONLY-NEXT: vpshufb %xmm11, %xmm10, %xmm5
3222 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm1, %xmm6
3223 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm7 = xmm9[0,1,2,3,4,5,5,7]
3224 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm6 = xmm7[1],xmm6[1]
3225 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1,2,3,4],xmm5[5,6,7]
3226 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm5
3227 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3,4],ymm5[5,6,7]
3228 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3229 ; AVX1-ONLY-NEXT: vmovaps %ymm0, 32(%rsi)
3230 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
3231 ; AVX1-ONLY-NEXT: vmovaps %ymm5, (%rsi)
3232 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3233 ; AVX1-ONLY-NEXT: vmovaps %ymm0, 32(%rdx)
3234 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3235 ; AVX1-ONLY-NEXT: vmovaps %ymm0, (%rdx)
3236 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3237 ; AVX1-ONLY-NEXT: vmovaps %ymm0, 32(%rcx)
3238 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3239 ; AVX1-ONLY-NEXT: vmovaps %ymm0, (%rcx)
3240 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3241 ; AVX1-ONLY-NEXT: vmovaps %ymm0, 32(%r8)
3242 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3243 ; AVX1-ONLY-NEXT: vmovaps %ymm0, (%r8)
3244 ; AVX1-ONLY-NEXT: vmovaps %ymm2, 32(%r9)
3245 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3246 ; AVX1-ONLY-NEXT: vmovaps %ymm0, (%r9)
3247 ; AVX1-ONLY-NEXT: movq {{[0-9]+}}(%rsp), %rax
3248 ; AVX1-ONLY-NEXT: vmovaps %ymm4, 32(%rax)
3249 ; AVX1-ONLY-NEXT: vmovaps %ymm3, (%rax)
3250 ; AVX1-ONLY-NEXT: addq $536, %rsp # imm = 0x218
3251 ; AVX1-ONLY-NEXT: vzeroupper
3252 ; AVX1-ONLY-NEXT: retq
3253 ;
3254 ; AVX2-SLOW-LABEL: load_i16_stride6_vf32:
3255 ; AVX2-SLOW: # %bb.0:
3256 ; AVX2-SLOW-NEXT: subq $488, %rsp # imm = 0x1E8
3257 ; AVX2-SLOW-NEXT: vmovdqa (%rdi), %ymm4
3258 ; AVX2-SLOW-NEXT: vmovdqu %ymm4, (%rsp) # 32-byte Spill
3259 ; AVX2-SLOW-NEXT: vmovdqa 32(%rdi), %ymm5
3260 ; AVX2-SLOW-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3261 ; AVX2-SLOW-NEXT: vmovdqa 64(%rdi), %ymm0
3262 ; AVX2-SLOW-NEXT: vmovdqa 96(%rdi), %ymm1
3263 ; AVX2-SLOW-NEXT: vmovdqa 224(%rdi), %ymm6
3264 ; AVX2-SLOW-NEXT: vmovdqa 192(%rdi), %ymm7
3265 ; AVX2-SLOW-NEXT: vmovdqa 288(%rdi), %ymm2
3266 ; AVX2-SLOW-NEXT: vmovdqa 256(%rdi), %ymm3
3267 ; AVX2-SLOW-NEXT: vperm2i128 {{.*#+}} ymm8 = ymm3[2,3],ymm2[2,3]
3268 ; AVX2-SLOW-NEXT: vperm2i128 {{.*#+}} ymm14 = ymm3[0,1],ymm2[0,1]
3269 ; AVX2-SLOW-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
3270 ; AVX2-SLOW-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3271 ; AVX2-SLOW-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[0,1],ymm1[0,1]
3272 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3273 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm9 = [8,9,4,5,4,5,6,7,0,1,4,5,0,1,12,13,24,25,20,21,20,21,22,23,16,17,20,21,16,17,28,29]
3274 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm0[1],ymm2[2,3,4,5],ymm0[6],ymm2[7]
3275 ; AVX2-SLOW-NEXT: vpshufb %ymm9, %ymm1, %ymm10
3276 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1],ymm4[2],ymm5[3,4],ymm4[5],ymm5[6,7]
3277 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} xmm12 = <8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15>
3278 ; AVX2-SLOW-NEXT: vpshufb %xmm12, %xmm2, %xmm0
3279 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm3
3280 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm11 = xmm3[2,2,2,2,4,5,6,7]
3281 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm11 = xmm0[0],xmm11[1],xmm0[2,3],xmm11[4],xmm0[5,6,7]
3282 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0]
3283 ; AVX2-SLOW-NEXT: vpblendvb %ymm0, %ymm11, %ymm10, %ymm4
3284 ; AVX2-SLOW-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3285 ; AVX2-SLOW-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3286 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm11 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7]
3287 ; AVX2-SLOW-NEXT: vpshufb %xmm12, %xmm11, %xmm10
3288 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm11, %xmm12
3289 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm13 = xmm12[2,2,2,2,4,5,6,7]
3290 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0],xmm13[1],xmm10[2,3],xmm13[4],xmm10[5,6,7]
3291 ; AVX2-SLOW-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3292 ; AVX2-SLOW-NEXT: vmovdqa %ymm14, %ymm4
3293 ; AVX2-SLOW-NEXT: vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3294 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm13 = ymm8[0],ymm14[1],ymm8[2,3,4,5],ymm14[6],ymm8[7]
3295 ; AVX2-SLOW-NEXT: vpshufb %ymm9, %ymm13, %ymm9
3296 ; AVX2-SLOW-NEXT: vpblendvb %ymm0, %ymm10, %ymm9, %ymm5
3297 ; AVX2-SLOW-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3298 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} xmm14 = <10,11,u,u,2,3,14,15,u,u,10,11,12,13,14,15>
3299 ; AVX2-SLOW-NEXT: vpshufb %xmm14, %xmm2, %xmm2
3300 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
3301 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,5,5]
3302 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3],xmm3[4],xmm2[5,6,7]
3303 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm3 = [10,11,6,7,4,5,6,7,6,7,6,7,2,3,14,15,26,27,22,23,20,21,22,23,22,23,22,23,18,19,30,31]
3304 ; AVX2-SLOW-NEXT: vpshufb %ymm3, %ymm1, %ymm1
3305 ; AVX2-SLOW-NEXT: vpblendvb %ymm0, %ymm2, %ymm1, %ymm1
3306 ; AVX2-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3307 ; AVX2-SLOW-NEXT: vpshufb %ymm3, %ymm13, %ymm1
3308 ; AVX2-SLOW-NEXT: vpshufb %xmm14, %xmm11, %xmm2
3309 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm3 = ymm7[0],ymm6[1],ymm7[2,3],ymm6[4],ymm7[5,6],ymm6[7]
3310 ; AVX2-SLOW-NEXT: vmovdqa %ymm6, %ymm10
3311 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm12[1,1,2,3]
3312 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,5,5,5,5]
3313 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm11[1],xmm2[2,3],xmm11[4],xmm2[5,6,7]
3314 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} xmm15 = <0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15>
3315 ; AVX2-SLOW-NEXT: vpblendvb %ymm0, %ymm2, %ymm1, %ymm0
3316 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3317 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm3, %xmm7
3318 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm7[0,2,0,3]
3319 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
3320 ; AVX2-SLOW-NEXT: vpshufb %xmm15, %xmm3, %xmm1
3321 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3],xmm0[4,5],xmm1[6,7]
3322 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm13 = <u,u,u,u,u,u,u,u,u,u,u,u,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21>
3323 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm6 = ymm4[0],ymm8[1],ymm4[2,3,4,5],ymm8[6],ymm4[7]
3324 ; AVX2-SLOW-NEXT: vpshufb %ymm13, %ymm6, %ymm4
3325 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm4[3,4,5,6,7]
3326 ; AVX2-SLOW-NEXT: vmovdqa 352(%rdi), %ymm1
3327 ; AVX2-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3328 ; AVX2-SLOW-NEXT: vmovdqa 320(%rdi), %ymm9
3329 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm9[0,1],ymm1[2],ymm9[3,4],ymm1[5],ymm9[6,7]
3330 ; AVX2-SLOW-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3331 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm1[2,2,2,2,4,5,6,7]
3332 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,1,2,2]
3333 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} xmm8 = <0,1,4,5,4,5,u,u,0,1,12,13,u,u,4,5>
3334 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm5
3335 ; AVX2-SLOW-NEXT: vpshufb %xmm8, %xmm5, %xmm12
3336 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm4 = xmm12[0,1,2],xmm4[3],xmm12[4,5],xmm4[6],xmm12[7]
3337 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
3338 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm4 = ymm0[0,1,2],ymm4[3,4,5,6,7],ymm0[8,9,10],ymm4[11,12,13,14,15]
3339 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7]
3340 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3341 ; AVX2-SLOW-NEXT: vmovdqa 160(%rdi), %ymm2
3342 ; AVX2-SLOW-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3343 ; AVX2-SLOW-NEXT: vmovdqa 128(%rdi), %ymm0
3344 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3345 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm2[2],ymm0[3,4],ymm2[5],ymm0[6,7]
3346 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm4
3347 ; AVX2-SLOW-NEXT: vpshufb %xmm8, %xmm4, %xmm8
3348 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm11 = xmm0[2,2,2,2,4,5,6,7]
3349 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[0,1,2,2]
3350 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2],xmm11[3],xmm8[4,5],xmm11[6],xmm8[7]
3351 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
3352 ; AVX2-SLOW-NEXT: vpblendd $109, (%rsp), %ymm2, %ymm11 # 32-byte Folded Reload
3353 ; AVX2-SLOW-NEXT: # ymm11 = mem[0],ymm2[1],mem[2,3],ymm2[4],mem[5,6],ymm2[7]
3354 ; AVX2-SLOW-NEXT: vpshufb %xmm15, %xmm11, %xmm15
3355 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm11, %xmm2
3356 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm2[0,2,0,3]
3357 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,6,6,7]
3358 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm12 = xmm15[0,1],xmm12[2],xmm15[3],xmm12[4,5],xmm15[6,7]
3359 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
3360 ; AVX2-SLOW-NEXT: vpblendd $66, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm15 # 32-byte Folded Reload
3361 ; AVX2-SLOW-NEXT: # ymm15 = ymm14[0],mem[1],ymm14[2,3,4,5],mem[6],ymm14[7]
3362 ; AVX2-SLOW-NEXT: vpshufb %ymm13, %ymm15, %ymm13
3363 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm12 = ymm12[0,1,2],ymm13[3,4,5,6,7]
3364 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
3365 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm8 = ymm12[0,1,2],ymm8[3,4,5,6,7],ymm12[8,9,10],ymm8[11,12,13,14,15]
3366 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm8 = ymm12[0,1,2,3],ymm8[4,5,6,7]
3367 ; AVX2-SLOW-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3368 ; AVX2-SLOW-NEXT: vpbroadcastq {{.*#+}} xmm12 = [2,3,14,15,10,11,0,0,2,3,14,15,10,11,0,0]
3369 ; AVX2-SLOW-NEXT: vpshufb %xmm12, %xmm7, %xmm7
3370 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} xmm13 = <2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15>
3371 ; AVX2-SLOW-NEXT: vpshufb %xmm13, %xmm3, %xmm3
3372 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm7[2],xmm3[3],xmm7[4,5],xmm3[6,7]
3373 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm7 = <u,u,u,u,u,u,u,u,u,u,u,u,10,11,6,7,18,19,30,31,26,27,26,27,30,31,26,27,26,27,22,23>
3374 ; AVX2-SLOW-NEXT: vpshufb %ymm7, %ymm6, %ymm6
3375 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2],ymm6[3,4,5,6,7]
3376 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} xmm6 = <6,7,2,3,4,5,u,u,2,3,14,15,u,u,6,7>
3377 ; AVX2-SLOW-NEXT: vpshufb %xmm6, %xmm5, %xmm5
3378 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5]
3379 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0,1,2],xmm1[3],xmm5[4,5],xmm1[6],xmm5[7]
3380 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
3381 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3,4,5,6,7],ymm3[8,9,10],ymm1[11,12,13,14,15]
3382 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
3383 ; AVX2-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3384 ; AVX2-SLOW-NEXT: vpshufb %xmm6, %xmm4, %xmm1
3385 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
3386 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3],xmm1[4,5],xmm0[6],xmm1[7]
3387 ; AVX2-SLOW-NEXT: vpshufb %xmm12, %xmm2, %xmm1
3388 ; AVX2-SLOW-NEXT: vpshufb %xmm13, %xmm11, %xmm2
3389 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2],xmm2[3],xmm1[4,5],xmm2[6,7]
3390 ; AVX2-SLOW-NEXT: vpshufb %ymm7, %ymm15, %ymm2
3391 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
3392 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
3393 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
3394 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
3395 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3396 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3397 ; AVX2-SLOW-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
3398 ; AVX2-SLOW-NEXT: # ymm2 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6],ymm0[7]
3399 ; AVX2-SLOW-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm1 # 32-byte Folded Reload
3400 ; AVX2-SLOW-NEXT: # ymm1 = mem[0,1],ymm9[2],mem[3,4],ymm9[5],mem[6,7]
3401 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm4
3402 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,1,2,1]
3403 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm1[2,1,0,3]
3404 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm6[0,0,0,0,4,5,6,7]
3405 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,6,7]
3406 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm5 = xmm4[0,1,2,3,6,5,6,4]
3407 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm5[4],xmm1[5,6],xmm5[7]
3408 ; AVX2-SLOW-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm5 # 32-byte Folded Reload
3409 ; AVX2-SLOW-NEXT: # ymm5 = mem[0,1],ymm10[2],mem[3,4],ymm10[5],mem[6,7]
3410 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[2,1,2,3]
3411 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm5, %xmm5
3412 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm5[0,3,2,1]
3413 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm11[0,0,2,3,4,5,6,7]
3414 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[0,1,3,3]
3415 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm12 = xmm7[2,1,2,0,4,5,6,7]
3416 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm5 = xmm12[0],xmm5[1,2],xmm12[3],xmm5[4,5,6,7]
3417 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
3418 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm13 = ymm2[4,5,0,1,12,13,14,15,8,9,12,13,0,1,u,u,20,21,16,17,28,29,30,31,24,25,28,29,16,17,u,u]
3419 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = ymm13[0,1,2],ymm1[3,4,5,6,7],ymm13[8,9,10],ymm1[11,12,13,14,15]
3420 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,4,6,5,4]
3421 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4],xmm13[5,6,7]
3422 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3],ymm1[4,5,6,7]
3423 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3424 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
3425 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
3426 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm10[0,1],ymm9[2],ymm10[3,4],ymm9[5],ymm10[6,7]
3427 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm5, %xmm13
3428 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[0,1,2,1]
3429 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm15 = xmm5[2,1,0,3]
3430 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm15[0,0,0,0,4,5,6,7]
3431 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,4,6,7]
3432 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm8 = xmm13[0,1,2,3,6,5,6,4]
3433 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3],xmm8[4],xmm5[5,6],xmm8[7]
3434 ; AVX2-SLOW-NEXT: vpblendd $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm8 # 32-byte Folded Reload
3435 ; AVX2-SLOW-NEXT: # ymm8 = ymm14[0,1],mem[2],ymm14[3],mem[4],ymm14[5,6],mem[7]
3436 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3437 ; AVX2-SLOW-NEXT: vpblendd $219, (%rsp), %ymm0, %ymm0 # 32-byte Folded Reload
3438 ; AVX2-SLOW-NEXT: # ymm0 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
3439 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,1,2,3]
3440 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm0
3441 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]
3442 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm0[0,0,2,3,4,5,6,7]
3443 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,1,3,3]
3444 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm12 = xmm1[2,1,2,0,4,5,6,7]
3445 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm12[0],xmm3[1,2],xmm12[3],xmm3[4,5,6,7]
3446 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
3447 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm12 = ymm8[4,5,0,1,12,13,14,15,8,9,12,13,0,1,u,u,20,21,16,17,28,29,30,31,24,25,28,29,16,17,u,u]
3448 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm5 = ymm12[0,1,2],ymm5[3,4,5,6,7],ymm12[8,9,10],ymm5[11,12,13,14,15]
3449 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,6,5,4]
3450 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm12[5,6,7]
3451 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm3[0,1,2,3],ymm5[4,5,6,7]
3452 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm4[0,1,2,3,7,5,6,5]
3453 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm6[1,1,1,1,4,5,6,7]
3454 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,7,7]
3455 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4],xmm4[5,6],xmm3[7]
3456 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm7[3,1,2,1,4,5,6,7]
3457 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm11[0,1,3,3,4,5,6,7]
3458 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,7,7,7,7]
3459 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm6[1,2],xmm4[3],xmm6[4,5,6,7]
3460 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm6 = <6,7,2,3,14,15,14,15,14,15,10,11,u,u,2,3,22,23,18,19,30,31,30,31,30,31,26,27,u,u,18,19>
3461 ; AVX2-SLOW-NEXT: vpshufb %ymm6, %ymm2, %ymm2
3462 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
3463 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = ymm2[0,1,2],ymm3[3,4,5,6,7],ymm2[8,9,10],ymm3[11,12,13,14,15]
3464 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,3,2]
3465 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3,4],xmm2[5,6,7]
3466 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
3467 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm13[0,1,2,3,7,5,6,5]
3468 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm15[1,1,1,1,4,5,6,7]
3469 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,7,7]
3470 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4],xmm4[5,6],xmm3[7]
3471 ; AVX2-SLOW-NEXT: vpshufb %ymm6, %ymm8, %ymm4
3472 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,1,4,5,6,7]
3473 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,3,3,4,5,6,7]
3474 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,7,7]
3475 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5,6,7]
3476 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm1
3477 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = ymm4[0,1,2],ymm1[3,4,5,6,7],ymm4[8,9,10],ymm1[11,12,13,14,15]
3478 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm4[0,1,3,2]
3479 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm3[5,6,7]
3480 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
3481 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm9[0],ymm10[1],ymm9[2,3],ymm10[4],ymm9[5,6],ymm10[7]
3482 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm3
3483 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,3,2,1]
3484 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm1[0,1,0,2,4,5,6,7]
3485 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,6,6,6,6]
3486 ; AVX2-SLOW-NEXT: vpbroadcastq {{.*#+}} xmm6 = [8,9,0,0,0,1,12,13,8,9,0,0,0,1,12,13]
3487 ; AVX2-SLOW-NEXT: vpshufb %xmm6, %xmm3, %xmm7
3488 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm7[4],xmm4[5],xmm7[6,7]
3489 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
3490 ; AVX2-SLOW-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
3491 ; AVX2-SLOW-NEXT: # ymm4 = mem[0,1,2,3,4],ymm4[5,6,7]
3492 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
3493 ; AVX2-SLOW-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
3494 ; AVX2-SLOW-NEXT: # ymm7 = ymm7[0],mem[1],ymm7[2,3],mem[4],ymm7[5,6],mem[7]
3495 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm7, %xmm8
3496 ; AVX2-SLOW-NEXT: vpshufb %xmm6, %xmm8, %xmm6
3497 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[0,3,2,1]
3498 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm9 = xmm7[0,1,0,2,4,5,6,7]
3499 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,6,6,6,6]
3500 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm6 = xmm9[0,1,2,3],xmm6[4],xmm9[5],xmm6[6,7]
3501 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
3502 ; AVX2-SLOW-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload
3503 ; AVX2-SLOW-NEXT: # ymm6 = mem[0,1,2,3,4],ymm6[5,6,7]
3504 ; AVX2-SLOW-NEXT: vpbroadcastq {{.*#+}} xmm9 = [10,11,0,0,2,3,14,15,10,11,0,0,2,3,14,15]
3505 ; AVX2-SLOW-NEXT: vpshufb %xmm9, %xmm3, %xmm3
3506 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]
3507 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,3,3]
3508 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4],xmm1[5],xmm3[6,7]
3509 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
3510 ; AVX2-SLOW-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
3511 ; AVX2-SLOW-NEXT: # ymm1 = mem[0,1,2,3,4],ymm1[5,6,7]
3512 ; AVX2-SLOW-NEXT: vpshufb %xmm9, %xmm8, %xmm3
3513 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,1,3,4,5,6,7]
3514 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[0,1,3,3]
3515 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm7[0,1,2,3],xmm3[4],xmm7[5],xmm3[6,7]
3516 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
3517 ; AVX2-SLOW-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
3518 ; AVX2-SLOW-NEXT: # ymm3 = mem[0,1,2,3,4],ymm3[5,6,7]
3519 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
3520 ; AVX2-SLOW-NEXT: vmovaps %ymm7, 32(%rsi)
3521 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
3522 ; AVX2-SLOW-NEXT: vmovaps %ymm7, (%rsi)
3523 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
3524 ; AVX2-SLOW-NEXT: vmovaps %ymm7, 32(%rdx)
3525 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
3526 ; AVX2-SLOW-NEXT: vmovaps %ymm7, (%rdx)
3527 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
3528 ; AVX2-SLOW-NEXT: vmovaps %ymm7, 32(%rcx)
3529 ; AVX2-SLOW-NEXT: vmovdqa %ymm5, (%rcx)
3530 ; AVX2-SLOW-NEXT: vmovdqa %ymm2, 32(%r8)
3531 ; AVX2-SLOW-NEXT: vmovdqa %ymm0, (%r8)
3532 ; AVX2-SLOW-NEXT: vmovdqa %ymm6, 32(%r9)
3533 ; AVX2-SLOW-NEXT: vmovdqa %ymm4, (%r9)
3534 ; AVX2-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
3535 ; AVX2-SLOW-NEXT: vmovdqa %ymm3, 32(%rax)
3536 ; AVX2-SLOW-NEXT: vmovdqa %ymm1, (%rax)
3537 ; AVX2-SLOW-NEXT: addq $488, %rsp # imm = 0x1E8
3538 ; AVX2-SLOW-NEXT: vzeroupper
3539 ; AVX2-SLOW-NEXT: retq
3540 ;
3541 ; AVX2-FAST-LABEL: load_i16_stride6_vf32:
3542 ; AVX2-FAST: # %bb.0:
3543 ; AVX2-FAST-NEXT: subq $488, %rsp # imm = 0x1E8
3544 ; AVX2-FAST-NEXT: vmovdqa (%rdi), %ymm4
3545 ; AVX2-FAST-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3546 ; AVX2-FAST-NEXT: vmovdqa 32(%rdi), %ymm5
3547 ; AVX2-FAST-NEXT: vmovdqu %ymm5, (%rsp) # 32-byte Spill
3548 ; AVX2-FAST-NEXT: vmovdqa 64(%rdi), %ymm0
3549 ; AVX2-FAST-NEXT: vmovdqa 96(%rdi), %ymm1
3550 ; AVX2-FAST-NEXT: vmovdqa 224(%rdi), %ymm6
3551 ; AVX2-FAST-NEXT: vmovdqa 192(%rdi), %ymm7
3552 ; AVX2-FAST-NEXT: vmovdqa 288(%rdi), %ymm2
3553 ; AVX2-FAST-NEXT: vmovdqa 256(%rdi), %ymm3
3554 ; AVX2-FAST-NEXT: vperm2i128 {{.*#+}} ymm8 = ymm3[2,3],ymm2[2,3]
3555 ; AVX2-FAST-NEXT: vperm2i128 {{.*#+}} ymm14 = ymm3[0,1],ymm2[0,1]
3556 ; AVX2-FAST-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
3557 ; AVX2-FAST-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3558 ; AVX2-FAST-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[0,1],ymm1[0,1]
3559 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3560 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm9 = [8,9,4,5,4,5,6,7,0,1,4,5,0,1,12,13,24,25,20,21,20,21,22,23,16,17,20,21,16,17,28,29]
3561 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm0[1],ymm2[2,3,4,5],ymm0[6],ymm2[7]
3562 ; AVX2-FAST-NEXT: vpshufb %ymm9, %ymm1, %ymm10
3563 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1],ymm4[2],ymm5[3,4],ymm4[5],ymm5[6,7]
3564 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm12 = <8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15>
3565 ; AVX2-FAST-NEXT: vpshufb %xmm12, %xmm2, %xmm0
3566 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm2, %xmm3
3567 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm11 = xmm3[2,2,2,2,4,5,6,7]
3568 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm11 = xmm0[0],xmm11[1],xmm0[2,3],xmm11[4],xmm0[5,6,7]
3569 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0]
3570 ; AVX2-FAST-NEXT: vpblendvb %ymm0, %ymm11, %ymm10, %ymm4
3571 ; AVX2-FAST-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3572 ; AVX2-FAST-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3573 ; AVX2-FAST-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3574 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm11 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7]
3575 ; AVX2-FAST-NEXT: vpshufb %xmm12, %xmm11, %xmm10
3576 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm11, %xmm12
3577 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm13 = xmm12[2,2,2,2,4,5,6,7]
3578 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0],xmm13[1],xmm10[2,3],xmm13[4],xmm10[5,6,7]
3579 ; AVX2-FAST-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3580 ; AVX2-FAST-NEXT: vmovdqa %ymm14, %ymm5
3581 ; AVX2-FAST-NEXT: vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3582 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm13 = ymm8[0],ymm14[1],ymm8[2,3,4,5],ymm14[6],ymm8[7]
3583 ; AVX2-FAST-NEXT: vpshufb %ymm9, %ymm13, %ymm9
3584 ; AVX2-FAST-NEXT: vpblendvb %ymm0, %ymm10, %ymm9, %ymm4
3585 ; AVX2-FAST-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3586 ; AVX2-FAST-NEXT: vpbroadcastd {{.*#+}} xmm14 = [10,11,6,7,10,11,6,7,10,11,6,7,10,11,6,7]
3587 ; AVX2-FAST-NEXT: vpshufb %xmm14, %xmm3, %xmm3
3588 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm15 = <10,11,u,u,2,3,14,15,u,u,10,11,12,13,14,15>
3589 ; AVX2-FAST-NEXT: vpshufb %xmm15, %xmm2, %xmm2
3590 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3],xmm3[4],xmm2[5,6,7]
3591 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm3 = [10,11,6,7,4,5,6,7,6,7,6,7,2,3,14,15,26,27,22,23,20,21,22,23,22,23,22,23,18,19,30,31]
3592 ; AVX2-FAST-NEXT: vpshufb %ymm3, %ymm1, %ymm1
3593 ; AVX2-FAST-NEXT: vpblendvb %ymm0, %ymm2, %ymm1, %ymm1
3594 ; AVX2-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3595 ; AVX2-FAST-NEXT: vpshufb %ymm3, %ymm13, %ymm1
3596 ; AVX2-FAST-NEXT: vpshufb %xmm14, %xmm12, %xmm2
3597 ; AVX2-FAST-NEXT: vpshufb %xmm15, %xmm11, %xmm3
3598 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm4 = ymm7[0],ymm6[1],ymm7[2,3],ymm6[4],ymm7[5,6],ymm6[7]
3599 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2,3],xmm2[4],xmm3[5,6,7]
3600 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm13 = <0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15>
3601 ; AVX2-FAST-NEXT: vpblendvb %ymm0, %ymm2, %ymm1, %ymm0
3602 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3603 ; AVX2-FAST-NEXT: vpshufb %xmm13, %xmm4, %xmm1
3604 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm4, %xmm2
3605 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm10 = xmm2[2,1,0,3]
3606 ; AVX2-FAST-NEXT: vpbroadcastq {{.*#+}} xmm15 = [8,9,12,13,0,1,0,0,8,9,12,13,0,1,0,0]
3607 ; AVX2-FAST-NEXT: vpshufb %xmm15, %xmm10, %xmm3
3608 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2],xmm1[3],xmm3[4,5],xmm1[6,7]
3609 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm7 = ymm5[0],ymm8[1],ymm5[2,3,4,5],ymm8[6],ymm5[7]
3610 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm3 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
3611 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2],ymm3[3,4,5,6,7]
3612 ; AVX2-FAST-NEXT: vmovdqa 352(%rdi), %ymm0
3613 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3614 ; AVX2-FAST-NEXT: vmovdqa 320(%rdi), %ymm11
3615 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm11[0,1],ymm0[2],ymm11[3,4],ymm0[5],ymm11[6,7]
3616 ; AVX2-FAST-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3617 ; AVX2-FAST-NEXT: vpbroadcastd {{.*#+}} xmm9 = [8,9,4,5,8,9,4,5,8,9,4,5,8,9,4,5]
3618 ; AVX2-FAST-NEXT: vpshufb %xmm9, %xmm2, %xmm6
3619 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm8 = <0,1,4,5,4,5,u,u,0,1,12,13,u,u,4,5>
3620 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm2, %xmm5
3621 ; AVX2-FAST-NEXT: vpshufb %xmm8, %xmm5, %xmm12
3622 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm6 = xmm12[0,1,2],xmm6[3],xmm12[4,5],xmm6[6],xmm12[7]
3623 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
3624 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm6 = ymm1[0,1,2],ymm6[3,4,5,6,7],ymm1[8,9,10],ymm6[11,12,13,14,15]
3625 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm6[4,5,6,7]
3626 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3627 ; AVX2-FAST-NEXT: vmovdqa 160(%rdi), %ymm6
3628 ; AVX2-FAST-NEXT: vmovdqa 128(%rdi), %ymm0
3629 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3630 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm6[2],ymm0[3,4],ymm6[5],ymm0[6,7]
3631 ; AVX2-FAST-NEXT: vpshufb %xmm9, %xmm0, %xmm9
3632 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm1
3633 ; AVX2-FAST-NEXT: vpshufb %xmm8, %xmm1, %xmm8
3634 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2],xmm9[3],xmm8[4,5],xmm9[6],xmm8[7]
3635 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
3636 ; AVX2-FAST-NEXT: vpblendd $146, (%rsp), %ymm3, %ymm9 # 32-byte Folded Reload
3637 ; AVX2-FAST-NEXT: # ymm9 = ymm3[0],mem[1],ymm3[2,3],mem[4],ymm3[5,6],mem[7]
3638 ; AVX2-FAST-NEXT: vpshufb %xmm13, %xmm9, %xmm3
3639 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm9, %xmm12
3640 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[2,1,0,3]
3641 ; AVX2-FAST-NEXT: vpshufb %xmm15, %xmm12, %xmm15
3642 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm15[2],xmm3[3],xmm15[4,5],xmm3[6,7]
3643 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
3644 ; AVX2-FAST-NEXT: vpblendd $66, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm15 # 32-byte Folded Reload
3645 ; AVX2-FAST-NEXT: # ymm15 = ymm14[0],mem[1],ymm14[2,3,4,5],mem[6],ymm14[7]
3646 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm13 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
3647 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2],ymm13[3,4,5,6,7]
3648 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
3649 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm8 = ymm3[0,1,2],ymm8[3,4,5,6,7],ymm3[8,9,10],ymm8[11,12,13,14,15]
3650 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm8[4,5,6,7]
3651 ; AVX2-FAST-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3652 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm13 = <2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15>
3653 ; AVX2-FAST-NEXT: vpshufb %xmm13, %xmm4, %xmm3
3654 ; AVX2-FAST-NEXT: vpbroadcastq {{.*#+}} xmm4 = [10,11,14,15,2,3,0,0,10,11,14,15,2,3,0,0]
3655 ; AVX2-FAST-NEXT: vpshufb %xmm4, %xmm10, %xmm10
3656 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm10[2],xmm3[3],xmm10[4,5],xmm3[6,7]
3657 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm10 = <u,u,u,u,u,u,u,u,u,u,u,u,10,11,6,7,18,19,30,31,26,27,26,27,30,31,26,27,26,27,22,23>
3658 ; AVX2-FAST-NEXT: vpshufb %ymm10, %ymm7, %ymm7
3659 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2],ymm7[3,4,5,6,7]
3660 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm7 = <6,7,2,3,4,5,u,u,2,3,14,15,u,u,6,7>
3661 ; AVX2-FAST-NEXT: vpshufb %xmm7, %xmm5, %xmm5
3662 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5]
3663 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm5[0,1,2],xmm2[3],xmm5[4,5],xmm2[6],xmm5[7]
3664 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
3665 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3,4,5,6,7],ymm3[8,9,10],ymm2[11,12,13,14,15]
3666 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
3667 ; AVX2-FAST-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3668 ; AVX2-FAST-NEXT: vpshufb %xmm7, %xmm1, %xmm1
3669 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
3670 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3],xmm1[4,5],xmm0[6],xmm1[7]
3671 ; AVX2-FAST-NEXT: vpshufb %xmm13, %xmm9, %xmm1
3672 ; AVX2-FAST-NEXT: vpshufb %xmm4, %xmm12, %xmm2
3673 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3],xmm2[4,5],xmm1[6,7]
3674 ; AVX2-FAST-NEXT: vpshufb %ymm10, %ymm15, %ymm2
3675 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
3676 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
3677 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
3678 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
3679 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3680 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3681 ; AVX2-FAST-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
3682 ; AVX2-FAST-NEXT: # ymm3 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6],ymm0[7]
3683 ; AVX2-FAST-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm1 # 32-byte Folded Reload
3684 ; AVX2-FAST-NEXT: # ymm1 = mem[0,1],ymm11[2],mem[3,4],ymm11[5],mem[6,7]
3685 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,1,0,3]
3686 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm1, %xmm1
3687 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[0,1,2,1]
3688 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm7 = <0,1,0,1,0,1,0,1,u,u,8,9,12,13,u,u>
3689 ; AVX2-FAST-NEXT: vpshufb %xmm7, %xmm5, %xmm4
3690 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm9 = xmm2[0,1,2,3,6,5,6,4]
3691 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm9[4],xmm4[5,6],xmm9[7]
3692 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3693 ; AVX2-FAST-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm9 # 32-byte Folded Reload
3694 ; AVX2-FAST-NEXT: # ymm9 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
3695 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm9, %xmm10
3696 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[0,3,2,1]
3697 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[2,1,2,3]
3698 ; AVX2-FAST-NEXT: vpbroadcastq {{.*#+}} xmm12 = [12,13,0,1,4,5,0,0,12,13,0,1,4,5,0,0]
3699 ; AVX2-FAST-NEXT: vpshufb %xmm12, %xmm10, %xmm13
3700 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm15 = xmm9[2,1,2,0,4,5,6,7]
3701 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm13 = xmm15[0],xmm13[1,2],xmm15[3],xmm13[4,5,6,7]
3702 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
3703 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm3[4,5,0,1,12,13,14,15,8,9,12,13,0,1,u,u,20,21,16,17,28,29,30,31,24,25,28,29,16,17,u,u]
3704 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm4 = ymm0[0,1,2],ymm4[3,4,5,6,7],ymm0[8,9,10],ymm4[11,12,13,14,15]
3705 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,5,4]
3706 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm13[0,1,2,3,4],xmm0[5,6,7]
3707 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7]
3708 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3709 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
3710 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1],ymm11[2],ymm6[3,4],ymm11[5],ymm6[6,7]
3711 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm13 = xmm0[2,1,0,3]
3712 ; AVX2-FAST-NEXT: vpshufb %xmm7, %xmm13, %xmm7
3713 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm0
3714 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,1,2,1]
3715 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm8 = xmm1[0,1,2,3,6,5,6,4]
3716 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3],xmm8[4],xmm7[5,6],xmm8[7]
3717 ; AVX2-FAST-NEXT: vpblendd $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm8 # 32-byte Folded Reload
3718 ; AVX2-FAST-NEXT: # ymm8 = ymm14[0,1],mem[2],ymm14[3],mem[4],ymm14[5,6],mem[7]
3719 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3720 ; AVX2-FAST-NEXT: vpblendd $36, (%rsp), %ymm0, %ymm0 # 32-byte Folded Reload
3721 ; AVX2-FAST-NEXT: # ymm0 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
3722 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm4
3723 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,3,2,1]
3724 ; AVX2-FAST-NEXT: vpshufb %xmm12, %xmm4, %xmm12
3725 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]
3726 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm15 = xmm0[2,1,2,0,4,5,6,7]
3727 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm12 = xmm15[0],xmm12[1,2],xmm15[3],xmm12[4,5,6,7]
3728 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
3729 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm15 = ymm8[4,5,0,1,12,13,14,15,8,9,12,13,0,1,u,u,20,21,16,17,28,29,30,31,24,25,28,29,16,17,u,u]
3730 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm7 = ymm15[0,1,2],ymm7[3,4,5,6,7],ymm15[8,9,10],ymm7[11,12,13,14,15]
3731 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,6,5,4]
3732 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1,2,3,4],xmm15[5,6,7]
3733 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm7 = ymm12[0,1,2,3],ymm7[4,5,6,7]
3734 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm12 = <2,3,2,3,2,3,2,3,u,u,10,11,14,15,u,u>
3735 ; AVX2-FAST-NEXT: vpshufb %xmm12, %xmm5, %xmm5
3736 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,5]
3737 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm5[0,1,2,3],xmm2[4],xmm5[5,6],xmm2[7]
3738 ; AVX2-FAST-NEXT: vpbroadcastq {{.*#+}} xmm5 = [14,15,2,3,6,7,0,0,14,15,2,3,6,7,0,0]
3739 ; AVX2-FAST-NEXT: vpshufb %xmm5, %xmm10, %xmm10
3740 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[3,1,2,1,4,5,6,7]
3741 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0],xmm10[1,2],xmm9[3],xmm10[4,5,6,7]
3742 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm10 = <6,7,2,3,14,15,14,15,14,15,10,11,u,u,2,3,22,23,18,19,30,31,30,31,30,31,26,27,u,u,18,19>
3743 ; AVX2-FAST-NEXT: vpshufb %ymm10, %ymm3, %ymm3
3744 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
3745 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3,4,5,6,7],ymm3[8,9,10],ymm2[11,12,13,14,15]
3746 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,1,3,2]
3747 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm9[0,1,2,3,4],xmm3[5,6,7]
3748 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
3749 ; AVX2-FAST-NEXT: vpshufb %xmm12, %xmm13, %xmm3
3750 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,5]
3751 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4],xmm3[5,6],xmm1[7]
3752 ; AVX2-FAST-NEXT: vpshufb %ymm10, %ymm8, %ymm3
3753 ; AVX2-FAST-NEXT: vpshufb %xmm5, %xmm4, %xmm4
3754 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[3,1,2,1,4,5,6,7]
3755 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm4[1,2],xmm0[3],xmm4[4,5,6,7]
3756 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
3757 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3,4,5,6,7],ymm3[8,9,10],ymm1[11,12,13,14,15]
3758 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,1,3,2]
3759 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm3[5,6,7]
3760 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
3761 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0],ymm6[1],ymm11[2,3],ymm6[4],ymm11[5,6],ymm6[7]
3762 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm1, %xmm3
3763 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,3,2,1]
3764 ; AVX2-FAST-NEXT: vpbroadcastq {{.*#+}} xmm4 = [8,9,0,0,0,1,12,13,8,9,0,0,0,1,12,13]
3765 ; AVX2-FAST-NEXT: vpshufb %xmm4, %xmm3, %xmm5
3766 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm6 = <0,1,2,3,0,1,4,5,u,u,12,13,u,u,u,u>
3767 ; AVX2-FAST-NEXT: vpshufb %xmm6, %xmm1, %xmm8
3768 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm5 = xmm8[0,1,2,3],xmm5[4],xmm8[5],xmm5[6,7]
3769 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
3770 ; AVX2-FAST-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
3771 ; AVX2-FAST-NEXT: # ymm5 = mem[0,1,2,3,4],ymm5[5,6,7]
3772 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
3773 ; AVX2-FAST-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
3774 ; AVX2-FAST-NEXT: # ymm8 = ymm8[0],mem[1],ymm8[2,3],mem[4],ymm8[5,6],mem[7]
3775 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm8, %xmm9
3776 ; AVX2-FAST-NEXT: vpshufb %xmm4, %xmm9, %xmm4
3777 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[0,3,2,1]
3778 ; AVX2-FAST-NEXT: vpshufb %xmm6, %xmm8, %xmm6
3779 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm4 = xmm6[0,1,2,3],xmm4[4],xmm6[5],xmm4[6,7]
3780 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
3781 ; AVX2-FAST-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
3782 ; AVX2-FAST-NEXT: # ymm4 = mem[0,1,2,3,4],ymm4[5,6,7]
3783 ; AVX2-FAST-NEXT: vpbroadcastq {{.*#+}} xmm6 = [10,11,0,0,2,3,14,15,10,11,0,0,2,3,14,15]
3784 ; AVX2-FAST-NEXT: vpshufb %xmm6, %xmm3, %xmm3
3785 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm10 = <0,1,2,3,2,3,6,7,u,u,14,15,u,u,u,u>
3786 ; AVX2-FAST-NEXT: vpshufb %xmm10, %xmm1, %xmm1
3787 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4],xmm1[5],xmm3[6,7]
3788 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
3789 ; AVX2-FAST-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
3790 ; AVX2-FAST-NEXT: # ymm1 = mem[0,1,2,3,4],ymm1[5,6,7]
3791 ; AVX2-FAST-NEXT: vpshufb %xmm6, %xmm9, %xmm3
3792 ; AVX2-FAST-NEXT: vpshufb %xmm10, %xmm8, %xmm6
3793 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm6[0,1,2,3],xmm3[4],xmm6[5],xmm3[6,7]
3794 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
3795 ; AVX2-FAST-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
3796 ; AVX2-FAST-NEXT: # ymm3 = mem[0,1,2,3,4],ymm3[5,6,7]
3797 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
3798 ; AVX2-FAST-NEXT: vmovaps %ymm6, 32(%rsi)
3799 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
3800 ; AVX2-FAST-NEXT: vmovaps %ymm6, (%rsi)
3801 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
3802 ; AVX2-FAST-NEXT: vmovaps %ymm6, 32(%rdx)
3803 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
3804 ; AVX2-FAST-NEXT: vmovaps %ymm6, (%rdx)
3805 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
3806 ; AVX2-FAST-NEXT: vmovaps %ymm6, 32(%rcx)
3807 ; AVX2-FAST-NEXT: vmovdqa %ymm7, (%rcx)
3808 ; AVX2-FAST-NEXT: vmovdqa %ymm2, 32(%r8)
3809 ; AVX2-FAST-NEXT: vmovdqa %ymm0, (%r8)
3810 ; AVX2-FAST-NEXT: vmovdqa %ymm4, 32(%r9)
3811 ; AVX2-FAST-NEXT: vmovdqa %ymm5, (%r9)
3812 ; AVX2-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
3813 ; AVX2-FAST-NEXT: vmovdqa %ymm3, 32(%rax)
3814 ; AVX2-FAST-NEXT: vmovdqa %ymm1, (%rax)
3815 ; AVX2-FAST-NEXT: addq $488, %rsp # imm = 0x1E8
3816 ; AVX2-FAST-NEXT: vzeroupper
3817 ; AVX2-FAST-NEXT: retq
3818 ;
3819 ; AVX2-FAST-PERLANE-LABEL: load_i16_stride6_vf32:
3820 ; AVX2-FAST-PERLANE: # %bb.0:
3821 ; AVX2-FAST-PERLANE-NEXT: subq $488, %rsp # imm = 0x1E8
3822 ; AVX2-FAST-PERLANE-NEXT: vmovdqa (%rdi), %ymm4
3823 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3824 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %ymm5
3825 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm5, (%rsp) # 32-byte Spill
3826 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 64(%rdi), %ymm0
3827 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 96(%rdi), %ymm1
3828 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 224(%rdi), %ymm6
3829 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 192(%rdi), %ymm7
3830 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 288(%rdi), %ymm2
3831 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 256(%rdi), %ymm3
3832 ; AVX2-FAST-PERLANE-NEXT: vperm2i128 {{.*#+}} ymm8 = ymm3[2,3],ymm2[2,3]
3833 ; AVX2-FAST-PERLANE-NEXT: vperm2i128 {{.*#+}} ymm14 = ymm3[0,1],ymm2[0,1]
3834 ; AVX2-FAST-PERLANE-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
3835 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3836 ; AVX2-FAST-PERLANE-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[0,1],ymm1[0,1]
3837 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3838 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} ymm9 = [8,9,4,5,4,5,6,7,0,1,4,5,0,1,12,13,24,25,20,21,20,21,22,23,16,17,20,21,16,17,28,29]
3839 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm0[1],ymm2[2,3,4,5],ymm0[6],ymm2[7]
3840 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm9, %ymm1, %ymm10
3841 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1],ymm4[2],ymm5[3,4],ymm4[5],ymm5[6,7]
3842 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm12 = <8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15>
3843 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm12, %xmm2, %xmm0
3844 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm2, %xmm3
3845 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm11 = xmm3[2,2,2,2,4,5,6,7]
3846 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm11 = xmm0[0],xmm11[1],xmm0[2,3],xmm11[4],xmm0[5,6,7]
3847 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0]
3848 ; AVX2-FAST-PERLANE-NEXT: vpblendvb %ymm0, %ymm11, %ymm10, %ymm4
3849 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3850 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3851 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3852 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm11 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7]
3853 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm12, %xmm11, %xmm10
3854 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm11, %xmm12
3855 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm13 = xmm12[2,2,2,2,4,5,6,7]
3856 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0],xmm13[1],xmm10[2,3],xmm13[4],xmm10[5,6,7]
3857 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3858 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm14, %ymm5
3859 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3860 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm13 = ymm8[0],ymm14[1],ymm8[2,3,4,5],ymm14[6],ymm8[7]
3861 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm9, %ymm13, %ymm9
3862 ; AVX2-FAST-PERLANE-NEXT: vpblendvb %ymm0, %ymm10, %ymm9, %ymm4
3863 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3864 ; AVX2-FAST-PERLANE-NEXT: vpbroadcastd {{.*#+}} xmm14 = [10,11,6,7,10,11,6,7,10,11,6,7,10,11,6,7]
3865 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm14, %xmm3, %xmm3
3866 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm15 = <10,11,u,u,2,3,14,15,u,u,10,11,12,13,14,15>
3867 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm15, %xmm2, %xmm2
3868 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3],xmm3[4],xmm2[5,6,7]
3869 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} ymm3 = [10,11,6,7,4,5,6,7,6,7,6,7,2,3,14,15,26,27,22,23,20,21,22,23,22,23,22,23,18,19,30,31]
3870 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm3, %ymm1, %ymm1
3871 ; AVX2-FAST-PERLANE-NEXT: vpblendvb %ymm0, %ymm2, %ymm1, %ymm1
3872 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3873 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm3, %ymm13, %ymm1
3874 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm14, %xmm12, %xmm2
3875 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm15, %xmm11, %xmm3
3876 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm4 = ymm7[0],ymm6[1],ymm7[2,3],ymm6[4],ymm7[5,6],ymm6[7]
3877 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2,3],xmm2[4],xmm3[5,6,7]
3878 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm13 = <0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15>
3879 ; AVX2-FAST-PERLANE-NEXT: vpblendvb %ymm0, %ymm2, %ymm1, %ymm0
3880 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3881 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm13, %xmm4, %xmm1
3882 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm4, %xmm2
3883 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm10 = xmm2[2,1,0,3]
3884 ; AVX2-FAST-PERLANE-NEXT: vpbroadcastq {{.*#+}} xmm15 = [8,9,12,13,0,1,0,0,8,9,12,13,0,1,0,0]
3885 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm15, %xmm10, %xmm3
3886 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2],xmm1[3],xmm3[4,5],xmm1[6,7]
3887 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm7 = ymm5[0],ymm8[1],ymm5[2,3,4,5],ymm8[6],ymm5[7]
3888 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} ymm3 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
3889 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2],ymm3[3,4,5,6,7]
3890 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 352(%rdi), %ymm0
3891 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3892 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 320(%rdi), %ymm11
3893 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm2 = ymm11[0,1],ymm0[2],ymm11[3,4],ymm0[5],ymm11[6,7]
3894 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3895 ; AVX2-FAST-PERLANE-NEXT: vpbroadcastd {{.*#+}} xmm9 = [8,9,4,5,8,9,4,5,8,9,4,5,8,9,4,5]
3896 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm9, %xmm2, %xmm6
3897 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm8 = <0,1,4,5,4,5,u,u,0,1,12,13,u,u,4,5>
3898 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm2, %xmm5
3899 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm8, %xmm5, %xmm12
3900 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm6 = xmm12[0,1,2],xmm6[3],xmm12[4,5],xmm6[6],xmm12[7]
3901 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
3902 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm6 = ymm1[0,1,2],ymm6[3,4,5,6,7],ymm1[8,9,10],ymm6[11,12,13,14,15]
3903 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm6[4,5,6,7]
3904 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3905 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 160(%rdi), %ymm6
3906 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 128(%rdi), %ymm0
3907 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3908 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm6[2],ymm0[3,4],ymm6[5],ymm0[6,7]
3909 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm9, %xmm0, %xmm9
3910 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm0, %xmm1
3911 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm8, %xmm1, %xmm8
3912 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2],xmm9[3],xmm8[4,5],xmm9[6],xmm8[7]
3913 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
3914 ; AVX2-FAST-PERLANE-NEXT: vpblendd $146, (%rsp), %ymm3, %ymm9 # 32-byte Folded Reload
3915 ; AVX2-FAST-PERLANE-NEXT: # ymm9 = ymm3[0],mem[1],ymm3[2,3],mem[4],ymm3[5,6],mem[7]
3916 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm13, %xmm9, %xmm3
3917 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm9, %xmm12
3918 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[2,1,0,3]
3919 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm15, %xmm12, %xmm15
3920 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm15[2],xmm3[3],xmm15[4,5],xmm3[6,7]
3921 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
3922 ; AVX2-FAST-PERLANE-NEXT: vpblendd $66, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm15 # 32-byte Folded Reload
3923 ; AVX2-FAST-PERLANE-NEXT: # ymm15 = ymm14[0],mem[1],ymm14[2,3,4,5],mem[6],ymm14[7]
3924 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} ymm13 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
3925 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2],ymm13[3,4,5,6,7]
3926 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
3927 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm8 = ymm3[0,1,2],ymm8[3,4,5,6,7],ymm3[8,9,10],ymm8[11,12,13,14,15]
3928 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm8[4,5,6,7]
3929 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3930 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm13 = <2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15>
3931 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm13, %xmm4, %xmm3
3932 ; AVX2-FAST-PERLANE-NEXT: vpbroadcastq {{.*#+}} xmm4 = [10,11,14,15,2,3,0,0,10,11,14,15,2,3,0,0]
3933 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm4, %xmm10, %xmm10
3934 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm10[2],xmm3[3],xmm10[4,5],xmm3[6,7]
3935 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} ymm10 = <u,u,u,u,u,u,u,u,u,u,u,u,10,11,6,7,18,19,30,31,26,27,26,27,30,31,26,27,26,27,22,23>
3936 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm10, %ymm7, %ymm7
3937 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2],ymm7[3,4,5,6,7]
3938 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm7 = <6,7,2,3,4,5,u,u,2,3,14,15,u,u,6,7>
3939 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm7, %xmm5, %xmm5
3940 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5]
3941 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm2 = xmm5[0,1,2],xmm2[3],xmm5[4,5],xmm2[6],xmm5[7]
3942 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
3943 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3,4,5,6,7],ymm3[8,9,10],ymm2[11,12,13,14,15]
3944 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
3945 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3946 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm7, %xmm1, %xmm1
3947 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
3948 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3],xmm1[4,5],xmm0[6],xmm1[7]
3949 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm13, %xmm9, %xmm1
3950 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm4, %xmm12, %xmm2
3951 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3],xmm2[4,5],xmm1[6,7]
3952 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm10, %ymm15, %ymm2
3953 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
3954 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
3955 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
3956 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
3957 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3958 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3959 ; AVX2-FAST-PERLANE-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
3960 ; AVX2-FAST-PERLANE-NEXT: # ymm3 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6],ymm0[7]
3961 ; AVX2-FAST-PERLANE-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm1 # 32-byte Folded Reload
3962 ; AVX2-FAST-PERLANE-NEXT: # ymm1 = mem[0,1],ymm11[2],mem[3,4],ymm11[5],mem[6,7]
3963 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,1,0,3]
3964 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm1, %xmm1
3965 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[0,1,2,1]
3966 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm7 = <0,1,0,1,0,1,0,1,u,u,8,9,12,13,u,u>
3967 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm7, %xmm5, %xmm4
3968 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm9 = xmm2[0,1,2,3,6,5,6,4]
3969 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm9[4],xmm4[5,6],xmm9[7]
3970 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3971 ; AVX2-FAST-PERLANE-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm9 # 32-byte Folded Reload
3972 ; AVX2-FAST-PERLANE-NEXT: # ymm9 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
3973 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm9, %xmm10
3974 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[0,3,2,1]
3975 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[2,1,2,3]
3976 ; AVX2-FAST-PERLANE-NEXT: vpbroadcastq {{.*#+}} xmm12 = [12,13,0,1,4,5,0,0,12,13,0,1,4,5,0,0]
3977 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm12, %xmm10, %xmm13
3978 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm15 = xmm9[2,1,2,0,4,5,6,7]
3979 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm13 = xmm15[0],xmm13[1,2],xmm15[3],xmm13[4,5,6,7]
3980 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
3981 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} ymm0 = ymm3[4,5,0,1,12,13,14,15,8,9,12,13,0,1,u,u,20,21,16,17,28,29,30,31,24,25,28,29,16,17,u,u]
3982 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm4 = ymm0[0,1,2],ymm4[3,4,5,6,7],ymm0[8,9,10],ymm4[11,12,13,14,15]
3983 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,5,4]
3984 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm13[0,1,2,3,4],xmm0[5,6,7]
3985 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7]
3986 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3987 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
3988 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1],ymm11[2],ymm6[3,4],ymm11[5],ymm6[6,7]
3989 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm13 = xmm0[2,1,0,3]
3990 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm7, %xmm13, %xmm7
3991 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm0, %xmm0
3992 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,1,2,1]
3993 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm8 = xmm1[0,1,2,3,6,5,6,4]
3994 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3],xmm8[4],xmm7[5,6],xmm8[7]
3995 ; AVX2-FAST-PERLANE-NEXT: vpblendd $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm8 # 32-byte Folded Reload
3996 ; AVX2-FAST-PERLANE-NEXT: # ymm8 = ymm14[0,1],mem[2],ymm14[3],mem[4],ymm14[5,6],mem[7]
3997 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3998 ; AVX2-FAST-PERLANE-NEXT: vpblendd $36, (%rsp), %ymm0, %ymm0 # 32-byte Folded Reload
3999 ; AVX2-FAST-PERLANE-NEXT: # ymm0 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
4000 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm0, %xmm4
4001 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,3,2,1]
4002 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm12, %xmm4, %xmm12
4003 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]
4004 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm15 = xmm0[2,1,2,0,4,5,6,7]
4005 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm12 = xmm15[0],xmm12[1,2],xmm15[3],xmm12[4,5,6,7]
4006 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
4007 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} ymm15 = ymm8[4,5,0,1,12,13,14,15,8,9,12,13,0,1,u,u,20,21,16,17,28,29,30,31,24,25,28,29,16,17,u,u]
4008 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm7 = ymm15[0,1,2],ymm7[3,4,5,6,7],ymm15[8,9,10],ymm7[11,12,13,14,15]
4009 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,6,5,4]
4010 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1,2,3,4],xmm15[5,6,7]
4011 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm7 = ymm12[0,1,2,3],ymm7[4,5,6,7]
4012 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm12 = <2,3,2,3,2,3,2,3,u,u,10,11,14,15,u,u>
4013 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm12, %xmm5, %xmm5
4014 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,5]
4015 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm2 = xmm5[0,1,2,3],xmm2[4],xmm5[5,6],xmm2[7]
4016 ; AVX2-FAST-PERLANE-NEXT: vpbroadcastq {{.*#+}} xmm5 = [14,15,2,3,6,7,0,0,14,15,2,3,6,7,0,0]
4017 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm5, %xmm10, %xmm10
4018 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[3,1,2,1,4,5,6,7]
4019 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0],xmm10[1,2],xmm9[3],xmm10[4,5,6,7]
4020 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} ymm10 = <6,7,2,3,14,15,14,15,14,15,10,11,u,u,2,3,22,23,18,19,30,31,30,31,30,31,26,27,u,u,18,19>
4021 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm10, %ymm3, %ymm3
4022 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
4023 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3,4,5,6,7],ymm3[8,9,10],ymm2[11,12,13,14,15]
4024 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,1,3,2]
4025 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm3 = xmm9[0,1,2,3,4],xmm3[5,6,7]
4026 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
4027 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm12, %xmm13, %xmm3
4028 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,5]
4029 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4],xmm3[5,6],xmm1[7]
4030 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm10, %ymm8, %ymm3
4031 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm5, %xmm4, %xmm4
4032 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[3,1,2,1,4,5,6,7]
4033 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm4[1,2],xmm0[3],xmm4[4,5,6,7]
4034 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
4035 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3,4,5,6,7],ymm3[8,9,10],ymm1[11,12,13,14,15]
4036 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,1,3,2]
4037 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm3[5,6,7]
4038 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
4039 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0],ymm6[1],ymm11[2,3],ymm6[4],ymm11[5,6],ymm6[7]
4040 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm1, %xmm3
4041 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,3,2,1]
4042 ; AVX2-FAST-PERLANE-NEXT: vpbroadcastq {{.*#+}} xmm4 = [8,9,0,0,0,1,12,13,8,9,0,0,0,1,12,13]
4043 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm4, %xmm3, %xmm5
4044 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm6 = <0,1,2,3,0,1,4,5,u,u,12,13,u,u,u,u>
4045 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm1, %xmm8
4046 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm5 = xmm8[0,1,2,3],xmm5[4],xmm8[5],xmm5[6,7]
4047 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
4048 ; AVX2-FAST-PERLANE-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
4049 ; AVX2-FAST-PERLANE-NEXT: # ymm5 = mem[0,1,2,3,4],ymm5[5,6,7]
4050 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
4051 ; AVX2-FAST-PERLANE-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
4052 ; AVX2-FAST-PERLANE-NEXT: # ymm8 = ymm8[0],mem[1],ymm8[2,3],mem[4],ymm8[5,6],mem[7]
4053 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm8, %xmm9
4054 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm4, %xmm9, %xmm4
4055 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[0,3,2,1]
4056 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm8, %xmm6
4057 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm4 = xmm6[0,1,2,3],xmm4[4],xmm6[5],xmm4[6,7]
4058 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
4059 ; AVX2-FAST-PERLANE-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
4060 ; AVX2-FAST-PERLANE-NEXT: # ymm4 = mem[0,1,2,3,4],ymm4[5,6,7]
4061 ; AVX2-FAST-PERLANE-NEXT: vpbroadcastq {{.*#+}} xmm6 = [10,11,0,0,2,3,14,15,10,11,0,0,2,3,14,15]
4062 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm3, %xmm3
4063 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm10 = <0,1,2,3,2,3,6,7,u,u,14,15,u,u,u,u>
4064 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm10, %xmm1, %xmm1
4065 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4],xmm1[5],xmm3[6,7]
4066 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
4067 ; AVX2-FAST-PERLANE-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
4068 ; AVX2-FAST-PERLANE-NEXT: # ymm1 = mem[0,1,2,3,4],ymm1[5,6,7]
4069 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm9, %xmm3
4070 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm10, %xmm8, %xmm6
4071 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm3 = xmm6[0,1,2,3],xmm3[4],xmm6[5],xmm3[6,7]
4072 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
4073 ; AVX2-FAST-PERLANE-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
4074 ; AVX2-FAST-PERLANE-NEXT: # ymm3 = mem[0,1,2,3,4],ymm3[5,6,7]
4075 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
4076 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm6, 32(%rsi)
4077 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
4078 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm6, (%rsi)
4079 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
4080 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm6, 32(%rdx)
4081 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
4082 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm6, (%rdx)
4083 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
4084 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm6, 32(%rcx)
4085 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm7, (%rcx)
4086 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm2, 32(%r8)
4087 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm0, (%r8)
4088 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm4, 32(%r9)
4089 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm5, (%r9)
4090 ; AVX2-FAST-PERLANE-NEXT: movq {{[0-9]+}}(%rsp), %rax
4091 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm3, 32(%rax)
4092 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm1, (%rax)
4093 ; AVX2-FAST-PERLANE-NEXT: addq $488, %rsp # imm = 0x1E8
4094 ; AVX2-FAST-PERLANE-NEXT: vzeroupper
4095 ; AVX2-FAST-PERLANE-NEXT: retq
4096 ;
4097 ; AVX512F-ONLY-SLOW-LABEL: load_i16_stride6_vf32:
4098 ; AVX512F-ONLY-SLOW: # %bb.0:
4099 ; AVX512F-ONLY-SLOW-NEXT: pushq %rax
4100 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} xmm0 = <0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15>
4101 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 224(%rdi), %ymm1
4102 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 192(%rdi), %ymm2
4103 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5,6],ymm1[7]
4104 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm2, %ymm17
4105 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm1, %ymm18
4106 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm0, %xmm4, %xmm1
4107 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm4, %xmm7
4108 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm7[0,2,0,3]
4109 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
4110 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3],xmm2[4,5],xmm1[6,7]
4111 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 160(%rdi), %ymm5
4112 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa (%rdi), %ymm10
4113 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 32(%rdi), %ymm11
4114 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 64(%rdi), %ymm3
4115 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 128(%rdi), %ymm14
4116 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm14[0,1],ymm5[2],ymm14[3,4],ymm5[5],ymm14[6,7]
4117 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm5, %ymm23
4118 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm6
4119 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} xmm8 = <0,1,4,5,4,5,u,u,0,1,12,13,u,u,4,5>
4120 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm8, %xmm6, %xmm5
4121 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm9 = xmm2[2,2,2,2,4,5,6,7]
4122 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[0,1,2,2]
4123 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2],xmm9[3],xmm5[4,5],xmm9[6],xmm5[7]
4124 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
4125 ; AVX512F-ONLY-SLOW-NEXT: vinserti32x4 $2, %xmm1, %zmm5, %zmm1
4126 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
4127 ; AVX512F-ONLY-SLOW-NEXT: vperm2i128 {{.*#+}} ymm12 = ymm3[2,3],mem[2,3]
4128 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm10[0],ymm11[1],ymm10[2,3],ymm11[4],ymm10[5,6],ymm11[7]
4129 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm11, %ymm25
4130 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm10, %ymm26
4131 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm0, %xmm1, %xmm0
4132 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm5
4133 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm5[0,2,0,3]
4134 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,6,6,7]
4135 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm9[2],xmm0[3],xmm9[4,5],xmm0[6,7]
4136 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, 96(%rdi), %ymm3, %ymm9
4137 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm3 = ymm9[0],ymm12[1],ymm9[2,3,4,5],ymm12[6],ymm9[7]
4138 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm9, %ymm29
4139 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa %ymm12, %ymm15
4140 ; AVX512F-ONLY-SLOW-NEXT: vpshufb {{.*#+}} ymm9 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
4141 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm9[3,4,5,6,7]
4142 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
4143 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 352(%rdi), %ymm9
4144 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 320(%rdi), %ymm10
4145 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm10[0,1],ymm9[2],ymm10[3,4],ymm9[5],ymm10[6,7]
4146 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm10, %ymm16
4147 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm9, %ymm24
4148 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm9
4149 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm8, %xmm9, %xmm8
4150 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm10 = xmm0[2,2,2,2,4,5,6,7]
4151 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[0,1,2,2]
4152 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2],xmm10[3],xmm8[4,5],xmm10[6],xmm8[7]
4153 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm10
4154 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 256(%rdi), %ymm8
4155 ; AVX512F-ONLY-SLOW-NEXT: vperm2i128 {{.*#+}} ymm11 = ymm8[2,3],mem[2,3]
4156 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, 288(%rdi), %ymm8, %ymm12
4157 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm8 = ymm12[0],ymm11[1],ymm12[2,3,4,5],ymm11[6],ymm12[7]
4158 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm12, %ymm20
4159 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm11, %ymm28
4160 ; AVX512F-ONLY-SLOW-NEXT: vpshufb {{.*#+}} ymm12 = ymm8[0,1,12,13,8,9,12,13,8,9,12,13,4,5,u,u,16,17,28,29,24,25,28,29,24,25,28,29,20,21,u,u]
4161 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} ymm10 = ymm12[0,1,2],ymm10[3,4,5,6,7],ymm12[8,9,10],ymm10[11,12,13,14,15]
4162 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,5,4,6]
4163 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]
4164 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm10, %ymm30
4165 ; AVX512F-ONLY-SLOW-NEXT: vpbroadcastq {{.*#+}} xmm10 = [2,3,14,15,10,11,0,0,2,3,14,15,10,11,0,0]
4166 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm10, %xmm7, %xmm7
4167 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} xmm12 = <2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15>
4168 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm12, %xmm4, %xmm4
4169 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm7[2],xmm4[3],xmm7[4,5],xmm4[6,7]
4170 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} xmm7 = <6,7,2,3,4,5,u,u,2,3,14,15,u,u,6,7>
4171 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm7, %xmm6, %xmm6
4172 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5]
4173 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm6[0,1,2],xmm2[3],xmm6[4,5],xmm2[6],xmm6[7]
4174 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
4175 ; AVX512F-ONLY-SLOW-NEXT: vinserti32x4 $2, %xmm4, %zmm2, %zmm21
4176 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm10, %xmm5, %xmm2
4177 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm12, %xmm1, %xmm1
4178 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3],xmm2[4,5],xmm1[6,7]
4179 ; AVX512F-ONLY-SLOW-NEXT: vpshufb {{.*#+}} ymm2 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,10,11,6,7,18,19,30,31,26,27,26,27,30,31,26,27,26,27,22,23]
4180 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm13 = ymm1[0,1,2],ymm2[3,4,5,6,7]
4181 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm7, %xmm9, %xmm1
4182 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
4183 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3],xmm1[4,5],xmm0[6],xmm1[7]
4184 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
4185 ; AVX512F-ONLY-SLOW-NEXT: vpshufb {{.*#+}} ymm1 = ymm8[2,3,14,15,10,11,10,11,14,15,10,11,u,u,6,7,18,19,30,31,26,27,26,27,30,31,26,27,u,u,22,23]
4186 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
4187 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,5,7]
4188 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
4189 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm0, %ymm19
4190 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm18, %ymm27
4191 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm17, %ymm0
4192 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm18, %ymm1
4193 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
4194 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm17, %ymm31
4195 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm1
4196 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm0[2,1,2,3]
4197 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm1[0,3,2,1]
4198 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm11[0,0,2,3,4,5,6,7]
4199 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,6,6,6]
4200 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm10[2,1,2,0,4,5,6,7]
4201 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5,6,7]
4202 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm23, %ymm1
4203 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm14[2],ymm1[3,4],ymm14[5],ymm1[6,7]
4204 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm14, %ymm22
4205 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm2
4206 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[2,1,0,3]
4207 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm9[0,0,0,0,4,5,6,7]
4208 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,6,7]
4209 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm2[0,1,2,1]
4210 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm8[0,1,2,3,6,5,6,4]
4211 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4],xmm1[5,6],xmm2[7]
4212 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
4213 ; AVX512F-ONLY-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm17
4214 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm25, %ymm0
4215 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm26, %ymm1
4216 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
4217 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm1
4218 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm0[2,1,2,3]
4219 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm1[0,3,2,1]
4220 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm6[0,0,2,3,4,5,6,7]
4221 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,3,3]
4222 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm7[2,1,2,0,4,5,6,7]
4223 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5,6,7]
4224 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm29, %ymm12
4225 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm12[0,1],ymm15[2],ymm12[3],ymm15[4],ymm12[5,6],ymm15[7]
4226 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm15, %ymm18
4227 ; AVX512F-ONLY-SLOW-NEXT: vpshufb {{.*#+}} ymm1 = ymm5[4,5,0,1,12,13,14,15,8,9,0,1,12,13,8,9,20,21,16,17,28,29,30,31,24,25,16,17,28,29,24,25]
4228 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm1[5,6,7]
4229 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0,1,2,3],ymm1[4,5,6,7]
4230 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm16, %ymm29
4231 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm24, %ymm0
4232 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm16, %ymm1
4233 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
4234 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm0
4235 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[2,1,0,3]
4236 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm4[0,0,0,0,4,5,6,7]
4237 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,6,7]
4238 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[0,1,2,1]
4239 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm14 = xmm3[0,1,2,3,6,5,6,4]
4240 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm14[4],xmm1[5,6],xmm14[7]
4241 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm14
4242 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm20, %ymm15
4243 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm28, %ymm0
4244 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm15[0,1],ymm0[2],ymm15[3],ymm0[4],ymm15[5,6],ymm0[7]
4245 ; AVX512F-ONLY-SLOW-NEXT: vpshufb {{.*#+}} ymm0 = ymm1[4,5,0,1,12,13,14,15,8,9,12,13,0,1,u,u,20,21,16,17,28,29,30,31,24,25,28,29,16,17,u,u]
4246 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} ymm14 = ymm0[0,1,2],ymm14[3,4,5,6,7],ymm0[8,9,10],ymm14[11,12,13,14,15]
4247 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,5,4]
4248 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm14[4,5,6,7]
4249 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm16
4250 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
4251 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $226, %zmm17, %zmm0, %zmm2
4252 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0]
4253 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm2, %zmm17, %zmm16
4254 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm10[3,1,2,1,4,5,6,7]
4255 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm10 = xmm11[0,1,3,3,4,5,6,7]
4256 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,7,7,7,7]
4257 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm10[1,2],xmm2[3],xmm10[4,5,6,7]
4258 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,7,5,6,5]
4259 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[1,1,1,1,4,5,6,7]
4260 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,5,7,7]
4261 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm8 = xmm9[0,1,2,3],xmm8[4],xmm9[5,6],xmm8[7]
4262 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
4263 ; AVX512F-ONLY-SLOW-NEXT: vinserti32x4 $2, %xmm2, %zmm8, %zmm2
4264 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm30, %zmm0, %zmm30
4265 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm19, %zmm0, %zmm19
4266 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[3,1,2,1,4,5,6,7]
4267 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,3,3,4,5,6,7]
4268 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,7,7,7,7]
4269 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0],xmm6[1,2],xmm7[3],xmm6[4,5,6,7]
4270 ; AVX512F-ONLY-SLOW-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[6,7,2,3,14,15,14,15,14,15,2,3,14,15,10,11,22,23,18,19,30,31,30,31,30,31,18,19,30,31,26,27]
4271 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm5[5,6,7]
4272 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3],ymm5[4,5,6,7]
4273 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,5]
4274 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[1,1,1,1,4,5,6,7]
4275 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,7,7]
4276 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4],xmm4[5,6],xmm3[7]
4277 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
4278 ; AVX512F-ONLY-SLOW-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[6,7,2,3,14,15,14,15,14,15,10,11,u,u,2,3,22,23,18,19,30,31,30,31,30,31,26,27,u,u,18,19]
4279 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = ymm1[0,1,2],ymm3[3,4,5,6,7],ymm1[8,9,10],ymm3[11,12,13,14,15]
4280 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,4,5]
4281 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
4282 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $226, %zmm2, %zmm0, %zmm5
4283 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm20
4284 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm5, %zmm17, %zmm20
4285 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} xmm5 = <8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15>
4286 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm27, %ymm0
4287 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm31, %ymm1
4288 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
4289 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm5, %xmm1, %xmm0
4290 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm6
4291 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm6[2,2,2,2,4,5,6,7]
4292 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3],xmm2[4],xmm0[5,6,7]
4293 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm23, %ymm2
4294 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm22, %ymm3
4295 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6],ymm2[7]
4296 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm7
4297 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,3,2,1]
4298 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm3[0,1,0,2,4,5,6,7]
4299 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,6,6,6]
4300 ; AVX512F-ONLY-SLOW-NEXT: vpbroadcastq {{.*#+}} xmm8 = [8,9,0,0,0,1,12,13,8,9,0,0,0,1,12,13]
4301 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm8, %xmm7, %xmm4
4302 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm4[4],xmm2[5],xmm4[6,7]
4303 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
4304 ; AVX512F-ONLY-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm2, %zmm4
4305 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm18, %ymm0
4306 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0],ymm12[1],ymm0[2,3,4,5],ymm12[6],ymm0[7]
4307 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm25, %ymm0
4308 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm26, %ymm9
4309 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm9[2],ymm0[3,4],ymm9[5],ymm0[6,7]
4310 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm5, %xmm0, %xmm5
4311 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm9
4312 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm10 = xmm9[2,2,2,2,4,5,6,7]
4313 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0],xmm10[1],xmm5[2,3],xmm10[4],xmm5[5,6,7]
4314 ; AVX512F-ONLY-SLOW-NEXT: vpshufb {{.*#+}} ymm10 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[4,5,0,1,12,13,24,25,20,21],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
4315 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} ymm11 = [65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
4316 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $236, %ymm11, %ymm10, %ymm5
4317 ; AVX512F-ONLY-SLOW-NEXT: movw $31, %ax
4318 ; AVX512F-ONLY-SLOW-NEXT: kmovw %eax, %k1
4319 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa32 %zmm5, %zmm4 {%k1}
4320 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm24, %ymm5
4321 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm29, %ymm10
4322 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0],ymm5[1],ymm10[2,3],ymm5[4],ymm10[5,6],ymm5[7]
4323 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm10, %xmm5
4324 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm8, %xmm5, %xmm12
4325 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm10[0,3,2,1]
4326 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm10 = xmm8[0,1,0,2,4,5,6,7]
4327 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,6,6,6,6]
4328 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1,2,3],xmm12[4],xmm10[5],xmm12[6,7]
4329 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
4330 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm28, %ymm12
4331 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm12 = ymm12[0],ymm15[1],ymm12[2,3,4,5],ymm15[6],ymm12[7]
4332 ; AVX512F-ONLY-SLOW-NEXT: vpshufb {{.*#+}} ymm14 = ymm12[8,9,4,5,4,5,6,7,0,1,4,5,0,1,12,13,24,25,20,21,u,u,u,u,u,u,u,u,u,u,u,u]
4333 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm10 = ymm14[0,1,2,3,4],ymm10[5,6,7]
4334 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm10, %zmm0, %zmm10
4335 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} xmm14 = <10,11,u,u,2,3,14,15,u,u,10,11,12,13,14,15>
4336 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm14, %xmm1, %xmm1
4337 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[1,1,2,3]
4338 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,5,5,5,5]
4339 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm6[1],xmm1[2,3],xmm6[4],xmm1[5,6,7]
4340 ; AVX512F-ONLY-SLOW-NEXT: vpbroadcastq {{.*#+}} xmm6 = [10,11,0,0,2,3,14,15,10,11,0,0,2,3,14,15]
4341 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm6, %xmm7, %xmm7
4342 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]
4343 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,1,3,3]
4344 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm7[4],xmm3[5],xmm7[6,7]
4345 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
4346 ; AVX512F-ONLY-SLOW-NEXT: vinserti32x4 $2, %xmm1, %zmm3, %zmm1
4347 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm14, %xmm0, %xmm0
4348 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm9[1,1,2,3]
4349 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,5,5]
4350 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2,3],xmm3[4],xmm0[5,6,7]
4351 ; AVX512F-ONLY-SLOW-NEXT: vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[6,7,2,3,14,15,26,27,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
4352 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $236, %ymm11, %ymm2, %ymm0
4353 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1}
4354 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm6, %xmm5, %xmm0
4355 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm8[0,1,1,3,4,5,6,7]
4356 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,3,3]
4357 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4],xmm2[5],xmm0[6,7]
4358 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
4359 ; AVX512F-ONLY-SLOW-NEXT: vpshufb {{.*#+}} ymm2 = ymm12[10,11,6,7,4,5,6,7,6,7,6,7,2,3,14,15,26,27,22,23,u,u,u,u,u,u,u,u,u,u,u,u]
4360 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5,6,7]
4361 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0
4362 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
4363 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
4364 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm3 # 64-byte Folded Reload
4365 ; AVX512F-ONLY-SLOW-NEXT: movw $-2048, %ax # imm = 0xF800
4366 ; AVX512F-ONLY-SLOW-NEXT: kmovw %eax, %k1
4367 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa32 %zmm30, %zmm3 {%k1}
4368 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm3, (%rsi)
4369 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $226, %zmm21, %zmm2, %zmm13
4370 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa32 %zmm19, %zmm13 {%k1}
4371 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm13, (%rdx)
4372 ; AVX512F-ONLY-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
4373 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm4, %zmm17, %zmm10
4374 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm1, %zmm17, %zmm0
4375 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm16, (%rcx)
4376 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm20, (%r8)
4377 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm10, (%r9)
4378 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm0, (%rax)
4379 ; AVX512F-ONLY-SLOW-NEXT: popq %rax
4380 ; AVX512F-ONLY-SLOW-NEXT: vzeroupper
4381 ; AVX512F-ONLY-SLOW-NEXT: retq
4382 ;
4383 ; AVX512F-ONLY-FAST-LABEL: load_i16_stride6_vf32:
4384 ; AVX512F-ONLY-FAST: # %bb.0:
4385 ; AVX512F-ONLY-FAST-NEXT: subq $136, %rsp
4386 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} xmm0 = <0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15>
4387 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 224(%rdi), %ymm1
4388 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 192(%rdi), %ymm2
4389 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm4 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5,6],ymm1[7]
4390 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm2, %ymm17
4391 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm1, %ymm20
4392 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm0, %xmm4, %xmm1
4393 ; AVX512F-ONLY-FAST-NEXT: vpbroadcastq {{.*#+}} xmm2 = [8,9,12,13,0,1,0,0,8,9,12,13,0,1,0,0]
4394 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm4, %xmm3
4395 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm9 = xmm3[2,1,0,3]
4396 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm2, %xmm9, %xmm3
4397 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2],xmm1[3],xmm3[4,5],xmm1[6,7]
4398 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 160(%rdi), %ymm13
4399 ; AVX512F-ONLY-FAST-NEXT: vmovdqa (%rdi), %ymm12
4400 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 128(%rdi), %ymm15
4401 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm15[0,1],ymm13[2],ymm15[3,4],ymm13[5],ymm15[6,7]
4402 ; AVX512F-ONLY-FAST-NEXT: vpbroadcastd {{.*#+}} xmm8 = [8,9,4,5,8,9,4,5,8,9,4,5,8,9,4,5]
4403 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm8, %xmm3, %xmm5
4404 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm3, %xmm7
4405 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} xmm10 = <0,1,4,5,4,5,u,u,0,1,12,13,u,u,4,5>
4406 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm10, %xmm7, %xmm6
4407 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3],xmm6[4,5],xmm5[6],xmm6[7]
4408 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 32(%rdi), %ymm6
4409 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 64(%rdi), %ymm11
4410 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
4411 ; AVX512F-ONLY-FAST-NEXT: vinserti32x4 $2, %xmm1, %zmm5, %zmm1
4412 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
4413 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm12[0],ymm6[1],ymm12[2,3],ymm6[4],ymm12[5,6],ymm6[7]
4414 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm6, %ymm16
4415 ; AVX512F-ONLY-FAST-NEXT: vmovdqa %ymm12, %ymm14
4416 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm0, %xmm1, %xmm0
4417 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm1, %xmm5
4418 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[2,1,0,3]
4419 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm2, %xmm6, %xmm2
4420 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2],xmm0[3],xmm2[4,5],xmm0[6,7]
4421 ; AVX512F-ONLY-FAST-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm11[2,3],mem[2,3]
4422 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, 96(%rdi), %ymm11, %ymm11
4423 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm5 = ymm11[0],ymm2[1],ymm11[2,3,4,5],ymm2[6],ymm11[7]
4424 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm11, %ymm22
4425 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm2, %ymm28
4426 ; AVX512F-ONLY-FAST-NEXT: vpshufb {{.*#+}} ymm2 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
4427 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3,4,5,6,7]
4428 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
4429 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 352(%rdi), %ymm2
4430 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 320(%rdi), %ymm11
4431 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1],ymm2[2],ymm11[3,4],ymm2[5],ymm11[6,7]
4432 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm11, %ymm24
4433 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm2, %ymm25
4434 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm8, %xmm0, %xmm2
4435 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm0, %xmm8
4436 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm10, %xmm8, %xmm10
4437 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm10[0,1,2],xmm2[3],xmm10[4,5],xmm2[6],xmm10[7]
4438 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm10
4439 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 256(%rdi), %ymm2
4440 ; AVX512F-ONLY-FAST-NEXT: vperm2i128 {{.*#+}} ymm11 = ymm2[2,3],mem[2,3]
4441 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, 288(%rdi), %ymm2, %ymm12
4442 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm12[0],ymm11[1],ymm12[2,3,4,5],ymm11[6],ymm12[7]
4443 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm12, %ymm26
4444 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm11, %ymm27
4445 ; AVX512F-ONLY-FAST-NEXT: vpshufb {{.*#+}} ymm12 = ymm2[0,1,12,13,8,9,12,13,8,9,12,13,4,5,u,u,16,17,28,29,24,25,28,29,24,25,28,29,20,21,u,u]
4446 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} ymm10 = ymm12[0,1,2],ymm10[3,4,5,6,7],ymm12[8,9,10],ymm10[11,12,13,14,15]
4447 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,5,4,6]
4448 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]
4449 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm10, %ymm29
4450 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} xmm10 = <2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15>
4451 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm10, %xmm4, %xmm4
4452 ; AVX512F-ONLY-FAST-NEXT: vpbroadcastq {{.*#+}} xmm12 = [10,11,14,15,2,3,0,0,10,11,14,15,2,3,0,0]
4453 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm12, %xmm9, %xmm9
4454 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm9[2],xmm4[3],xmm9[4,5],xmm4[6,7]
4455 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} xmm9 = <6,7,2,3,4,5,u,u,2,3,14,15,u,u,6,7>
4456 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm9, %xmm7, %xmm7
4457 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,5,5]
4458 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm7[0,1,2],xmm3[3],xmm7[4,5],xmm3[6],xmm7[7]
4459 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
4460 ; AVX512F-ONLY-FAST-NEXT: vinserti32x4 $2, %xmm4, %zmm3, %zmm3
4461 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
4462 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm10, %xmm1, %xmm1
4463 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm12, %xmm6, %xmm3
4464 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2],xmm1[3],xmm3[4,5],xmm1[6,7]
4465 ; AVX512F-ONLY-FAST-NEXT: vpshufb {{.*#+}} ymm3 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,10,11,6,7,18,19,30,31,26,27,26,27,30,31,26,27,26,27,22,23]
4466 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2],ymm3[3,4,5,6,7]
4467 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
4468 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm9, %xmm8, %xmm1
4469 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
4470 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3],xmm1[4,5],xmm0[6],xmm1[7]
4471 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
4472 ; AVX512F-ONLY-FAST-NEXT: vpshufb {{.*#+}} ymm1 = ymm2[2,3,14,15,10,11,10,11,14,15,10,11,u,u,6,7,18,19,30,31,26,27,26,27,30,31,26,27,u,u,22,23]
4473 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
4474 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,5,7]
4475 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
4476 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm0, %ymm18
4477 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm17, %ymm0
4478 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm20, %ymm1
4479 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
4480 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm20, %ymm30
4481 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm17, %ymm31
4482 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm0, %xmm1
4483 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm10 = xmm0[2,1,2,3]
4484 ; AVX512F-ONLY-FAST-NEXT: vpshuflw {{.*#+}} xmm0 = xmm10[2,1,2,0,4,5,6,7]
4485 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm11 = xmm1[0,3,2,1]
4486 ; AVX512F-ONLY-FAST-NEXT: vpshufb {{.*#+}} xmm1 = xmm11[u,u,0,1,4,5,u,u,12,13,12,13,12,13,12,13]
4487 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3],xmm1[4,5,6,7]
4488 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm13[0,1],ymm15[2],ymm13[3,4],ymm15[5],ymm13[6,7]
4489 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm13, %ymm21
4490 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm15, %ymm19
4491 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
4492 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm8 = xmm1[2,1,0,3]
4493 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} xmm9 = <0,1,0,1,0,1,0,1,u,u,8,9,12,13,u,u>
4494 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm9, %xmm8, %xmm1
4495 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[0,1,2,1]
4496 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm2 = xmm5[0,1,2,3,6,5,6,4]
4497 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4],xmm1[5,6],xmm2[7]
4498 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
4499 ; AVX512F-ONLY-FAST-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm17
4500 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm16, %ymm23
4501 ; AVX512F-ONLY-FAST-NEXT: vmovdqa %ymm14, %ymm15
4502 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm16, %ymm0
4503 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm14[0,1],ymm0[2],ymm14[3,4],ymm0[5],ymm14[6,7]
4504 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm0, %xmm1
4505 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,1,2,3]
4506 ; AVX512F-ONLY-FAST-NEXT: vpshuflw {{.*#+}} xmm0 = xmm4[2,1,2,0,4,5,6,7]
4507 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm7 = xmm1[0,3,2,1]
4508 ; AVX512F-ONLY-FAST-NEXT: vpshufb {{.*#+}} xmm1 = xmm7[u,u,0,1,4,5,u,u,12,13,u,u,u,u,u,u]
4509 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3],xmm1[4,5,6,7]
4510 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm22, %ymm14
4511 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm28, %ymm13
4512 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm6 = ymm14[0,1],ymm13[2],ymm14[3],ymm13[4],ymm14[5,6],ymm13[7]
4513 ; AVX512F-ONLY-FAST-NEXT: vpshufb {{.*#+}} ymm1 = ymm6[4,5,0,1,12,13,14,15,8,9,0,1,12,13,8,9,20,21,16,17,28,29,30,31,24,25,16,17,28,29,24,25]
4514 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm1[5,6,7]
4515 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0,1,2,3],ymm1[4,5,6,7]
4516 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm24, %ymm0
4517 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm25, %ymm1
4518 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
4519 ; AVX512F-ONLY-FAST-NEXT: vextracti32x4 $1, %ymm1, %xmm16
4520 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,1,0,3]
4521 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm9, %xmm3, %xmm1
4522 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm9 = xmm16[0,1,2,1]
4523 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm12 = xmm9[0,1,2,3,6,5,6,4]
4524 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm12[4],xmm1[5,6],xmm12[7]
4525 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm12
4526 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm26, %ymm0
4527 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm27, %ymm1
4528 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1],ymm1[2],ymm0[3],ymm1[4],ymm0[5,6],ymm1[7]
4529 ; AVX512F-ONLY-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm1[4,5,0,1,12,13,14,15,8,9,12,13,0,1,u,u,20,21,16,17,28,29,30,31,24,25,28,29,16,17,u,u]
4530 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} ymm12 = ymm0[0,1,2],ymm12[3,4,5,6,7],ymm0[8,9,10],ymm12[11,12,13,14,15]
4531 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,5,4]
4532 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm12[4,5,6,7]
4533 ; AVX512F-ONLY-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm16
4534 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
4535 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $226, %zmm17, %zmm0, %zmm2
4536 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0]
4537 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm2, %zmm17, %zmm16
4538 ; AVX512F-ONLY-FAST-NEXT: vpshuflw {{.*#+}} xmm2 = xmm10[3,1,2,1,4,5,6,7]
4539 ; AVX512F-ONLY-FAST-NEXT: vpshufb {{.*#+}} xmm10 = xmm11[u,u,2,3,6,7,u,u,14,15,14,15,14,15,14,15]
4540 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm10[1,2],xmm2[3],xmm10[4,5,6,7]
4541 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} xmm10 = <2,3,2,3,2,3,2,3,u,u,10,11,14,15,u,u>
4542 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm10, %xmm8, %xmm8
4543 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,7,5,6,5]
4544 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm5 = xmm8[0,1,2,3],xmm5[4],xmm8[5,6],xmm5[7]
4545 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
4546 ; AVX512F-ONLY-FAST-NEXT: vinserti32x4 $2, %xmm2, %zmm5, %zmm2
4547 ; AVX512F-ONLY-FAST-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[3,1,2,1,4,5,6,7]
4548 ; AVX512F-ONLY-FAST-NEXT: vpshufb {{.*#+}} xmm5 = xmm7[u,u,2,3,6,7,u,u,14,15,u,u,u,u,u,u]
4549 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm5[1,2],xmm4[3],xmm5[4,5,6,7]
4550 ; AVX512F-ONLY-FAST-NEXT: vinserti64x4 $1, %ymm29, %zmm0, %zmm28
4551 ; AVX512F-ONLY-FAST-NEXT: vinserti64x4 $1, %ymm18, %zmm0, %zmm22
4552 ; AVX512F-ONLY-FAST-NEXT: vpshufb {{.*#+}} ymm5 = ymm6[6,7,2,3,14,15,14,15,14,15,2,3,14,15,10,11,22,23,18,19,30,31,30,31,30,31,18,19,30,31,26,27]
4553 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm5[5,6,7]
4554 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
4555 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm10, %xmm3, %xmm3
4556 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm5 = xmm9[0,1,2,3,7,5,6,5]
4557 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm5[4],xmm3[5,6],xmm5[7]
4558 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
4559 ; AVX512F-ONLY-FAST-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[6,7,2,3,14,15,14,15,14,15,10,11,u,u,2,3,22,23,18,19,30,31,30,31,30,31,26,27,u,u,18,19]
4560 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} ymm3 = ymm1[0,1,2],ymm3[3,4,5,6,7],ymm1[8,9,10],ymm3[11,12,13,14,15]
4561 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,4,5]
4562 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
4563 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $226, %zmm2, %zmm0, %zmm4
4564 ; AVX512F-ONLY-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm20
4565 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm4, %zmm17, %zmm20
4566 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} xmm0 = <8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15>
4567 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm30, %ymm1
4568 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm31, %ymm2
4569 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm4 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]
4570 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm0, %xmm4, %xmm1
4571 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm4, %xmm6
4572 ; AVX512F-ONLY-FAST-NEXT: vpshuflw {{.*#+}} xmm2 = xmm6[2,2,2,2,4,5,6,7]
4573 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm1[0],xmm2[1],xmm1[2,3],xmm2[4],xmm1[5,6,7]
4574 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm21, %ymm1
4575 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm19, %ymm3
4576 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0],ymm1[1],ymm3[2,3],ymm1[4],ymm3[5,6],ymm1[7]
4577 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm1, %xmm8
4578 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,3,2,1]
4579 ; AVX512F-ONLY-FAST-NEXT: vpbroadcastq {{.*#+}} xmm5 = [8,9,0,0,0,1,12,13,8,9,0,0,0,1,12,13]
4580 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm5, %xmm8, %xmm3
4581 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} xmm10 = <0,1,2,3,0,1,4,5,u,u,12,13,u,u,u,u>
4582 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm10, %xmm1, %xmm9
4583 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm9[0,1,2,3],xmm3[4],xmm9[5],xmm3[6,7]
4584 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
4585 ; AVX512F-ONLY-FAST-NEXT: vinserti32x4 $2, %xmm2, %zmm3, %zmm3
4586 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm13[0],ymm14[1],ymm13[2,3,4,5],ymm14[6],ymm13[7]
4587 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm23, %ymm7
4588 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm9 = ymm7[0,1],ymm15[2],ymm7[3,4],ymm15[5],ymm7[6,7]
4589 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm0, %xmm9, %xmm0
4590 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm9, %xmm11
4591 ; AVX512F-ONLY-FAST-NEXT: vpshuflw {{.*#+}} xmm12 = xmm11[2,2,2,2,4,5,6,7]
4592 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm12[1],xmm0[2,3],xmm12[4],xmm0[5,6,7]
4593 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 {{.*#+}} ymm23 = [65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
4594 ; AVX512F-ONLY-FAST-NEXT: vpshufb {{.*#+}} ymm12 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[4,5,0,1,12,13,24,25,20,21],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
4595 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $236, %ymm23, %ymm12, %ymm0
4596 ; AVX512F-ONLY-FAST-NEXT: movw $31, %ax
4597 ; AVX512F-ONLY-FAST-NEXT: kmovw %eax, %k1
4598 ; AVX512F-ONLY-FAST-NEXT: vmovdqa32 %zmm0, %zmm3 {%k1}
4599 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm24, %ymm0
4600 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm25, %ymm7
4601 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm12 = ymm0[0],ymm7[1],ymm0[2,3],ymm7[4],ymm0[5,6],ymm7[7]
4602 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm12, %xmm0
4603 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm5, %xmm0, %xmm7
4604 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm5 = xmm12[0,3,2,1]
4605 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm10, %xmm5, %xmm10
4606 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm7 = xmm10[0,1,2,3],xmm7[4],xmm10[5],xmm7[6,7]
4607 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
4608 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm26, %ymm10
4609 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm27, %ymm12
4610 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm12 = ymm12[0],ymm10[1],ymm12[2,3,4,5],ymm10[6],ymm12[7]
4611 ; AVX512F-ONLY-FAST-NEXT: vpshufb {{.*#+}} ymm10 = ymm12[8,9,4,5,4,5,6,7,0,1,4,5,0,1,12,13,24,25,20,21,u,u,u,u,u,u,u,u,u,u,u,u]
4612 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm7 = ymm10[0,1,2,3,4],ymm7[5,6,7]
4613 ; AVX512F-ONLY-FAST-NEXT: vinserti64x4 $1, %ymm7, %zmm0, %zmm7
4614 ; AVX512F-ONLY-FAST-NEXT: vpbroadcastd {{.*#+}} xmm10 = [10,11,6,7,10,11,6,7,10,11,6,7,10,11,6,7]
4615 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm10, %xmm11, %xmm11
4616 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} xmm15 = <10,11,u,u,2,3,14,15,u,u,10,11,12,13,14,15>
4617 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm15, %xmm9, %xmm9
4618 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0],xmm11[1],xmm9[2,3],xmm11[4],xmm9[5,6,7]
4619 ; AVX512F-ONLY-FAST-NEXT: vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[6,7,2,3,14,15,26,27,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
4620 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $248, %ymm23, %ymm9, %ymm2
4621 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm10, %xmm6, %xmm6
4622 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm15, %xmm4, %xmm4
4623 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm6[1],xmm4[2,3],xmm6[4],xmm4[5,6,7]
4624 ; AVX512F-ONLY-FAST-NEXT: vpbroadcastq {{.*#+}} xmm6 = [10,11,0,0,2,3,14,15,10,11,0,0,2,3,14,15]
4625 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm6, %xmm8, %xmm8
4626 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} xmm9 = <0,1,2,3,2,3,6,7,u,u,14,15,u,u,u,u>
4627 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm9, %xmm1, %xmm1
4628 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm8[4],xmm1[5],xmm8[6,7]
4629 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
4630 ; AVX512F-ONLY-FAST-NEXT: vinserti32x4 $2, %xmm4, %zmm1, %zmm1
4631 ; AVX512F-ONLY-FAST-NEXT: vmovdqa32 %zmm2, %zmm1 {%k1}
4632 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm6, %xmm0, %xmm0
4633 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm9, %xmm5, %xmm2
4634 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4],xmm2[5],xmm0[6,7]
4635 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
4636 ; AVX512F-ONLY-FAST-NEXT: vpshufb {{.*#+}} ymm2 = ymm12[10,11,6,7,4,5,6,7,6,7,6,7,2,3,14,15,26,27,22,23,u,u,u,u,u,u,u,u,u,u,u,u]
4637 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5,6,7]
4638 ; AVX512F-ONLY-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0
4639 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
4640 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
4641 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm4 # 64-byte Folded Reload
4642 ; AVX512F-ONLY-FAST-NEXT: movw $-2048, %ax # imm = 0xF800
4643 ; AVX512F-ONLY-FAST-NEXT: kmovw %eax, %k1
4644 ; AVX512F-ONLY-FAST-NEXT: vmovdqa32 %zmm28, %zmm4 {%k1}
4645 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm4, (%rsi)
4646 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 (%rsp), %zmm4 # 64-byte Reload
4647 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm4 # 64-byte Folded Reload
4648 ; AVX512F-ONLY-FAST-NEXT: vmovdqa32 %zmm22, %zmm4 {%k1}
4649 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm4, (%rdx)
4650 ; AVX512F-ONLY-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
4651 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm3, %zmm17, %zmm7
4652 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm1, %zmm17, %zmm0
4653 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm16, (%rcx)
4654 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm20, (%r8)
4655 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm7, (%r9)
4656 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm0, (%rax)
4657 ; AVX512F-ONLY-FAST-NEXT: addq $136, %rsp
4658 ; AVX512F-ONLY-FAST-NEXT: vzeroupper
4659 ; AVX512F-ONLY-FAST-NEXT: retq
4660 ;
4661 ; AVX512DQ-SLOW-LABEL: load_i16_stride6_vf32:
4662 ; AVX512DQ-SLOW: # %bb.0:
4663 ; AVX512DQ-SLOW-NEXT: pushq %rax
4664 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} xmm1 = <0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15>
4665 ; AVX512DQ-SLOW-NEXT: vmovdqa 224(%rdi), %ymm13
4666 ; AVX512DQ-SLOW-NEXT: vmovdqa 192(%rdi), %ymm2
4667 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm2[0],ymm13[1],ymm2[2,3],ymm13[4],ymm2[5,6],ymm13[7]
4668 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm2, %ymm18
4669 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm1, %xmm4, %xmm0
4670 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm4, %xmm10
4671 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm10[0,2,0,3]
4672 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
4673 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2],xmm0[3],xmm2[4,5],xmm0[6,7]
4674 ; AVX512DQ-SLOW-NEXT: vmovdqa 160(%rdi), %ymm2
4675 ; AVX512DQ-SLOW-NEXT: vmovdqa (%rdi), %ymm5
4676 ; AVX512DQ-SLOW-NEXT: vmovdqa 32(%rdi), %ymm9
4677 ; AVX512DQ-SLOW-NEXT: vmovdqa 64(%rdi), %ymm3
4678 ; AVX512DQ-SLOW-NEXT: vmovdqa 128(%rdi), %ymm6
4679 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm15 = ymm6[0,1],ymm2[2],ymm6[3,4],ymm2[5],ymm6[6,7]
4680 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm6, %ymm19
4681 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm2, %ymm21
4682 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm15, %xmm7
4683 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} xmm6 = <0,1,4,5,4,5,u,u,0,1,12,13,u,u,4,5>
4684 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm6, %xmm7, %xmm2
4685 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm15[2,2,2,2,4,5,6,7]
4686 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[0,1,2,2]
4687 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm8[3],xmm2[4,5],xmm8[6],xmm2[7]
4688 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
4689 ; AVX512DQ-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm2, %zmm16
4690 ; AVX512DQ-SLOW-NEXT: vperm2i128 {{.*#+}} ymm12 = ymm3[2,3],mem[2,3]
4691 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0],ymm9[1],ymm5[2,3],ymm9[4],ymm5[5,6],ymm9[7]
4692 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm9, %ymm22
4693 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm5, %ymm24
4694 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm1, %xmm2, %xmm1
4695 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm5
4696 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm5[0,2,0,3]
4697 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,6,6,7]
4698 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm9[2],xmm1[3],xmm9[4,5],xmm1[6,7]
4699 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, 96(%rdi), %ymm3, %ymm8
4700 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm3 = ymm8[0],ymm12[1],ymm8[2,3,4,5],ymm12[6],ymm8[7]
4701 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm8, %ymm29
4702 ; AVX512DQ-SLOW-NEXT: vpshufb {{.*#+}} ymm9 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
4703 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm9 = ymm1[0,1,2],ymm9[3,4,5,6,7]
4704 ; AVX512DQ-SLOW-NEXT: vmovdqa 352(%rdi), %ymm0
4705 ; AVX512DQ-SLOW-NEXT: vmovdqa 320(%rdi), %ymm1
4706 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm14 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
4707 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm1, %ymm23
4708 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm0, %ymm25
4709 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm14, %xmm1
4710 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm6, %xmm1, %xmm6
4711 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm11 = xmm14[2,2,2,2,4,5,6,7]
4712 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[0,1,2,2]
4713 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2],xmm11[3],xmm6[4,5],xmm11[6],xmm6[7]
4714 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm8
4715 ; AVX512DQ-SLOW-NEXT: vmovdqa 256(%rdi), %ymm6
4716 ; AVX512DQ-SLOW-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm6[2,3],mem[2,3]
4717 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, 288(%rdi), %ymm6, %ymm11
4718 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm6 = ymm11[0],ymm0[1],ymm11[2,3,4,5],ymm0[6],ymm11[7]
4719 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm11, %ymm26
4720 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm0, %ymm27
4721 ; AVX512DQ-SLOW-NEXT: vpshufb {{.*#+}} ymm0 = ymm6[0,1,12,13,8,9,12,13,8,9,12,13,4,5,u,u,16,17,28,29,24,25,28,29,24,25,28,29,20,21,u,u]
4722 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} ymm8 = ymm0[0,1,2],ymm8[3,4,5,6,7],ymm0[8,9,10],ymm8[11,12,13,14,15]
4723 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,4,6]
4724 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
4725 ; AVX512DQ-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
4726 ; AVX512DQ-SLOW-NEXT: vpternlogq $226, %zmm16, %zmm17, %zmm9
4727 ; AVX512DQ-SLOW-NEXT: movw $-2048, %ax # imm = 0xF800
4728 ; AVX512DQ-SLOW-NEXT: kmovw %eax, %k1
4729 ; AVX512DQ-SLOW-NEXT: vinserti32x8 $1, %ymm0, %zmm0, %zmm9 {%k1}
4730 ; AVX512DQ-SLOW-NEXT: vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
4731 ; AVX512DQ-SLOW-NEXT: vpbroadcastq {{.*#+}} xmm0 = [2,3,14,15,10,11,0,0,2,3,14,15,10,11,0,0]
4732 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm0, %xmm10, %xmm8
4733 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} xmm10 = <2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15>
4734 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm10, %xmm4, %xmm4
4735 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm8[2],xmm4[3],xmm8[4,5],xmm4[6,7]
4736 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} xmm8 = <6,7,2,3,4,5,u,u,2,3,14,15,u,u,6,7>
4737 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm8, %xmm7, %xmm7
4738 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,5,5,5,5]
4739 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2],xmm15[3],xmm7[4,5],xmm15[6],xmm7[7]
4740 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
4741 ; AVX512DQ-SLOW-NEXT: vinserti32x4 $2, %xmm4, %zmm7, %zmm4
4742 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm0, %xmm5, %xmm0
4743 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm10, %xmm2, %xmm2
4744 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3],xmm0[4,5],xmm2[6,7]
4745 ; AVX512DQ-SLOW-NEXT: vpshufb {{.*#+}} ymm2 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,10,11,6,7,18,19,30,31,26,27,26,27,30,31,26,27,26,27,22,23]
4746 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0,1,2],ymm2[3,4,5,6,7]
4747 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm8, %xmm1, %xmm0
4748 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm14[0,1,2,3,5,5,5,5]
4749 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]
4750 ; AVX512DQ-SLOW-NEXT: vpternlogq $226, %zmm4, %zmm17, %zmm2
4751 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
4752 ; AVX512DQ-SLOW-NEXT: vpshufb {{.*#+}} ymm1 = ymm6[2,3,14,15,10,11,10,11,14,15,10,11,u,u,6,7,18,19,30,31,26,27,26,27,30,31,26,27,u,u,22,23]
4753 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
4754 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,5,7]
4755 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
4756 ; AVX512DQ-SLOW-NEXT: vinserti32x8 $1, %ymm0, %zmm0, %zmm2 {%k1}
4757 ; AVX512DQ-SLOW-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
4758 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm18, %ymm20
4759 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm18, %ymm0
4760 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm13[2],ymm0[3,4],ymm13[5],ymm0[6,7]
4761 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm13, %ymm30
4762 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm15 = xmm0[2,1,2,3]
4763 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm0
4764 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm0[0,3,2,1]
4765 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm10[0,0,2,3,4,5,6,7]
4766 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,6,6,6]
4767 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm15[2,1,2,0,4,5,6,7]
4768 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5,6,7]
4769 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm21, %ymm13
4770 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm19, %ymm1
4771 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm13[0,1],ymm1[2],ymm13[3,4],ymm1[5],ymm13[6,7]
4772 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm2
4773 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm14 = xmm1[2,1,0,3]
4774 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm14[0,0,0,0,4,5,6,7]
4775 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,6,7]
4776 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm2[0,1,2,1]
4777 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm7[0,1,2,3,6,5,6,4]
4778 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4],xmm1[5,6],xmm2[7]
4779 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
4780 ; AVX512DQ-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm16
4781 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm22, %ymm0
4782 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm24, %ymm1
4783 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
4784 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm1
4785 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,1,2,3]
4786 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm1[0,3,2,1]
4787 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm6[0,0,2,3,4,5,6,7]
4788 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,3,3]
4789 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm4[2,1,2,0,4,5,6,7]
4790 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5,6,7]
4791 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm29, %ymm11
4792 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm3 = ymm11[0,1],ymm12[2],ymm11[3],ymm12[4],ymm11[5,6],ymm12[7]
4793 ; AVX512DQ-SLOW-NEXT: vpshufb {{.*#+}} ymm1 = ymm3[4,5,0,1,12,13,14,15,8,9,0,1,12,13,8,9,20,21,16,17,28,29,30,31,24,25,16,17,28,29,24,25]
4794 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm1[5,6,7]
4795 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm8 = ymm0[0,1,2,3],ymm1[4,5,6,7]
4796 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm23, %ymm0
4797 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm25, %ymm1
4798 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
4799 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm1
4800 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[2,1,0,3]
4801 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm5[0,0,0,0,4,5,6,7]
4802 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,6,7]
4803 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]
4804 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm9 = xmm1[0,1,2,3,6,5,6,4]
4805 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm9[4],xmm0[5,6],xmm9[7]
4806 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm9
4807 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm26, %ymm0
4808 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm27, %ymm2
4809 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0,1],ymm2[2],ymm0[3],ymm2[4],ymm0[5,6],ymm2[7]
4810 ; AVX512DQ-SLOW-NEXT: vpshufb {{.*#+}} ymm0 = ymm2[4,5,0,1,12,13,14,15,8,9,12,13,0,1,u,u,20,21,16,17,28,29,30,31,24,25,28,29,16,17,u,u]
4811 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} ymm9 = ymm0[0,1,2],ymm9[3,4,5,6,7],ymm0[8,9,10],ymm9[11,12,13,14,15]
4812 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,5,4]
4813 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm9[4,5,6,7]
4814 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm18
4815 ; AVX512DQ-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
4816 ; AVX512DQ-SLOW-NEXT: vpternlogq $226, %zmm16, %zmm0, %zmm8
4817 ; AVX512DQ-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0]
4818 ; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm8, %zmm17, %zmm18
4819 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm15[3,1,2,1,4,5,6,7]
4820 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm9 = xmm10[0,1,3,3,4,5,6,7]
4821 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,7,7,7,7]
4822 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0],xmm9[1,2],xmm8[3],xmm9[4,5,6,7]
4823 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,7,5,6,5]
4824 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm9 = xmm14[1,1,1,1,4,5,6,7]
4825 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,5,7,7]
4826 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm7 = xmm9[0,1,2,3],xmm7[4],xmm9[5,6],xmm7[7]
4827 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
4828 ; AVX512DQ-SLOW-NEXT: vinserti32x4 $2, %xmm8, %zmm7, %zmm7
4829 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[3,1,2,1,4,5,6,7]
4830 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,3,3,4,5,6,7]
4831 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,7,7,7,7]
4832 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm6[1,2],xmm4[3],xmm6[4,5,6,7]
4833 ; AVX512DQ-SLOW-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[6,7,2,3,14,15,14,15,14,15,2,3,14,15,10,11,22,23,18,19,30,31,30,31,30,31,18,19,30,31,26,27]
4834 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
4835 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
4836 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,5]
4837 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm5[1,1,1,1,4,5,6,7]
4838 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,7,7]
4839 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm4[0,1,2,3],xmm1[4],xmm4[5,6],xmm1[7]
4840 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
4841 ; AVX512DQ-SLOW-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[6,7,2,3,14,15,14,15,14,15,10,11,u,u,2,3,22,23,18,19,30,31,30,31,30,31,26,27,u,u,18,19]
4842 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7],ymm2[8,9,10],ymm1[11,12,13,14,15]
4843 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,4,5]
4844 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
4845 ; AVX512DQ-SLOW-NEXT: vpternlogq $226, %zmm7, %zmm0, %zmm3
4846 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm16
4847 ; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm3, %zmm17, %zmm16
4848 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} xmm7 = <8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15>
4849 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm20, %ymm0
4850 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm30, %ymm1
4851 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
4852 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm7, %xmm0, %xmm1
4853 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm5
4854 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm5[2,2,2,2,4,5,6,7]
4855 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2,3],xmm2[4],xmm1[5,6,7]
4856 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm19, %ymm2
4857 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0],ymm13[1],ymm2[2,3],ymm13[4],ymm2[5,6],ymm13[7]
4858 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm6
4859 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,3,2,1]
4860 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[0,1,0,2,4,5,6,7]
4861 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,6,6,6]
4862 ; AVX512DQ-SLOW-NEXT: vpbroadcastq {{.*#+}} xmm8 = [8,9,0,0,0,1,12,13,8,9,0,0,0,1,12,13]
4863 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm8, %xmm6, %xmm4
4864 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm4[4],xmm3[5],xmm4[6,7]
4865 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
4866 ; AVX512DQ-SLOW-NEXT: vinserti32x4 $2, %xmm1, %zmm3, %zmm3
4867 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm12[0],ymm11[1],ymm12[2,3,4,5],ymm11[6],ymm12[7]
4868 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm22, %ymm4
4869 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm24, %ymm9
4870 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm9[2],ymm4[3,4],ymm9[5],ymm4[6,7]
4871 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm7, %xmm4, %xmm7
4872 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm4, %xmm13
4873 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm9 = xmm13[2,2,2,2,4,5,6,7]
4874 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0],xmm9[1],xmm7[2,3],xmm9[4],xmm7[5,6,7]
4875 ; AVX512DQ-SLOW-NEXT: vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[4,5,0,1,12,13,24,25,20,21],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
4876 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} ymm10 = [65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
4877 ; AVX512DQ-SLOW-NEXT: vpternlogq $236, %ymm10, %ymm9, %ymm7
4878 ; AVX512DQ-SLOW-NEXT: movw $31, %ax
4879 ; AVX512DQ-SLOW-NEXT: kmovw %eax, %k1
4880 ; AVX512DQ-SLOW-NEXT: vinserti32x8 $0, %ymm7, %zmm0, %zmm3 {%k1}
4881 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm23, %ymm7
4882 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm25, %ymm9
4883 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0],ymm9[1],ymm7[2,3],ymm9[4],ymm7[5,6],ymm9[7]
4884 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm7, %xmm9
4885 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm8, %xmm9, %xmm8
4886 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[0,3,2,1]
4887 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm14 = xmm7[0,1,0,2,4,5,6,7]
4888 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,6,6,6,6]
4889 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm8 = xmm14[0,1,2,3],xmm8[4],xmm14[5],xmm8[6,7]
4890 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
4891 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm26, %ymm11
4892 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm27, %ymm14
4893 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm11 = ymm14[0],ymm11[1],ymm14[2,3,4,5],ymm11[6],ymm14[7]
4894 ; AVX512DQ-SLOW-NEXT: vpshufb {{.*#+}} ymm14 = ymm11[8,9,4,5,4,5,6,7,0,1,4,5,0,1,12,13,24,25,20,21,u,u,u,u,u,u,u,u,u,u,u,u]
4895 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4],ymm8[5,6,7]
4896 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm8, %zmm0, %zmm8
4897 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} xmm14 = <10,11,u,u,2,3,14,15,u,u,10,11,12,13,14,15>
4898 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm14, %xmm0, %xmm0
4899 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,3]
4900 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,5,5]
4901 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm5[1],xmm0[2,3],xmm5[4],xmm0[5,6,7]
4902 ; AVX512DQ-SLOW-NEXT: vpbroadcastq {{.*#+}} xmm5 = [10,11,0,0,2,3,14,15,10,11,0,0,2,3,14,15]
4903 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm5, %xmm6, %xmm6
4904 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,1,3,4,5,6,7]
4905 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,3,3]
4906 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm6[4],xmm2[5],xmm6[6,7]
4907 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
4908 ; AVX512DQ-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm2, %zmm0
4909 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm14, %xmm4, %xmm2
4910 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm13[1,1,2,3]
4911 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,5,5,5]
4912 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm4[1],xmm2[2,3],xmm4[4],xmm2[5,6,7]
4913 ; AVX512DQ-SLOW-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[6,7,2,3,14,15,26,27,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
4914 ; AVX512DQ-SLOW-NEXT: vpternlogq $236, %ymm10, %ymm1, %ymm2
4915 ; AVX512DQ-SLOW-NEXT: vinserti32x8 $0, %ymm2, %zmm0, %zmm0 {%k1}
4916 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm5, %xmm9, %xmm1
4917 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm7[0,1,1,3,4,5,6,7]
4918 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,3,3]
4919 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4],xmm2[5],xmm1[6,7]
4920 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
4921 ; AVX512DQ-SLOW-NEXT: vpshufb {{.*#+}} ymm2 = ymm11[10,11,6,7,4,5,6,7,6,7,6,7,2,3,14,15,26,27,22,23,u,u,u,u,u,u,u,u,u,u,u,u]
4922 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4],ymm1[5,6,7]
4923 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm1
4924 ; AVX512DQ-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
4925 ; AVX512DQ-SLOW-NEXT: vmovaps %zmm2, (%rsi)
4926 ; AVX512DQ-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
4927 ; AVX512DQ-SLOW-NEXT: vmovaps %zmm2, (%rdx)
4928 ; AVX512DQ-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
4929 ; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm3, %zmm17, %zmm8
4930 ; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm0, %zmm17, %zmm1
4931 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm18, (%rcx)
4932 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm16, (%r8)
4933 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm8, (%r9)
4934 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm1, (%rax)
4935 ; AVX512DQ-SLOW-NEXT: popq %rax
4936 ; AVX512DQ-SLOW-NEXT: vzeroupper
4937 ; AVX512DQ-SLOW-NEXT: retq
4938 ;
4939 ; AVX512DQ-FAST-LABEL: load_i16_stride6_vf32:
4940 ; AVX512DQ-FAST: # %bb.0:
4941 ; AVX512DQ-FAST-NEXT: pushq %rax
4942 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} xmm0 = <0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15>
4943 ; AVX512DQ-FAST-NEXT: vmovdqa 224(%rdi), %ymm1
4944 ; AVX512DQ-FAST-NEXT: vmovdqa 192(%rdi), %ymm2
4945 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm5 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5,6],ymm1[7]
4946 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm2, %ymm20
4947 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm1, %ymm21
4948 ; AVX512DQ-FAST-NEXT: vpshufb %xmm0, %xmm5, %xmm1
4949 ; AVX512DQ-FAST-NEXT: vpbroadcastq {{.*#+}} xmm2 = [8,9,12,13,0,1,0,0,8,9,12,13,0,1,0,0]
4950 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm5, %xmm3
4951 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm14 = xmm3[2,1,0,3]
4952 ; AVX512DQ-FAST-NEXT: vpshufb %xmm2, %xmm14, %xmm3
4953 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2],xmm1[3],xmm3[4,5],xmm1[6,7]
4954 ; AVX512DQ-FAST-NEXT: vmovdqa 160(%rdi), %ymm12
4955 ; AVX512DQ-FAST-NEXT: vmovdqa (%rdi), %ymm7
4956 ; AVX512DQ-FAST-NEXT: vmovdqa 128(%rdi), %ymm13
4957 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm15 = ymm13[0,1],ymm12[2],ymm13[3,4],ymm12[5],ymm13[6,7]
4958 ; AVX512DQ-FAST-NEXT: vpbroadcastd {{.*#+}} xmm11 = [8,9,4,5,8,9,4,5,8,9,4,5,8,9,4,5]
4959 ; AVX512DQ-FAST-NEXT: vpshufb %xmm11, %xmm15, %xmm4
4960 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm15, %xmm10
4961 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} xmm8 = <0,1,4,5,4,5,u,u,0,1,12,13,u,u,4,5>
4962 ; AVX512DQ-FAST-NEXT: vpshufb %xmm8, %xmm10, %xmm6
4963 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm4 = xmm6[0,1,2],xmm4[3],xmm6[4,5],xmm4[6],xmm6[7]
4964 ; AVX512DQ-FAST-NEXT: vmovdqa 32(%rdi), %ymm6
4965 ; AVX512DQ-FAST-NEXT: vmovdqa 64(%rdi), %ymm9
4966 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
4967 ; AVX512DQ-FAST-NEXT: vinserti32x4 $2, %xmm1, %zmm4, %zmm16
4968 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm7[0],ymm6[1],ymm7[2,3],ymm6[4],ymm7[5,6],ymm6[7]
4969 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm6, %ymm18
4970 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm7, %ymm19
4971 ; AVX512DQ-FAST-NEXT: vpshufb %xmm0, %xmm3, %xmm0
4972 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm3, %xmm4
4973 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[2,1,0,3]
4974 ; AVX512DQ-FAST-NEXT: vpshufb %xmm2, %xmm7, %xmm2
4975 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2],xmm0[3],xmm2[4,5],xmm0[6,7]
4976 ; AVX512DQ-FAST-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm9[2,3],mem[2,3]
4977 ; AVX512DQ-FAST-NEXT: vinserti128 $1, 96(%rdi), %ymm9, %ymm2
4978 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm6 = ymm2[0],ymm1[1],ymm2[2,3,4,5],ymm1[6],ymm2[7]
4979 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm2, %ymm26
4980 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm1, %ymm27
4981 ; AVX512DQ-FAST-NEXT: vpshufb {{.*#+}} ymm2 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
4982 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm9 = ymm0[0,1,2],ymm2[3,4,5,6,7]
4983 ; AVX512DQ-FAST-NEXT: vmovdqa 352(%rdi), %ymm0
4984 ; AVX512DQ-FAST-NEXT: vmovdqa 320(%rdi), %ymm2
4985 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm0[2],ymm2[3,4],ymm0[5],ymm2[6,7]
4986 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm2, %ymm22
4987 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm0, %ymm23
4988 ; AVX512DQ-FAST-NEXT: vpshufb %xmm11, %xmm1, %xmm2
4989 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm1, %xmm4
4990 ; AVX512DQ-FAST-NEXT: vpshufb %xmm8, %xmm4, %xmm8
4991 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm8[0,1,2],xmm2[3],xmm8[4,5],xmm2[6],xmm8[7]
4992 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm8
4993 ; AVX512DQ-FAST-NEXT: vmovdqa 256(%rdi), %ymm2
4994 ; AVX512DQ-FAST-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm2[2,3],mem[2,3]
4995 ; AVX512DQ-FAST-NEXT: vinserti128 $1, 288(%rdi), %ymm2, %ymm11
4996 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm11[0],ymm0[1],ymm11[2,3,4,5],ymm0[6],ymm11[7]
4997 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm11, %ymm24
4998 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm0, %ymm25
4999 ; AVX512DQ-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm2[0,1,12,13,8,9,12,13,8,9,12,13,4,5,u,u,16,17,28,29,24,25,28,29,24,25,28,29,20,21,u,u]
5000 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} ymm8 = ymm0[0,1,2],ymm8[3,4,5,6,7],ymm0[8,9,10],ymm8[11,12,13,14,15]
5001 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,4,6]
5002 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
5003 ; AVX512DQ-FAST-NEXT: vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
5004 ; AVX512DQ-FAST-NEXT: vpternlogq $226, %zmm16, %zmm17, %zmm9
5005 ; AVX512DQ-FAST-NEXT: movw $-2048, %ax # imm = 0xF800
5006 ; AVX512DQ-FAST-NEXT: kmovw %eax, %k1
5007 ; AVX512DQ-FAST-NEXT: vinserti32x8 $1, %ymm0, %zmm0, %zmm9 {%k1}
5008 ; AVX512DQ-FAST-NEXT: vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
5009 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} xmm0 = <2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15>
5010 ; AVX512DQ-FAST-NEXT: vpshufb %xmm0, %xmm5, %xmm5
5011 ; AVX512DQ-FAST-NEXT: vpbroadcastq {{.*#+}} xmm8 = [10,11,14,15,2,3,0,0,10,11,14,15,2,3,0,0]
5012 ; AVX512DQ-FAST-NEXT: vpshufb %xmm8, %xmm14, %xmm14
5013 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1],xmm14[2],xmm5[3],xmm14[4,5],xmm5[6,7]
5014 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} xmm14 = <6,7,2,3,4,5,u,u,2,3,14,15,u,u,6,7>
5015 ; AVX512DQ-FAST-NEXT: vpshufb %xmm14, %xmm10, %xmm10
5016 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,5,5,5,5]
5017 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1,2],xmm15[3],xmm10[4,5],xmm15[6],xmm10[7]
5018 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
5019 ; AVX512DQ-FAST-NEXT: vinserti32x4 $2, %xmm5, %zmm10, %zmm5
5020 ; AVX512DQ-FAST-NEXT: vpshufb %xmm0, %xmm3, %xmm0
5021 ; AVX512DQ-FAST-NEXT: vpshufb %xmm8, %xmm7, %xmm3
5022 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2],xmm0[3],xmm3[4,5],xmm0[6,7]
5023 ; AVX512DQ-FAST-NEXT: vpshufb {{.*#+}} ymm3 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,10,11,6,7,18,19,30,31,26,27,26,27,30,31,26,27,26,27,22,23]
5024 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm0[0,1,2],ymm3[3,4,5,6,7]
5025 ; AVX512DQ-FAST-NEXT: vpshufb %xmm14, %xmm4, %xmm0
5026 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5]
5027 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]
5028 ; AVX512DQ-FAST-NEXT: vpternlogq $226, %zmm5, %zmm17, %zmm3
5029 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
5030 ; AVX512DQ-FAST-NEXT: vpshufb {{.*#+}} ymm1 = ymm2[2,3,14,15,10,11,10,11,14,15,10,11,u,u,6,7,18,19,30,31,26,27,26,27,30,31,26,27,u,u,22,23]
5031 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
5032 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,5,7]
5033 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
5034 ; AVX512DQ-FAST-NEXT: vinserti32x8 $1, %ymm0, %zmm0, %zmm3 {%k1}
5035 ; AVX512DQ-FAST-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
5036 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm20, %ymm0
5037 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm21, %ymm1
5038 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
5039 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm20, %ymm28
5040 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm15 = xmm0[2,1,2,3]
5041 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm0, %xmm0
5042 ; AVX512DQ-FAST-NEXT: vpshuflw {{.*#+}} xmm1 = xmm15[2,1,2,0,4,5,6,7]
5043 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm8 = xmm0[0,3,2,1]
5044 ; AVX512DQ-FAST-NEXT: vpshufb {{.*#+}} xmm0 = xmm8[u,u,0,1,4,5,u,u,12,13,12,13,12,13,12,13]
5045 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5,6,7]
5046 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm12[0,1],ymm13[2],ymm12[3,4],ymm13[5],ymm12[6,7]
5047 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm12, %ymm29
5048 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm13, %ymm30
5049 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
5050 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,1,0,3]
5051 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} xmm6 = <0,1,0,1,0,1,0,1,u,u,8,9,12,13,u,u>
5052 ; AVX512DQ-FAST-NEXT: vpshufb %xmm6, %xmm5, %xmm3
5053 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,2,1]
5054 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm4 = xmm2[0,1,2,3,6,5,6,4]
5055 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm4[4],xmm3[5,6],xmm4[7]
5056 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
5057 ; AVX512DQ-FAST-NEXT: vinserti32x4 $2, %xmm0, %zmm3, %zmm16
5058 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm18, %ymm20
5059 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm19, %ymm13
5060 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm18, %ymm0
5061 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1],ymm0[2],ymm13[3,4],ymm0[5],ymm13[6,7]
5062 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
5063 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,1,2,3]
5064 ; AVX512DQ-FAST-NEXT: vpshuflw {{.*#+}} xmm0 = xmm4[2,1,2,0,4,5,6,7]
5065 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,3,2,1]
5066 ; AVX512DQ-FAST-NEXT: vpshufb {{.*#+}} xmm7 = xmm3[u,u,0,1,4,5,u,u,12,13,u,u,u,u,u,u]
5067 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm7[1,2],xmm0[3],xmm7[4,5,6,7]
5068 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm26, %ymm12
5069 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm27, %ymm9
5070 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm14 = ymm12[0,1],ymm9[2],ymm12[3],ymm9[4],ymm12[5,6],ymm9[7]
5071 ; AVX512DQ-FAST-NEXT: vpshufb {{.*#+}} ymm7 = ymm14[4,5,0,1,12,13,14,15,8,9,0,1,12,13,8,9,20,21,16,17,28,29,30,31,24,25,16,17,28,29,24,25]
5072 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm7[5,6,7]
5073 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm10 = ymm0[0,1,2,3],ymm7[4,5,6,7]
5074 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm22, %ymm0
5075 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm23, %ymm1
5076 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
5077 ; AVX512DQ-FAST-NEXT: vextracti32x4 $1, %ymm0, %xmm17
5078 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm7 = xmm0[2,1,0,3]
5079 ; AVX512DQ-FAST-NEXT: vpshufb %xmm6, %xmm7, %xmm0
5080 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm6 = xmm17[0,1,2,1]
5081 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm11 = xmm6[0,1,2,3,6,5,6,4]
5082 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm11[4],xmm0[5,6],xmm11[7]
5083 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm11
5084 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm24, %ymm0
5085 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm25, %ymm1
5086 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1],ymm1[2],ymm0[3],ymm1[4],ymm0[5,6],ymm1[7]
5087 ; AVX512DQ-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm1[4,5,0,1,12,13,14,15,8,9,12,13,0,1,u,u,20,21,16,17,28,29,30,31,24,25,28,29,16,17,u,u]
5088 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} ymm11 = ymm0[0,1,2],ymm11[3,4,5,6,7],ymm0[8,9,10],ymm11[11,12,13,14,15]
5089 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,5,4]
5090 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm11[4,5,6,7]
5091 ; AVX512DQ-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm18
5092 ; AVX512DQ-FAST-NEXT: vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
5093 ; AVX512DQ-FAST-NEXT: vpternlogq $226, %zmm16, %zmm0, %zmm10
5094 ; AVX512DQ-FAST-NEXT: vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0]
5095 ; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm10, %zmm17, %zmm18
5096 ; AVX512DQ-FAST-NEXT: vpshuflw {{.*#+}} xmm10 = xmm15[3,1,2,1,4,5,6,7]
5097 ; AVX512DQ-FAST-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[u,u,2,3,6,7,u,u,14,15,14,15,14,15,14,15]
5098 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm8 = xmm10[0],xmm8[1,2],xmm10[3],xmm8[4,5,6,7]
5099 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} xmm10 = <2,3,2,3,2,3,2,3,u,u,10,11,14,15,u,u>
5100 ; AVX512DQ-FAST-NEXT: vpshufb %xmm10, %xmm5, %xmm5
5101 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,5]
5102 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm5[0,1,2,3],xmm2[4],xmm5[5,6],xmm2[7]
5103 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5104 ; AVX512DQ-FAST-NEXT: vinserti32x4 $2, %xmm8, %zmm2, %zmm2
5105 ; AVX512DQ-FAST-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[3,1,2,1,4,5,6,7]
5106 ; AVX512DQ-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u,u,2,3,6,7,u,u,14,15,u,u,u,u,u,u]
5107 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1,2],xmm4[3],xmm3[4,5,6,7]
5108 ; AVX512DQ-FAST-NEXT: vpshufb {{.*#+}} ymm4 = ymm14[6,7,2,3,14,15,14,15,14,15,2,3,14,15,10,11,22,23,18,19,30,31,30,31,30,31,18,19,30,31,26,27]
5109 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm4[5,6,7]
5110 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4,5,6,7]
5111 ; AVX512DQ-FAST-NEXT: vpshufb %xmm10, %xmm7, %xmm4
5112 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm5 = xmm6[0,1,2,3,7,5,6,5]
5113 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm5[4],xmm4[5,6],xmm5[7]
5114 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
5115 ; AVX512DQ-FAST-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[6,7,2,3,14,15,14,15,14,15,10,11,u,u,2,3,22,23,18,19,30,31,30,31,30,31,26,27,u,u,18,19]
5116 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} ymm4 = ymm1[0,1,2],ymm4[3,4,5,6,7],ymm1[8,9,10],ymm4[11,12,13,14,15]
5117 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,4,5]
5118 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm4[4,5,6,7]
5119 ; AVX512DQ-FAST-NEXT: vpternlogq $226, %zmm2, %zmm0, %zmm3
5120 ; AVX512DQ-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm16
5121 ; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm3, %zmm17, %zmm16
5122 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} xmm4 = <8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15>
5123 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm21, %ymm0
5124 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm28, %ymm1
5125 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
5126 ; AVX512DQ-FAST-NEXT: vpshufb %xmm4, %xmm3, %xmm0
5127 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm3, %xmm15
5128 ; AVX512DQ-FAST-NEXT: vpshuflw {{.*#+}} xmm1 = xmm15[2,2,2,2,4,5,6,7]
5129 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6,7]
5130 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm29, %ymm0
5131 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm30, %ymm2
5132 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7]
5133 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm0, %xmm6
5134 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]
5135 ; AVX512DQ-FAST-NEXT: vpbroadcastq {{.*#+}} xmm5 = [8,9,0,0,0,1,12,13,8,9,0,0,0,1,12,13]
5136 ; AVX512DQ-FAST-NEXT: vpshufb %xmm5, %xmm6, %xmm2
5137 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} xmm7 = <0,1,2,3,0,1,4,5,u,u,12,13,u,u,u,u>
5138 ; AVX512DQ-FAST-NEXT: vpshufb %xmm7, %xmm0, %xmm8
5139 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm8[0,1,2,3],xmm2[4],xmm8[5],xmm2[6,7]
5140 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
5141 ; AVX512DQ-FAST-NEXT: vinserti32x4 $2, %xmm1, %zmm2, %zmm2
5142 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm9[0],ymm12[1],ymm9[2,3,4,5],ymm12[6],ymm9[7]
5143 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm20, %ymm8
5144 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1],ymm13[2],ymm8[3,4],ymm13[5],ymm8[6,7]
5145 ; AVX512DQ-FAST-NEXT: vpshufb %xmm4, %xmm8, %xmm4
5146 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm8, %xmm10
5147 ; AVX512DQ-FAST-NEXT: vpshuflw {{.*#+}} xmm11 = xmm10[2,2,2,2,4,5,6,7]
5148 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm11[1],xmm4[2,3],xmm11[4],xmm4[5,6,7]
5149 ; AVX512DQ-FAST-NEXT: vmovdqa64 {{.*#+}} ymm19 = [65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
5150 ; AVX512DQ-FAST-NEXT: vpshufb {{.*#+}} ymm13 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[4,5,0,1,12,13,24,25,20,21],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
5151 ; AVX512DQ-FAST-NEXT: vpternlogq $236, %ymm19, %ymm13, %ymm4
5152 ; AVX512DQ-FAST-NEXT: movw $31, %ax
5153 ; AVX512DQ-FAST-NEXT: kmovw %eax, %k1
5154 ; AVX512DQ-FAST-NEXT: vinserti32x8 $0, %ymm4, %zmm0, %zmm2 {%k1}
5155 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm22, %ymm4
5156 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm23, %ymm11
5157 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm13 = ymm4[0],ymm11[1],ymm4[2,3],ymm11[4],ymm4[5,6],ymm11[7]
5158 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm13, %xmm4
5159 ; AVX512DQ-FAST-NEXT: vpshufb %xmm5, %xmm4, %xmm14
5160 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm5 = xmm13[0,3,2,1]
5161 ; AVX512DQ-FAST-NEXT: vpshufb %xmm7, %xmm5, %xmm7
5162 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3],xmm14[4],xmm7[5],xmm14[6,7]
5163 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
5164 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm24, %ymm11
5165 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm25, %ymm12
5166 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm13 = ymm12[0],ymm11[1],ymm12[2,3,4,5],ymm11[6],ymm12[7]
5167 ; AVX512DQ-FAST-NEXT: vpshufb {{.*#+}} ymm14 = ymm13[8,9,4,5,4,5,6,7,0,1,4,5,0,1,12,13,24,25,20,21,u,u,u,u,u,u,u,u,u,u,u,u]
5168 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm7 = ymm14[0,1,2,3,4],ymm7[5,6,7]
5169 ; AVX512DQ-FAST-NEXT: vinserti64x4 $1, %ymm7, %zmm0, %zmm7
5170 ; AVX512DQ-FAST-NEXT: vpbroadcastd {{.*#+}} xmm14 = [10,11,6,7,10,11,6,7,10,11,6,7,10,11,6,7]
5171 ; AVX512DQ-FAST-NEXT: vpshufb %xmm14, %xmm10, %xmm10
5172 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} xmm11 = <10,11,u,u,2,3,14,15,u,u,10,11,12,13,14,15>
5173 ; AVX512DQ-FAST-NEXT: vpshufb %xmm11, %xmm8, %xmm8
5174 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0],xmm10[1],xmm8[2,3],xmm10[4],xmm8[5,6,7]
5175 ; AVX512DQ-FAST-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[6,7,2,3,14,15,26,27,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
5176 ; AVX512DQ-FAST-NEXT: vpternlogq $248, %ymm19, %ymm8, %ymm1
5177 ; AVX512DQ-FAST-NEXT: vpshufb %xmm14, %xmm15, %xmm8
5178 ; AVX512DQ-FAST-NEXT: vpshufb %xmm11, %xmm3, %xmm3
5179 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm8[1],xmm3[2,3],xmm8[4],xmm3[5,6,7]
5180 ; AVX512DQ-FAST-NEXT: vpbroadcastq {{.*#+}} xmm8 = [10,11,0,0,2,3,14,15,10,11,0,0,2,3,14,15]
5181 ; AVX512DQ-FAST-NEXT: vpshufb %xmm8, %xmm6, %xmm6
5182 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} xmm10 = <0,1,2,3,2,3,6,7,u,u,14,15,u,u,u,u>
5183 ; AVX512DQ-FAST-NEXT: vpshufb %xmm10, %xmm0, %xmm0
5184 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm6[4],xmm0[5],xmm6[6,7]
5185 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
5186 ; AVX512DQ-FAST-NEXT: vinserti32x4 $2, %xmm3, %zmm0, %zmm0
5187 ; AVX512DQ-FAST-NEXT: vinserti32x8 $0, %ymm1, %zmm0, %zmm0 {%k1}
5188 ; AVX512DQ-FAST-NEXT: vpshufb %xmm8, %xmm4, %xmm1
5189 ; AVX512DQ-FAST-NEXT: vpshufb %xmm10, %xmm5, %xmm3
5190 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4],xmm3[5],xmm1[6,7]
5191 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
5192 ; AVX512DQ-FAST-NEXT: vpshufb {{.*#+}} ymm3 = ymm13[10,11,6,7,4,5,6,7,6,7,6,7,2,3,14,15,26,27,22,23,u,u,u,u,u,u,u,u,u,u,u,u]
5193 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4],ymm1[5,6,7]
5194 ; AVX512DQ-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm1
5195 ; AVX512DQ-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
5196 ; AVX512DQ-FAST-NEXT: vmovaps %zmm3, (%rsi)
5197 ; AVX512DQ-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
5198 ; AVX512DQ-FAST-NEXT: vmovaps %zmm3, (%rdx)
5199 ; AVX512DQ-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
5200 ; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm2, %zmm17, %zmm7
5201 ; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm0, %zmm17, %zmm1
5202 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm18, (%rcx)
5203 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm16, (%r8)
5204 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm7, (%r9)
5205 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm1, (%rax)
5206 ; AVX512DQ-FAST-NEXT: popq %rax
5207 ; AVX512DQ-FAST-NEXT: vzeroupper
5208 ; AVX512DQ-FAST-NEXT: retq
5209 ;
5210 ; AVX512BW-LABEL: load_i16_stride6_vf32:
5211 ; AVX512BW: # %bb.0:
5212 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
5213 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm3
5214 ; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm5
5215 ; AVX512BW-NEXT: vmovdqa64 128(%rdi), %zmm2
5216 ; AVX512BW-NEXT: vmovdqa64 192(%rdi), %zmm4
5217 ; AVX512BW-NEXT: vmovdqa64 320(%rdi), %zmm0
5218 ; AVX512BW-NEXT: vmovdqa64 256(%rdi), %zmm1
5219 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [0,0,0,0,0,0,4,10,16,22,28,34,40,46,52,58,0,0,0,0,0,0,4,10,16,22,28,34,40,46,52,58]
5220 ; AVX512BW-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]
5221 ; AVX512BW-NEXT: vpermi2w %zmm0, %zmm1, %zmm7
5222 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm8 = [0,6,12,18,24,30,0,0,0,0,0,34,40,46,52,58,0,6,12,18,24,30,0,0,0,0,0,34,40,46,52,58]
5223 ; AVX512BW-NEXT: # zmm8 = mem[0,1,2,3,0,1,2,3]
5224 ; AVX512BW-NEXT: vpermi2w %zmm2, %zmm4, %zmm8
5225 ; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm6 = <0,6,12,18,24,30,36,42,48,54,60,u,u,u,u,u>
5226 ; AVX512BW-NEXT: vpermi2w %zmm5, %zmm3, %zmm6
5227 ; AVX512BW-NEXT: movl $4192256, %edi # imm = 0x3FF800
5228 ; AVX512BW-NEXT: kmovd %edi, %k1
5229 ; AVX512BW-NEXT: vmovdqu16 %zmm8, %zmm6 {%k1}
5230 ; AVX512BW-NEXT: movw $-2048, %di # imm = 0xF800
5231 ; AVX512BW-NEXT: kmovd %edi, %k2
5232 ; AVX512BW-NEXT: vmovdqa32 %zmm7, %zmm6 {%k2}
5233 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm8 = [0,0,0,0,0,0,5,11,17,23,29,35,41,47,53,59,0,0,0,0,0,0,5,11,17,23,29,35,41,47,53,59]
5234 ; AVX512BW-NEXT: # zmm8 = mem[0,1,2,3,0,1,2,3]
5235 ; AVX512BW-NEXT: vpermi2w %zmm0, %zmm1, %zmm8
5236 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm9 = [1,7,13,19,25,31,0,0,0,0,0,35,41,47,53,59,1,7,13,19,25,31,0,0,0,0,0,35,41,47,53,59]
5237 ; AVX512BW-NEXT: # zmm9 = mem[0,1,2,3,0,1,2,3]
5238 ; AVX512BW-NEXT: vpermi2w %zmm2, %zmm4, %zmm9
5239 ; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm7 = <1,7,13,19,25,31,37,43,49,55,61,u,u,u,u,u>
5240 ; AVX512BW-NEXT: vpermi2w %zmm5, %zmm3, %zmm7
5241 ; AVX512BW-NEXT: vmovdqu16 %zmm9, %zmm7 {%k1}
5242 ; AVX512BW-NEXT: vmovdqa32 %zmm8, %zmm7 {%k2}
5243 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm8 = [0,0,0,0,0,0,6,12,18,24,30,36,42,48,54,60,0,0,0,0,0,0,6,12,18,24,30,36,42,48,54,60]
5244 ; AVX512BW-NEXT: # zmm8 = mem[0,1,2,3,0,1,2,3]
5245 ; AVX512BW-NEXT: vpermi2w %zmm0, %zmm1, %zmm8
5246 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm9 = [34,40,46,52,58,0,0,0,0,0,0,4,10,16,22,28,34,40,46,52,58,0,0,0,0,0,0,4,10,16,22,28]
5247 ; AVX512BW-NEXT: # zmm9 = mem[0,1,2,3,0,1,2,3]
5248 ; AVX512BW-NEXT: vpermi2w %zmm4, %zmm2, %zmm9
5249 ; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm10 = <34,40,46,52,58,0,6,12,18,24,30,u,u,u,u,u>
5250 ; AVX512BW-NEXT: vpermi2w %zmm3, %zmm5, %zmm10
5251 ; AVX512BW-NEXT: movl $2095104, %edi # imm = 0x1FF800
5252 ; AVX512BW-NEXT: kmovd %edi, %k2
5253 ; AVX512BW-NEXT: vmovdqu16 %zmm9, %zmm10 {%k2}
5254 ; AVX512BW-NEXT: movl $-2097152, %edi # imm = 0xFFE00000
5255 ; AVX512BW-NEXT: kmovd %edi, %k1
5256 ; AVX512BW-NEXT: vmovdqu16 %zmm8, %zmm10 {%k1}
5257 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm8 = [0,0,0,0,0,1,7,13,19,25,31,37,43,49,55,61,0,0,0,0,0,1,7,13,19,25,31,37,43,49,55,61]
5258 ; AVX512BW-NEXT: # zmm8 = mem[0,1,2,3,0,1,2,3]
5259 ; AVX512BW-NEXT: vpermi2w %zmm0, %zmm1, %zmm8
5260 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm9 = [35,41,47,53,59,0,0,0,0,0,0,5,11,17,23,29,35,41,47,53,59,0,0,0,0,0,0,5,11,17,23,29]
5261 ; AVX512BW-NEXT: # zmm9 = mem[0,1,2,3,0,1,2,3]
5262 ; AVX512BW-NEXT: vpermi2w %zmm4, %zmm2, %zmm9
5263 ; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm11 = <35,41,47,53,59,1,7,13,19,25,31,u,u,u,u,u>
5264 ; AVX512BW-NEXT: vpermi2w %zmm3, %zmm5, %zmm11
5265 ; AVX512BW-NEXT: vmovdqu16 %zmm9, %zmm11 {%k2}
5266 ; AVX512BW-NEXT: vmovdqu16 %zmm8, %zmm11 {%k1}
5267 ; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm8 = <4,10,16,22,28,34,40,46,52,58,u,u,u,u,u,u>
5268 ; AVX512BW-NEXT: vpermi2w %zmm5, %zmm3, %zmm8
5269 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm9 = [36,42,48,54,60,0,0,0,0,0,0,6,12,18,24,30,36,42,48,54,60,0,0,0,0,0,0,6,12,18,24,30]
5270 ; AVX512BW-NEXT: # zmm9 = mem[0,1,2,3,0,1,2,3]
5271 ; AVX512BW-NEXT: vpermi2w %zmm4, %zmm2, %zmm9
5272 ; AVX512BW-NEXT: movw $31, %di
5273 ; AVX512BW-NEXT: kmovd %edi, %k2
5274 ; AVX512BW-NEXT: vmovdqa32 %zmm8, %zmm9 {%k2}
5275 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm8 = [0,0,0,0,0,34,40,46,52,58,0,6,12,18,24,30,0,0,0,0,0,34,40,46,52,58,0,6,12,18,24,30]
5276 ; AVX512BW-NEXT: # zmm8 = mem[0,1,2,3,0,1,2,3]
5277 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm8
5278 ; AVX512BW-NEXT: vmovdqu16 %zmm8, %zmm9 {%k1}
5279 ; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm8 = <5,11,17,23,29,35,41,47,53,59,u,u,u,u,u,u>
5280 ; AVX512BW-NEXT: vpermi2w %zmm5, %zmm3, %zmm8
5281 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm3 = [37,43,49,55,61,0,0,0,0,0,1,7,13,19,25,31,37,43,49,55,61,0,0,0,0,0,1,7,13,19,25,31]
5282 ; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3]
5283 ; AVX512BW-NEXT: vpermi2w %zmm4, %zmm2, %zmm3
5284 ; AVX512BW-NEXT: vmovdqa32 %zmm8, %zmm3 {%k2}
5285 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm2 = [0,0,0,0,0,35,41,47,53,59,1,7,13,19,25,31,0,0,0,0,0,35,41,47,53,59,1,7,13,19,25,31]
5286 ; AVX512BW-NEXT: # zmm2 = mem[0,1,2,3,0,1,2,3]
5287 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
5288 ; AVX512BW-NEXT: vmovdqu16 %zmm2, %zmm3 {%k1}
5289 ; AVX512BW-NEXT: vmovdqa64 %zmm6, (%rsi)
5290 ; AVX512BW-NEXT: vmovdqa64 %zmm7, (%rdx)
5291 ; AVX512BW-NEXT: vmovdqa64 %zmm10, (%rcx)
5292 ; AVX512BW-NEXT: vmovdqa64 %zmm11, (%r8)
5293 ; AVX512BW-NEXT: vmovdqa64 %zmm9, (%r9)
5294 ; AVX512BW-NEXT: vmovdqa64 %zmm3, (%rax)
5295 ; AVX512BW-NEXT: vzeroupper
5296 ; AVX512BW-NEXT: retq
5297 %wide.vec = load <192 x i16>, ptr %in.vec, align 64
5298 %strided.vec0 = shufflevector <192 x i16> %wide.vec, <192 x i16> poison, <32 x i32> <i32 0, i32 6, i32 12, i32 18, i32 24, i32 30, i32 36, i32 42, i32 48, i32 54, i32 60, i32 66, i32 72, i32 78, i32 84, i32 90, i32 96, i32 102, i32 108, i32 114, i32 120, i32 126, i32 132, i32 138, i32 144, i32 150, i32 156, i32 162, i32 168, i32 174, i32 180, i32 186>
5299 %strided.vec1 = shufflevector <192 x i16> %wide.vec, <192 x i16> poison, <32 x i32> <i32 1, i32 7, i32 13, i32 19, i32 25, i32 31, i32 37, i32 43, i32 49, i32 55, i32 61, i32 67, i32 73, i32 79, i32 85, i32 91, i32 97, i32 103, i32 109, i32 115, i32 121, i32 127, i32 133, i32 139, i32 145, i32 151, i32 157, i32 163, i32 169, i32 175, i32 181, i32 187>
5300 %strided.vec2 = shufflevector <192 x i16> %wide.vec, <192 x i16> poison, <32 x i32> <i32 2, i32 8, i32 14, i32 20, i32 26, i32 32, i32 38, i32 44, i32 50, i32 56, i32 62, i32 68, i32 74, i32 80, i32 86, i32 92, i32 98, i32 104, i32 110, i32 116, i32 122, i32 128, i32 134, i32 140, i32 146, i32 152, i32 158, i32 164, i32 170, i32 176, i32 182, i32 188>
5301 %strided.vec3 = shufflevector <192 x i16> %wide.vec, <192 x i16> poison, <32 x i32> <i32 3, i32 9, i32 15, i32 21, i32 27, i32 33, i32 39, i32 45, i32 51, i32 57, i32 63, i32 69, i32 75, i32 81, i32 87, i32 93, i32 99, i32 105, i32 111, i32 117, i32 123, i32 129, i32 135, i32 141, i32 147, i32 153, i32 159, i32 165, i32 171, i32 177, i32 183, i32 189>
5302 %strided.vec4 = shufflevector <192 x i16> %wide.vec, <192 x i16> poison, <32 x i32> <i32 4, i32 10, i32 16, i32 22, i32 28, i32 34, i32 40, i32 46, i32 52, i32 58, i32 64, i32 70, i32 76, i32 82, i32 88, i32 94, i32 100, i32 106, i32 112, i32 118, i32 124, i32 130, i32 136, i32 142, i32 148, i32 154, i32 160, i32 166, i32 172, i32 178, i32 184, i32 190>
5303 %strided.vec5 = shufflevector <192 x i16> %wide.vec, <192 x i16> poison, <32 x i32> <i32 5, i32 11, i32 17, i32 23, i32 29, i32 35, i32 41, i32 47, i32 53, i32 59, i32 65, i32 71, i32 77, i32 83, i32 89, i32 95, i32 101, i32 107, i32 113, i32 119, i32 125, i32 131, i32 137, i32 143, i32 149, i32 155, i32 161, i32 167, i32 173, i32 179, i32 185, i32 191>
5304 store <32 x i16> %strided.vec0, ptr %out.vec0, align 64
5305 store <32 x i16> %strided.vec1, ptr %out.vec1, align 64
5306 store <32 x i16> %strided.vec2, ptr %out.vec2, align 64
5307 store <32 x i16> %strided.vec3, ptr %out.vec3, align 64
5308 store <32 x i16> %strided.vec4, ptr %out.vec4, align 64
5309 store <32 x i16> %strided.vec5, ptr %out.vec5, align 64
5310 ret void
5311 }
5313 define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4, ptr %out.vec5) nounwind {
5314 ; SSE-LABEL: load_i16_stride6_vf64:
5315 ; SSE: # %bb.0:
5316 ; SSE-NEXT: subq $1160, %rsp # imm = 0x488
5317 ; SSE-NEXT: movdqa 496(%rdi), %xmm5
5318 ; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5319 ; SSE-NEXT: movdqa 512(%rdi), %xmm10
5320 ; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5321 ; SSE-NEXT: movdqa 144(%rdi), %xmm8
5322 ; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5323 ; SSE-NEXT: movdqa 160(%rdi), %xmm3
5324 ; SSE-NEXT: movdqa 176(%rdi), %xmm0
5325 ; SSE-NEXT: movdqa 112(%rdi), %xmm7
5326 ; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5327 ; SSE-NEXT: movdqa 96(%rdi), %xmm4
5328 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5329 ; SSE-NEXT: movdqa 128(%rdi), %xmm1
5330 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5331 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,2,4,5,6,7]
5332 ; SSE-NEXT: movdqa {{.*#+}} xmm12 = [65535,65535,65535,0,0,0,65535,65535]
5333 ; SSE-NEXT: movdqa %xmm12, %xmm2
5334 ; SSE-NEXT: pandn %xmm1, %xmm2
5335 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm4[0,1,0,3]
5336 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5337 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
5338 ; SSE-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm7[2],xmm1[3],xmm7[3]
5339 ; SSE-NEXT: pand %xmm12, %xmm1
5340 ; SSE-NEXT: por %xmm2, %xmm1
5341 ; SSE-NEXT: movdqa %xmm1, %xmm2
5342 ; SSE-NEXT: movdqa %xmm3, %xmm1
5343 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm3[2,2,3,3]
5344 ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
5345 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5346 ; SSE-NEXT: movdqa %xmm0, %xmm4
5347 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm3[3,0]
5348 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5349 ; SSE-NEXT: movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5350 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,0],xmm0[0,0]
5351 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm0[2,3]
5352 ; SSE-NEXT: movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5353 ; SSE-NEXT: pslld $16, %xmm0
5354 ; SSE-NEXT: psrldq {{.*#+}} xmm1 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
5355 ; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
5356 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm8[0,3,2,3]
5357 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5358 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
5359 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,1],xmm0[1,3]
5360 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,0]
5361 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5362 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm10[0,1,1,2,4,5,6,7]
5363 ; SSE-NEXT: movdqa %xmm12, %xmm1
5364 ; SSE-NEXT: pandn %xmm0, %xmm1
5365 ; SSE-NEXT: movdqa 480(%rdi), %xmm0
5366 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5367 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
5368 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5369 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
5370 ; SSE-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm5[2],xmm0[3],xmm5[3]
5371 ; SSE-NEXT: pand %xmm12, %xmm0
5372 ; SSE-NEXT: por %xmm1, %xmm0
5373 ; SSE-NEXT: movdqa %xmm0, %xmm2
5374 ; SSE-NEXT: movdqa 544(%rdi), %xmm4
5375 ; SSE-NEXT: movdqa 560(%rdi), %xmm1
5376 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm4[2,2,3,3]
5377 ; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
5378 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5379 ; SSE-NEXT: movdqa %xmm1, %xmm0
5380 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm4[3,0]
5381 ; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5382 ; SSE-NEXT: movdqa %xmm4, %xmm0
5383 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5384 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[1,0],xmm1[0,0]
5385 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm1[2,3]
5386 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5387 ; SSE-NEXT: pslld $16, %xmm1
5388 ; SSE-NEXT: psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
5389 ; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
5390 ; SSE-NEXT: movdqa 528(%rdi), %xmm1
5391 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5392 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3]
5393 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5394 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,2,4,5,6,7]
5395 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,1],xmm1[1,3]
5396 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm0[2,0]
5397 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5398 ; SSE-NEXT: movdqa 32(%rdi), %xmm0
5399 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5400 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,2,4,5,6,7]
5401 ; SSE-NEXT: movdqa %xmm12, %xmm1
5402 ; SSE-NEXT: pandn %xmm0, %xmm1
5403 ; SSE-NEXT: movdqa (%rdi), %xmm0
5404 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5405 ; SSE-NEXT: movdqa 16(%rdi), %xmm6
5406 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
5407 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5408 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
5409 ; SSE-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm6[2],xmm0[3],xmm6[3]
5410 ; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5411 ; SSE-NEXT: pand %xmm12, %xmm0
5412 ; SSE-NEXT: por %xmm1, %xmm0
5413 ; SSE-NEXT: movdqa %xmm0, %xmm2
5414 ; SSE-NEXT: movdqa 64(%rdi), %xmm3
5415 ; SSE-NEXT: movdqa 80(%rdi), %xmm0
5416 ; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm3[2,2,3,3]
5417 ; SSE-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm0[0],xmm11[1],xmm0[1],xmm11[2],xmm0[2],xmm11[3],xmm0[3]
5418 ; SSE-NEXT: movdqa %xmm0, %xmm1
5419 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm3[3,0]
5420 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5421 ; SSE-NEXT: movdqa %xmm3, %xmm1
5422 ; SSE-NEXT: movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5423 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,0],xmm0[0,0]
5424 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm0[2,3]
5425 ; SSE-NEXT: movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5426 ; SSE-NEXT: pslld $16, %xmm0
5427 ; SSE-NEXT: psrldq {{.*#+}} xmm1 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
5428 ; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
5429 ; SSE-NEXT: movdqa 48(%rdi), %xmm0
5430 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5431 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3]
5432 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5433 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
5434 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,1],xmm0[1,3]
5435 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,0]
5436 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5437 ; SSE-NEXT: movdqa 416(%rdi), %xmm0
5438 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5439 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,2,4,5,6,7]
5440 ; SSE-NEXT: movdqa %xmm12, %xmm1
5441 ; SSE-NEXT: pandn %xmm0, %xmm1
5442 ; SSE-NEXT: movdqa 400(%rdi), %xmm2
5443 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5444 ; SSE-NEXT: movdqa 384(%rdi), %xmm0
5445 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5446 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
5447 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5448 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
5449 ; SSE-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
5450 ; SSE-NEXT: pand %xmm12, %xmm0
5451 ; SSE-NEXT: por %xmm1, %xmm0
5452 ; SSE-NEXT: movdqa %xmm0, %xmm2
5453 ; SSE-NEXT: movdqa 448(%rdi), %xmm4
5454 ; SSE-NEXT: movdqa 464(%rdi), %xmm0
5455 ; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm4[2,2,3,3]
5456 ; SSE-NEXT: punpcklwd {{.*#+}} xmm14 = xmm14[0],xmm0[0],xmm14[1],xmm0[1],xmm14[2],xmm0[2],xmm14[3],xmm0[3]
5457 ; SSE-NEXT: movdqa %xmm0, %xmm1
5458 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm4[3,0]
5459 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5460 ; SSE-NEXT: movdqa %xmm4, %xmm1
5461 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5462 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[1,0],xmm0[0,0]
5463 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm0[2,3]
5464 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5465 ; SSE-NEXT: pslld $16, %xmm0
5466 ; SSE-NEXT: psrldq {{.*#+}} xmm1 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
5467 ; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
5468 ; SSE-NEXT: movdqa 432(%rdi), %xmm0
5469 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5470 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3]
5471 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5472 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
5473 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,1],xmm0[1,3]
5474 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,0]
5475 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5476 ; SSE-NEXT: movdqa 320(%rdi), %xmm0
5477 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5478 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,2,4,5,6,7]
5479 ; SSE-NEXT: movdqa %xmm12, %xmm1
5480 ; SSE-NEXT: pandn %xmm0, %xmm1
5481 ; SSE-NEXT: movdqa 304(%rdi), %xmm2
5482 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5483 ; SSE-NEXT: movdqa 288(%rdi), %xmm0
5484 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5485 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
5486 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5487 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
5488 ; SSE-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
5489 ; SSE-NEXT: pand %xmm12, %xmm0
5490 ; SSE-NEXT: por %xmm1, %xmm0
5491 ; SSE-NEXT: movdqa %xmm0, %xmm2
5492 ; SSE-NEXT: movdqa 352(%rdi), %xmm4
5493 ; SSE-NEXT: movdqa 368(%rdi), %xmm0
5494 ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm4[2,2,3,3]
5495 ; SSE-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3]
5496 ; SSE-NEXT: movdqa %xmm0, %xmm1
5497 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm4[3,0]
5498 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5499 ; SSE-NEXT: movdqa %xmm4, %xmm1
5500 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5501 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[1,0],xmm0[0,0]
5502 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm0[2,3]
5503 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5504 ; SSE-NEXT: pslld $16, %xmm0
5505 ; SSE-NEXT: psrldq {{.*#+}} xmm1 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
5506 ; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
5507 ; SSE-NEXT: movdqa 336(%rdi), %xmm0
5508 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5509 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3]
5510 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5511 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
5512 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,1],xmm0[1,3]
5513 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,0]
5514 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5515 ; SSE-NEXT: movdqa 704(%rdi), %xmm0
5516 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5517 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,2,4,5,6,7]
5518 ; SSE-NEXT: movdqa %xmm12, %xmm1
5519 ; SSE-NEXT: pandn %xmm0, %xmm1
5520 ; SSE-NEXT: movdqa 688(%rdi), %xmm2
5521 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5522 ; SSE-NEXT: movdqa 672(%rdi), %xmm0
5523 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5524 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
5525 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5526 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
5527 ; SSE-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
5528 ; SSE-NEXT: pand %xmm12, %xmm0
5529 ; SSE-NEXT: por %xmm1, %xmm0
5530 ; SSE-NEXT: movdqa %xmm0, %xmm2
5531 ; SSE-NEXT: movdqa 736(%rdi), %xmm4
5532 ; SSE-NEXT: movdqa 752(%rdi), %xmm0
5533 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm4[2,2,3,3]
5534 ; SSE-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
5535 ; SSE-NEXT: movdqa %xmm0, %xmm1
5536 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm4[3,0]
5537 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5538 ; SSE-NEXT: movdqa %xmm4, %xmm1
5539 ; SSE-NEXT: movaps %xmm4, (%rsp) # 16-byte Spill
5540 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[1,0],xmm0[0,0]
5541 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm0[2,3]
5542 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5543 ; SSE-NEXT: pslld $16, %xmm0
5544 ; SSE-NEXT: psrldq {{.*#+}} xmm1 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
5545 ; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
5546 ; SSE-NEXT: movdqa 720(%rdi), %xmm0
5547 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5548 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3]
5549 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5550 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
5551 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,1],xmm0[1,3]
5552 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,0]
5553 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5554 ; SSE-NEXT: movdqa 224(%rdi), %xmm0
5555 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5556 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,2,4,5,6,7]
5557 ; SSE-NEXT: movdqa %xmm12, %xmm1
5558 ; SSE-NEXT: pandn %xmm0, %xmm1
5559 ; SSE-NEXT: movdqa 208(%rdi), %xmm2
5560 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5561 ; SSE-NEXT: movdqa 192(%rdi), %xmm0
5562 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5563 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
5564 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5565 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
5566 ; SSE-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
5567 ; SSE-NEXT: pand %xmm12, %xmm0
5568 ; SSE-NEXT: por %xmm1, %xmm0
5569 ; SSE-NEXT: movdqa %xmm0, %xmm1
5570 ; SSE-NEXT: movdqa 256(%rdi), %xmm3
5571 ; SSE-NEXT: movdqa 272(%rdi), %xmm0
5572 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,2,3,3]
5573 ; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
5574 ; SSE-NEXT: movdqa %xmm0, %xmm4
5575 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm3[3,0]
5576 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5577 ; SSE-NEXT: movdqa %xmm3, %xmm4
5578 ; SSE-NEXT: movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5579 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,0],xmm0[0,0]
5580 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm0[2,3]
5581 ; SSE-NEXT: movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5582 ; SSE-NEXT: pslld $16, %xmm0
5583 ; SSE-NEXT: psrldq {{.*#+}} xmm4 = xmm4[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
5584 ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
5585 ; SSE-NEXT: movdqa 240(%rdi), %xmm0
5586 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5587 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3]
5588 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5589 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
5590 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[3,1],xmm0[1,3]
5591 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,0]
5592 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5593 ; SSE-NEXT: movdqa 608(%rdi), %xmm0
5594 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5595 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,2,4,5,6,7]
5596 ; SSE-NEXT: movdqa %xmm12, %xmm4
5597 ; SSE-NEXT: pandn %xmm0, %xmm4
5598 ; SSE-NEXT: movdqa 592(%rdi), %xmm1
5599 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5600 ; SSE-NEXT: movdqa 576(%rdi), %xmm0
5601 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5602 ; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm0[0,1,0,3]
5603 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm15[0,1,2,3,4,6,6,7]
5604 ; SSE-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
5605 ; SSE-NEXT: pand %xmm12, %xmm0
5606 ; SSE-NEXT: por %xmm4, %xmm0
5607 ; SSE-NEXT: movdqa %xmm0, %xmm1
5608 ; SSE-NEXT: movdqa 640(%rdi), %xmm7
5609 ; SSE-NEXT: movdqa 656(%rdi), %xmm4
5610 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm7[2,2,3,3]
5611 ; SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
5612 ; SSE-NEXT: movdqa %xmm4, %xmm0
5613 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm7[3,0]
5614 ; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5615 ; SSE-NEXT: movdqa %xmm7, %xmm0
5616 ; SSE-NEXT: movaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5617 ; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[1,0],xmm4[0,0]
5618 ; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[2,0],xmm4[2,3]
5619 ; SSE-NEXT: movaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5620 ; SSE-NEXT: pslld $16, %xmm4
5621 ; SSE-NEXT: psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
5622 ; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
5623 ; SSE-NEXT: movdqa 624(%rdi), %xmm4
5624 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5625 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm4[0,3,2,3]
5626 ; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm9[0,1,0,2,4,5,6,7]
5627 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,1],xmm13[1,3]
5628 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,0]
5629 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5630 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
5631 ; SSE-NEXT: movdqa %xmm10, %xmm0
5632 ; SSE-NEXT: psrld $16, %xmm0
5633 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
5634 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,5,7,6,7]
5635 ; SSE-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
5636 ; SSE-NEXT: movdqa %xmm12, %xmm13
5637 ; SSE-NEXT: movdqa %xmm12, %xmm0
5638 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5639 ; SSE-NEXT: pandn %xmm4, %xmm0
5640 ; SSE-NEXT: pand %xmm12, %xmm1
5641 ; SSE-NEXT: por %xmm0, %xmm1
5642 ; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
5643 ; SSE-NEXT: # xmm0 = mem[0,1,1,3,4,5,6,7]
5644 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
5645 ; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[3,1],xmm0[1,3]
5646 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm12[2,0]
5647 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5648 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5649 ; SSE-NEXT: psrld $16, %xmm0
5650 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
5651 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,5,7,6,7]
5652 ; SSE-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
5653 ; SSE-NEXT: movdqa %xmm13, %xmm0
5654 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
5655 ; SSE-NEXT: pand %xmm13, %xmm1
5656 ; SSE-NEXT: por %xmm0, %xmm1
5657 ; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
5658 ; SSE-NEXT: # xmm0 = mem[0,1,1,3,4,5,6,7]
5659 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
5660 ; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[3,1],xmm0[1,3]
5661 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm7[2,0]
5662 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5663 ; SSE-NEXT: psrld $16, %xmm6
5664 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
5665 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,5,7,6,7]
5666 ; SSE-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm6[2],xmm1[3],xmm6[3]
5667 ; SSE-NEXT: movdqa %xmm13, %xmm0
5668 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
5669 ; SSE-NEXT: pandn %xmm6, %xmm0
5670 ; SSE-NEXT: pand %xmm13, %xmm1
5671 ; SSE-NEXT: por %xmm0, %xmm1
5672 ; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
5673 ; SSE-NEXT: # xmm0 = mem[0,1,1,3,4,5,6,7]
5674 ; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[3,1],xmm0[1,3]
5675 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm11[2,0]
5676 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5677 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5678 ; SSE-NEXT: psrld $16, %xmm0
5679 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
5680 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,5,7,6,7]
5681 ; SSE-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
5682 ; SSE-NEXT: movdqa %xmm13, %xmm0
5683 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
5684 ; SSE-NEXT: pandn %xmm11, %xmm0
5685 ; SSE-NEXT: pand %xmm13, %xmm1
5686 ; SSE-NEXT: por %xmm0, %xmm1
5687 ; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
5688 ; SSE-NEXT: # xmm0 = mem[0,1,1,3,4,5,6,7]
5689 ; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[3,1],xmm0[1,3]
5690 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm14[2,0]
5691 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5692 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5693 ; SSE-NEXT: psrld $16, %xmm0
5694 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
5695 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,5,7,6,7]
5696 ; SSE-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
5697 ; SSE-NEXT: movdqa %xmm13, %xmm0
5698 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
5699 ; SSE-NEXT: pandn %xmm14, %xmm0
5700 ; SSE-NEXT: pand %xmm13, %xmm1
5701 ; SSE-NEXT: por %xmm0, %xmm1
5702 ; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
5703 ; SSE-NEXT: # xmm0 = mem[0,1,1,3,4,5,6,7]
5704 ; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[3,1],xmm0[1,3]
5705 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm8[2,0]
5706 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5707 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5708 ; SSE-NEXT: psrld $16, %xmm0
5709 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
5710 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,5,7,6,7]
5711 ; SSE-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
5712 ; SSE-NEXT: movdqa %xmm13, %xmm0
5713 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
5714 ; SSE-NEXT: pandn %xmm12, %xmm0
5715 ; SSE-NEXT: pand %xmm13, %xmm1
5716 ; SSE-NEXT: por %xmm0, %xmm1
5717 ; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
5718 ; SSE-NEXT: # xmm0 = mem[0,1,1,3,4,5,6,7]
5719 ; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[3,1],xmm0[1,3]
5720 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm5[2,0]
5721 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5722 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5723 ; SSE-NEXT: psrld $16, %xmm0
5724 ; SSE-NEXT: pshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
5725 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,5,7,6,7]
5726 ; SSE-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
5727 ; SSE-NEXT: movdqa %xmm13, %xmm0
5728 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
5729 ; SSE-NEXT: pandn %xmm7, %xmm0
5730 ; SSE-NEXT: pand %xmm13, %xmm1
5731 ; SSE-NEXT: por %xmm0, %xmm1
5732 ; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
5733 ; SSE-NEXT: # xmm0 = mem[0,1,1,3,4,5,6,7]
5734 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,1],xmm0[1,3]
5735 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,0]
5736 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5737 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5738 ; SSE-NEXT: psrld $16, %xmm0
5739 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm15[0,1,2,3,5,7,6,7]
5740 ; SSE-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm0[2],xmm2[3],xmm0[3]
5741 ; SSE-NEXT: pand %xmm13, %xmm2
5742 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload
5743 ; SSE-NEXT: por %xmm2, %xmm13
5744 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm9[0,1,1,3,4,5,6,7]
5745 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,1],xmm0[1,3]
5746 ; SSE-NEXT: shufps {{.*#+}} xmm13 = xmm13[0,1],xmm3[2,0]
5747 ; SSE-NEXT: movaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5748 ; SSE-NEXT: movdqa %xmm4, %xmm0
5749 ; SSE-NEXT: psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
5750 ; SSE-NEXT: pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
5751 ; SSE-NEXT: # xmm1 = mem[1,1,1,1]
5752 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
5753 ; SSE-NEXT: movdqa {{.*#+}} xmm4 = [65535,65535,65535,0,0,65535,65535,65535]
5754 ; SSE-NEXT: movdqa %xmm4, %xmm1
5755 ; SSE-NEXT: pandn %xmm0, %xmm1
5756 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
5757 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,0],xmm10[0,0]
5758 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm10[2,3]
5759 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm3[0,2,2,3,4,5,6,7]
5760 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3]
5761 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm0[1,0,2,3,4,5,6,7]
5762 ; SSE-NEXT: pand %xmm4, %xmm2
5763 ; SSE-NEXT: por %xmm1, %xmm2
5764 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5765 ; SSE-NEXT: shufps $132, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
5766 ; SSE-NEXT: # xmm0 = xmm0[0,1],mem[0,2]
5767 ; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5768 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
5769 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,0]
5770 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,6,5,4]
5771 ; SSE-NEXT: movdqa {{.*#+}} xmm15 = [65535,65535,65535,65535,65535,0,0,0]
5772 ; SSE-NEXT: movdqa %xmm15, %xmm5
5773 ; SSE-NEXT: pandn %xmm1, %xmm5
5774 ; SSE-NEXT: pand %xmm15, %xmm2
5775 ; SSE-NEXT: por %xmm2, %xmm5
5776 ; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5777 ; SSE-NEXT: psrldq {{.*#+}} xmm6 = xmm6[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
5778 ; SSE-NEXT: pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
5779 ; SSE-NEXT: # xmm2 = mem[1,1,1,1]
5780 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm2[0]
5781 ; SSE-NEXT: movdqa %xmm4, %xmm2
5782 ; SSE-NEXT: pandn %xmm6, %xmm2
5783 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
5784 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5785 ; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[1,0],xmm0[0,0]
5786 ; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[2,0],xmm0[2,3]
5787 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm8[0,2,2,3,4,5,6,7]
5788 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3]
5789 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,0,2,3,4,5,6,7]
5790 ; SSE-NEXT: pand %xmm4, %xmm1
5791 ; SSE-NEXT: por %xmm2, %xmm1
5792 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5793 ; SSE-NEXT: shufps $132, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
5794 ; SSE-NEXT: # xmm0 = xmm0[0,1],mem[0,2]
5795 ; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5796 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,6,6,7]
5797 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,0]
5798 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,5,4]
5799 ; SSE-NEXT: movdqa %xmm15, %xmm5
5800 ; SSE-NEXT: pandn %xmm2, %xmm5
5801 ; SSE-NEXT: pand %xmm15, %xmm1
5802 ; SSE-NEXT: por %xmm1, %xmm5
5803 ; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5804 ; SSE-NEXT: movdqa %xmm14, %xmm1
5805 ; SSE-NEXT: psrldq {{.*#+}} xmm1 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
5806 ; SSE-NEXT: pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
5807 ; SSE-NEXT: # xmm2 = mem[1,1,1,1]
5808 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
5809 ; SSE-NEXT: movdqa %xmm4, %xmm2
5810 ; SSE-NEXT: pandn %xmm1, %xmm2
5811 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5812 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
5813 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,0],xmm5[0,0]
5814 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm5[2,3]
5815 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm1[0,2,2,3,4,5,6,7]
5816 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,3,2,3]
5817 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[1,0,2,3,4,5,6,7]
5818 ; SSE-NEXT: pand %xmm4, %xmm5
5819 ; SSE-NEXT: por %xmm2, %xmm5
5820 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5821 ; SSE-NEXT: shufps $132, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
5822 ; SSE-NEXT: # xmm0 = xmm0[0,1],mem[0,2]
5823 ; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5824 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,6,6,7]
5825 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,0]
5826 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,5,4]
5827 ; SSE-NEXT: movdqa %xmm15, %xmm6
5828 ; SSE-NEXT: pandn %xmm2, %xmm6
5829 ; SSE-NEXT: pand %xmm15, %xmm5
5830 ; SSE-NEXT: por %xmm5, %xmm6
5831 ; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5832 ; SSE-NEXT: psrldq {{.*#+}} xmm7 = xmm7[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
5833 ; SSE-NEXT: pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
5834 ; SSE-NEXT: # xmm5 = mem[1,1,1,1]
5835 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm7 = xmm7[0],xmm5[0]
5836 ; SSE-NEXT: movdqa %xmm4, %xmm5
5837 ; SSE-NEXT: pandn %xmm7, %xmm5
5838 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
5839 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5840 ; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[1,0],xmm0[0,0]
5841 ; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[2,0],xmm0[2,3]
5842 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm9[0,2,2,3,4,5,6,7]
5843 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]
5844 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,0,2,3,4,5,6,7]
5845 ; SSE-NEXT: pand %xmm4, %xmm2
5846 ; SSE-NEXT: por %xmm5, %xmm2
5847 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5848 ; SSE-NEXT: shufps $132, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
5849 ; SSE-NEXT: # xmm0 = xmm0[0,1],mem[0,2]
5850 ; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5851 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm0[0,1,2,3,4,6,6,7]
5852 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,1,2,0]
5853 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,6,5,4]
5854 ; SSE-NEXT: movdqa %xmm15, %xmm6
5855 ; SSE-NEXT: pandn %xmm5, %xmm6
5856 ; SSE-NEXT: pand %xmm15, %xmm2
5857 ; SSE-NEXT: por %xmm2, %xmm6
5858 ; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5859 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
5860 ; SSE-NEXT: movdqa %xmm14, %xmm2
5861 ; SSE-NEXT: psrldq {{.*#+}} xmm2 = xmm2[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
5862 ; SSE-NEXT: pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
5863 ; SSE-NEXT: # xmm5 = mem[1,1,1,1]
5864 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm5[0]
5865 ; SSE-NEXT: movdqa %xmm4, %xmm5
5866 ; SSE-NEXT: pandn %xmm2, %xmm5
5867 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
5868 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5869 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,0],xmm0[0,0]
5870 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[2,3]
5871 ; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm2[0,2,2,3,4,5,6,7]
5872 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,3,2,3]
5873 ; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[1,0,2,3,4,5,6,7]
5874 ; SSE-NEXT: pand %xmm4, %xmm6
5875 ; SSE-NEXT: por %xmm5, %xmm6
5876 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5877 ; SSE-NEXT: shufps $132, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
5878 ; SSE-NEXT: # xmm0 = xmm0[0,1],mem[0,2]
5879 ; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5880 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm0[0,1,2,3,4,6,6,7]
5881 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,1,2,0]
5882 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,6,5,4]
5883 ; SSE-NEXT: movdqa %xmm15, %xmm7
5884 ; SSE-NEXT: pandn %xmm5, %xmm7
5885 ; SSE-NEXT: pand %xmm15, %xmm6
5886 ; SSE-NEXT: por %xmm6, %xmm7
5887 ; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5888 ; SSE-NEXT: movdqa %xmm11, %xmm5
5889 ; SSE-NEXT: psrldq {{.*#+}} xmm5 = xmm5[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
5890 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
5891 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm10[1,1,1,1]
5892 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm6[0]
5893 ; SSE-NEXT: movdqa %xmm4, %xmm6
5894 ; SSE-NEXT: pandn %xmm5, %xmm6
5895 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
5896 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5897 ; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[1,0],xmm0[0,0]
5898 ; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[2,0],xmm0[2,3]
5899 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm11[0,2,2,3,4,5,6,7]
5900 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,3,2,3]
5901 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[1,0,2,3,4,5,6,7]
5902 ; SSE-NEXT: pand %xmm4, %xmm5
5903 ; SSE-NEXT: por %xmm6, %xmm5
5904 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5905 ; SSE-NEXT: shufps $132, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
5906 ; SSE-NEXT: # xmm0 = xmm0[0,1],mem[0,2]
5907 ; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5908 ; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm0[0,1,2,3,4,6,6,7]
5909 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,1,2,0]
5910 ; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,6,5,4]
5911 ; SSE-NEXT: movdqa %xmm15, %xmm7
5912 ; SSE-NEXT: pandn %xmm6, %xmm7
5913 ; SSE-NEXT: pand %xmm15, %xmm5
5914 ; SSE-NEXT: por %xmm5, %xmm7
5915 ; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5916 ; SSE-NEXT: movdqa %xmm12, %xmm5
5917 ; SSE-NEXT: psrldq {{.*#+}} xmm5 = xmm5[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
5918 ; SSE-NEXT: pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
5919 ; SSE-NEXT: # xmm6 = mem[1,1,1,1]
5920 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm6[0]
5921 ; SSE-NEXT: movdqa %xmm4, %xmm7
5922 ; SSE-NEXT: pandn %xmm5, %xmm7
5923 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
5924 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
5925 ; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[1,0],xmm5[0,0]
5926 ; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[2,0],xmm5[2,3]
5927 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm6[0,2,2,3,4,5,6,7]
5928 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,3,2,3]
5929 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[1,0,2,3,4,5,6,7]
5930 ; SSE-NEXT: pand %xmm4, %xmm5
5931 ; SSE-NEXT: por %xmm7, %xmm5
5932 ; SSE-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
5933 ; SSE-NEXT: shufps $132, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
5934 ; SSE-NEXT: # xmm0 = xmm0[0,1],mem[0,2]
5935 ; SSE-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill
5936 ; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm0[0,1,2,3,4,6,6,7]
5937 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,1,2,0]
5938 ; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,6,5,4]
5939 ; SSE-NEXT: movdqa %xmm15, %xmm12
5940 ; SSE-NEXT: pandn %xmm7, %xmm12
5941 ; SSE-NEXT: pand %xmm15, %xmm5
5942 ; SSE-NEXT: por %xmm5, %xmm12
5943 ; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5944 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
5945 ; SSE-NEXT: movdqa %xmm13, %xmm5
5946 ; SSE-NEXT: psrldq {{.*#+}} xmm5 = xmm5[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
5947 ; SSE-NEXT: pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
5948 ; SSE-NEXT: # xmm7 = mem[1,1,1,1]
5949 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm7[0]
5950 ; SSE-NEXT: movdqa %xmm4, %xmm7
5951 ; SSE-NEXT: pandn %xmm5, %xmm7
5952 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
5953 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5954 ; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[1,0],xmm0[0,0]
5955 ; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[2,0],xmm0[2,3]
5956 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm12[0,2,2,3,4,5,6,7]
5957 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,3,2,3]
5958 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[1,0,2,3,4,5,6,7]
5959 ; SSE-NEXT: pand %xmm4, %xmm5
5960 ; SSE-NEXT: por %xmm7, %xmm5
5961 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5962 ; SSE-NEXT: shufps $132, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
5963 ; SSE-NEXT: # xmm0 = xmm0[0,1],mem[0,2]
5964 ; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5965 ; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm0[0,1,2,3,4,6,6,7]
5966 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,1,2,0]
5967 ; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,6,5,4]
5968 ; SSE-NEXT: movdqa %xmm15, %xmm0
5969 ; SSE-NEXT: pandn %xmm7, %xmm0
5970 ; SSE-NEXT: pand %xmm15, %xmm5
5971 ; SSE-NEXT: por %xmm5, %xmm0
5972 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5973 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
5974 ; SSE-NEXT: psrlq $48, %xmm5
5975 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5976 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm0[2,2,3,3]
5977 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm7 = xmm7[0],xmm5[0]
5978 ; SSE-NEXT: movdqa %xmm4, %xmm5
5979 ; SSE-NEXT: pandn %xmm7, %xmm5
5980 ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm8[3,1,2,3,4,5,6,7]
5981 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,3,2,3]
5982 ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,3,3,4,5,6,7]
5983 ; SSE-NEXT: pand %xmm4, %xmm7
5984 ; SSE-NEXT: por %xmm5, %xmm7
5985 ; SSE-NEXT: pshufhw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
5986 ; SSE-NEXT: # xmm5 = mem[0,1,2,3,7,5,6,7]
5987 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,1,0,2]
5988 ; SSE-NEXT: movdqa %xmm15, %xmm8
5989 ; SSE-NEXT: pandn %xmm5, %xmm8
5990 ; SSE-NEXT: pand %xmm15, %xmm7
5991 ; SSE-NEXT: por %xmm7, %xmm8
5992 ; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5993 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
5994 ; SSE-NEXT: movdqa %xmm8, %xmm5
5995 ; SSE-NEXT: psrlq $48, %xmm5
5996 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
5997 ; SSE-NEXT: # xmm7 = mem[2,2,3,3]
5998 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm7 = xmm7[0],xmm5[0]
5999 ; SSE-NEXT: movdqa %xmm4, %xmm5
6000 ; SSE-NEXT: pandn %xmm7, %xmm5
6001 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7]
6002 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,3,2,3]
6003 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,3,4,5,6,7]
6004 ; SSE-NEXT: pand %xmm4, %xmm3
6005 ; SSE-NEXT: por %xmm5, %xmm3
6006 ; SSE-NEXT: pshufhw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
6007 ; SSE-NEXT: # xmm5 = mem[0,1,2,3,7,5,6,7]
6008 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,1,0,2]
6009 ; SSE-NEXT: movdqa %xmm15, %xmm7
6010 ; SSE-NEXT: pandn %xmm5, %xmm7
6011 ; SSE-NEXT: pand %xmm15, %xmm3
6012 ; SSE-NEXT: por %xmm3, %xmm7
6013 ; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6014 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
6015 ; SSE-NEXT: movdqa %xmm7, %xmm3
6016 ; SSE-NEXT: psrlq $48, %xmm3
6017 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
6018 ; SSE-NEXT: # xmm5 = mem[2,2,3,3]
6019 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm3[0]
6020 ; SSE-NEXT: movdqa %xmm4, %xmm3
6021 ; SSE-NEXT: pandn %xmm5, %xmm3
6022 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm9[3,1,2,3,4,5,6,7]
6023 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,3,2,3]
6024 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[0,1,3,3,4,5,6,7]
6025 ; SSE-NEXT: pand %xmm4, %xmm5
6026 ; SSE-NEXT: por %xmm3, %xmm5
6027 ; SSE-NEXT: pshufhw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
6028 ; SSE-NEXT: # xmm3 = mem[0,1,2,3,7,5,6,7]
6029 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,2]
6030 ; SSE-NEXT: movdqa %xmm15, %xmm9
6031 ; SSE-NEXT: pandn %xmm3, %xmm9
6032 ; SSE-NEXT: pand %xmm15, %xmm5
6033 ; SSE-NEXT: por %xmm5, %xmm9
6034 ; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6035 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
6036 ; SSE-NEXT: psrlq $48, %xmm3
6037 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
6038 ; SSE-NEXT: # xmm5 = mem[2,2,3,3]
6039 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm3[0]
6040 ; SSE-NEXT: movdqa %xmm4, %xmm3
6041 ; SSE-NEXT: pandn %xmm5, %xmm3
6042 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]
6043 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3]
6044 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,3,3,4,5,6,7]
6045 ; SSE-NEXT: pand %xmm4, %xmm1
6046 ; SSE-NEXT: por %xmm3, %xmm1
6047 ; SSE-NEXT: pshufhw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
6048 ; SSE-NEXT: # xmm3 = mem[0,1,2,3,7,5,6,7]
6049 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,2]
6050 ; SSE-NEXT: movdqa %xmm15, %xmm5
6051 ; SSE-NEXT: pandn %xmm3, %xmm5
6052 ; SSE-NEXT: pand %xmm15, %xmm1
6053 ; SSE-NEXT: por %xmm1, %xmm5
6054 ; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6055 ; SSE-NEXT: psrlq $48, %xmm10
6056 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
6057 ; SSE-NEXT: # xmm3 = mem[2,2,3,3]
6058 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm10[0]
6059 ; SSE-NEXT: movdqa %xmm4, %xmm1
6060 ; SSE-NEXT: pandn %xmm3, %xmm1
6061 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm11[3,1,2,3,4,5,6,7]
6062 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,3,2,3]
6063 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,3,4,5,6,7]
6064 ; SSE-NEXT: pand %xmm4, %xmm3
6065 ; SSE-NEXT: por %xmm1, %xmm3
6066 ; SSE-NEXT: pshufhw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
6067 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,7,5,6,7]
6068 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,2]
6069 ; SSE-NEXT: movdqa %xmm15, %xmm5
6070 ; SSE-NEXT: pandn %xmm1, %xmm5
6071 ; SSE-NEXT: pand %xmm15, %xmm3
6072 ; SSE-NEXT: por %xmm3, %xmm5
6073 ; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6074 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
6075 ; SSE-NEXT: movdqa %xmm9, %xmm1
6076 ; SSE-NEXT: psrlq $48, %xmm1
6077 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm14[2,2,3,3]
6078 ; SSE-NEXT: movdqa %xmm14, %xmm11
6079 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm1[0]
6080 ; SSE-NEXT: movdqa %xmm4, %xmm1
6081 ; SSE-NEXT: pandn %xmm3, %xmm1
6082 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]
6083 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]
6084 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,3,4,5,6,7]
6085 ; SSE-NEXT: pand %xmm4, %xmm2
6086 ; SSE-NEXT: por %xmm1, %xmm2
6087 ; SSE-NEXT: pshufhw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
6088 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,7,5,6,7]
6089 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,2]
6090 ; SSE-NEXT: movdqa %xmm15, %xmm3
6091 ; SSE-NEXT: pandn %xmm1, %xmm3
6092 ; SSE-NEXT: pand %xmm15, %xmm2
6093 ; SSE-NEXT: por %xmm2, %xmm3
6094 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6095 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
6096 ; SSE-NEXT: movdqa %xmm5, %xmm1
6097 ; SSE-NEXT: psrlq $48, %xmm1
6098 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm13[2,2,3,3]
6099 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
6100 ; SSE-NEXT: movdqa %xmm4, %xmm1
6101 ; SSE-NEXT: pandn %xmm2, %xmm1
6102 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm12[3,1,2,3,4,5,6,7]
6103 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]
6104 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,3,4,5,6,7]
6105 ; SSE-NEXT: pand %xmm4, %xmm2
6106 ; SSE-NEXT: por %xmm1, %xmm2
6107 ; SSE-NEXT: pshufhw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
6108 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,7,5,6,7]
6109 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,2]
6110 ; SSE-NEXT: movdqa %xmm15, %xmm3
6111 ; SSE-NEXT: pandn %xmm1, %xmm3
6112 ; SSE-NEXT: pand %xmm15, %xmm2
6113 ; SSE-NEXT: por %xmm2, %xmm3
6114 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6115 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
6116 ; SSE-NEXT: movdqa %xmm10, %xmm1
6117 ; SSE-NEXT: psrlq $48, %xmm1
6118 ; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
6119 ; SSE-NEXT: # xmm2 = mem[2,2,3,3]
6120 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
6121 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm6[3,1,2,3,4,5,6,7]
6122 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3]
6123 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,3,3,4,5,6,7]
6124 ; SSE-NEXT: pand %xmm4, %xmm1
6125 ; SSE-NEXT: pandn %xmm2, %xmm4
6126 ; SSE-NEXT: por %xmm1, %xmm4
6127 ; SSE-NEXT: pshufhw $231, (%rsp), %xmm1 # 16-byte Folded Reload
6128 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,7,5,6,7]
6129 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,2]
6130 ; SSE-NEXT: movdqa %xmm15, %xmm2
6131 ; SSE-NEXT: pandn %xmm1, %xmm2
6132 ; SSE-NEXT: pand %xmm15, %xmm4
6133 ; SSE-NEXT: por %xmm4, %xmm2
6134 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6135 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
6136 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm13[1,1,1,1]
6137 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
6138 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm6[2,3,2,3]
6139 ; SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
6140 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,1,0,3]
6141 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm2[0,1,2,3,4,5,4,6]
6142 ; SSE-NEXT: punpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
6143 ; SSE-NEXT: # xmm1 = xmm1[1],mem[1]
6144 ; SSE-NEXT: movss {{.*#+}} xmm1 = xmm3[0],xmm1[1,2,3]
6145 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
6146 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm14[0,2,2,3,4,5,6,7]
6147 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,3]
6148 ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,4,6]
6149 ; SSE-NEXT: movdqa %xmm15, %xmm0
6150 ; SSE-NEXT: pandn %xmm3, %xmm0
6151 ; SSE-NEXT: andps %xmm15, %xmm1
6152 ; SSE-NEXT: por %xmm1, %xmm0
6153 ; SSE-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
6154 ; SSE-NEXT: pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
6155 ; SSE-NEXT: # xmm1 = mem[1,1,1,1]
6156 ; SSE-NEXT: pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
6157 ; SSE-NEXT: # xmm3 = mem[2,3,2,3]
6158 ; SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
6159 ; SSE-NEXT: pshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
6160 ; SSE-NEXT: # xmm1 = mem[0,1,0,3]
6161 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm1[0,1,2,3,4,5,4,6]
6162 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm4 = xmm4[1],xmm8[1]
6163 ; SSE-NEXT: movss {{.*#+}} xmm4 = xmm3[0],xmm4[1,2,3]
6164 ; SSE-NEXT: pshuflw $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
6165 ; SSE-NEXT: # xmm3 = mem[0,2,2,3,4,5,6,7]
6166 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,3]
6167 ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,4,6]
6168 ; SSE-NEXT: movdqa %xmm15, %xmm0
6169 ; SSE-NEXT: pandn %xmm3, %xmm0
6170 ; SSE-NEXT: andps %xmm15, %xmm4
6171 ; SSE-NEXT: por %xmm4, %xmm0
6172 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6173 ; SSE-NEXT: pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
6174 ; SSE-NEXT: # xmm3 = mem[1,1,1,1]
6175 ; SSE-NEXT: pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
6176 ; SSE-NEXT: # xmm4 = mem[2,3,2,3]
6177 ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
6178 ; SSE-NEXT: pshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
6179 ; SSE-NEXT: # xmm0 = mem[0,1,0,3]
6180 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6181 ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm0[0,1,2,3,4,5,4,6]
6182 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm7[1]
6183 ; SSE-NEXT: movss {{.*#+}} xmm3 = xmm4[0],xmm3[1,2,3]
6184 ; SSE-NEXT: pshuflw $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
6185 ; SSE-NEXT: # xmm4 = mem[0,2,2,3,4,5,6,7]
6186 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,3]
6187 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,4,6]
6188 ; SSE-NEXT: movdqa %xmm15, %xmm0
6189 ; SSE-NEXT: pandn %xmm4, %xmm0
6190 ; SSE-NEXT: andps %xmm15, %xmm3
6191 ; SSE-NEXT: por %xmm3, %xmm0
6192 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6193 ; SSE-NEXT: pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
6194 ; SSE-NEXT: # xmm3 = mem[1,1,1,1]
6195 ; SSE-NEXT: pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
6196 ; SSE-NEXT: # xmm4 = mem[2,3,2,3]
6197 ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
6198 ; SSE-NEXT: pshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
6199 ; SSE-NEXT: # xmm0 = mem[0,1,0,3]
6200 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6201 ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm0[0,1,2,3,4,5,4,6]
6202 ; SSE-NEXT: punpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
6203 ; SSE-NEXT: # xmm3 = xmm3[1],mem[1]
6204 ; SSE-NEXT: movss {{.*#+}} xmm3 = xmm4[0],xmm3[1,2,3]
6205 ; SSE-NEXT: pshuflw $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
6206 ; SSE-NEXT: # xmm4 = mem[0,2,2,3,4,5,6,7]
6207 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,3]
6208 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,4,6]
6209 ; SSE-NEXT: movdqa %xmm15, %xmm0
6210 ; SSE-NEXT: pandn %xmm4, %xmm0
6211 ; SSE-NEXT: andps %xmm15, %xmm3
6212 ; SSE-NEXT: por %xmm3, %xmm0
6213 ; SSE-NEXT: pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
6214 ; SSE-NEXT: # xmm3 = mem[1,1,1,1]
6215 ; SSE-NEXT: pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
6216 ; SSE-NEXT: # xmm4 = mem[2,3,2,3]
6217 ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
6218 ; SSE-NEXT: pshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
6219 ; SSE-NEXT: # xmm3 = mem[0,1,0,3]
6220 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6221 ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,4,6]
6222 ; SSE-NEXT: punpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
6223 ; SSE-NEXT: # xmm3 = xmm3[1],mem[1]
6224 ; SSE-NEXT: movss {{.*#+}} xmm3 = xmm4[0],xmm3[1,2,3]
6225 ; SSE-NEXT: pshuflw $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
6226 ; SSE-NEXT: # xmm4 = mem[0,2,2,3,4,5,6,7]
6227 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,3]
6228 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,4,6]
6229 ; SSE-NEXT: movdqa %xmm15, %xmm12
6230 ; SSE-NEXT: pandn %xmm4, %xmm12
6231 ; SSE-NEXT: andps %xmm15, %xmm3
6232 ; SSE-NEXT: por %xmm3, %xmm12
6233 ; SSE-NEXT: pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
6234 ; SSE-NEXT: # xmm3 = mem[1,1,1,1]
6235 ; SSE-NEXT: pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
6236 ; SSE-NEXT: # xmm4 = mem[2,3,2,3]
6237 ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
6238 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm11[0,1,0,3]
6239 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6240 ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,4,6]
6241 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm9[1]
6242 ; SSE-NEXT: movss {{.*#+}} xmm3 = xmm4[0],xmm3[1,2,3]
6243 ; SSE-NEXT: pshuflw $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
6244 ; SSE-NEXT: # xmm4 = mem[0,2,2,3,4,5,6,7]
6245 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,3]
6246 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,4,6]
6247 ; SSE-NEXT: movdqa %xmm15, %xmm11
6248 ; SSE-NEXT: pandn %xmm4, %xmm11
6249 ; SSE-NEXT: andps %xmm15, %xmm3
6250 ; SSE-NEXT: por %xmm3, %xmm11
6251 ; SSE-NEXT: pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
6252 ; SSE-NEXT: # xmm3 = mem[1,1,1,1]
6253 ; SSE-NEXT: pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
6254 ; SSE-NEXT: # xmm4 = mem[2,3,2,3]
6255 ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
6256 ; SSE-NEXT: pshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
6257 ; SSE-NEXT: # xmm3 = mem[0,1,0,3]
6258 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6259 ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,4,6]
6260 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm5[1]
6261 ; SSE-NEXT: movss {{.*#+}} xmm3 = xmm4[0],xmm3[1,2,3]
6262 ; SSE-NEXT: pshuflw $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
6263 ; SSE-NEXT: # xmm4 = mem[0,2,2,3,4,5,6,7]
6264 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,3]
6265 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,4,6]
6266 ; SSE-NEXT: movdqa %xmm15, %xmm9
6267 ; SSE-NEXT: pandn %xmm4, %xmm9
6268 ; SSE-NEXT: andps %xmm15, %xmm3
6269 ; SSE-NEXT: por %xmm3, %xmm9
6270 ; SSE-NEXT: pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
6271 ; SSE-NEXT: # xmm3 = mem[1,1,1,1]
6272 ; SSE-NEXT: pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
6273 ; SSE-NEXT: # xmm4 = mem[2,3,2,3]
6274 ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
6275 ; SSE-NEXT: pshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
6276 ; SSE-NEXT: # xmm3 = mem[0,1,0,3]
6277 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6278 ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,4,6]
6279 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm10[1]
6280 ; SSE-NEXT: movss {{.*#+}} xmm3 = xmm4[0],xmm3[1,2,3]
6281 ; SSE-NEXT: pshuflw $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
6282 ; SSE-NEXT: # xmm4 = mem[0,2,2,3,4,5,6,7]
6283 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,3]
6284 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,4,6]
6285 ; SSE-NEXT: movdqa %xmm15, %xmm8
6286 ; SSE-NEXT: pandn %xmm4, %xmm8
6287 ; SSE-NEXT: andps %xmm15, %xmm3
6288 ; SSE-NEXT: por %xmm3, %xmm8
6289 ; SSE-NEXT: psrlq $48, %xmm13
6290 ; SSE-NEXT: movdqa %xmm6, %xmm3
6291 ; SSE-NEXT: psrldq {{.*#+}} xmm3 = xmm3[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
6292 ; SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm13[0],xmm3[1],xmm13[1],xmm3[2],xmm13[2],xmm3[3],xmm13[3]
6293 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
6294 ; SSE-NEXT: psrld $16, %xmm4
6295 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,5,7]
6296 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm4[1]
6297 ; SSE-NEXT: movss {{.*#+}} xmm2 = xmm3[0],xmm2[1,2,3]
6298 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm14[3,1,2,3,4,5,6,7]
6299 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,3]
6300 ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,5,7]
6301 ; SSE-NEXT: movdqa %xmm15, %xmm7
6302 ; SSE-NEXT: pandn %xmm3, %xmm7
6303 ; SSE-NEXT: andps %xmm15, %xmm2
6304 ; SSE-NEXT: por %xmm2, %xmm7
6305 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
6306 ; SSE-NEXT: psrlq $48, %xmm3
6307 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
6308 ; SSE-NEXT: psrldq {{.*#+}} xmm2 = xmm2[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
6309 ; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
6310 ; SSE-NEXT: movdqa %xmm2, %xmm3
6311 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
6312 ; SSE-NEXT: psrld $16, %xmm2
6313 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,5,7]
6314 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm1 = xmm1[1],xmm2[1]
6315 ; SSE-NEXT: movss {{.*#+}} xmm1 = xmm3[0],xmm1[1,2,3]
6316 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
6317 ; SSE-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7]
6318 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,3]
6319 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,5,7]
6320 ; SSE-NEXT: movdqa %xmm15, %xmm6
6321 ; SSE-NEXT: pandn %xmm2, %xmm6
6322 ; SSE-NEXT: andps %xmm15, %xmm1
6323 ; SSE-NEXT: por %xmm1, %xmm6
6324 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
6325 ; SSE-NEXT: psrlq $48, %xmm2
6326 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6327 ; SSE-NEXT: psrldq {{.*#+}} xmm1 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
6328 ; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
6329 ; SSE-NEXT: movdqa %xmm1, %xmm2
6330 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
6331 ; SSE-NEXT: psrld $16, %xmm3
6332 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
6333 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,4,5,5,7]
6334 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm1 = xmm1[1],xmm3[1]
6335 ; SSE-NEXT: movss {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
6336 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
6337 ; SSE-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7]
6338 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,3]
6339 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,5,7]
6340 ; SSE-NEXT: movdqa %xmm15, %xmm10
6341 ; SSE-NEXT: pandn %xmm2, %xmm10
6342 ; SSE-NEXT: andps %xmm15, %xmm1
6343 ; SSE-NEXT: por %xmm1, %xmm10
6344 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
6345 ; SSE-NEXT: psrlq $48, %xmm2
6346 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6347 ; SSE-NEXT: psrldq {{.*#+}} xmm1 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
6348 ; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
6349 ; SSE-NEXT: movdqa %xmm1, %xmm2
6350 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
6351 ; SSE-NEXT: psrld $16, %xmm3
6352 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
6353 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,4,5,5,7]
6354 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm1 = xmm1[1],xmm3[1]
6355 ; SSE-NEXT: movss {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
6356 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
6357 ; SSE-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7]
6358 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,3]
6359 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,5,7]
6360 ; SSE-NEXT: movdqa %xmm15, %xmm14
6361 ; SSE-NEXT: pandn %xmm2, %xmm14
6362 ; SSE-NEXT: andps %xmm15, %xmm1
6363 ; SSE-NEXT: por %xmm1, %xmm14
6364 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
6365 ; SSE-NEXT: psrlq $48, %xmm2
6366 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6367 ; SSE-NEXT: psrldq {{.*#+}} xmm1 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
6368 ; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
6369 ; SSE-NEXT: movdqa %xmm1, %xmm2
6370 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
6371 ; SSE-NEXT: psrld $16, %xmm3
6372 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
6373 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,4,5,5,7]
6374 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm1 = xmm1[1],xmm3[1]
6375 ; SSE-NEXT: movss {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
6376 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
6377 ; SSE-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7]
6378 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,3]
6379 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,5,7]
6380 ; SSE-NEXT: movdqa %xmm15, %xmm13
6381 ; SSE-NEXT: pandn %xmm2, %xmm13
6382 ; SSE-NEXT: andps %xmm15, %xmm1
6383 ; SSE-NEXT: por %xmm1, %xmm13
6384 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
6385 ; SSE-NEXT: psrlq $48, %xmm2
6386 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6387 ; SSE-NEXT: psrldq {{.*#+}} xmm1 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
6388 ; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
6389 ; SSE-NEXT: movdqa %xmm1, %xmm2
6390 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
6391 ; SSE-NEXT: psrld $16, %xmm3
6392 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
6393 ; SSE-NEXT: # xmm1 = mem[0,1,2,3,4,5,5,7]
6394 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm1 = xmm1[1],xmm3[1]
6395 ; SSE-NEXT: movss {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
6396 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
6397 ; SSE-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7]
6398 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,3]
6399 ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm2[0,1,2,3,4,4,5,7]
6400 ; SSE-NEXT: movdqa %xmm15, %xmm5
6401 ; SSE-NEXT: pandn %xmm3, %xmm5
6402 ; SSE-NEXT: andps %xmm15, %xmm1
6403 ; SSE-NEXT: por %xmm1, %xmm5
6404 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
6405 ; SSE-NEXT: psrlq $48, %xmm2
6406 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6407 ; SSE-NEXT: psrldq {{.*#+}} xmm1 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
6408 ; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
6409 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
6410 ; SSE-NEXT: psrld $16, %xmm2
6411 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
6412 ; SSE-NEXT: # xmm3 = mem[0,1,2,3,4,5,5,7]
6413 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm2[1]
6414 ; SSE-NEXT: movss {{.*#+}} xmm3 = xmm1[0],xmm3[1,2,3]
6415 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
6416 ; SSE-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]
6417 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3]
6418 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm1[0,1,2,3,4,4,5,7]
6419 ; SSE-NEXT: movdqa %xmm15, %xmm2
6420 ; SSE-NEXT: pandn %xmm4, %xmm2
6421 ; SSE-NEXT: andps %xmm15, %xmm3
6422 ; SSE-NEXT: por %xmm3, %xmm2
6423 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6424 ; SSE-NEXT: psrlq $48, %xmm1
6425 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
6426 ; SSE-NEXT: psrldq {{.*#+}} xmm3 = xmm3[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
6427 ; SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
6428 ; SSE-NEXT: movdqa %xmm3, %xmm4
6429 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6430 ; SSE-NEXT: psrld $16, %xmm1
6431 ; SSE-NEXT: pshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
6432 ; SSE-NEXT: # xmm3 = mem[0,1,2,3,4,5,5,7]
6433 ; SSE-NEXT: punpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm1[1]
6434 ; SSE-NEXT: movss {{.*#+}} xmm3 = xmm4[0],xmm3[1,2,3]
6435 ; SSE-NEXT: andps %xmm15, %xmm3
6436 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
6437 ; SSE-NEXT: # xmm4 = mem[3,1,2,3,4,5,6,7]
6438 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,3]
6439 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,4,5,7]
6440 ; SSE-NEXT: pandn %xmm4, %xmm15
6441 ; SSE-NEXT: por %xmm3, %xmm15
6442 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6443 ; SSE-NEXT: movaps %xmm1, 96(%rsi)
6444 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
6445 ; SSE-NEXT: movaps %xmm3, 32(%rsi)
6446 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
6447 ; SSE-NEXT: movaps %xmm3, 112(%rsi)
6448 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
6449 ; SSE-NEXT: movaps %xmm3, 48(%rsi)
6450 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
6451 ; SSE-NEXT: movaps %xmm3, 64(%rsi)
6452 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
6453 ; SSE-NEXT: movaps %xmm3, (%rsi)
6454 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
6455 ; SSE-NEXT: movaps %xmm3, 80(%rsi)
6456 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
6457 ; SSE-NEXT: movaps %xmm3, 16(%rsi)
6458 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6459 ; SSE-NEXT: movaps %xmm1, 96(%rdx)
6460 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6461 ; SSE-NEXT: movaps %xmm1, 32(%rdx)
6462 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6463 ; SSE-NEXT: movaps %xmm1, 112(%rdx)
6464 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6465 ; SSE-NEXT: movaps %xmm1, 48(%rdx)
6466 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6467 ; SSE-NEXT: movaps %xmm1, 64(%rdx)
6468 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6469 ; SSE-NEXT: movaps %xmm1, (%rdx)
6470 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6471 ; SSE-NEXT: movaps %xmm1, 80(%rdx)
6472 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6473 ; SSE-NEXT: movaps %xmm1, 16(%rdx)
6474 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6475 ; SSE-NEXT: movaps %xmm1, 96(%rcx)
6476 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6477 ; SSE-NEXT: movaps %xmm1, 112(%rcx)
6478 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6479 ; SSE-NEXT: movaps %xmm1, 64(%rcx)
6480 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6481 ; SSE-NEXT: movaps %xmm1, 80(%rcx)
6482 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6483 ; SSE-NEXT: movaps %xmm1, 32(%rcx)
6484 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6485 ; SSE-NEXT: movaps %xmm1, 48(%rcx)
6486 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6487 ; SSE-NEXT: movaps %xmm1, (%rcx)
6488 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6489 ; SSE-NEXT: movaps %xmm1, 16(%rcx)
6490 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6491 ; SSE-NEXT: movaps %xmm1, 112(%r8)
6492 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6493 ; SSE-NEXT: movaps %xmm1, 96(%r8)
6494 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6495 ; SSE-NEXT: movaps %xmm1, 80(%r8)
6496 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6497 ; SSE-NEXT: movaps %xmm1, 64(%r8)
6498 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6499 ; SSE-NEXT: movaps %xmm1, 48(%r8)
6500 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6501 ; SSE-NEXT: movaps %xmm1, 32(%r8)
6502 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6503 ; SSE-NEXT: movaps %xmm1, 16(%r8)
6504 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6505 ; SSE-NEXT: movaps %xmm1, (%r8)
6506 ; SSE-NEXT: movdqa %xmm8, 112(%r9)
6507 ; SSE-NEXT: movdqa %xmm9, 96(%r9)
6508 ; SSE-NEXT: movdqa %xmm11, 80(%r9)
6509 ; SSE-NEXT: movdqa %xmm12, 64(%r9)
6510 ; SSE-NEXT: movdqa %xmm0, 48(%r9)
6511 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
6512 ; SSE-NEXT: movaps %xmm0, 32(%r9)
6513 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
6514 ; SSE-NEXT: movaps %xmm0, 16(%r9)
6515 ; SSE-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
6516 ; SSE-NEXT: movaps %xmm0, (%r9)
6517 ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
6518 ; SSE-NEXT: movdqa %xmm15, 112(%rax)
6519 ; SSE-NEXT: movdqa %xmm2, 96(%rax)
6520 ; SSE-NEXT: movdqa %xmm5, 80(%rax)
6521 ; SSE-NEXT: movdqa %xmm13, 64(%rax)
6522 ; SSE-NEXT: movdqa %xmm14, 48(%rax)
6523 ; SSE-NEXT: movdqa %xmm10, 32(%rax)
6524 ; SSE-NEXT: movdqa %xmm6, 16(%rax)
6525 ; SSE-NEXT: movdqa %xmm7, (%rax)
6526 ; SSE-NEXT: addq $1160, %rsp # imm = 0x488
6527 ; SSE-NEXT: retq
6528 ;
6529 ; AVX1-ONLY-LABEL: load_i16_stride6_vf64:
6530 ; AVX1-ONLY: # %bb.0:
6531 ; AVX1-ONLY-NEXT: subq $1368, %rsp # imm = 0x558
6532 ; AVX1-ONLY-NEXT: vmovdqa 96(%rdi), %xmm0
6533 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6534 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
6535 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6536 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
6537 ; AVX1-ONLY-NEXT: vmovdqa 112(%rdi), %xmm1
6538 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6539 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
6540 ; AVX1-ONLY-NEXT: vmovdqa 80(%rdi), %xmm1
6541 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6542 ; AVX1-ONLY-NEXT: vpslld $16, %xmm1, %xmm1
6543 ; AVX1-ONLY-NEXT: vmovdqa 64(%rdi), %xmm2
6544 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6545 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
6546 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
6547 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
6548 ; AVX1-ONLY-NEXT: vmovdqa (%rdi), %xmm3
6549 ; AVX1-ONLY-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6550 ; AVX1-ONLY-NEXT: vmovdqa 16(%rdi), %xmm4
6551 ; AVX1-ONLY-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6552 ; AVX1-ONLY-NEXT: vmovdqa 32(%rdi), %xmm1
6553 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6554 ; AVX1-ONLY-NEXT: vmovdqa 48(%rdi), %xmm2
6555 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6556 ; AVX1-ONLY-NEXT: vpsrlq $16, %xmm1, %xmm1
6557 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[0,3,2,3]
6558 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm2 = xmm5[0,1,0,2,4,5,6,7]
6559 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
6560 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[0,1,0,3]
6561 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6562 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
6563 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm2[2],xmm4[2],xmm2[3],xmm4[3]
6564 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3,4,5],xmm2[6,7]
6565 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
6566 ; AVX1-ONLY-NEXT: vmovdqa 176(%rdi), %xmm1
6567 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6568 ; AVX1-ONLY-NEXT: vpslld $16, %xmm1, %xmm1
6569 ; AVX1-ONLY-NEXT: vmovdqa 160(%rdi), %xmm2
6570 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6571 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
6572 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
6573 ; AVX1-ONLY-NEXT: vmovdqa 128(%rdi), %xmm2
6574 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6575 ; AVX1-ONLY-NEXT: vpsrlq $16, %xmm2, %xmm2
6576 ; AVX1-ONLY-NEXT: vmovdqa 144(%rdi), %xmm3
6577 ; AVX1-ONLY-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6578 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,3,2,3]
6579 ; AVX1-ONLY-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6580 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,0,2,4,5,6,7]
6581 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
6582 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5],xmm1[6,7]
6583 ; AVX1-ONLY-NEXT: vmovaps {{.*#+}} ymm10 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0]
6584 ; AVX1-ONLY-NEXT: vandps %ymm0, %ymm10, %ymm0
6585 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2
6586 ; AVX1-ONLY-NEXT: vandnps %ymm2, %ymm10, %ymm2
6587 ; AVX1-ONLY-NEXT: vorps %ymm2, %ymm0, %ymm0
6588 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6589 ; AVX1-ONLY-NEXT: vmovdqa 464(%rdi), %xmm0
6590 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6591 ; AVX1-ONLY-NEXT: vpslld $16, %xmm0, %xmm0
6592 ; AVX1-ONLY-NEXT: vmovdqa 448(%rdi), %xmm1
6593 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6594 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm2 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
6595 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
6596 ; AVX1-ONLY-NEXT: vmovdqa 480(%rdi), %xmm1
6597 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6598 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm12 = xmm1[0,1,0,3]
6599 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm2 = xmm12[0,1,2,3,4,6,6,7]
6600 ; AVX1-ONLY-NEXT: vmovdqa 496(%rdi), %xmm1
6601 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6602 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
6603 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
6604 ; AVX1-ONLY-NEXT: vmovdqa 416(%rdi), %xmm1
6605 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6606 ; AVX1-ONLY-NEXT: vpsrlq $16, %xmm1, %xmm2
6607 ; AVX1-ONLY-NEXT: vmovdqa 432(%rdi), %xmm1
6608 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6609 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,3,2,3]
6610 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6611 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm3 = xmm1[0,1,0,2,4,5,6,7]
6612 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
6613 ; AVX1-ONLY-NEXT: vmovdqa 384(%rdi), %xmm1
6614 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6615 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm13 = xmm1[0,1,0,3]
6616 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm3 = xmm13[0,1,2,3,4,6,6,7]
6617 ; AVX1-ONLY-NEXT: vmovdqa 400(%rdi), %xmm1
6618 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6619 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm3 = xmm3[2],xmm1[2],xmm3[3],xmm1[3]
6620 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3,4,5],xmm3[6,7]
6621 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
6622 ; AVX1-ONLY-NEXT: vmovdqa 560(%rdi), %xmm1
6623 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6624 ; AVX1-ONLY-NEXT: vpslld $16, %xmm1, %xmm2
6625 ; AVX1-ONLY-NEXT: vmovdqa 544(%rdi), %xmm1
6626 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6627 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm3 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
6628 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
6629 ; AVX1-ONLY-NEXT: vmovdqa 512(%rdi), %xmm1
6630 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6631 ; AVX1-ONLY-NEXT: vpsrlq $16, %xmm1, %xmm3
6632 ; AVX1-ONLY-NEXT: vmovdqa 528(%rdi), %xmm1
6633 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6634 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm11 = xmm1[0,3,2,3]
6635 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm4 = xmm11[0,1,0,2,4,5,6,7]
6636 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
6637 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3,4,5],xmm2[6,7]
6638 ; AVX1-ONLY-NEXT: vandps %ymm0, %ymm10, %ymm0
6639 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2
6640 ; AVX1-ONLY-NEXT: vandnps %ymm2, %ymm10, %ymm2
6641 ; AVX1-ONLY-NEXT: vorps %ymm2, %ymm0, %ymm0
6642 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6643 ; AVX1-ONLY-NEXT: vmovdqa 272(%rdi), %xmm0
6644 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6645 ; AVX1-ONLY-NEXT: vpslld $16, %xmm0, %xmm0
6646 ; AVX1-ONLY-NEXT: vmovdqa 256(%rdi), %xmm1
6647 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, (%rsp) # 16-byte Spill
6648 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm2 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
6649 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
6650 ; AVX1-ONLY-NEXT: vmovdqa 288(%rdi), %xmm1
6651 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6652 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm8 = xmm1[0,1,0,3]
6653 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm2 = xmm8[0,1,2,3,4,6,6,7]
6654 ; AVX1-ONLY-NEXT: vmovdqa 304(%rdi), %xmm1
6655 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6656 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
6657 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
6658 ; AVX1-ONLY-NEXT: vmovdqa 224(%rdi), %xmm1
6659 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6660 ; AVX1-ONLY-NEXT: vpsrlq $16, %xmm1, %xmm2
6661 ; AVX1-ONLY-NEXT: vmovdqa 240(%rdi), %xmm1
6662 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6663 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,3,2,3]
6664 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6665 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm3 = xmm1[0,1,0,2,4,5,6,7]
6666 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
6667 ; AVX1-ONLY-NEXT: vmovdqa 192(%rdi), %xmm1
6668 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6669 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[0,1,0,3]
6670 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm3 = xmm9[0,1,2,3,4,6,6,7]
6671 ; AVX1-ONLY-NEXT: vmovdqa 208(%rdi), %xmm1
6672 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6673 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm3 = xmm3[2],xmm1[2],xmm3[3],xmm1[3]
6674 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3,4,5],xmm3[6,7]
6675 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
6676 ; AVX1-ONLY-NEXT: vmovdqa 368(%rdi), %xmm1
6677 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6678 ; AVX1-ONLY-NEXT: vpslld $16, %xmm1, %xmm2
6679 ; AVX1-ONLY-NEXT: vmovdqa 352(%rdi), %xmm6
6680 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm3 = xmm6[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
6681 ; AVX1-ONLY-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6682 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
6683 ; AVX1-ONLY-NEXT: vmovdqa 320(%rdi), %xmm1
6684 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6685 ; AVX1-ONLY-NEXT: vpsrlq $16, %xmm1, %xmm3
6686 ; AVX1-ONLY-NEXT: vmovdqa 336(%rdi), %xmm1
6687 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6688 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm7 = xmm1[0,3,2,3]
6689 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm4 = xmm7[0,1,0,2,4,5,6,7]
6690 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
6691 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3,4,5],xmm2[6,7]
6692 ; AVX1-ONLY-NEXT: vandps %ymm0, %ymm10, %ymm0
6693 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2
6694 ; AVX1-ONLY-NEXT: vandnps %ymm2, %ymm10, %ymm2
6695 ; AVX1-ONLY-NEXT: vorps %ymm2, %ymm0, %ymm0
6696 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6697 ; AVX1-ONLY-NEXT: vmovdqa 656(%rdi), %xmm0
6698 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6699 ; AVX1-ONLY-NEXT: vpslld $16, %xmm0, %xmm0
6700 ; AVX1-ONLY-NEXT: vmovdqa 640(%rdi), %xmm1
6701 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6702 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm2 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
6703 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
6704 ; AVX1-ONLY-NEXT: vmovdqa 672(%rdi), %xmm1
6705 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6706 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[0,1,0,3]
6707 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm3 = xmm4[0,1,2,3,4,6,6,7]
6708 ; AVX1-ONLY-NEXT: vmovdqa 688(%rdi), %xmm1
6709 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6710 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm3 = xmm3[2],xmm1[2],xmm3[3],xmm1[3]
6711 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm1
6712 ; AVX1-ONLY-NEXT: vmovdqa 608(%rdi), %xmm0
6713 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6714 ; AVX1-ONLY-NEXT: vpsrlq $16, %xmm0, %xmm3
6715 ; AVX1-ONLY-NEXT: vmovdqa 624(%rdi), %xmm0
6716 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6717 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,3,2,3]
6718 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6719 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm14 = xmm0[0,1,0,2,4,5,6,7]
6720 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm14 = xmm14[0],xmm3[0],xmm14[1],xmm3[1]
6721 ; AVX1-ONLY-NEXT: vmovdqa 576(%rdi), %xmm0
6722 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6723 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[0,1,0,3]
6724 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm0 = xmm3[0,1,2,3,4,6,6,7]
6725 ; AVX1-ONLY-NEXT: vmovdqa 592(%rdi), %xmm2
6726 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6727 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
6728 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm14[3,4,5],xmm0[6,7]
6729 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7]
6730 ; AVX1-ONLY-NEXT: vmovdqa 752(%rdi), %xmm0
6731 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6732 ; AVX1-ONLY-NEXT: vpslld $16, %xmm0, %xmm0
6733 ; AVX1-ONLY-NEXT: vmovdqa 736(%rdi), %xmm2
6734 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6735 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm14 = xmm2[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
6736 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm0[0],xmm14[1],xmm0[1],xmm14[2],xmm0[2],xmm14[3],xmm0[3]
6737 ; AVX1-ONLY-NEXT: vmovdqa 704(%rdi), %xmm0
6738 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6739 ; AVX1-ONLY-NEXT: vpsrlq $16, %xmm0, %xmm0
6740 ; AVX1-ONLY-NEXT: vmovdqa 720(%rdi), %xmm2
6741 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6742 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]
6743 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm15 = xmm2[0,1,0,2,4,5,6,7]
6744 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]
6745 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm14[6,7]
6746 ; AVX1-ONLY-NEXT: vandps %ymm1, %ymm10, %ymm1
6747 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
6748 ; AVX1-ONLY-NEXT: vandnps %ymm0, %ymm10, %ymm0
6749 ; AVX1-ONLY-NEXT: vorps %ymm0, %ymm1, %ymm0
6750 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6751 ; AVX1-ONLY-NEXT: vpshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
6752 ; AVX1-ONLY-NEXT: # xmm0 = mem[0,1,2,3,5,7,6,7]
6753 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6754 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm1, %xmm1
6755 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
6756 ; AVX1-ONLY-NEXT: vpshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
6757 ; AVX1-ONLY-NEXT: # xmm1 = mem[2,2,3,3]
6758 ; AVX1-ONLY-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
6759 ; AVX1-ONLY-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
6760 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
6761 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm5[0,1,1,3,4,5,6,7]
6762 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
6763 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm14 = xmm5[1,1,1,1]
6764 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm14[0],xmm1[1],xmm14[1]
6765 ; AVX1-ONLY-NEXT: vpshufhw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload
6766 ; AVX1-ONLY-NEXT: # xmm14 = mem[0,1,2,3,5,7,6,7]
6767 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
6768 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm15, %xmm15
6769 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm14 = xmm14[2],xmm15[2],xmm14[3],xmm15[3]
6770 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm14[0,1,2],xmm1[3,4,5],xmm14[6,7]
6771 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
6772 ; AVX1-ONLY-NEXT: vpshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
6773 ; AVX1-ONLY-NEXT: # xmm1 = mem[0,1,1,3,4,5,6,7]
6774 ; AVX1-ONLY-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload
6775 ; AVX1-ONLY-NEXT: # xmm14 = mem[1,1,1,1]
6776 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm14[0],xmm1[1],xmm14[1]
6777 ; AVX1-ONLY-NEXT: vpshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload
6778 ; AVX1-ONLY-NEXT: # xmm14 = mem[2,2,3,3]
6779 ; AVX1-ONLY-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm14, %xmm14 # 16-byte Folded Reload
6780 ; AVX1-ONLY-NEXT: # xmm14 = xmm14[0],mem[0],xmm14[1],mem[1],xmm14[2],mem[2],xmm14[3],mem[3]
6781 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm14[6,7]
6782 ; AVX1-ONLY-NEXT: vandps %ymm0, %ymm10, %ymm0
6783 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
6784 ; AVX1-ONLY-NEXT: vandnps %ymm1, %ymm10, %ymm1
6785 ; AVX1-ONLY-NEXT: vorps %ymm1, %ymm0, %ymm0
6786 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6787 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm0 = xmm12[0,1,2,3,5,7,6,7]
6788 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6789 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm1, %xmm1
6790 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
6791 ; AVX1-ONLY-NEXT: vpshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
6792 ; AVX1-ONLY-NEXT: # xmm1 = mem[2,2,3,3]
6793 ; AVX1-ONLY-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
6794 ; AVX1-ONLY-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
6795 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
6796 ; AVX1-ONLY-NEXT: vpshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
6797 ; AVX1-ONLY-NEXT: # xmm1 = mem[0,1,1,3,4,5,6,7]
6798 ; AVX1-ONLY-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
6799 ; AVX1-ONLY-NEXT: # xmm12 = mem[1,1,1,1]
6800 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm12[0],xmm1[1],xmm12[1]
6801 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm12 = xmm13[0,1,2,3,5,7,6,7]
6802 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
6803 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm13, %xmm13
6804 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm12 = xmm12[2],xmm13[2],xmm12[3],xmm13[3]
6805 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm12[0,1,2],xmm1[3,4,5],xmm12[6,7]
6806 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
6807 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm11[0,1,1,3,4,5,6,7]
6808 ; AVX1-ONLY-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
6809 ; AVX1-ONLY-NEXT: # xmm11 = mem[1,1,1,1]
6810 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm11[0],xmm1[1],xmm11[1]
6811 ; AVX1-ONLY-NEXT: vpshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
6812 ; AVX1-ONLY-NEXT: # xmm11 = mem[2,2,3,3]
6813 ; AVX1-ONLY-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm11, %xmm11 # 16-byte Folded Reload
6814 ; AVX1-ONLY-NEXT: # xmm11 = xmm11[0],mem[0],xmm11[1],mem[1],xmm11[2],mem[2],xmm11[3],mem[3]
6815 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm11[6,7]
6816 ; AVX1-ONLY-NEXT: vandps %ymm0, %ymm10, %ymm0
6817 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
6818 ; AVX1-ONLY-NEXT: vandnps %ymm1, %ymm10, %ymm1
6819 ; AVX1-ONLY-NEXT: vorps %ymm1, %ymm0, %ymm0
6820 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6821 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm0 = xmm8[0,1,2,3,5,7,6,7]
6822 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6823 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm1, %xmm1
6824 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
6825 ; AVX1-ONLY-NEXT: vpshufd $250, (%rsp), %xmm1 # 16-byte Folded Reload
6826 ; AVX1-ONLY-NEXT: # xmm1 = mem[2,2,3,3]
6827 ; AVX1-ONLY-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
6828 ; AVX1-ONLY-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
6829 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
6830 ; AVX1-ONLY-NEXT: vpshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
6831 ; AVX1-ONLY-NEXT: # xmm1 = mem[0,1,1,3,4,5,6,7]
6832 ; AVX1-ONLY-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
6833 ; AVX1-ONLY-NEXT: # xmm8 = mem[1,1,1,1]
6834 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1]
6835 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm8 = xmm9[0,1,2,3,5,7,6,7]
6836 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
6837 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm9, %xmm9
6838 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm8 = xmm8[2],xmm9[2],xmm8[3],xmm9[3]
6839 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm8[0,1,2],xmm1[3,4,5],xmm8[6,7]
6840 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
6841 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm7[0,1,1,3,4,5,6,7]
6842 ; AVX1-ONLY-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
6843 ; AVX1-ONLY-NEXT: # xmm7 = mem[1,1,1,1]
6844 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm7[0],xmm1[1],xmm7[1]
6845 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[2,2,3,3]
6846 ; AVX1-ONLY-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm7 # 16-byte Folded Reload
6847 ; AVX1-ONLY-NEXT: # xmm7 = xmm7[0],mem[0],xmm7[1],mem[1],xmm7[2],mem[2],xmm7[3],mem[3]
6848 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm7[6,7]
6849 ; AVX1-ONLY-NEXT: vandps %ymm0, %ymm10, %ymm0
6850 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
6851 ; AVX1-ONLY-NEXT: vandnps %ymm1, %ymm10, %ymm1
6852 ; AVX1-ONLY-NEXT: vorps %ymm1, %ymm0, %ymm0
6853 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6854 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm0 = xmm4[0,1,2,3,5,7,6,7]
6855 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
6856 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm12, %xmm1
6857 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
6858 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
6859 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm1 = xmm13[2,2,3,3]
6860 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
6861 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm11[0],xmm1[1],xmm11[1],xmm1[2],xmm11[2],xmm1[3],xmm11[3]
6862 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
6863 ; AVX1-ONLY-NEXT: vpshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
6864 ; AVX1-ONLY-NEXT: # xmm1 = mem[0,1,1,3,4,5,6,7]
6865 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
6866 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm4 = xmm7[1,1,1,1]
6867 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
6868 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,7,6,7]
6869 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
6870 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm6, %xmm4
6871 ; AVX1-ONLY-NEXT: vpunpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
6872 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3,4,5],xmm3[6,7]
6873 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
6874 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm2[0,1,1,3,4,5,6,7]
6875 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
6876 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm4[1,1,1,1]
6877 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
6878 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
6879 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm8[2,2,3,3]
6880 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
6881 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
6882 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm2[6,7]
6883 ; AVX1-ONLY-NEXT: vandps %ymm0, %ymm10, %ymm0
6884 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
6885 ; AVX1-ONLY-NEXT: vandnps %ymm1, %ymm10, %ymm1
6886 ; AVX1-ONLY-NEXT: vorps %ymm1, %ymm0, %ymm0
6887 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6888 ; AVX1-ONLY-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
6889 ; AVX1-ONLY-NEXT: # xmm0 = mem[1,1,1,1]
6890 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm1 = xmm5[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
6891 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
6892 ; AVX1-ONLY-NEXT: vmovdqa {{.*#+}} xmm15 = [4,5,0,1,12,13,14,15,8,9,10,11,12,13,14,15]
6893 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6894 ; AVX1-ONLY-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
6895 ; AVX1-ONLY-NEXT: # xmm1 = xmm1[0,1],mem[2,3],xmm1[4,5,6,7]
6896 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6897 ; AVX1-ONLY-NEXT: vpshufb %xmm15, %xmm1, %xmm1
6898 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[3,4],xmm1[5,6,7]
6899 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
6900 ; AVX1-ONLY-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
6901 ; AVX1-ONLY-NEXT: # xmm0 = xmm0[0,1],mem[2,3],xmm0[4,5,6,7]
6902 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6903 ; AVX1-ONLY-NEXT: vpshufb %xmm15, %xmm0, %xmm0
6904 ; AVX1-ONLY-NEXT: vmovdqa {{.*#+}} xmm14 = [0,1,2,3,4,5,6,7,8,9,0,1,12,13,8,9]
6905 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
6906 ; AVX1-ONLY-NEXT: vpblendw $207, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload
6907 ; AVX1-ONLY-NEXT: # xmm2 = mem[0,1,2,3],xmm2[4,5],mem[6,7]
6908 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6909 ; AVX1-ONLY-NEXT: vpshufb %xmm14, %xmm2, %xmm2
6910 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm2
6911 ; AVX1-ONLY-NEXT: vmovaps {{.*#+}} ymm5 = [0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
6912 ; AVX1-ONLY-NEXT: vandnps %ymm1, %ymm5, %ymm1
6913 ; AVX1-ONLY-NEXT: vandps %ymm5, %ymm2, %ymm2
6914 ; AVX1-ONLY-NEXT: vorps %ymm1, %ymm2, %ymm1
6915 ; AVX1-ONLY-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
6916 ; AVX1-ONLY-NEXT: # xmm2 = mem[1,1,1,1]
6917 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
6918 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
6919 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
6920 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
6921 ; AVX1-ONLY-NEXT: vpblendw $48, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
6922 ; AVX1-ONLY-NEXT: # xmm0 = xmm0[0,1,2,3],mem[4,5],xmm0[6,7]
6923 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6924 ; AVX1-ONLY-NEXT: vpshufb %xmm14, %xmm0, %xmm3
6925 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3,4],xmm3[5,6,7]
6926 ; AVX1-ONLY-NEXT: vandps %ymm1, %ymm10, %ymm1
6927 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2
6928 ; AVX1-ONLY-NEXT: vandnps %ymm2, %ymm10, %ymm2
6929 ; AVX1-ONLY-NEXT: vorps %ymm2, %ymm1, %ymm0
6930 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6931 ; AVX1-ONLY-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
6932 ; AVX1-ONLY-NEXT: # xmm1 = mem[1,1,1,1]
6933 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
6934 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm2 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
6935 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
6936 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
6937 ; AVX1-ONLY-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
6938 ; AVX1-ONLY-NEXT: # xmm0 = xmm0[0,1],mem[2,3],xmm0[4,5,6,7]
6939 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6940 ; AVX1-ONLY-NEXT: vpshufb %xmm15, %xmm0, %xmm2
6941 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3,4],xmm2[5,6,7]
6942 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
6943 ; AVX1-ONLY-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
6944 ; AVX1-ONLY-NEXT: # xmm0 = xmm0[0,1],mem[2,3],xmm0[4,5,6,7]
6945 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6946 ; AVX1-ONLY-NEXT: vpshufb %xmm15, %xmm0, %xmm2
6947 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
6948 ; AVX1-ONLY-NEXT: vpblendw $48, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
6949 ; AVX1-ONLY-NEXT: # xmm0 = xmm0[0,1,2,3],mem[4,5],xmm0[6,7]
6950 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6951 ; AVX1-ONLY-NEXT: vpshufb %xmm14, %xmm0, %xmm3
6952 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2
6953 ; AVX1-ONLY-NEXT: vandnps %ymm1, %ymm5, %ymm1
6954 ; AVX1-ONLY-NEXT: vandps %ymm5, %ymm2, %ymm2
6955 ; AVX1-ONLY-NEXT: vorps %ymm1, %ymm2, %ymm1
6956 ; AVX1-ONLY-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
6957 ; AVX1-ONLY-NEXT: # xmm2 = mem[1,1,1,1]
6958 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
6959 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
6960 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
6961 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
6962 ; AVX1-ONLY-NEXT: vpblendw $48, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
6963 ; AVX1-ONLY-NEXT: # xmm0 = xmm0[0,1,2,3],mem[4,5],xmm0[6,7]
6964 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6965 ; AVX1-ONLY-NEXT: vpshufb %xmm14, %xmm0, %xmm3
6966 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3,4],xmm3[5,6,7]
6967 ; AVX1-ONLY-NEXT: vandps %ymm1, %ymm10, %ymm1
6968 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2
6969 ; AVX1-ONLY-NEXT: vandnps %ymm2, %ymm10, %ymm2
6970 ; AVX1-ONLY-NEXT: vorps %ymm2, %ymm1, %ymm0
6971 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6972 ; AVX1-ONLY-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
6973 ; AVX1-ONLY-NEXT: # xmm1 = mem[1,1,1,1]
6974 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm2 = xmm7[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
6975 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
6976 ; AVX1-ONLY-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm0 # 16-byte Folded Reload
6977 ; AVX1-ONLY-NEXT: # xmm0 = xmm6[0,1],mem[2,3],xmm6[4,5,6,7]
6978 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6979 ; AVX1-ONLY-NEXT: vpshufb %xmm15, %xmm0, %xmm2
6980 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3,4],xmm2[5,6,7]
6981 ; AVX1-ONLY-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12, %xmm0 # 16-byte Folded Reload
6982 ; AVX1-ONLY-NEXT: # xmm0 = xmm12[0,1],mem[2,3],xmm12[4,5,6,7]
6983 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6984 ; AVX1-ONLY-NEXT: vpshufb %xmm15, %xmm0, %xmm2
6985 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm13[0,1,2,3],xmm11[4,5],xmm13[6,7]
6986 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6987 ; AVX1-ONLY-NEXT: vpshufb %xmm14, %xmm0, %xmm3
6988 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2
6989 ; AVX1-ONLY-NEXT: vandnps %ymm1, %ymm5, %ymm1
6990 ; AVX1-ONLY-NEXT: vandps %ymm5, %ymm2, %ymm2
6991 ; AVX1-ONLY-NEXT: vorps %ymm1, %ymm2, %ymm2
6992 ; AVX1-ONLY-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
6993 ; AVX1-ONLY-NEXT: # xmm1 = mem[1,1,1,1]
6994 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm3 = xmm4[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
6995 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm1[0]
6996 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm8[0,1,2,3],xmm9[4,5],xmm8[6,7]
6997 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6998 ; AVX1-ONLY-NEXT: vpshufb %xmm14, %xmm0, %xmm4
6999 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0,1,2],xmm3[3,4],xmm4[5,6,7]
7000 ; AVX1-ONLY-NEXT: vandps %ymm2, %ymm10, %ymm2
7001 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3
7002 ; AVX1-ONLY-NEXT: vandnps %ymm3, %ymm10, %ymm3
7003 ; AVX1-ONLY-NEXT: vorps %ymm3, %ymm2, %ymm0
7004 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7005 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
7006 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm9[1,1,1,1]
7007 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
7008 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm3 = xmm4[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
7009 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
7010 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
7011 ; AVX1-ONLY-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
7012 ; AVX1-ONLY-NEXT: # xmm0 = xmm0[0,1],mem[2,3],xmm0[4,5,6,7]
7013 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7014 ; AVX1-ONLY-NEXT: vpshufb %xmm15, %xmm0, %xmm3
7015 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3,4],xmm3[5,6,7]
7016 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
7017 ; AVX1-ONLY-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm12 # 16-byte Folded Reload
7018 ; AVX1-ONLY-NEXT: # xmm12 = xmm0[0,1],mem[2,3],xmm0[4,5,6,7]
7019 ; AVX1-ONLY-NEXT: vpshufb %xmm15, %xmm12, %xmm0
7020 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
7021 ; AVX1-ONLY-NEXT: vpblendw $207, (%rsp), %xmm1, %xmm3 # 16-byte Folded Reload
7022 ; AVX1-ONLY-NEXT: # xmm3 = mem[0,1,2,3],xmm1[4,5],mem[6,7]
7023 ; AVX1-ONLY-NEXT: vpshufb %xmm14, %xmm3, %xmm1
7024 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
7025 ; AVX1-ONLY-NEXT: vandnps %ymm2, %ymm5, %ymm1
7026 ; AVX1-ONLY-NEXT: vandps %ymm5, %ymm0, %ymm0
7027 ; AVX1-ONLY-NEXT: vorps %ymm1, %ymm0, %ymm0
7028 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
7029 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm1 = xmm11[1,1,1,1]
7030 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
7031 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm2 = xmm15[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
7032 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
7033 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
7034 ; AVX1-ONLY-NEXT: vpblendw $48, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload
7035 ; AVX1-ONLY-NEXT: # xmm2 = xmm2[0,1,2,3],mem[4,5],xmm2[6,7]
7036 ; AVX1-ONLY-NEXT: vpshufb %xmm14, %xmm2, %xmm14
7037 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm14[0,1,2],xmm1[3,4],xmm14[5,6,7]
7038 ; AVX1-ONLY-NEXT: vandps %ymm0, %ymm10, %ymm0
7039 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
7040 ; AVX1-ONLY-NEXT: vandnps %ymm1, %ymm10, %ymm1
7041 ; AVX1-ONLY-NEXT: vorps %ymm1, %ymm0, %ymm0
7042 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7043 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
7044 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm0, %xmm0
7045 ; AVX1-ONLY-NEXT: vpshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
7046 ; AVX1-ONLY-NEXT: # xmm1 = mem[2,2,3,3]
7047 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
7048 ; AVX1-ONLY-NEXT: vmovdqa {{.*#+}} xmm14 = [6,7,2,3,14,15,14,15,8,9,10,11,12,13,14,15]
7049 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
7050 ; AVX1-ONLY-NEXT: vpshufb %xmm14, %xmm1, %xmm1
7051 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3,4],xmm1[5,6,7]
7052 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
7053 ; AVX1-ONLY-NEXT: vpshufb %xmm14, %xmm1, %xmm1
7054 ; AVX1-ONLY-NEXT: vmovdqa {{.*#+}} xmm6 = [0,1,2,3,4,5,6,7,14,15,2,3,14,15,10,11]
7055 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
7056 ; AVX1-ONLY-NEXT: vpshufb %xmm6, %xmm7, %xmm8
7057 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm1, %ymm8, %ymm1
7058 ; AVX1-ONLY-NEXT: vandnps %ymm0, %ymm5, %ymm0
7059 ; AVX1-ONLY-NEXT: vandps %ymm5, %ymm1, %ymm1
7060 ; AVX1-ONLY-NEXT: vorps %ymm0, %ymm1, %ymm0
7061 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
7062 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm1, %xmm1
7063 ; AVX1-ONLY-NEXT: vpshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
7064 ; AVX1-ONLY-NEXT: # xmm8 = mem[2,2,3,3]
7065 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm8[0],xmm1[0]
7066 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
7067 ; AVX1-ONLY-NEXT: vpshufb %xmm6, %xmm7, %xmm8
7068 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm8[0,1,2],xmm1[3,4],xmm8[5,6,7]
7069 ; AVX1-ONLY-NEXT: vandps %ymm0, %ymm10, %ymm0
7070 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
7071 ; AVX1-ONLY-NEXT: vandnps %ymm1, %ymm10, %ymm1
7072 ; AVX1-ONLY-NEXT: vorps %ymm1, %ymm0, %ymm0
7073 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7074 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
7075 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm8, %xmm0
7076 ; AVX1-ONLY-NEXT: vpshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
7077 ; AVX1-ONLY-NEXT: # xmm1 = mem[2,2,3,3]
7078 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
7079 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
7080 ; AVX1-ONLY-NEXT: vpshufb %xmm14, %xmm1, %xmm1
7081 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3,4],xmm1[5,6,7]
7082 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
7083 ; AVX1-ONLY-NEXT: vpshufb %xmm14, %xmm1, %xmm1
7084 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
7085 ; AVX1-ONLY-NEXT: vpshufb %xmm6, %xmm7, %xmm7
7086 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm1, %ymm7, %ymm1
7087 ; AVX1-ONLY-NEXT: vandnps %ymm0, %ymm5, %ymm0
7088 ; AVX1-ONLY-NEXT: vandps %ymm5, %ymm1, %ymm1
7089 ; AVX1-ONLY-NEXT: vorps %ymm0, %ymm1, %ymm0
7090 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
7091 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm1, %xmm1
7092 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
7093 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm7 = xmm13[2,2,3,3]
7094 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm7[0],xmm1[0]
7095 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
7096 ; AVX1-ONLY-NEXT: vpshufb %xmm6, %xmm7, %xmm7
7097 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm7[0,1,2],xmm1[3,4],xmm7[5,6,7]
7098 ; AVX1-ONLY-NEXT: vandps %ymm0, %ymm10, %ymm0
7099 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
7100 ; AVX1-ONLY-NEXT: vandnps %ymm1, %ymm10, %ymm1
7101 ; AVX1-ONLY-NEXT: vorps %ymm1, %ymm0, %ymm0
7102 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7103 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm9, %xmm0
7104 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm1 = xmm4[2,2,3,3]
7105 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
7106 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
7107 ; AVX1-ONLY-NEXT: vpshufb %xmm14, %xmm1, %xmm1
7108 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3,4],xmm1[5,6,7]
7109 ; AVX1-ONLY-NEXT: vpshufb %xmm14, %xmm12, %xmm1
7110 ; AVX1-ONLY-NEXT: vpshufb %xmm6, %xmm3, %xmm3
7111 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm1
7112 ; AVX1-ONLY-NEXT: vandnps %ymm0, %ymm5, %ymm0
7113 ; AVX1-ONLY-NEXT: vandps %ymm5, %ymm1, %ymm1
7114 ; AVX1-ONLY-NEXT: vorps %ymm0, %ymm1, %ymm0
7115 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm11, %xmm1
7116 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm3 = xmm15[2,2,3,3]
7117 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0]
7118 ; AVX1-ONLY-NEXT: vpshufb %xmm6, %xmm2, %xmm2
7119 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3,4],xmm2[5,6,7]
7120 ; AVX1-ONLY-NEXT: vandps %ymm0, %ymm10, %ymm0
7121 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
7122 ; AVX1-ONLY-NEXT: vandnps %ymm1, %ymm10, %ymm1
7123 ; AVX1-ONLY-NEXT: vorps %ymm1, %ymm0, %ymm0
7124 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7125 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
7126 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm0, %xmm0
7127 ; AVX1-ONLY-NEXT: vpshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
7128 ; AVX1-ONLY-NEXT: # xmm1 = mem[2,2,3,3]
7129 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
7130 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
7131 ; AVX1-ONLY-NEXT: vpshufb %xmm14, %xmm1, %xmm1
7132 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3,4],xmm1[5,6,7]
7133 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
7134 ; AVX1-ONLY-NEXT: vpshufb %xmm14, %xmm1, %xmm1
7135 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
7136 ; AVX1-ONLY-NEXT: vpshufb %xmm6, %xmm2, %xmm2
7137 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1
7138 ; AVX1-ONLY-NEXT: vandnps %ymm0, %ymm5, %ymm0
7139 ; AVX1-ONLY-NEXT: vandps %ymm5, %ymm1, %ymm1
7140 ; AVX1-ONLY-NEXT: vorps %ymm0, %ymm1, %ymm0
7141 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
7142 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm1, %xmm1
7143 ; AVX1-ONLY-NEXT: vpshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
7144 ; AVX1-ONLY-NEXT: # xmm2 = mem[2,2,3,3]
7145 ; AVX1-ONLY-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
7146 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
7147 ; AVX1-ONLY-NEXT: vpshufb %xmm6, %xmm2, %xmm2
7148 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3,4],xmm2[5,6,7]
7149 ; AVX1-ONLY-NEXT: vandps %ymm0, %ymm10, %ymm0
7150 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
7151 ; AVX1-ONLY-NEXT: vandnps %ymm1, %ymm10, %ymm1
7152 ; AVX1-ONLY-NEXT: vorps %ymm1, %ymm0, %ymm0
7153 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7154 ; AVX1-ONLY-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
7155 ; AVX1-ONLY-NEXT: # xmm0 = mem[1,1,1,1]
7156 ; AVX1-ONLY-NEXT: vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
7157 ; AVX1-ONLY-NEXT: # xmm1 = mem[2,3,2,3]
7158 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
7159 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
7160 ; AVX1-ONLY-NEXT: vpblendw $243, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm2 # 16-byte Folded Reload
7161 ; AVX1-ONLY-NEXT: # xmm2 = mem[0,1],xmm1[2,3],mem[4,5,6,7]
7162 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7163 ; AVX1-ONLY-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,4,5,4,5,6,7,0,1,4,5,0,1,12,13]
7164 ; AVX1-ONLY-NEXT: vpshufb %xmm1, %xmm2, %xmm2
7165 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0
7166 ; AVX1-ONLY-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
7167 ; AVX1-ONLY-NEXT: # xmm2 = mem[1,1,1,1]
7168 ; AVX1-ONLY-NEXT: vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
7169 ; AVX1-ONLY-NEXT: # xmm3 = mem[2,3,2,3]
7170 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
7171 ; AVX1-ONLY-NEXT: vpshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
7172 ; AVX1-ONLY-NEXT: # xmm3 = mem[0,1,0,3]
7173 ; AVX1-ONLY-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7174 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,4,6]
7175 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm8[1]
7176 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3,4,5,6,7]
7177 ; AVX1-ONLY-NEXT: vmovaps {{.*#+}} ymm5 = [65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
7178 ; AVX1-ONLY-NEXT: vandnps %ymm0, %ymm5, %ymm0
7179 ; AVX1-ONLY-NEXT: vandps %ymm5, %ymm2, %ymm2
7180 ; AVX1-ONLY-NEXT: vorps %ymm0, %ymm2, %ymm0
7181 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
7182 ; AVX1-ONLY-NEXT: vpblendw $243, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm4 # 16-byte Folded Reload
7183 ; AVX1-ONLY-NEXT: # xmm4 = mem[0,1],xmm2[2,3],mem[4,5,6,7]
7184 ; AVX1-ONLY-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7185 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm13[0,1,0,3]
7186 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7187 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,6]
7188 ; AVX1-ONLY-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload
7189 ; AVX1-ONLY-NEXT: # xmm2 = xmm2[1],mem[1]
7190 ; AVX1-ONLY-NEXT: vpshufb %xmm1, %xmm4, %xmm3
7191 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4],xmm3[5,6,7]
7192 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2
7193 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
7194 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7195 ; AVX1-ONLY-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
7196 ; AVX1-ONLY-NEXT: # xmm0 = mem[1,1,1,1]
7197 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
7198 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]
7199 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
7200 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
7201 ; AVX1-ONLY-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm13 # 16-byte Folded Reload
7202 ; AVX1-ONLY-NEXT: # xmm13 = xmm2[0,1],mem[2,3],xmm2[4,5,6,7]
7203 ; AVX1-ONLY-NEXT: vpshufb %xmm1, %xmm13, %xmm2
7204 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0
7205 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
7206 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm12[1,1,1,1]
7207 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
7208 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm3 = xmm11[2,3,2,3]
7209 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
7210 ; AVX1-ONLY-NEXT: vpshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
7211 ; AVX1-ONLY-NEXT: # xmm3 = mem[0,1,0,3]
7212 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm6 = xmm3[0,1,2,3,4,5,4,6]
7213 ; AVX1-ONLY-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm6 # 16-byte Folded Reload
7214 ; AVX1-ONLY-NEXT: # xmm6 = xmm6[1],mem[1]
7215 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm6[2,3,4,5,6,7]
7216 ; AVX1-ONLY-NEXT: vandnps %ymm0, %ymm5, %ymm0
7217 ; AVX1-ONLY-NEXT: vandps %ymm5, %ymm2, %ymm2
7218 ; AVX1-ONLY-NEXT: vorps %ymm0, %ymm2, %ymm0
7219 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
7220 ; AVX1-ONLY-NEXT: vpblendw $243, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm8 # 16-byte Folded Reload
7221 ; AVX1-ONLY-NEXT: # xmm8 = mem[0,1],xmm2[2,3],mem[4,5,6,7]
7222 ; AVX1-ONLY-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7223 ; AVX1-ONLY-NEXT: vpshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
7224 ; AVX1-ONLY-NEXT: # xmm6 = mem[0,1,0,3]
7225 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm7 = xmm6[0,1,2,3,4,5,4,6]
7226 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
7227 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm7 = xmm7[1],xmm2[1]
7228 ; AVX1-ONLY-NEXT: vpshufb %xmm1, %xmm8, %xmm10
7229 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4],xmm10[5,6,7]
7230 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm7, %ymm0, %ymm7
7231 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm7[5,6,7]
7232 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7233 ; AVX1-ONLY-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
7234 ; AVX1-ONLY-NEXT: # xmm0 = mem[1,1,1,1]
7235 ; AVX1-ONLY-NEXT: vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
7236 ; AVX1-ONLY-NEXT: # xmm7 = mem[2,3,2,3]
7237 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3]
7238 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
7239 ; AVX1-ONLY-NEXT: vpblendw $12, (%rsp), %xmm7, %xmm7 # 16-byte Folded Reload
7240 ; AVX1-ONLY-NEXT: # xmm7 = xmm7[0,1],mem[2,3],xmm7[4,5,6,7]
7241 ; AVX1-ONLY-NEXT: vmovdqa %xmm7, (%rsp) # 16-byte Spill
7242 ; AVX1-ONLY-NEXT: vpshufb %xmm1, %xmm7, %xmm7
7243 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm7, %ymm0
7244 ; AVX1-ONLY-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
7245 ; AVX1-ONLY-NEXT: # xmm7 = mem[1,1,1,1]
7246 ; AVX1-ONLY-NEXT: vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
7247 ; AVX1-ONLY-NEXT: # xmm10 = mem[2,3,2,3]
7248 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm10[0],xmm7[0],xmm10[1],xmm7[1],xmm10[2],xmm7[2],xmm10[3],xmm7[3]
7249 ; AVX1-ONLY-NEXT: vpshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
7250 ; AVX1-ONLY-NEXT: # xmm8 = mem[0,1,0,3]
7251 ; AVX1-ONLY-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7252 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm10 = xmm8[0,1,2,3,4,5,4,6]
7253 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm10 = xmm10[1],xmm9[1]
7254 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1],xmm10[2,3,4,5,6,7]
7255 ; AVX1-ONLY-NEXT: vandnps %ymm0, %ymm5, %ymm0
7256 ; AVX1-ONLY-NEXT: vandps %ymm5, %ymm7, %ymm7
7257 ; AVX1-ONLY-NEXT: vorps %ymm0, %ymm7, %ymm0
7258 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
7259 ; AVX1-ONLY-NEXT: vpblendw $243, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm8 # 16-byte Folded Reload
7260 ; AVX1-ONLY-NEXT: # xmm8 = mem[0,1],xmm7[2,3],mem[4,5,6,7]
7261 ; AVX1-ONLY-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7262 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm7 = xmm15[0,1,0,3]
7263 ; AVX1-ONLY-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7264 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5,4,6]
7265 ; AVX1-ONLY-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm7 # 16-byte Folded Reload
7266 ; AVX1-ONLY-NEXT: # xmm7 = xmm7[1],mem[1]
7267 ; AVX1-ONLY-NEXT: vpshufb %xmm1, %xmm8, %xmm14
7268 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4],xmm14[5,6,7]
7269 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm7, %ymm0, %ymm7
7270 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm7[5,6,7]
7271 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7272 ; AVX1-ONLY-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
7273 ; AVX1-ONLY-NEXT: # xmm0 = mem[1,1,1,1]
7274 ; AVX1-ONLY-NEXT: vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
7275 ; AVX1-ONLY-NEXT: # xmm7 = mem[2,3,2,3]
7276 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3]
7277 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
7278 ; AVX1-ONLY-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm10 # 16-byte Folded Reload
7279 ; AVX1-ONLY-NEXT: # xmm10 = xmm7[0,1],mem[2,3],xmm7[4,5,6,7]
7280 ; AVX1-ONLY-NEXT: vpshufb %xmm1, %xmm10, %xmm7
7281 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm7, %ymm0
7282 ; AVX1-ONLY-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
7283 ; AVX1-ONLY-NEXT: # xmm7 = mem[1,1,1,1]
7284 ; AVX1-ONLY-NEXT: vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
7285 ; AVX1-ONLY-NEXT: # xmm15 = mem[2,3,2,3]
7286 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm15[0],xmm7[0],xmm15[1],xmm7[1],xmm15[2],xmm7[2],xmm15[3],xmm7[3]
7287 ; AVX1-ONLY-NEXT: vpshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
7288 ; AVX1-ONLY-NEXT: # xmm15 = mem[0,1,0,3]
7289 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm9 = xmm15[0,1,2,3,4,5,4,6]
7290 ; AVX1-ONLY-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm9, %xmm9 # 16-byte Folded Reload
7291 ; AVX1-ONLY-NEXT: # xmm9 = xmm9[1],mem[1]
7292 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1],xmm9[2,3,4,5,6,7]
7293 ; AVX1-ONLY-NEXT: vandnps %ymm0, %ymm5, %ymm0
7294 ; AVX1-ONLY-NEXT: vandps %ymm5, %ymm7, %ymm7
7295 ; AVX1-ONLY-NEXT: vorps %ymm0, %ymm7, %ymm0
7296 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
7297 ; AVX1-ONLY-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm7 # 16-byte Folded Reload
7298 ; AVX1-ONLY-NEXT: # xmm7 = xmm7[0,1],mem[2,3],xmm7[4,5,6,7]
7299 ; AVX1-ONLY-NEXT: vpshufb %xmm1, %xmm7, %xmm9
7300 ; AVX1-ONLY-NEXT: vpshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload
7301 ; AVX1-ONLY-NEXT: # xmm14 = mem[0,1,0,3]
7302 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm8 = xmm14[0,1,2,3,4,5,4,6]
7303 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
7304 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm8 = xmm8[1],xmm1[1]
7305 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7]
7306 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm8, %ymm0, %ymm8
7307 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm8[5,6,7]
7308 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7309 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
7310 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm0, %xmm0
7311 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm8 = xmm4[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
7312 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3]
7313 ; AVX1-ONLY-NEXT: vmovdqa {{.*#+}} xmm0 = [6,7,2,3,4,5,6,7,6,7,6,7,2,3,14,15]
7314 ; AVX1-ONLY-NEXT: vpshufb %xmm0, %xmm13, %xmm4
7315 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm8, %ymm4, %ymm4
7316 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm12, %xmm8
7317 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm9 = xmm11[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
7318 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
7319 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
7320 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm9, %xmm9
7321 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,5,7]
7322 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm9[1]
7323 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm3 = xmm8[0,1],xmm3[2,3,4,5,6,7]
7324 ; AVX1-ONLY-NEXT: vandnps %ymm4, %ymm5, %ymm4
7325 ; AVX1-ONLY-NEXT: vandps %ymm5, %ymm3, %ymm3
7326 ; AVX1-ONLY-NEXT: vorps %ymm4, %ymm3, %ymm3
7327 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm2, %xmm4
7328 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm2 = xmm6[0,1,2,3,4,5,5,7]
7329 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm4[1]
7330 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
7331 ; AVX1-ONLY-NEXT: vpshufb %xmm0, %xmm4, %xmm4
7332 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4],xmm4[5,6,7]
7333 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2
7334 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm2[5,6,7]
7335 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
7336 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm2, %xmm2
7337 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
7338 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm4 = xmm4[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
7339 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
7340 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
7341 ; AVX1-ONLY-NEXT: vpshufb %xmm0, %xmm4, %xmm4
7342 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm2, %ymm4, %ymm2
7343 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
7344 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm4, %xmm4
7345 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
7346 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm6 = xmm6[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
7347 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
7348 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
7349 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm6, %xmm6
7350 ; AVX1-ONLY-NEXT: vpshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
7351 ; AVX1-ONLY-NEXT: # xmm8 = mem[0,1,2,3,4,5,5,7]
7352 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm6 = xmm8[1],xmm6[1]
7353 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm6[2,3,4,5,6,7]
7354 ; AVX1-ONLY-NEXT: vandnps %ymm2, %ymm5, %ymm2
7355 ; AVX1-ONLY-NEXT: vandps %ymm5, %ymm4, %ymm4
7356 ; AVX1-ONLY-NEXT: vorps %ymm2, %ymm4, %ymm2
7357 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
7358 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm4, %xmm4
7359 ; AVX1-ONLY-NEXT: vpshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
7360 ; AVX1-ONLY-NEXT: # xmm6 = mem[0,1,2,3,4,5,5,7]
7361 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm4 = xmm6[1],xmm4[1]
7362 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
7363 ; AVX1-ONLY-NEXT: vpshufb %xmm0, %xmm6, %xmm6
7364 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm6[5,6,7]
7365 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4
7366 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm4[5,6,7]
7367 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
7368 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm4, %xmm4
7369 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
7370 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm6 = xmm6[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
7371 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
7372 ; AVX1-ONLY-NEXT: vmovdqa (%rsp), %xmm6 # 16-byte Reload
7373 ; AVX1-ONLY-NEXT: vpshufb %xmm0, %xmm6, %xmm6
7374 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm4
7375 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
7376 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm6, %xmm6
7377 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
7378 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm8 = xmm8[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
7379 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]
7380 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
7381 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm8, %xmm8
7382 ; AVX1-ONLY-NEXT: vpshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload
7383 ; AVX1-ONLY-NEXT: # xmm9 = mem[0,1,2,3,4,5,5,7]
7384 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm8 = xmm9[1],xmm8[1]
7385 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm8[2,3,4,5,6,7]
7386 ; AVX1-ONLY-NEXT: vandnps %ymm4, %ymm5, %ymm4
7387 ; AVX1-ONLY-NEXT: vandps %ymm5, %ymm6, %ymm6
7388 ; AVX1-ONLY-NEXT: vorps %ymm4, %ymm6, %ymm4
7389 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
7390 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm6, %xmm6
7391 ; AVX1-ONLY-NEXT: vpshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
7392 ; AVX1-ONLY-NEXT: # xmm8 = mem[0,1,2,3,4,5,5,7]
7393 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm6 = xmm8[1],xmm6[1]
7394 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
7395 ; AVX1-ONLY-NEXT: vpshufb %xmm0, %xmm8, %xmm8
7396 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm8[5,6,7]
7397 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6
7398 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3,4],ymm6[5,6,7]
7399 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
7400 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm6, %xmm6
7401 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
7402 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm8 = xmm8[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
7403 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]
7404 ; AVX1-ONLY-NEXT: vpshufb %xmm0, %xmm10, %xmm8
7405 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm6, %ymm8, %ymm6
7406 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
7407 ; AVX1-ONLY-NEXT: vpsrlq $48, %xmm8, %xmm8
7408 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
7409 ; AVX1-ONLY-NEXT: vpsrldq {{.*#+}} xmm9 = xmm9[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
7410 ; AVX1-ONLY-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
7411 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
7412 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm9, %xmm9
7413 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm10 = xmm15[0,1,2,3,4,5,5,7]
7414 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm9 = xmm10[1],xmm9[1]
7415 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1],xmm9[2,3,4,5,6,7]
7416 ; AVX1-ONLY-NEXT: vandnps %ymm6, %ymm5, %ymm6
7417 ; AVX1-ONLY-NEXT: vandps %ymm5, %ymm8, %ymm5
7418 ; AVX1-ONLY-NEXT: vorps %ymm6, %ymm5, %ymm5
7419 ; AVX1-ONLY-NEXT: vpshufb %xmm0, %xmm7, %xmm0
7420 ; AVX1-ONLY-NEXT: vpsrld $16, %xmm1, %xmm6
7421 ; AVX1-ONLY-NEXT: vpshufhw {{.*#+}} xmm1 = xmm14[0,1,2,3,4,5,5,7]
7422 ; AVX1-ONLY-NEXT: vpunpckhqdq {{.*#+}} xmm1 = xmm1[1],xmm6[1]
7423 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4],xmm0[5,6,7]
7424 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
7425 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm5[0,1,2,3,4],ymm0[5,6,7]
7426 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7427 ; AVX1-ONLY-NEXT: vmovaps %ymm1, 96(%rsi)
7428 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7429 ; AVX1-ONLY-NEXT: vmovaps %ymm1, 32(%rsi)
7430 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7431 ; AVX1-ONLY-NEXT: vmovaps %ymm1, 64(%rsi)
7432 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7433 ; AVX1-ONLY-NEXT: vmovaps %ymm1, (%rsi)
7434 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7435 ; AVX1-ONLY-NEXT: vmovaps %ymm1, 96(%rdx)
7436 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7437 ; AVX1-ONLY-NEXT: vmovaps %ymm1, 32(%rdx)
7438 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7439 ; AVX1-ONLY-NEXT: vmovaps %ymm1, 64(%rdx)
7440 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7441 ; AVX1-ONLY-NEXT: vmovaps %ymm1, (%rdx)
7442 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7443 ; AVX1-ONLY-NEXT: vmovaps %ymm1, 32(%rcx)
7444 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7445 ; AVX1-ONLY-NEXT: vmovaps %ymm1, 96(%rcx)
7446 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7447 ; AVX1-ONLY-NEXT: vmovaps %ymm1, 64(%rcx)
7448 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7449 ; AVX1-ONLY-NEXT: vmovaps %ymm1, (%rcx)
7450 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7451 ; AVX1-ONLY-NEXT: vmovaps %ymm1, 96(%r8)
7452 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7453 ; AVX1-ONLY-NEXT: vmovaps %ymm1, 32(%r8)
7454 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7455 ; AVX1-ONLY-NEXT: vmovaps %ymm1, 64(%r8)
7456 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7457 ; AVX1-ONLY-NEXT: vmovaps %ymm1, (%r8)
7458 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7459 ; AVX1-ONLY-NEXT: vmovaps %ymm1, 96(%r9)
7460 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7461 ; AVX1-ONLY-NEXT: vmovaps %ymm1, 32(%r9)
7462 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7463 ; AVX1-ONLY-NEXT: vmovaps %ymm1, (%r9)
7464 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7465 ; AVX1-ONLY-NEXT: vmovaps %ymm1, 64(%r9)
7466 ; AVX1-ONLY-NEXT: movq {{[0-9]+}}(%rsp), %rax
7467 ; AVX1-ONLY-NEXT: vmovaps %ymm0, 96(%rax)
7468 ; AVX1-ONLY-NEXT: vmovaps %ymm4, 32(%rax)
7469 ; AVX1-ONLY-NEXT: vmovaps %ymm2, 64(%rax)
7470 ; AVX1-ONLY-NEXT: vmovaps %ymm3, (%rax)
7471 ; AVX1-ONLY-NEXT: addq $1368, %rsp # imm = 0x558
7472 ; AVX1-ONLY-NEXT: vzeroupper
7473 ; AVX1-ONLY-NEXT: retq
7474 ;
7475 ; AVX2-SLOW-LABEL: load_i16_stride6_vf64:
7476 ; AVX2-SLOW: # %bb.0:
7477 ; AVX2-SLOW-NEXT: subq $1256, %rsp # imm = 0x4E8
7478 ; AVX2-SLOW-NEXT: vmovdqa 64(%rdi), %ymm0
7479 ; AVX2-SLOW-NEXT: vmovdqa 96(%rdi), %ymm1
7480 ; AVX2-SLOW-NEXT: vmovaps 672(%rdi), %ymm2
7481 ; AVX2-SLOW-NEXT: vmovaps 640(%rdi), %ymm3
7482 ; AVX2-SLOW-NEXT: vmovdqa 288(%rdi), %ymm4
7483 ; AVX2-SLOW-NEXT: vmovdqa 256(%rdi), %ymm5
7484 ; AVX2-SLOW-NEXT: vmovdqa 416(%rdi), %ymm8
7485 ; AVX2-SLOW-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7486 ; AVX2-SLOW-NEXT: vmovdqa 384(%rdi), %ymm10
7487 ; AVX2-SLOW-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7488 ; AVX2-SLOW-NEXT: vmovdqa 480(%rdi), %ymm6
7489 ; AVX2-SLOW-NEXT: vmovdqa 448(%rdi), %ymm7
7490 ; AVX2-SLOW-NEXT: vperm2i128 {{.*#+}} ymm13 = ymm7[2,3],ymm6[2,3]
7491 ; AVX2-SLOW-NEXT: vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7492 ; AVX2-SLOW-NEXT: vperm2i128 {{.*#+}} ymm6 = ymm7[0,1],ymm6[0,1]
7493 ; AVX2-SLOW-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7494 ; AVX2-SLOW-NEXT: vperm2i128 {{.*#+}} ymm7 = ymm5[2,3],ymm4[2,3]
7495 ; AVX2-SLOW-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7496 ; AVX2-SLOW-NEXT: vperm2i128 {{.*#+}} ymm11 = ymm5[0,1],ymm4[0,1]
7497 ; AVX2-SLOW-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm3[2,3],ymm2[2,3]
7498 ; AVX2-SLOW-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7499 ; AVX2-SLOW-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm3[0,1],ymm2[0,1]
7500 ; AVX2-SLOW-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7501 ; AVX2-SLOW-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm0[2,3],ymm1[2,3]
7502 ; AVX2-SLOW-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7503 ; AVX2-SLOW-NEXT: vperm2i128 {{.*#+}} ymm3 = ymm0[0,1],ymm1[0,1]
7504 ; AVX2-SLOW-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7505 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm9 = [8,9,4,5,4,5,6,7,0,1,4,5,0,1,12,13,24,25,20,21,20,21,22,23,16,17,20,21,16,17,28,29]
7506 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1],ymm10[2],ymm8[3,4],ymm10[5],ymm8[6,7]
7507 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} xmm12 = <8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15>
7508 ; AVX2-SLOW-NEXT: vpshufb %xmm12, %xmm4, %xmm0
7509 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm4, %xmm1
7510 ; AVX2-SLOW-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7511 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,2,2,2,4,5,6,7]
7512 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6,7]
7513 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm13[0],ymm6[1],ymm13[2,3,4,5],ymm6[6],ymm13[7]
7514 ; AVX2-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7515 ; AVX2-SLOW-NEXT: vpshufb %ymm9, %ymm1, %ymm1
7516 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} xmm14 = [255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0]
7517 ; AVX2-SLOW-NEXT: vpblendvb %ymm14, %ymm0, %ymm1, %ymm0
7518 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7519 ; AVX2-SLOW-NEXT: vmovdqa (%rdi), %ymm0
7520 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7521 ; AVX2-SLOW-NEXT: vmovdqa 32(%rdi), %ymm1
7522 ; AVX2-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7523 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm15 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
7524 ; AVX2-SLOW-NEXT: vpshufb %xmm12, %xmm15, %xmm0
7525 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm15, %xmm2
7526 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm2[2,2,2,2,4,5,6,7]
7527 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6,7]
7528 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0],ymm3[1],ymm5[2,3,4,5],ymm3[6],ymm5[7]
7529 ; AVX2-SLOW-NEXT: vpshufb %ymm9, %ymm3, %ymm1
7530 ; AVX2-SLOW-NEXT: vpblendvb %ymm14, %ymm0, %ymm1, %ymm0
7531 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7532 ; AVX2-SLOW-NEXT: vmovdqa 224(%rdi), %ymm0
7533 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7534 ; AVX2-SLOW-NEXT: vmovdqa 192(%rdi), %ymm1
7535 ; AVX2-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7536 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
7537 ; AVX2-SLOW-NEXT: vpshufb %xmm12, %xmm5, %xmm0
7538 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm5, %xmm7
7539 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm7[2,2,2,2,4,5,6,7]
7540 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6,7]
7541 ; AVX2-SLOW-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7542 ; AVX2-SLOW-NEXT: vpblendd $189, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm6 # 32-byte Folded Reload
7543 ; AVX2-SLOW-NEXT: # ymm6 = mem[0],ymm11[1],mem[2,3,4,5],ymm11[6],mem[7]
7544 ; AVX2-SLOW-NEXT: vpshufb %ymm9, %ymm6, %ymm1
7545 ; AVX2-SLOW-NEXT: vpblendvb %ymm14, %ymm0, %ymm1, %ymm0
7546 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7547 ; AVX2-SLOW-NEXT: vmovdqa 608(%rdi), %ymm1
7548 ; AVX2-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7549 ; AVX2-SLOW-NEXT: vmovdqa 576(%rdi), %ymm0
7550 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7551 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
7552 ; AVX2-SLOW-NEXT: vpshufb %xmm12, %xmm0, %xmm1
7553 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm12
7554 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm13 = xmm12[2,2,2,2,4,5,6,7]
7555 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm13 = xmm1[0],xmm13[1],xmm1[2,3],xmm13[4],xmm1[5,6,7]
7556 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
7557 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
7558 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm8[0],ymm10[1],ymm8[2,3,4,5],ymm10[6],ymm8[7]
7559 ; AVX2-SLOW-NEXT: vpshufb %ymm9, %ymm1, %ymm9
7560 ; AVX2-SLOW-NEXT: vpblendvb %ymm14, %ymm13, %ymm9, %ymm9
7561 ; AVX2-SLOW-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7562 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} xmm9 = <10,11,u,u,2,3,14,15,u,u,10,11,12,13,14,15>
7563 ; AVX2-SLOW-NEXT: vpshufb %xmm9, %xmm15, %xmm13
7564 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
7565 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5]
7566 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0],xmm2[1],xmm13[2,3],xmm2[4],xmm13[5,6,7]
7567 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm2 = [10,11,6,7,4,5,6,7,6,7,6,7,2,3,14,15,26,27,22,23,20,21,22,23,22,23,22,23,18,19,30,31]
7568 ; AVX2-SLOW-NEXT: vpshufb %ymm2, %ymm3, %ymm3
7569 ; AVX2-SLOW-NEXT: vpblendvb %ymm14, %ymm13, %ymm3, %ymm3
7570 ; AVX2-SLOW-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7571 ; AVX2-SLOW-NEXT: vpshufb %xmm9, %xmm4, %xmm3
7572 ; AVX2-SLOW-NEXT: vpshufd $229, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
7573 ; AVX2-SLOW-NEXT: # xmm4 = mem[1,1,2,3]
7574 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,5,5,5]
7575 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm4[1],xmm3[2,3],xmm4[4],xmm3[5,6,7]
7576 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
7577 ; AVX2-SLOW-NEXT: vpshufb %ymm2, %ymm4, %ymm4
7578 ; AVX2-SLOW-NEXT: vpblendvb %ymm14, %ymm3, %ymm4, %ymm3
7579 ; AVX2-SLOW-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7580 ; AVX2-SLOW-NEXT: vpshufb %xmm9, %xmm5, %xmm3
7581 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm7[1,1,2,3]
7582 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,5,5,5]
7583 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm4[1],xmm3[2,3],xmm4[4],xmm3[5,6,7]
7584 ; AVX2-SLOW-NEXT: vpshufb %ymm2, %ymm6, %ymm4
7585 ; AVX2-SLOW-NEXT: vpblendvb %ymm14, %ymm3, %ymm4, %ymm3
7586 ; AVX2-SLOW-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7587 ; AVX2-SLOW-NEXT: vpshufb %ymm2, %ymm1, %ymm1
7588 ; AVX2-SLOW-NEXT: vpshufb %xmm9, %xmm0, %xmm0
7589 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm12[1,1,2,3]
7590 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5]
7591 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3],xmm2[4],xmm0[5,6,7]
7592 ; AVX2-SLOW-NEXT: vpblendvb %ymm14, %ymm0, %ymm1, %ymm0
7593 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7594 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
7595 ; AVX2-SLOW-NEXT: vpblendd $109, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm6 # 32-byte Folded Reload
7596 ; AVX2-SLOW-NEXT: # ymm6 = mem[0],ymm0[1],mem[2,3],ymm0[4],mem[5,6],ymm0[7]
7597 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm6, %xmm0
7598 ; AVX2-SLOW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7599 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,0,3]
7600 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
7601 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} xmm9 = <0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15>
7602 ; AVX2-SLOW-NEXT: vpshufb %xmm9, %xmm6, %xmm1
7603 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3],xmm0[4,5],xmm1[6,7]
7604 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm2 = <u,u,u,u,u,u,u,u,u,u,u,u,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21>
7605 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7606 ; AVX2-SLOW-NEXT: vpblendd $189, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
7607 ; AVX2-SLOW-NEXT: # ymm1 = mem[0],ymm1[1],mem[2,3,4,5],ymm1[6],mem[7]
7608 ; AVX2-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7609 ; AVX2-SLOW-NEXT: vpshufb %ymm2, %ymm1, %ymm1
7610 ; AVX2-SLOW-NEXT: vmovdqa %ymm2, %ymm5
7611 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
7612 ; AVX2-SLOW-NEXT: vmovdqa 544(%rdi), %ymm1
7613 ; AVX2-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7614 ; AVX2-SLOW-NEXT: vmovdqa 512(%rdi), %ymm2
7615 ; AVX2-SLOW-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7616 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm3 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
7617 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm3[2,2,2,2,4,5,6,7]
7618 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,2]
7619 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} xmm7 = <0,1,4,5,4,5,u,u,0,1,12,13,u,u,4,5>
7620 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm3, %xmm2
7621 ; AVX2-SLOW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7622 ; AVX2-SLOW-NEXT: vpshufb %xmm7, %xmm2, %xmm4
7623 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm4[0,1,2],xmm1[3],xmm4[4,5],xmm1[6],xmm4[7]
7624 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
7625 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
7626 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
7627 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7628 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
7629 ; AVX2-SLOW-NEXT: vpblendd $109, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 32-byte Folded Reload
7630 ; AVX2-SLOW-NEXT: # ymm4 = mem[0],ymm0[1],mem[2,3],ymm0[4],mem[5,6],ymm0[7]
7631 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm4, %xmm0
7632 ; AVX2-SLOW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7633 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,0,3]
7634 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
7635 ; AVX2-SLOW-NEXT: vpshufb %xmm9, %xmm4, %xmm1
7636 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3],xmm0[4,5],xmm1[6,7]
7637 ; AVX2-SLOW-NEXT: vpblendd $66, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm1 # 32-byte Folded Reload
7638 ; AVX2-SLOW-NEXT: # ymm1 = ymm11[0],mem[1],ymm11[2,3,4,5],mem[6],ymm11[7]
7639 ; AVX2-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7640 ; AVX2-SLOW-NEXT: vpshufb %ymm5, %ymm1, %ymm1
7641 ; AVX2-SLOW-NEXT: vmovdqa %ymm5, %ymm11
7642 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
7643 ; AVX2-SLOW-NEXT: vmovdqa 352(%rdi), %ymm1
7644 ; AVX2-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7645 ; AVX2-SLOW-NEXT: vmovdqa 320(%rdi), %ymm2
7646 ; AVX2-SLOW-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7647 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
7648 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm2[2,2,2,2,4,5,6,7]
7649 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm15 = xmm5[0,1,2,2]
7650 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm1
7651 ; AVX2-SLOW-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7652 ; AVX2-SLOW-NEXT: vpshufb %xmm7, %xmm1, %xmm14
7653 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm14 = xmm14[0,1,2],xmm15[3],xmm14[4,5],xmm15[6],xmm14[7]
7654 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
7655 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm14 = ymm0[0,1,2],ymm14[3,4,5,6,7],ymm0[8,9,10],ymm14[11,12,13,14,15]
7656 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm14[4,5,6,7]
7657 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7658 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
7659 ; AVX2-SLOW-NEXT: vpblendd $109, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
7660 ; AVX2-SLOW-NEXT: # ymm1 = mem[0],ymm0[1],mem[2,3],ymm0[4],mem[5,6],ymm0[7]
7661 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm0
7662 ; AVX2-SLOW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7663 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm15 = xmm0[0,2,0,3]
7664 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,6,6,7]
7665 ; AVX2-SLOW-NEXT: vpshufb %xmm9, %xmm1, %xmm13
7666 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0,1],xmm15[2],xmm13[3],xmm15[4,5],xmm13[6,7]
7667 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm10[0],ymm8[1],ymm10[2,3,4,5],ymm8[6],ymm10[7]
7668 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7669 ; AVX2-SLOW-NEXT: vpshufb %ymm11, %ymm0, %ymm12
7670 ; AVX2-SLOW-NEXT: vmovdqa %ymm11, %ymm14
7671 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm12 = ymm13[0,1,2],ymm12[3,4,5,6,7]
7672 ; AVX2-SLOW-NEXT: vmovdqa 736(%rdi), %ymm0
7673 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7674 ; AVX2-SLOW-NEXT: vmovdqa 704(%rdi), %ymm5
7675 ; AVX2-SLOW-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7676 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm13 = ymm5[0,1],ymm0[2],ymm5[3,4],ymm0[5],ymm5[6,7]
7677 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm11 = xmm13[2,2,2,2,4,5,6,7]
7678 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[0,1,2,2]
7679 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm13, %xmm8
7680 ; AVX2-SLOW-NEXT: vpshufb %xmm7, %xmm8, %xmm10
7681 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1,2],xmm11[3],xmm10[4,5],xmm11[6],xmm10[7]
7682 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
7683 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm10 = ymm12[0,1,2],ymm10[3,4,5,6,7],ymm12[8,9,10],ymm10[11,12,13,14,15]
7684 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm12[0,1,2,3],ymm10[4,5,6,7]
7685 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7686 ; AVX2-SLOW-NEXT: vmovdqa 160(%rdi), %ymm0
7687 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7688 ; AVX2-SLOW-NEXT: vmovdqa 128(%rdi), %ymm5
7689 ; AVX2-SLOW-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7690 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm10 = ymm5[0,1],ymm0[2],ymm5[3,4],ymm0[5],ymm5[6,7]
7691 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm10, %xmm11
7692 ; AVX2-SLOW-NEXT: vpshufb %xmm7, %xmm11, %xmm7
7693 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm12 = xmm10[2,2,2,2,4,5,6,7]
7694 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[0,1,2,2]
7695 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2],xmm12[3],xmm7[4,5],xmm12[6],xmm7[7]
7696 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
7697 ; AVX2-SLOW-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm12 # 32-byte Folded Reload
7698 ; AVX2-SLOW-NEXT: # ymm12 = ymm0[0],mem[1],ymm0[2,3],mem[4],ymm0[5,6],mem[7]
7699 ; AVX2-SLOW-NEXT: vpshufb %xmm9, %xmm12, %xmm0
7700 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm12, %xmm5
7701 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm5[0,2,0,3]
7702 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,6,6,7]
7703 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm9[2],xmm0[3],xmm9[4,5],xmm0[6,7]
7704 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
7705 ; AVX2-SLOW-NEXT: vpblendd $189, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
7706 ; AVX2-SLOW-NEXT: # ymm9 = mem[0],ymm9[1],mem[2,3,4,5],ymm9[6],mem[7]
7707 ; AVX2-SLOW-NEXT: vpshufb %ymm14, %ymm9, %ymm14
7708 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm14[3,4,5,6,7]
7709 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
7710 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm7 = ymm0[0,1,2],ymm7[3,4,5,6,7],ymm0[8,9,10],ymm7[11,12,13,14,15]
7711 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm7[4,5,6,7]
7712 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7713 ; AVX2-SLOW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
7714 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[u,u,u,u,10,11,u,u,2,3,14,15,u,u,u,u]
7715 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} xmm14 = <2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15>
7716 ; AVX2-SLOW-NEXT: vpshufb %xmm14, %xmm6, %xmm6
7717 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm7[2],xmm6[3],xmm7[4,5],xmm6[6,7]
7718 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm7 = <u,u,u,u,u,u,u,u,u,u,u,u,10,11,6,7,18,19,30,31,26,27,26,27,30,31,26,27,26,27,22,23>
7719 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
7720 ; AVX2-SLOW-NEXT: vpshufb %ymm7, %ymm0, %ymm15
7721 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2],ymm15[3,4,5,6,7]
7722 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} xmm15 = <6,7,2,3,4,5,u,u,2,3,14,15,u,u,6,7>
7723 ; AVX2-SLOW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
7724 ; AVX2-SLOW-NEXT: vpshufb %xmm15, %xmm0, %xmm0
7725 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,5,5]
7726 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[3],xmm0[4,5],xmm3[6],xmm0[7]
7727 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
7728 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm0 = ymm6[0,1,2],ymm0[3,4,5,6,7],ymm6[8,9,10],ymm0[11,12,13,14,15]
7729 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1,2,3],ymm0[4,5,6,7]
7730 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7731 ; AVX2-SLOW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
7732 ; AVX2-SLOW-NEXT: vpbroadcastq {{.*#+}} xmm6 = [2,3,14,15,10,11,0,0,2,3,14,15,10,11,0,0]
7733 ; AVX2-SLOW-NEXT: vpshufb %xmm6, %xmm0, %xmm0
7734 ; AVX2-SLOW-NEXT: vpshufb %xmm14, %xmm4, %xmm3
7735 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1],xmm0[2],xmm3[3],xmm0[4,5],xmm3[6,7]
7736 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
7737 ; AVX2-SLOW-NEXT: vpshufb %ymm7, %ymm3, %ymm3
7738 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm3[3,4,5,6,7]
7739 ; AVX2-SLOW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
7740 ; AVX2-SLOW-NEXT: vpshufb %xmm15, %xmm3, %xmm3
7741 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5]
7742 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3],xmm3[4,5],xmm2[6],xmm3[7]
7743 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
7744 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = ymm0[0,1,2],ymm2[3,4,5,6,7],ymm0[8,9,10],ymm2[11,12,13,14,15]
7745 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
7746 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7747 ; AVX2-SLOW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
7748 ; AVX2-SLOW-NEXT: vpshufb %xmm6, %xmm0, %xmm0
7749 ; AVX2-SLOW-NEXT: vpshufb %xmm14, %xmm1, %xmm1
7750 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3],xmm0[4,5],xmm1[6,7]
7751 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7752 ; AVX2-SLOW-NEXT: vpshufb %ymm7, %ymm1, %ymm1
7753 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
7754 ; AVX2-SLOW-NEXT: vpshufb %xmm15, %xmm8, %xmm1
7755 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm13[0,1,2,3,5,5,5,5]
7756 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3],xmm1[4,5],xmm2[6],xmm1[7]
7757 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
7758 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
7759 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
7760 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7761 ; AVX2-SLOW-NEXT: vpshufb %xmm15, %xmm11, %xmm0
7762 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm10[0,1,2,3,5,5,5,5]
7763 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]
7764 ; AVX2-SLOW-NEXT: vpshufb %xmm6, %xmm5, %xmm1
7765 ; AVX2-SLOW-NEXT: vpshufb %xmm14, %xmm12, %xmm2
7766 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2],xmm2[3],xmm1[4,5],xmm2[6,7]
7767 ; AVX2-SLOW-NEXT: vpshufb %ymm7, %ymm9, %ymm2
7768 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
7769 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
7770 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
7771 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
7772 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7773 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
7774 ; AVX2-SLOW-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
7775 ; AVX2-SLOW-NEXT: # ymm2 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6],ymm0[7]
7776 ; AVX2-SLOW-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7777 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
7778 ; AVX2-SLOW-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
7779 ; AVX2-SLOW-NEXT: # ymm0 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
7780 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm1
7781 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]
7782 ; AVX2-SLOW-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7783 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm14 = xmm0[2,1,0,3]
7784 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm14[0,0,0,0,4,5,6,7]
7785 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,6,7]
7786 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,4]
7787 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4],xmm0[5,6],xmm1[7]
7788 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7789 ; AVX2-SLOW-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
7790 ; AVX2-SLOW-NEXT: # ymm1 = mem[0,1],ymm1[2],mem[3,4],ymm1[5],mem[6,7]
7791 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm1[2,1,2,3]
7792 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm1
7793 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[0,3,2,1]
7794 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm4[0,0,2,3,4,5,6,7]
7795 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[0,1,3,3]
7796 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm7[2,1,2,0,4,5,6,7]
7797 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0],xmm5[1,2],xmm6[3],xmm5[4,5,6,7]
7798 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
7799 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm1 = <4,5,0,1,12,13,14,15,8,9,12,13,0,1,u,u,20,21,16,17,28,29,30,31,24,25,28,29,16,17,u,u>
7800 ; AVX2-SLOW-NEXT: vpshufb %ymm1, %ymm2, %ymm6
7801 ; AVX2-SLOW-NEXT: vmovdqa %ymm1, %ymm8
7802 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm0 = ymm6[0,1,2],ymm0[3,4,5,6,7],ymm6[8,9,10],ymm0[11,12,13,14,15]
7803 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,6,5,4]
7804 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4],xmm6[5,6,7]
7805 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3],ymm0[4,5,6,7]
7806 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7807 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
7808 ; AVX2-SLOW-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
7809 ; AVX2-SLOW-NEXT: # ymm2 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6],ymm0[7]
7810 ; AVX2-SLOW-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7811 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
7812 ; AVX2-SLOW-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
7813 ; AVX2-SLOW-NEXT: # ymm0 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
7814 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm5
7815 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm5[0,1,2,1]
7816 ; AVX2-SLOW-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7817 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,1,0,3]
7818 ; AVX2-SLOW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7819 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
7820 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,6,7]
7821 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm5 = xmm1[0,1,2,3,6,5,6,4]
7822 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm5[4],xmm0[5,6],xmm5[7]
7823 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7824 ; AVX2-SLOW-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm10 # 32-byte Folded Reload
7825 ; AVX2-SLOW-NEXT: # ymm10 = mem[0,1],ymm1[2],mem[3,4],ymm1[5],mem[6,7]
7826 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm10[2,1,2,3]
7827 ; AVX2-SLOW-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7828 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm10, %xmm10
7829 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm10[0,3,2,1]
7830 ; AVX2-SLOW-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7831 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm10 = xmm3[0,0,2,3,4,5,6,7]
7832 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[0,1,3,3]
7833 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm12 = xmm1[2,1,2,0,4,5,6,7]
7834 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm10 = xmm12[0],xmm10[1,2],xmm12[3],xmm10[4,5,6,7]
7835 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
7836 ; AVX2-SLOW-NEXT: vpshufb %ymm8, %ymm2, %ymm12
7837 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm0 = ymm12[0,1,2],ymm0[3,4,5,6,7],ymm12[8,9,10],ymm0[11,12,13,14,15]
7838 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,6,5,4]
7839 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1,2,3,4],xmm12[5,6,7]
7840 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7]
7841 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7842 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
7843 ; AVX2-SLOW-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
7844 ; AVX2-SLOW-NEXT: # ymm3 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6],ymm0[7]
7845 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
7846 ; AVX2-SLOW-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm10 # 32-byte Folded Reload
7847 ; AVX2-SLOW-NEXT: # ymm10 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
7848 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm10, %xmm12
7849 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[0,1,2,1]
7850 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[2,1,0,3]
7851 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm13 = xmm10[0,0,0,0,4,5,6,7]
7852 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,4,4,6,7]
7853 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm15 = xmm12[0,1,2,3,6,5,6,4]
7854 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0,1,2,3],xmm15[4],xmm13[5,6],xmm15[7]
7855 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
7856 ; AVX2-SLOW-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
7857 ; AVX2-SLOW-NEXT: # ymm15 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
7858 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm15[2,1,2,3]
7859 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm15, %xmm15
7860 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm15 = xmm15[0,3,2,1]
7861 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm11 = xmm15[0,0,2,3,4,5,6,7]
7862 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[0,1,3,3]
7863 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm1[2,1,2,0,4,5,6,7]
7864 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0],xmm11[1,2],xmm6[3],xmm11[4,5,6,7]
7865 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm11
7866 ; AVX2-SLOW-NEXT: vpshufb %ymm8, %ymm3, %ymm13
7867 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm11 = ymm13[0,1,2],ymm11[3,4,5,6,7],ymm13[8,9,10],ymm11[11,12,13,14,15]
7868 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,4,6,5,4]
7869 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm13[5,6,7]
7870 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1,2,3],ymm11[4,5,6,7]
7871 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7872 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
7873 ; AVX2-SLOW-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm6 # 32-byte Folded Reload
7874 ; AVX2-SLOW-NEXT: # ymm6 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
7875 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm6, %xmm11
7876 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm11[0,1,2,1]
7877 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[2,1,0,3]
7878 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm11 = xmm6[0,0,0,0,4,5,6,7]
7879 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,4,4,6,7]
7880 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm9 = xmm13[0,1,2,3,6,5,6,4]
7881 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm9 = xmm11[0,1,2,3],xmm9[4],xmm11[5,6],xmm9[7]
7882 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
7883 ; AVX2-SLOW-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm11 # 32-byte Folded Reload
7884 ; AVX2-SLOW-NEXT: # ymm11 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6],ymm0[7]
7885 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
7886 ; AVX2-SLOW-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
7887 ; AVX2-SLOW-NEXT: # ymm0 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
7888 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,1,2,3]
7889 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm0
7890 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]
7891 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm0[0,0,2,3,4,5,6,7]
7892 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[0,1,3,3]
7893 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm2[2,1,2,0,4,5,6,7]
7894 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm5 = xmm8[0],xmm5[1,2],xmm8[3],xmm5[4,5,6,7]
7895 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm8
7896 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm9 = ymm11[4,5,0,1,12,13,14,15,8,9,12,13,0,1,u,u,20,21,16,17,28,29,30,31,24,25,28,29,16,17,u,u]
7897 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm8 = ymm9[0,1,2],ymm8[3,4,5,6,7],ymm9[8,9,10],ymm8[11,12,13,14,15]
7898 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,6,5,4]
7899 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4],xmm9[5,6,7]
7900 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm8[4,5,6,7]
7901 ; AVX2-SLOW-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7902 ; AVX2-SLOW-NEXT: vpshufhw $103, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
7903 ; AVX2-SLOW-NEXT: # xmm5 = mem[0,1,2,3,7,5,6,5]
7904 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm14[1,1,1,1,4,5,6,7]
7905 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5,7,7]
7906 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm5 = xmm8[0,1,2,3],xmm5[4],xmm8[5,6],xmm5[7]
7907 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[3,1,2,1,4,5,6,7]
7908 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,3,4,5,6,7]
7909 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,7,7,7,7]
7910 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm4 = xmm7[0],xmm4[1,2],xmm7[3],xmm4[4,5,6,7]
7911 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm8 = <6,7,2,3,14,15,14,15,14,15,10,11,u,u,2,3,22,23,18,19,30,31,30,31,30,31,26,27,u,u,18,19>
7912 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
7913 ; AVX2-SLOW-NEXT: vpshufb %ymm8, %ymm7, %ymm7
7914 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
7915 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm5 = ymm7[0,1,2],ymm5[3,4,5,6,7],ymm7[8,9,10],ymm5[11,12,13,14,15]
7916 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[0,1,3,2]
7917 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm7[5,6,7]
7918 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
7919 ; AVX2-SLOW-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7920 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm5 = xmm12[0,1,2,3,7,5,6,5]
7921 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm10[1,1,1,1,4,5,6,7]
7922 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5,7,7]
7923 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm5 = xmm7[0,1,2,3],xmm5[4],xmm7[5,6],xmm5[7]
7924 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,1,4,5,6,7]
7925 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm15[0,1,3,3,4,5,6,7]
7926 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,7,7,7,7]
7927 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm7[1,2],xmm1[3],xmm7[4,5,6,7]
7928 ; AVX2-SLOW-NEXT: vpshufb %ymm8, %ymm3, %ymm3
7929 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
7930 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm5 = ymm3[0,1,2],ymm5[3,4,5,6,7],ymm3[8,9,10],ymm5[11,12,13,14,15]
7931 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,1,3,2]
7932 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm3[5,6,7]
7933 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm7 = ymm1[0,1,2,3],ymm5[4,5,6,7]
7934 ; AVX2-SLOW-NEXT: vpshufhw $103, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
7935 ; AVX2-SLOW-NEXT: # xmm1 = mem[0,1,2,3,7,5,6,5]
7936 ; AVX2-SLOW-NEXT: vpshuflw $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
7937 ; AVX2-SLOW-NEXT: # xmm3 = mem[1,1,1,1,4,5,6,7]
7938 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,7,7]
7939 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4],xmm3[5,6],xmm1[7]
7940 ; AVX2-SLOW-NEXT: vpshuflw $103, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
7941 ; AVX2-SLOW-NEXT: # xmm3 = mem[3,1,2,1,4,5,6,7]
7942 ; AVX2-SLOW-NEXT: vpshuflw $244, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
7943 ; AVX2-SLOW-NEXT: # xmm5 = mem[0,1,3,3,4,5,6,7]
7944 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,7,7,7,7]
7945 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm5[1,2],xmm3[3],xmm5[4,5,6,7]
7946 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
7947 ; AVX2-SLOW-NEXT: vpshufb %ymm8, %ymm4, %ymm5
7948 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
7949 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = ymm5[0,1,2],ymm1[3,4,5,6,7],ymm5[8,9,10],ymm1[11,12,13,14,15]
7950 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[0,1,3,2]
7951 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm5[5,6,7]
7952 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
7953 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm13[0,1,2,3,7,5,6,5]
7954 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm6[1,1,1,1,4,5,6,7]
7955 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,7,7]
7956 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm5[0,1,2,3],xmm3[4],xmm5[5,6],xmm3[7]
7957 ; AVX2-SLOW-NEXT: vpshufb %ymm8, %ymm11, %ymm5
7958 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,1,4,5,6,7]
7959 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,3,3,4,5,6,7]
7960 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,7,7]
7961 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1,2],xmm2[3],xmm0[4,5,6,7]
7962 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm2
7963 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = ymm5[0,1,2],ymm2[3,4,5,6,7],ymm5[8,9,10],ymm2[11,12,13,14,15]
7964 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm5[0,1,3,2]
7965 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm3[5,6,7]
7966 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0,1,2,3],ymm2[4,5,6,7]
7967 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
7968 ; AVX2-SLOW-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
7969 ; AVX2-SLOW-NEXT: # ymm0 = ymm0[0],mem[1],ymm0[2,3],mem[4],ymm0[5,6],mem[7]
7970 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm5
7971 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]
7972 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm0[0,1,0,2,4,5,6,7]
7973 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,6,6,6]
7974 ; AVX2-SLOW-NEXT: vpbroadcastq {{.*#+}} xmm6 = [8,9,0,0,0,1,12,13,8,9,0,0,0,1,12,13]
7975 ; AVX2-SLOW-NEXT: vpshufb %xmm6, %xmm5, %xmm8
7976 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm8[4],xmm3[5],xmm8[6,7]
7977 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
7978 ; AVX2-SLOW-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
7979 ; AVX2-SLOW-NEXT: # ymm3 = mem[0,1,2,3,4],ymm3[5,6,7]
7980 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
7981 ; AVX2-SLOW-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm8 # 32-byte Folded Reload
7982 ; AVX2-SLOW-NEXT: # ymm8 = ymm4[0],mem[1],ymm4[2,3],mem[4],ymm4[5,6],mem[7]
7983 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm8, %xmm9
7984 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[0,3,2,1]
7985 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm10 = xmm8[0,1,0,2,4,5,6,7]
7986 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,6,6,6,6]
7987 ; AVX2-SLOW-NEXT: vpshufb %xmm6, %xmm9, %xmm11
7988 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1,2,3],xmm11[4],xmm10[5],xmm11[6,7]
7989 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
7990 ; AVX2-SLOW-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload
7991 ; AVX2-SLOW-NEXT: # ymm10 = mem[0,1,2,3,4],ymm10[5,6,7]
7992 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
7993 ; AVX2-SLOW-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm11 # 32-byte Folded Reload
7994 ; AVX2-SLOW-NEXT: # ymm11 = ymm4[0],mem[1],ymm4[2,3],mem[4],ymm4[5,6],mem[7]
7995 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm11, %xmm12
7996 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[0,3,2,1]
7997 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm13 = xmm11[0,1,0,2,4,5,6,7]
7998 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,6,6,6,6]
7999 ; AVX2-SLOW-NEXT: vpshufb %xmm6, %xmm12, %xmm14
8000 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0,1,2,3],xmm14[4],xmm13[5],xmm14[6,7]
8001 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
8002 ; AVX2-SLOW-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm13 # 32-byte Folded Reload
8003 ; AVX2-SLOW-NEXT: # ymm13 = mem[0,1,2,3,4],ymm13[5,6,7]
8004 ; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
8005 ; AVX2-SLOW-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm14 # 32-byte Folded Reload
8006 ; AVX2-SLOW-NEXT: # ymm14 = ymm4[0],mem[1],ymm4[2,3],mem[4],ymm4[5,6],mem[7]
8007 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm14, %xmm15
8008 ; AVX2-SLOW-NEXT: vpshufb %xmm6, %xmm15, %xmm6
8009 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm14 = xmm14[0,3,2,1]
8010 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm14[0,1,0,2,4,5,6,7]
8011 ; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,6,6,6,6]
8012 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm6[4],xmm4[5],xmm6[6,7]
8013 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
8014 ; AVX2-SLOW-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
8015 ; AVX2-SLOW-NEXT: # ymm4 = mem[0,1,2,3,4],ymm4[5,6,7]
8016 ; AVX2-SLOW-NEXT: vpbroadcastq {{.*#+}} xmm6 = [10,11,0,0,2,3,14,15,10,11,0,0,2,3,14,15]
8017 ; AVX2-SLOW-NEXT: vpshufb %xmm6, %xmm9, %xmm9
8018 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[0,1,1,3,4,5,6,7]
8019 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[0,1,3,3]
8020 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2,3],xmm9[4],xmm8[5],xmm9[6,7]
8021 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
8022 ; AVX2-SLOW-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
8023 ; AVX2-SLOW-NEXT: # ymm8 = mem[0,1,2,3,4],ymm8[5,6,7]
8024 ; AVX2-SLOW-NEXT: vpshufb %xmm6, %xmm5, %xmm5
8025 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]
8026 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,3,3]
8027 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm5[4],xmm0[5],xmm5[6,7]
8028 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
8029 ; AVX2-SLOW-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
8030 ; AVX2-SLOW-NEXT: # ymm0 = mem[0,1,2,3,4],ymm0[5,6,7]
8031 ; AVX2-SLOW-NEXT: vpshufb %xmm6, %xmm12, %xmm5
8032 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm9 = xmm11[0,1,1,3,4,5,6,7]
8033 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[0,1,3,3]
8034 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm5 = xmm9[0,1,2,3],xmm5[4],xmm9[5],xmm5[6,7]
8035 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
8036 ; AVX2-SLOW-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
8037 ; AVX2-SLOW-NEXT: # ymm5 = mem[0,1,2,3,4],ymm5[5,6,7]
8038 ; AVX2-SLOW-NEXT: vpshufb %xmm6, %xmm15, %xmm6
8039 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm9 = xmm14[0,1,1,3,4,5,6,7]
8040 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[0,1,3,3]
8041 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm6 = xmm9[0,1,2,3],xmm6[4],xmm9[5],xmm6[6,7]
8042 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
8043 ; AVX2-SLOW-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload
8044 ; AVX2-SLOW-NEXT: # ymm6 = mem[0,1,2,3,4],ymm6[5,6,7]
8045 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
8046 ; AVX2-SLOW-NEXT: vmovaps %ymm9, 96(%rsi)
8047 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
8048 ; AVX2-SLOW-NEXT: vmovaps %ymm9, 32(%rsi)
8049 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
8050 ; AVX2-SLOW-NEXT: vmovaps %ymm9, 64(%rsi)
8051 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
8052 ; AVX2-SLOW-NEXT: vmovaps %ymm9, (%rsi)
8053 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
8054 ; AVX2-SLOW-NEXT: vmovaps %ymm9, 96(%rdx)
8055 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
8056 ; AVX2-SLOW-NEXT: vmovaps %ymm9, 32(%rdx)
8057 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
8058 ; AVX2-SLOW-NEXT: vmovaps %ymm9, 64(%rdx)
8059 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
8060 ; AVX2-SLOW-NEXT: vmovaps %ymm9, (%rdx)
8061 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
8062 ; AVX2-SLOW-NEXT: vmovaps %ymm9, 32(%rcx)
8063 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
8064 ; AVX2-SLOW-NEXT: vmovaps %ymm9, 96(%rcx)
8065 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
8066 ; AVX2-SLOW-NEXT: vmovaps %ymm9, 64(%rcx)
8067 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
8068 ; AVX2-SLOW-NEXT: vmovaps %ymm9, (%rcx)
8069 ; AVX2-SLOW-NEXT: vmovdqa %ymm1, 96(%r8)
8070 ; AVX2-SLOW-NEXT: vmovdqa %ymm7, 32(%r8)
8071 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8072 ; AVX2-SLOW-NEXT: vmovaps %ymm1, 64(%r8)
8073 ; AVX2-SLOW-NEXT: vmovdqa %ymm2, (%r8)
8074 ; AVX2-SLOW-NEXT: vmovdqa %ymm4, 96(%r9)
8075 ; AVX2-SLOW-NEXT: vmovdqa %ymm13, 32(%r9)
8076 ; AVX2-SLOW-NEXT: vmovdqa %ymm10, (%r9)
8077 ; AVX2-SLOW-NEXT: vmovdqa %ymm3, 64(%r9)
8078 ; AVX2-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
8079 ; AVX2-SLOW-NEXT: vmovdqa %ymm6, 96(%rax)
8080 ; AVX2-SLOW-NEXT: vmovdqa %ymm5, 32(%rax)
8081 ; AVX2-SLOW-NEXT: vmovdqa %ymm0, 64(%rax)
8082 ; AVX2-SLOW-NEXT: vmovdqa %ymm8, (%rax)
8083 ; AVX2-SLOW-NEXT: addq $1256, %rsp # imm = 0x4E8
8084 ; AVX2-SLOW-NEXT: vzeroupper
8085 ; AVX2-SLOW-NEXT: retq
8086 ;
8087 ; AVX2-FAST-LABEL: load_i16_stride6_vf64:
8088 ; AVX2-FAST: # %bb.0:
8089 ; AVX2-FAST-NEXT: subq $1272, %rsp # imm = 0x4F8
8090 ; AVX2-FAST-NEXT: vmovdqa 64(%rdi), %ymm0
8091 ; AVX2-FAST-NEXT: vmovdqa 96(%rdi), %ymm1
8092 ; AVX2-FAST-NEXT: vmovaps 672(%rdi), %ymm2
8093 ; AVX2-FAST-NEXT: vmovaps 640(%rdi), %ymm3
8094 ; AVX2-FAST-NEXT: vmovdqa 288(%rdi), %ymm4
8095 ; AVX2-FAST-NEXT: vmovdqa 256(%rdi), %ymm5
8096 ; AVX2-FAST-NEXT: vmovdqa 416(%rdi), %ymm8
8097 ; AVX2-FAST-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8098 ; AVX2-FAST-NEXT: vmovdqa 384(%rdi), %ymm10
8099 ; AVX2-FAST-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8100 ; AVX2-FAST-NEXT: vmovdqa 480(%rdi), %ymm6
8101 ; AVX2-FAST-NEXT: vmovdqa 448(%rdi), %ymm7
8102 ; AVX2-FAST-NEXT: vperm2i128 {{.*#+}} ymm11 = ymm7[2,3],ymm6[2,3]
8103 ; AVX2-FAST-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8104 ; AVX2-FAST-NEXT: vperm2i128 {{.*#+}} ymm7 = ymm7[0,1],ymm6[0,1]
8105 ; AVX2-FAST-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8106 ; AVX2-FAST-NEXT: vperm2i128 {{.*#+}} ymm12 = ymm5[2,3],ymm4[2,3]
8107 ; AVX2-FAST-NEXT: vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8108 ; AVX2-FAST-NEXT: vperm2i128 {{.*#+}} ymm14 = ymm5[0,1],ymm4[0,1]
8109 ; AVX2-FAST-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm3[2,3],ymm2[2,3]
8110 ; AVX2-FAST-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8111 ; AVX2-FAST-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm3[0,1],ymm2[0,1]
8112 ; AVX2-FAST-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8113 ; AVX2-FAST-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm0[2,3],ymm1[2,3]
8114 ; AVX2-FAST-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8115 ; AVX2-FAST-NEXT: vperm2i128 {{.*#+}} ymm3 = ymm0[0,1],ymm1[0,1]
8116 ; AVX2-FAST-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8117 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm9 = [8,9,4,5,4,5,6,7,0,1,4,5,0,1,12,13,24,25,20,21,20,21,22,23,16,17,20,21,16,17,28,29]
8118 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1],ymm10[2],ymm8[3,4],ymm10[5],ymm8[6,7]
8119 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm6 = <8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15>
8120 ; AVX2-FAST-NEXT: vpshufb %xmm6, %xmm4, %xmm0
8121 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm4, %xmm1
8122 ; AVX2-FAST-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8123 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,2,2,2,4,5,6,7]
8124 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6,7]
8125 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0],ymm7[1],ymm11[2,3,4,5],ymm7[6],ymm11[7]
8126 ; AVX2-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8127 ; AVX2-FAST-NEXT: vpshufb %ymm9, %ymm1, %ymm1
8128 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0]
8129 ; AVX2-FAST-NEXT: vpblendvb %ymm10, %ymm0, %ymm1, %ymm0
8130 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8131 ; AVX2-FAST-NEXT: vmovdqa (%rdi), %ymm0
8132 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8133 ; AVX2-FAST-NEXT: vmovdqa 32(%rdi), %ymm1
8134 ; AVX2-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8135 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm15 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
8136 ; AVX2-FAST-NEXT: vpshufb %xmm6, %xmm15, %xmm0
8137 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm15, %xmm2
8138 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm1 = xmm2[2,2,2,2,4,5,6,7]
8139 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6,7]
8140 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0],ymm3[1],ymm5[2,3,4,5],ymm3[6],ymm5[7]
8141 ; AVX2-FAST-NEXT: vpshufb %ymm9, %ymm3, %ymm1
8142 ; AVX2-FAST-NEXT: vpblendvb %ymm10, %ymm0, %ymm1, %ymm0
8143 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8144 ; AVX2-FAST-NEXT: vmovdqa 224(%rdi), %ymm0
8145 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8146 ; AVX2-FAST-NEXT: vmovdqa 192(%rdi), %ymm1
8147 ; AVX2-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8148 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm5 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
8149 ; AVX2-FAST-NEXT: vpshufb %xmm6, %xmm5, %xmm0
8150 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm5, %xmm7
8151 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm1 = xmm7[2,2,2,2,4,5,6,7]
8152 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6,7]
8153 ; AVX2-FAST-NEXT: vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8154 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm8 = ymm12[0],ymm14[1],ymm12[2,3,4,5],ymm14[6],ymm12[7]
8155 ; AVX2-FAST-NEXT: vpshufb %ymm9, %ymm8, %ymm1
8156 ; AVX2-FAST-NEXT: vpblendvb %ymm10, %ymm0, %ymm1, %ymm0
8157 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8158 ; AVX2-FAST-NEXT: vmovdqa 608(%rdi), %ymm1
8159 ; AVX2-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8160 ; AVX2-FAST-NEXT: vmovdqa 576(%rdi), %ymm0
8161 ; AVX2-FAST-NEXT: vmovdqu %ymm0, (%rsp) # 32-byte Spill
8162 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
8163 ; AVX2-FAST-NEXT: vpshufb %xmm6, %xmm0, %xmm1
8164 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm6
8165 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm13 = xmm6[2,2,2,2,4,5,6,7]
8166 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm13 = xmm1[0],xmm13[1],xmm1[2,3],xmm13[4],xmm1[5,6,7]
8167 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
8168 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
8169 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0],ymm12[1],ymm11[2,3,4,5],ymm12[6],ymm11[7]
8170 ; AVX2-FAST-NEXT: vpshufb %ymm9, %ymm1, %ymm9
8171 ; AVX2-FAST-NEXT: vpblendvb %ymm10, %ymm13, %ymm9, %ymm9
8172 ; AVX2-FAST-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8173 ; AVX2-FAST-NEXT: vpbroadcastd {{.*#+}} xmm9 = [10,11,6,7,10,11,6,7,10,11,6,7,10,11,6,7]
8174 ; AVX2-FAST-NEXT: vpshufb %xmm9, %xmm2, %xmm13
8175 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm2 = <10,11,u,u,2,3,14,15,u,u,10,11,12,13,14,15>
8176 ; AVX2-FAST-NEXT: vpshufb %xmm2, %xmm15, %xmm15
8177 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm13 = xmm15[0],xmm13[1],xmm15[2,3],xmm13[4],xmm15[5,6,7]
8178 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm15 = [10,11,6,7,4,5,6,7,6,7,6,7,2,3,14,15,26,27,22,23,20,21,22,23,22,23,22,23,18,19,30,31]
8179 ; AVX2-FAST-NEXT: vpshufb %ymm15, %ymm3, %ymm3
8180 ; AVX2-FAST-NEXT: vpblendvb %ymm10, %ymm13, %ymm3, %ymm3
8181 ; AVX2-FAST-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8182 ; AVX2-FAST-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
8183 ; AVX2-FAST-NEXT: vpshufb %xmm9, %xmm3, %xmm3
8184 ; AVX2-FAST-NEXT: vpshufb %xmm2, %xmm4, %xmm4
8185 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6,7]
8186 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
8187 ; AVX2-FAST-NEXT: vpshufb %ymm15, %ymm4, %ymm4
8188 ; AVX2-FAST-NEXT: vpblendvb %ymm10, %ymm3, %ymm4, %ymm3
8189 ; AVX2-FAST-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8190 ; AVX2-FAST-NEXT: vpshufb %xmm9, %xmm7, %xmm3
8191 ; AVX2-FAST-NEXT: vpshufb %xmm2, %xmm5, %xmm4
8192 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6,7]
8193 ; AVX2-FAST-NEXT: vpshufb %ymm15, %ymm8, %ymm4
8194 ; AVX2-FAST-NEXT: vpblendvb %ymm10, %ymm3, %ymm4, %ymm3
8195 ; AVX2-FAST-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8196 ; AVX2-FAST-NEXT: vpshufb %ymm15, %ymm1, %ymm1
8197 ; AVX2-FAST-NEXT: vpshufb %xmm9, %xmm6, %xmm3
8198 ; AVX2-FAST-NEXT: vpshufb %xmm2, %xmm0, %xmm0
8199 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2,3],xmm3[4],xmm0[5,6,7]
8200 ; AVX2-FAST-NEXT: vpblendvb %ymm10, %ymm0, %ymm1, %ymm0
8201 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8202 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8203 ; AVX2-FAST-NEXT: vpblendd $109, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm7 # 32-byte Folded Reload
8204 ; AVX2-FAST-NEXT: # ymm7 = mem[0],ymm0[1],mem[2,3],ymm0[4],mem[5,6],ymm0[7]
8205 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm7, %xmm0
8206 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,1,0,3]
8207 ; AVX2-FAST-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8208 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm2 = <0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15>
8209 ; AVX2-FAST-NEXT: vpshufb %xmm2, %xmm7, %xmm0
8210 ; AVX2-FAST-NEXT: vmovdqa %xmm2, %xmm13
8211 ; AVX2-FAST-NEXT: vpbroadcastq {{.*#+}} xmm3 = [8,9,12,13,0,1,0,0,8,9,12,13,0,1,0,0]
8212 ; AVX2-FAST-NEXT: vpshufb %xmm3, %xmm1, %xmm2
8213 ; AVX2-FAST-NEXT: vmovdqa %xmm3, %xmm8
8214 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2],xmm0[3],xmm2[4,5],xmm0[6,7]
8215 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm2 = <u,u,u,u,u,u,u,u,u,u,u,u,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21>
8216 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8217 ; AVX2-FAST-NEXT: vpblendd $189, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
8218 ; AVX2-FAST-NEXT: # ymm1 = mem[0],ymm1[1],mem[2,3,4,5],ymm1[6],mem[7]
8219 ; AVX2-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8220 ; AVX2-FAST-NEXT: vpshufb %ymm2, %ymm1, %ymm3
8221 ; AVX2-FAST-NEXT: vmovdqa %ymm2, %ymm4
8222 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm3[3,4,5,6,7]
8223 ; AVX2-FAST-NEXT: vmovdqa 544(%rdi), %ymm1
8224 ; AVX2-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8225 ; AVX2-FAST-NEXT: vmovdqa 512(%rdi), %ymm2
8226 ; AVX2-FAST-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8227 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
8228 ; AVX2-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8229 ; AVX2-FAST-NEXT: vpbroadcastd {{.*#+}} xmm9 = [8,9,4,5,8,9,4,5,8,9,4,5,8,9,4,5]
8230 ; AVX2-FAST-NEXT: vpshufb %xmm9, %xmm1, %xmm5
8231 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm10 = <0,1,4,5,4,5,u,u,0,1,12,13,u,u,4,5>
8232 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm1, %xmm1
8233 ; AVX2-FAST-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8234 ; AVX2-FAST-NEXT: vpshufb %xmm10, %xmm1, %xmm6
8235 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3],xmm6[4,5],xmm5[6],xmm6[7]
8236 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
8237 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm5 = ymm0[0,1,2],ymm5[3,4,5,6,7],ymm0[8,9,10],ymm5[11,12,13,14,15]
8238 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
8239 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8240 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8241 ; AVX2-FAST-NEXT: vpblendd $109, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
8242 ; AVX2-FAST-NEXT: # ymm5 = mem[0],ymm0[1],mem[2,3],ymm0[4],mem[5,6],ymm0[7]
8243 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm5, %xmm0
8244 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,1,0,3]
8245 ; AVX2-FAST-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8246 ; AVX2-FAST-NEXT: vpshufb %xmm13, %xmm5, %xmm0
8247 ; AVX2-FAST-NEXT: vmovdqa %xmm8, %xmm6
8248 ; AVX2-FAST-NEXT: vpshufb %xmm8, %xmm1, %xmm8
8249 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm8[2],xmm0[3],xmm8[4,5],xmm0[6,7]
8250 ; AVX2-FAST-NEXT: vpblendd $66, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm1 # 32-byte Folded Reload
8251 ; AVX2-FAST-NEXT: # ymm1 = ymm14[0],mem[1],ymm14[2,3,4,5],mem[6],ymm14[7]
8252 ; AVX2-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8253 ; AVX2-FAST-NEXT: vmovdqa %ymm4, %ymm2
8254 ; AVX2-FAST-NEXT: vpshufb %ymm4, %ymm1, %ymm15
8255 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm15 = ymm0[0,1,2],ymm15[3,4,5,6,7]
8256 ; AVX2-FAST-NEXT: vmovdqa 352(%rdi), %ymm0
8257 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8258 ; AVX2-FAST-NEXT: vmovdqa 320(%rdi), %ymm1
8259 ; AVX2-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8260 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm4 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
8261 ; AVX2-FAST-NEXT: vpshufb %xmm9, %xmm4, %xmm0
8262 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm4, %xmm1
8263 ; AVX2-FAST-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8264 ; AVX2-FAST-NEXT: vpshufb %xmm10, %xmm1, %xmm14
8265 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm14[0,1,2],xmm0[3],xmm14[4,5],xmm0[6],xmm14[7]
8266 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
8267 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm0 = ymm15[0,1,2],ymm0[3,4,5,6,7],ymm15[8,9,10],ymm0[11,12,13,14,15]
8268 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm15[0,1,2,3],ymm0[4,5,6,7]
8269 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8270 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8271 ; AVX2-FAST-NEXT: vpblendd $109, (%rsp), %ymm0, %ymm3 # 32-byte Folded Reload
8272 ; AVX2-FAST-NEXT: # ymm3 = mem[0],ymm0[1],mem[2,3],ymm0[4],mem[5,6],ymm0[7]
8273 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm3, %xmm14
8274 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm14[2,1,0,3]
8275 ; AVX2-FAST-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8276 ; AVX2-FAST-NEXT: vpshufb %xmm13, %xmm3, %xmm15
8277 ; AVX2-FAST-NEXT: vmovdqa %xmm13, %xmm8
8278 ; AVX2-FAST-NEXT: vpshufb %xmm6, %xmm0, %xmm13
8279 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm13 = xmm15[0,1],xmm13[2],xmm15[3],xmm13[4,5],xmm15[6,7]
8280 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm12[0],ymm11[1],ymm12[2,3,4,5],ymm11[6],ymm12[7]
8281 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8282 ; AVX2-FAST-NEXT: vpshufb %ymm2, %ymm0, %ymm12
8283 ; AVX2-FAST-NEXT: vmovdqa %ymm2, %ymm14
8284 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm12 = ymm13[0,1,2],ymm12[3,4,5,6,7]
8285 ; AVX2-FAST-NEXT: vmovdqa 736(%rdi), %ymm0
8286 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8287 ; AVX2-FAST-NEXT: vmovdqa 704(%rdi), %ymm1
8288 ; AVX2-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8289 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm13 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
8290 ; AVX2-FAST-NEXT: vpshufb %xmm9, %xmm13, %xmm0
8291 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm13, %xmm6
8292 ; AVX2-FAST-NEXT: vpshufb %xmm10, %xmm6, %xmm11
8293 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm11[0,1,2],xmm0[3],xmm11[4,5],xmm0[6],xmm11[7]
8294 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
8295 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm0 = ymm12[0,1,2],ymm0[3,4,5,6,7],ymm12[8,9,10],ymm0[11,12,13,14,15]
8296 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm12[0,1,2,3],ymm0[4,5,6,7]
8297 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8298 ; AVX2-FAST-NEXT: vmovdqa 160(%rdi), %ymm0
8299 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8300 ; AVX2-FAST-NEXT: vmovdqa 128(%rdi), %ymm1
8301 ; AVX2-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8302 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
8303 ; AVX2-FAST-NEXT: vpshufb %xmm9, %xmm2, %xmm9
8304 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm2, %xmm11
8305 ; AVX2-FAST-NEXT: vpshufb %xmm10, %xmm11, %xmm10
8306 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3],xmm10[4,5],xmm9[6],xmm10[7]
8307 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8308 ; AVX2-FAST-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm10 # 32-byte Folded Reload
8309 ; AVX2-FAST-NEXT: # ymm10 = ymm0[0],mem[1],ymm0[2,3],mem[4],ymm0[5,6],mem[7]
8310 ; AVX2-FAST-NEXT: vpshufb %xmm8, %xmm10, %xmm1
8311 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm10, %xmm12
8312 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[2,1,0,3]
8313 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm8 = xmm12[u,u,u,u,0,1,u,u,8,9,12,13,u,u,u,u]
8314 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm8[2],xmm1[3],xmm8[4,5],xmm1[6,7]
8315 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8316 ; AVX2-FAST-NEXT: vpblendd $189, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
8317 ; AVX2-FAST-NEXT: # ymm8 = mem[0],ymm0[1],mem[2,3,4,5],ymm0[6],mem[7]
8318 ; AVX2-FAST-NEXT: vpshufb %ymm14, %ymm8, %ymm14
8319 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2],ymm14[3,4,5,6,7]
8320 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
8321 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm9 = ymm1[0,1,2],ymm9[3,4,5,6,7],ymm1[8,9,10],ymm9[11,12,13,14,15]
8322 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm9[4,5,6,7]
8323 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8324 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15]
8325 ; AVX2-FAST-NEXT: vpbroadcastq {{.*#+}} xmm9 = [10,11,14,15,2,3,0,0,10,11,14,15,2,3,0,0]
8326 ; AVX2-FAST-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
8327 ; AVX2-FAST-NEXT: vpshufb %xmm9, %xmm0, %xmm14
8328 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1],xmm14[2],xmm7[3],xmm14[4,5],xmm7[6,7]
8329 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm14 = <u,u,u,u,u,u,u,u,u,u,u,u,10,11,6,7,18,19,30,31,26,27,26,27,30,31,26,27,26,27,22,23>
8330 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8331 ; AVX2-FAST-NEXT: vpshufb %ymm14, %ymm0, %ymm15
8332 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2],ymm15[3,4,5,6,7]
8333 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm15 = <6,7,2,3,4,5,u,u,2,3,14,15,u,u,6,7>
8334 ; AVX2-FAST-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
8335 ; AVX2-FAST-NEXT: vpshufb %xmm15, %xmm0, %xmm0
8336 ; AVX2-FAST-NEXT: vpshufhw $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
8337 ; AVX2-FAST-NEXT: # xmm1 = mem[0,1,2,3,5,5,5,5]
8338 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]
8339 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
8340 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm0 = ymm7[0,1,2],ymm0[3,4,5,6,7],ymm7[8,9,10],ymm0[11,12,13,14,15]
8341 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm0[4,5,6,7]
8342 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8343 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm7 = <2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15>
8344 ; AVX2-FAST-NEXT: vpshufb %xmm7, %xmm5, %xmm0
8345 ; AVX2-FAST-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
8346 ; AVX2-FAST-NEXT: vpshufb %xmm9, %xmm1, %xmm1
8347 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3],xmm1[4,5],xmm0[6,7]
8348 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8349 ; AVX2-FAST-NEXT: vpshufb %ymm14, %ymm1, %ymm1
8350 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
8351 ; AVX2-FAST-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
8352 ; AVX2-FAST-NEXT: vpshufb %xmm15, %xmm1, %xmm1
8353 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,5,5,5]
8354 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3],xmm1[4,5],xmm4[6],xmm1[7]
8355 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
8356 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
8357 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
8358 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8359 ; AVX2-FAST-NEXT: vpshufb %xmm7, %xmm3, %xmm0
8360 ; AVX2-FAST-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
8361 ; AVX2-FAST-NEXT: vpshufb %xmm9, %xmm1, %xmm1
8362 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3],xmm1[4,5],xmm0[6,7]
8363 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8364 ; AVX2-FAST-NEXT: vpshufb %ymm14, %ymm1, %ymm1
8365 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
8366 ; AVX2-FAST-NEXT: vpshufb %xmm15, %xmm6, %xmm1
8367 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm3 = xmm13[0,1,2,3,5,5,5,5]
8368 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm3[3],xmm1[4,5],xmm3[6],xmm1[7]
8369 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
8370 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
8371 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
8372 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8373 ; AVX2-FAST-NEXT: vpshufb %xmm15, %xmm11, %xmm0
8374 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm1 = xmm2[0,1,2,3,5,5,5,5]
8375 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]
8376 ; AVX2-FAST-NEXT: vpshufb %xmm7, %xmm10, %xmm1
8377 ; AVX2-FAST-NEXT: vpshufb %xmm9, %xmm12, %xmm2
8378 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3],xmm2[4,5],xmm1[6,7]
8379 ; AVX2-FAST-NEXT: vpshufb %ymm14, %ymm8, %ymm2
8380 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
8381 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
8382 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
8383 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
8384 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8385 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8386 ; AVX2-FAST-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
8387 ; AVX2-FAST-NEXT: # ymm3 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6],ymm0[7]
8388 ; AVX2-FAST-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8389 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8390 ; AVX2-FAST-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
8391 ; AVX2-FAST-NEXT: # ymm0 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
8392 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,1,0,3]
8393 ; AVX2-FAST-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8394 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm0
8395 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,1,2,1]
8396 ; AVX2-FAST-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8397 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm8 = <0,1,0,1,0,1,0,1,u,u,8,9,12,13,u,u>
8398 ; AVX2-FAST-NEXT: vpshufb %xmm8, %xmm1, %xmm0
8399 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm1 = xmm2[0,1,2,3,6,5,6,4]
8400 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0,1,2,3],xmm1[4],xmm0[5,6],xmm1[7]
8401 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8402 ; AVX2-FAST-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
8403 ; AVX2-FAST-NEXT: # ymm1 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
8404 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm1, %xmm0
8405 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm7 = xmm0[0,3,2,1]
8406 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[2,1,2,3]
8407 ; AVX2-FAST-NEXT: vpbroadcastq {{.*#+}} xmm0 = [12,13,0,1,4,5,0,0,12,13,0,1,4,5,0,0]
8408 ; AVX2-FAST-NEXT: vpshufb %xmm0, %xmm7, %xmm5
8409 ; AVX2-FAST-NEXT: vmovdqa %xmm0, %xmm10
8410 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm6 = xmm4[2,1,2,0,4,5,6,7]
8411 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0],xmm5[1,2],xmm6[3],xmm5[4,5,6,7]
8412 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
8413 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm0 = <4,5,0,1,12,13,14,15,8,9,12,13,0,1,u,u,20,21,16,17,28,29,30,31,24,25,28,29,16,17,u,u>
8414 ; AVX2-FAST-NEXT: vpshufb %ymm0, %ymm3, %ymm6
8415 ; AVX2-FAST-NEXT: vmovdqa %ymm0, %ymm9
8416 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm2 = ymm6[0,1,2],ymm2[3,4,5,6,7],ymm6[8,9,10],ymm2[11,12,13,14,15]
8417 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,6,5,4]
8418 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4],xmm6[5,6,7]
8419 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3],ymm2[4,5,6,7]
8420 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8421 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8422 ; AVX2-FAST-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
8423 ; AVX2-FAST-NEXT: # ymm1 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6],ymm0[7]
8424 ; AVX2-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8425 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8426 ; AVX2-FAST-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
8427 ; AVX2-FAST-NEXT: # ymm2 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
8428 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm2[2,1,0,3]
8429 ; AVX2-FAST-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8430 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm2, %xmm2
8431 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,1,2,1]
8432 ; AVX2-FAST-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8433 ; AVX2-FAST-NEXT: vpshufb %xmm8, %xmm0, %xmm2
8434 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm12 = xmm3[0,1,2,3,6,5,6,4]
8435 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm12 = xmm2[0,1,2,3],xmm12[4],xmm2[5,6],xmm12[7]
8436 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8437 ; AVX2-FAST-NEXT: vpblendd $219, (%rsp), %ymm0, %ymm2 # 32-byte Folded Reload
8438 ; AVX2-FAST-NEXT: # ymm2 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
8439 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm2, %xmm13
8440 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm13[0,3,2,1]
8441 ; AVX2-FAST-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill
8442 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,1,2,3]
8443 ; AVX2-FAST-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8444 ; AVX2-FAST-NEXT: vpshufb %xmm10, %xmm0, %xmm13
8445 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm15 = xmm2[2,1,2,0,4,5,6,7]
8446 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm13 = xmm15[0],xmm13[1,2],xmm15[3],xmm13[4,5,6,7]
8447 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm12
8448 ; AVX2-FAST-NEXT: vpshufb %ymm9, %ymm1, %ymm15
8449 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm12 = ymm15[0,1,2],ymm12[3,4,5,6,7],ymm15[8,9,10],ymm12[11,12,13,14,15]
8450 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,6,5,4]
8451 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0,1,2,3,4],xmm15[5,6,7]
8452 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3],ymm12[4,5,6,7]
8453 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8454 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8455 ; AVX2-FAST-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
8456 ; AVX2-FAST-NEXT: # ymm14 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6],ymm0[7]
8457 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8458 ; AVX2-FAST-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm13 # 32-byte Folded Reload
8459 ; AVX2-FAST-NEXT: # ymm13 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
8460 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm15 = xmm13[2,1,0,3]
8461 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm13, %xmm13
8462 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[0,1,2,1]
8463 ; AVX2-FAST-NEXT: vpshufb %xmm8, %xmm15, %xmm0
8464 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm11 = xmm13[0,1,2,3,6,5,6,4]
8465 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1,2,3],xmm11[4],xmm0[5,6],xmm11[7]
8466 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8467 ; AVX2-FAST-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm11 # 32-byte Folded Reload
8468 ; AVX2-FAST-NEXT: # ymm11 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
8469 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm11, %xmm6
8470 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[0,3,2,1]
8471 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[2,1,2,3]
8472 ; AVX2-FAST-NEXT: vpshufb %xmm10, %xmm6, %xmm0
8473 ; AVX2-FAST-NEXT: vmovdqa %xmm10, %xmm3
8474 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm5 = xmm11[2,1,2,0,4,5,6,7]
8475 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm5[0],xmm0[1,2],xmm5[3],xmm0[4,5,6,7]
8476 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
8477 ; AVX2-FAST-NEXT: vpshufb %ymm9, %ymm14, %ymm5
8478 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm1 = ymm5[0,1,2],ymm1[3,4,5,6,7],ymm5[8,9,10],ymm1[11,12,13,14,15]
8479 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,6,5,4]
8480 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm5[5,6,7]
8481 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
8482 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8483 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8484 ; AVX2-FAST-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
8485 ; AVX2-FAST-NEXT: # ymm0 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
8486 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[2,1,0,3]
8487 ; AVX2-FAST-NEXT: vpshufb %xmm8, %xmm12, %xmm1
8488 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm0
8489 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm8 = xmm0[0,1,2,1]
8490 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm0 = xmm8[0,1,2,3,6,5,6,4]
8491 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm1[0,1,2,3],xmm0[4],xmm1[5,6],xmm0[7]
8492 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8493 ; AVX2-FAST-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
8494 ; AVX2-FAST-NEXT: # ymm5 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6],ymm0[7]
8495 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8496 ; AVX2-FAST-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
8497 ; AVX2-FAST-NEXT: # ymm0 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
8498 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm10
8499 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[0,3,2,1]
8500 ; AVX2-FAST-NEXT: vpshufb %xmm3, %xmm10, %xmm1
8501 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,1,2,3]
8502 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm9 = xmm3[2,1,2,0,4,5,6,7]
8503 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm9[0],xmm1[1,2],xmm9[3],xmm1[4,5,6,7]
8504 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
8505 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm9 = ymm5[4,5,0,1,12,13,14,15,8,9,12,13,0,1,u,u,20,21,16,17,28,29,30,31,24,25,28,29,16,17,u,u]
8506 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm2 = ymm9[0,1,2],ymm2[3,4,5,6,7],ymm9[8,9,10],ymm2[11,12,13,14,15]
8507 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,6,5,4]
8508 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm9[5,6,7]
8509 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm2[4,5,6,7]
8510 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8511 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm2 = <2,3,2,3,2,3,2,3,u,u,10,11,14,15,u,u>
8512 ; AVX2-FAST-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
8513 ; AVX2-FAST-NEXT: vpshufb %xmm2, %xmm0, %xmm1
8514 ; AVX2-FAST-NEXT: vpshufhw $103, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload
8515 ; AVX2-FAST-NEXT: # xmm9 = mem[0,1,2,3,7,5,6,5]
8516 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm9[4],xmm1[5,6],xmm9[7]
8517 ; AVX2-FAST-NEXT: vpbroadcastq {{.*#+}} xmm9 = [14,15,2,3,6,7,0,0,14,15,2,3,6,7,0,0]
8518 ; AVX2-FAST-NEXT: vpshufb %xmm9, %xmm7, %xmm7
8519 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[3,1,2,1,4,5,6,7]
8520 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm7[1,2],xmm4[3],xmm7[4,5,6,7]
8521 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm7 = <6,7,2,3,14,15,14,15,14,15,10,11,u,u,2,3,22,23,18,19,30,31,30,31,30,31,26,27,u,u,18,19>
8522 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8523 ; AVX2-FAST-NEXT: vpshufb %ymm7, %ymm0, %ymm0
8524 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
8525 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
8526 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,3,2]
8527 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm4[0,1,2,3,4],xmm0[5,6,7]
8528 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
8529 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8530 ; AVX2-FAST-NEXT: vpshufb %xmm2, %xmm15, %xmm0
8531 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm1 = xmm13[0,1,2,3,7,5,6,5]
8532 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4],xmm0[5,6],xmm1[7]
8533 ; AVX2-FAST-NEXT: vpshufb %xmm9, %xmm6, %xmm1
8534 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm6 = xmm11[3,1,2,1,4,5,6,7]
8535 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm6[0],xmm1[1,2],xmm6[3],xmm1[4,5,6,7]
8536 ; AVX2-FAST-NEXT: vpshufb %ymm7, %ymm14, %ymm6
8537 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
8538 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm0 = ymm6[0,1,2],ymm0[3,4,5,6,7],ymm6[8,9,10],ymm0[11,12,13,14,15]
8539 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[0,1,3,2]
8540 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm6[5,6,7]
8541 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
8542 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8543 ; AVX2-FAST-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
8544 ; AVX2-FAST-NEXT: vpshufb %xmm2, %xmm0, %xmm0
8545 ; AVX2-FAST-NEXT: vpshufhw $103, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
8546 ; AVX2-FAST-NEXT: # xmm6 = mem[0,1,2,3,7,5,6,5]
8547 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm6[4],xmm0[5,6],xmm6[7]
8548 ; AVX2-FAST-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload
8549 ; AVX2-FAST-NEXT: vpshufb %xmm9, %xmm1, %xmm6
8550 ; AVX2-FAST-NEXT: vpshuflw $103, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
8551 ; AVX2-FAST-NEXT: # xmm11 = mem[3,1,2,1,4,5,6,7]
8552 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm6 = xmm11[0],xmm6[1,2],xmm11[3],xmm6[4,5,6,7]
8553 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8554 ; AVX2-FAST-NEXT: vpshufb %ymm7, %ymm1, %ymm11
8555 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
8556 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm0 = ymm11[0,1,2],ymm0[3,4,5,6,7],ymm11[8,9,10],ymm0[11,12,13,14,15]
8557 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[0,1,3,2]
8558 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm11[5,6,7]
8559 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm0[4,5,6,7]
8560 ; AVX2-FAST-NEXT: vpshufb %xmm2, %xmm12, %xmm0
8561 ; AVX2-FAST-NEXT: vpshufhw {{.*#+}} xmm2 = xmm8[0,1,2,3,7,5,6,5]
8562 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4],xmm0[5,6],xmm2[7]
8563 ; AVX2-FAST-NEXT: vpshufb %ymm7, %ymm5, %ymm2
8564 ; AVX2-FAST-NEXT: vpshufb %xmm9, %xmm10, %xmm5
8565 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[3,1,2,1,4,5,6,7]
8566 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm5[1,2],xmm3[3],xmm5[4,5,6,7]
8567 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
8568 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7],ymm2[8,9,10],ymm0[11,12,13,14,15]
8569 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,3,2]
8570 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3,4],xmm2[5,6,7]
8571 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm4 = ymm2[0,1,2,3],ymm0[4,5,6,7]
8572 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8573 ; AVX2-FAST-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
8574 ; AVX2-FAST-NEXT: # ymm0 = ymm0[0],mem[1],ymm0[2,3],mem[4],ymm0[5,6],mem[7]
8575 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm5
8576 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,3,2,1]
8577 ; AVX2-FAST-NEXT: vpbroadcastq {{.*#+}} xmm7 = [8,9,0,0,0,1,12,13,8,9,0,0,0,1,12,13]
8578 ; AVX2-FAST-NEXT: vpshufb %xmm7, %xmm5, %xmm3
8579 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm8 = <0,1,2,3,0,1,4,5,u,u,12,13,u,u,u,u>
8580 ; AVX2-FAST-NEXT: vpshufb %xmm8, %xmm2, %xmm9
8581 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm9[0,1,2,3],xmm3[4],xmm9[5],xmm3[6,7]
8582 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
8583 ; AVX2-FAST-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
8584 ; AVX2-FAST-NEXT: # ymm3 = mem[0,1,2,3,4],ymm3[5,6,7]
8585 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8586 ; AVX2-FAST-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm9 # 32-byte Folded Reload
8587 ; AVX2-FAST-NEXT: # ymm9 = ymm0[0],mem[1],ymm0[2,3],mem[4],ymm0[5,6],mem[7]
8588 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm9, %xmm10
8589 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[0,3,2,1]
8590 ; AVX2-FAST-NEXT: vpshufb %xmm7, %xmm10, %xmm11
8591 ; AVX2-FAST-NEXT: vpshufb %xmm8, %xmm9, %xmm12
8592 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm11 = xmm12[0,1,2,3],xmm11[4],xmm12[5],xmm11[6,7]
8593 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
8594 ; AVX2-FAST-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm11 # 32-byte Folded Reload
8595 ; AVX2-FAST-NEXT: # ymm11 = mem[0,1,2,3,4],ymm11[5,6,7]
8596 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8597 ; AVX2-FAST-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm12 # 32-byte Folded Reload
8598 ; AVX2-FAST-NEXT: # ymm12 = ymm0[0],mem[1],ymm0[2,3],mem[4],ymm0[5,6],mem[7]
8599 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm12, %xmm13
8600 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[0,3,2,1]
8601 ; AVX2-FAST-NEXT: vpshufb %xmm7, %xmm13, %xmm14
8602 ; AVX2-FAST-NEXT: vpshufb %xmm8, %xmm12, %xmm15
8603 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm14 = xmm15[0,1,2,3],xmm14[4],xmm15[5],xmm14[6,7]
8604 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
8605 ; AVX2-FAST-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload
8606 ; AVX2-FAST-NEXT: # ymm14 = mem[0,1,2,3,4],ymm14[5,6,7]
8607 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8608 ; AVX2-FAST-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
8609 ; AVX2-FAST-NEXT: # ymm15 = ymm0[0],mem[1],ymm0[2,3],mem[4],ymm0[5,6],mem[7]
8610 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm15, %xmm0
8611 ; AVX2-FAST-NEXT: vpshufb %xmm7, %xmm0, %xmm7
8612 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm15 = xmm15[0,3,2,1]
8613 ; AVX2-FAST-NEXT: vpshufb %xmm8, %xmm15, %xmm8
8614 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm7 = xmm8[0,1,2,3],xmm7[4],xmm8[5],xmm7[6,7]
8615 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
8616 ; AVX2-FAST-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
8617 ; AVX2-FAST-NEXT: # ymm7 = mem[0,1,2,3,4],ymm7[5,6,7]
8618 ; AVX2-FAST-NEXT: vpbroadcastq {{.*#+}} xmm8 = [10,11,0,0,2,3,14,15,10,11,0,0,2,3,14,15]
8619 ; AVX2-FAST-NEXT: vpshufb %xmm8, %xmm10, %xmm10
8620 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm1 = <0,1,2,3,2,3,6,7,u,u,14,15,u,u,u,u>
8621 ; AVX2-FAST-NEXT: vpshufb %xmm1, %xmm9, %xmm9
8622 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1,2,3],xmm10[4],xmm9[5],xmm10[6,7]
8623 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
8624 ; AVX2-FAST-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
8625 ; AVX2-FAST-NEXT: # ymm9 = mem[0,1,2,3,4],ymm9[5,6,7]
8626 ; AVX2-FAST-NEXT: vpshufb %xmm8, %xmm5, %xmm5
8627 ; AVX2-FAST-NEXT: vpshufb %xmm1, %xmm2, %xmm2
8628 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm5[4],xmm2[5],xmm5[6,7]
8629 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
8630 ; AVX2-FAST-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
8631 ; AVX2-FAST-NEXT: # ymm2 = mem[0,1,2,3,4],ymm2[5,6,7]
8632 ; AVX2-FAST-NEXT: vpshufb %xmm8, %xmm13, %xmm5
8633 ; AVX2-FAST-NEXT: vpshufb %xmm1, %xmm12, %xmm10
8634 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm5 = xmm10[0,1,2,3],xmm5[4],xmm10[5],xmm5[6,7]
8635 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
8636 ; AVX2-FAST-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
8637 ; AVX2-FAST-NEXT: # ymm5 = mem[0,1,2,3,4],ymm5[5,6,7]
8638 ; AVX2-FAST-NEXT: vpshufb %xmm8, %xmm0, %xmm0
8639 ; AVX2-FAST-NEXT: vpshufb %xmm1, %xmm15, %xmm1
8640 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4],xmm1[5],xmm0[6,7]
8641 ; AVX2-FAST-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
8642 ; AVX2-FAST-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
8643 ; AVX2-FAST-NEXT: # ymm0 = mem[0,1,2,3,4],ymm0[5,6,7]
8644 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8645 ; AVX2-FAST-NEXT: vmovaps %ymm1, 96(%rsi)
8646 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8647 ; AVX2-FAST-NEXT: vmovaps %ymm1, 32(%rsi)
8648 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8649 ; AVX2-FAST-NEXT: vmovaps %ymm1, 64(%rsi)
8650 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8651 ; AVX2-FAST-NEXT: vmovaps %ymm1, (%rsi)
8652 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8653 ; AVX2-FAST-NEXT: vmovaps %ymm1, 96(%rdx)
8654 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8655 ; AVX2-FAST-NEXT: vmovaps %ymm1, 32(%rdx)
8656 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8657 ; AVX2-FAST-NEXT: vmovaps %ymm1, 64(%rdx)
8658 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8659 ; AVX2-FAST-NEXT: vmovaps %ymm1, (%rdx)
8660 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8661 ; AVX2-FAST-NEXT: vmovaps %ymm1, 32(%rcx)
8662 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8663 ; AVX2-FAST-NEXT: vmovaps %ymm1, 96(%rcx)
8664 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8665 ; AVX2-FAST-NEXT: vmovaps %ymm1, 64(%rcx)
8666 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8667 ; AVX2-FAST-NEXT: vmovaps %ymm1, (%rcx)
8668 ; AVX2-FAST-NEXT: vmovdqa %ymm6, 96(%r8)
8669 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8670 ; AVX2-FAST-NEXT: vmovaps %ymm1, 32(%r8)
8671 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8672 ; AVX2-FAST-NEXT: vmovaps %ymm1, 64(%r8)
8673 ; AVX2-FAST-NEXT: vmovdqa %ymm4, (%r8)
8674 ; AVX2-FAST-NEXT: vmovdqa %ymm7, 96(%r9)
8675 ; AVX2-FAST-NEXT: vmovdqa %ymm14, 32(%r9)
8676 ; AVX2-FAST-NEXT: vmovdqa %ymm11, (%r9)
8677 ; AVX2-FAST-NEXT: vmovdqa %ymm3, 64(%r9)
8678 ; AVX2-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
8679 ; AVX2-FAST-NEXT: vmovdqa %ymm0, 96(%rax)
8680 ; AVX2-FAST-NEXT: vmovdqa %ymm5, 32(%rax)
8681 ; AVX2-FAST-NEXT: vmovdqa %ymm2, 64(%rax)
8682 ; AVX2-FAST-NEXT: vmovdqa %ymm9, (%rax)
8683 ; AVX2-FAST-NEXT: addq $1272, %rsp # imm = 0x4F8
8684 ; AVX2-FAST-NEXT: vzeroupper
8685 ; AVX2-FAST-NEXT: retq
8686 ;
8687 ; AVX2-FAST-PERLANE-LABEL: load_i16_stride6_vf64:
8688 ; AVX2-FAST-PERLANE: # %bb.0:
8689 ; AVX2-FAST-PERLANE-NEXT: subq $1272, %rsp # imm = 0x4F8
8690 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 64(%rdi), %ymm0
8691 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 96(%rdi), %ymm1
8692 ; AVX2-FAST-PERLANE-NEXT: vmovaps 672(%rdi), %ymm2
8693 ; AVX2-FAST-PERLANE-NEXT: vmovaps 640(%rdi), %ymm3
8694 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 288(%rdi), %ymm4
8695 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 256(%rdi), %ymm5
8696 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 416(%rdi), %ymm8
8697 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8698 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 384(%rdi), %ymm10
8699 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8700 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 480(%rdi), %ymm6
8701 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 448(%rdi), %ymm7
8702 ; AVX2-FAST-PERLANE-NEXT: vperm2i128 {{.*#+}} ymm11 = ymm7[2,3],ymm6[2,3]
8703 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8704 ; AVX2-FAST-PERLANE-NEXT: vperm2i128 {{.*#+}} ymm7 = ymm7[0,1],ymm6[0,1]
8705 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8706 ; AVX2-FAST-PERLANE-NEXT: vperm2i128 {{.*#+}} ymm12 = ymm5[2,3],ymm4[2,3]
8707 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8708 ; AVX2-FAST-PERLANE-NEXT: vperm2i128 {{.*#+}} ymm14 = ymm5[0,1],ymm4[0,1]
8709 ; AVX2-FAST-PERLANE-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm3[2,3],ymm2[2,3]
8710 ; AVX2-FAST-PERLANE-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8711 ; AVX2-FAST-PERLANE-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm3[0,1],ymm2[0,1]
8712 ; AVX2-FAST-PERLANE-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8713 ; AVX2-FAST-PERLANE-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm0[2,3],ymm1[2,3]
8714 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8715 ; AVX2-FAST-PERLANE-NEXT: vperm2i128 {{.*#+}} ymm3 = ymm0[0,1],ymm1[0,1]
8716 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8717 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} ymm9 = [8,9,4,5,4,5,6,7,0,1,4,5,0,1,12,13,24,25,20,21,20,21,22,23,16,17,20,21,16,17,28,29]
8718 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1],ymm10[2],ymm8[3,4],ymm10[5],ymm8[6,7]
8719 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm6 = <8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15>
8720 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm4, %xmm0
8721 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm4, %xmm1
8722 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8723 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,2,2,2,4,5,6,7]
8724 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6,7]
8725 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0],ymm7[1],ymm11[2,3,4,5],ymm7[6],ymm11[7]
8726 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8727 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm9, %ymm1, %ymm1
8728 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0]
8729 ; AVX2-FAST-PERLANE-NEXT: vpblendvb %ymm10, %ymm0, %ymm1, %ymm0
8730 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8731 ; AVX2-FAST-PERLANE-NEXT: vmovdqa (%rdi), %ymm0
8732 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8733 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %ymm1
8734 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8735 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm15 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
8736 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm15, %xmm0
8737 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm15, %xmm2
8738 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm2[2,2,2,2,4,5,6,7]
8739 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6,7]
8740 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0],ymm3[1],ymm5[2,3,4,5],ymm3[6],ymm5[7]
8741 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm9, %ymm3, %ymm1
8742 ; AVX2-FAST-PERLANE-NEXT: vpblendvb %ymm10, %ymm0, %ymm1, %ymm0
8743 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8744 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 224(%rdi), %ymm0
8745 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8746 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 192(%rdi), %ymm1
8747 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8748 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm5 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
8749 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm5, %xmm0
8750 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm5, %xmm7
8751 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm7[2,2,2,2,4,5,6,7]
8752 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6,7]
8753 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8754 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm8 = ymm12[0],ymm14[1],ymm12[2,3,4,5],ymm14[6],ymm12[7]
8755 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm9, %ymm8, %ymm1
8756 ; AVX2-FAST-PERLANE-NEXT: vpblendvb %ymm10, %ymm0, %ymm1, %ymm0
8757 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8758 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 608(%rdi), %ymm1
8759 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8760 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 576(%rdi), %ymm0
8761 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, (%rsp) # 32-byte Spill
8762 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
8763 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm0, %xmm1
8764 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm0, %xmm6
8765 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm13 = xmm6[2,2,2,2,4,5,6,7]
8766 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm13 = xmm1[0],xmm13[1],xmm1[2,3],xmm13[4],xmm1[5,6,7]
8767 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
8768 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
8769 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0],ymm12[1],ymm11[2,3,4,5],ymm12[6],ymm11[7]
8770 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm9, %ymm1, %ymm9
8771 ; AVX2-FAST-PERLANE-NEXT: vpblendvb %ymm10, %ymm13, %ymm9, %ymm9
8772 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8773 ; AVX2-FAST-PERLANE-NEXT: vpbroadcastd {{.*#+}} xmm9 = [10,11,6,7,10,11,6,7,10,11,6,7,10,11,6,7]
8774 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm9, %xmm2, %xmm13
8775 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm2 = <10,11,u,u,2,3,14,15,u,u,10,11,12,13,14,15>
8776 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm2, %xmm15, %xmm15
8777 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm13 = xmm15[0],xmm13[1],xmm15[2,3],xmm13[4],xmm15[5,6,7]
8778 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} ymm15 = [10,11,6,7,4,5,6,7,6,7,6,7,2,3,14,15,26,27,22,23,20,21,22,23,22,23,22,23,18,19,30,31]
8779 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm15, %ymm3, %ymm3
8780 ; AVX2-FAST-PERLANE-NEXT: vpblendvb %ymm10, %ymm13, %ymm3, %ymm3
8781 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8782 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
8783 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm9, %xmm3, %xmm3
8784 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm2, %xmm4, %xmm4
8785 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6,7]
8786 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
8787 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm15, %ymm4, %ymm4
8788 ; AVX2-FAST-PERLANE-NEXT: vpblendvb %ymm10, %ymm3, %ymm4, %ymm3
8789 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8790 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm9, %xmm7, %xmm3
8791 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm2, %xmm5, %xmm4
8792 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6,7]
8793 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm15, %ymm8, %ymm4
8794 ; AVX2-FAST-PERLANE-NEXT: vpblendvb %ymm10, %ymm3, %ymm4, %ymm3
8795 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8796 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm15, %ymm1, %ymm1
8797 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm9, %xmm6, %xmm3
8798 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm2, %xmm0, %xmm0
8799 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2,3],xmm3[4],xmm0[5,6,7]
8800 ; AVX2-FAST-PERLANE-NEXT: vpblendvb %ymm10, %ymm0, %ymm1, %ymm0
8801 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8802 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8803 ; AVX2-FAST-PERLANE-NEXT: vpblendd $109, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm7 # 32-byte Folded Reload
8804 ; AVX2-FAST-PERLANE-NEXT: # ymm7 = mem[0],ymm0[1],mem[2,3],ymm0[4],mem[5,6],ymm0[7]
8805 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm7, %xmm0
8806 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,1,0,3]
8807 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8808 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm2 = <0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15>
8809 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm2, %xmm7, %xmm0
8810 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm2, %xmm13
8811 ; AVX2-FAST-PERLANE-NEXT: vpbroadcastq {{.*#+}} xmm3 = [8,9,12,13,0,1,0,0,8,9,12,13,0,1,0,0]
8812 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm3, %xmm1, %xmm2
8813 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm3, %xmm8
8814 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2],xmm0[3],xmm2[4,5],xmm0[6,7]
8815 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} ymm2 = <u,u,u,u,u,u,u,u,u,u,u,u,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21>
8816 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8817 ; AVX2-FAST-PERLANE-NEXT: vpblendd $189, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
8818 ; AVX2-FAST-PERLANE-NEXT: # ymm1 = mem[0],ymm1[1],mem[2,3,4,5],ymm1[6],mem[7]
8819 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8820 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm2, %ymm1, %ymm3
8821 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm2, %ymm4
8822 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm3[3,4,5,6,7]
8823 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 544(%rdi), %ymm1
8824 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8825 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 512(%rdi), %ymm2
8826 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8827 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
8828 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8829 ; AVX2-FAST-PERLANE-NEXT: vpbroadcastd {{.*#+}} xmm9 = [8,9,4,5,8,9,4,5,8,9,4,5,8,9,4,5]
8830 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm9, %xmm1, %xmm5
8831 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm10 = <0,1,4,5,4,5,u,u,0,1,12,13,u,u,4,5>
8832 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm1, %xmm1
8833 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8834 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm10, %xmm1, %xmm6
8835 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3],xmm6[4,5],xmm5[6],xmm6[7]
8836 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
8837 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm5 = ymm0[0,1,2],ymm5[3,4,5,6,7],ymm0[8,9,10],ymm5[11,12,13,14,15]
8838 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
8839 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8840 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8841 ; AVX2-FAST-PERLANE-NEXT: vpblendd $109, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
8842 ; AVX2-FAST-PERLANE-NEXT: # ymm5 = mem[0],ymm0[1],mem[2,3],ymm0[4],mem[5,6],ymm0[7]
8843 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm5, %xmm0
8844 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,1,0,3]
8845 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8846 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm13, %xmm5, %xmm0
8847 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm8, %xmm6
8848 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm8, %xmm1, %xmm8
8849 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm8[2],xmm0[3],xmm8[4,5],xmm0[6,7]
8850 ; AVX2-FAST-PERLANE-NEXT: vpblendd $66, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm1 # 32-byte Folded Reload
8851 ; AVX2-FAST-PERLANE-NEXT: # ymm1 = ymm14[0],mem[1],ymm14[2,3,4,5],mem[6],ymm14[7]
8852 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8853 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm4, %ymm2
8854 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm4, %ymm1, %ymm15
8855 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm15 = ymm0[0,1,2],ymm15[3,4,5,6,7]
8856 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 352(%rdi), %ymm0
8857 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8858 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 320(%rdi), %ymm1
8859 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8860 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm4 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
8861 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm9, %xmm4, %xmm0
8862 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm4, %xmm1
8863 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8864 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm10, %xmm1, %xmm14
8865 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm14[0,1,2],xmm0[3],xmm14[4,5],xmm0[6],xmm14[7]
8866 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
8867 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm0 = ymm15[0,1,2],ymm0[3,4,5,6,7],ymm15[8,9,10],ymm0[11,12,13,14,15]
8868 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm15[0,1,2,3],ymm0[4,5,6,7]
8869 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8870 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8871 ; AVX2-FAST-PERLANE-NEXT: vpblendd $109, (%rsp), %ymm0, %ymm3 # 32-byte Folded Reload
8872 ; AVX2-FAST-PERLANE-NEXT: # ymm3 = mem[0],ymm0[1],mem[2,3],ymm0[4],mem[5,6],ymm0[7]
8873 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm3, %xmm14
8874 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm0 = xmm14[2,1,0,3]
8875 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8876 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm13, %xmm3, %xmm15
8877 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm13, %xmm8
8878 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm0, %xmm13
8879 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm13 = xmm15[0,1],xmm13[2],xmm15[3],xmm13[4,5],xmm15[6,7]
8880 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm12[0],ymm11[1],ymm12[2,3,4,5],ymm11[6],ymm12[7]
8881 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8882 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm2, %ymm0, %ymm12
8883 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm2, %ymm14
8884 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm12 = ymm13[0,1,2],ymm12[3,4,5,6,7]
8885 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 736(%rdi), %ymm0
8886 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8887 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 704(%rdi), %ymm1
8888 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8889 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm13 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
8890 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm9, %xmm13, %xmm0
8891 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm13, %xmm6
8892 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm10, %xmm6, %xmm11
8893 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm11[0,1,2],xmm0[3],xmm11[4,5],xmm0[6],xmm11[7]
8894 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
8895 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm0 = ymm12[0,1,2],ymm0[3,4,5,6,7],ymm12[8,9,10],ymm0[11,12,13,14,15]
8896 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm12[0,1,2,3],ymm0[4,5,6,7]
8897 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8898 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 160(%rdi), %ymm0
8899 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8900 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 128(%rdi), %ymm1
8901 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8902 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
8903 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm9, %xmm2, %xmm9
8904 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm2, %xmm11
8905 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm10, %xmm11, %xmm10
8906 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3],xmm10[4,5],xmm9[6],xmm10[7]
8907 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8908 ; AVX2-FAST-PERLANE-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm10 # 32-byte Folded Reload
8909 ; AVX2-FAST-PERLANE-NEXT: # ymm10 = ymm0[0],mem[1],ymm0[2,3],mem[4],ymm0[5,6],mem[7]
8910 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm8, %xmm10, %xmm1
8911 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm10, %xmm12
8912 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[2,1,0,3]
8913 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm8 = xmm12[u,u,u,u,0,1,u,u,8,9,12,13,u,u,u,u]
8914 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm8[2],xmm1[3],xmm8[4,5],xmm1[6,7]
8915 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8916 ; AVX2-FAST-PERLANE-NEXT: vpblendd $189, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
8917 ; AVX2-FAST-PERLANE-NEXT: # ymm8 = mem[0],ymm0[1],mem[2,3,4,5],ymm0[6],mem[7]
8918 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm14, %ymm8, %ymm14
8919 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2],ymm14[3,4,5,6,7]
8920 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
8921 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm9 = ymm1[0,1,2],ymm9[3,4,5,6,7],ymm1[8,9,10],ymm9[11,12,13,14,15]
8922 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm9[4,5,6,7]
8923 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8924 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15]
8925 ; AVX2-FAST-PERLANE-NEXT: vpbroadcastq {{.*#+}} xmm9 = [10,11,14,15,2,3,0,0,10,11,14,15,2,3,0,0]
8926 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
8927 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm9, %xmm0, %xmm14
8928 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1],xmm14[2],xmm7[3],xmm14[4,5],xmm7[6,7]
8929 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} ymm14 = <u,u,u,u,u,u,u,u,u,u,u,u,10,11,6,7,18,19,30,31,26,27,26,27,30,31,26,27,26,27,22,23>
8930 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8931 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm14, %ymm0, %ymm15
8932 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2],ymm15[3,4,5,6,7]
8933 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm15 = <6,7,2,3,4,5,u,u,2,3,14,15,u,u,6,7>
8934 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
8935 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm15, %xmm0, %xmm0
8936 ; AVX2-FAST-PERLANE-NEXT: vpshufhw $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
8937 ; AVX2-FAST-PERLANE-NEXT: # xmm1 = mem[0,1,2,3,5,5,5,5]
8938 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]
8939 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
8940 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm0 = ymm7[0,1,2],ymm0[3,4,5,6,7],ymm7[8,9,10],ymm0[11,12,13,14,15]
8941 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm0[4,5,6,7]
8942 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8943 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm7 = <2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15>
8944 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm7, %xmm5, %xmm0
8945 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
8946 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm9, %xmm1, %xmm1
8947 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3],xmm1[4,5],xmm0[6,7]
8948 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8949 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm14, %ymm1, %ymm1
8950 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
8951 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
8952 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm15, %xmm1, %xmm1
8953 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,5,5,5]
8954 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3],xmm1[4,5],xmm4[6],xmm1[7]
8955 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
8956 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
8957 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
8958 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8959 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm7, %xmm3, %xmm0
8960 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
8961 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm9, %xmm1, %xmm1
8962 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3],xmm1[4,5],xmm0[6,7]
8963 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8964 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm14, %ymm1, %ymm1
8965 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
8966 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm15, %xmm6, %xmm1
8967 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm3 = xmm13[0,1,2,3,5,5,5,5]
8968 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm3[3],xmm1[4,5],xmm3[6],xmm1[7]
8969 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
8970 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
8971 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
8972 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8973 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm15, %xmm11, %xmm0
8974 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm1 = xmm2[0,1,2,3,5,5,5,5]
8975 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]
8976 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm7, %xmm10, %xmm1
8977 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm9, %xmm12, %xmm2
8978 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3],xmm2[4,5],xmm1[6,7]
8979 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm14, %ymm8, %ymm2
8980 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
8981 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
8982 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
8983 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
8984 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8985 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8986 ; AVX2-FAST-PERLANE-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
8987 ; AVX2-FAST-PERLANE-NEXT: # ymm3 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6],ymm0[7]
8988 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8989 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
8990 ; AVX2-FAST-PERLANE-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
8991 ; AVX2-FAST-PERLANE-NEXT: # ymm0 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
8992 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,1,0,3]
8993 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8994 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm0, %xmm0
8995 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,1,2,1]
8996 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8997 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm8 = <0,1,0,1,0,1,0,1,u,u,8,9,12,13,u,u>
8998 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm8, %xmm1, %xmm0
8999 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm1 = xmm2[0,1,2,3,6,5,6,4]
9000 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0,1,2,3],xmm1[4],xmm0[5,6],xmm1[7]
9001 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9002 ; AVX2-FAST-PERLANE-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
9003 ; AVX2-FAST-PERLANE-NEXT: # ymm1 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
9004 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm1, %xmm0
9005 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm7 = xmm0[0,3,2,1]
9006 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[2,1,2,3]
9007 ; AVX2-FAST-PERLANE-NEXT: vpbroadcastq {{.*#+}} xmm0 = [12,13,0,1,4,5,0,0,12,13,0,1,4,5,0,0]
9008 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm0, %xmm7, %xmm5
9009 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm0, %xmm10
9010 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm6 = xmm4[2,1,2,0,4,5,6,7]
9011 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0],xmm5[1,2],xmm6[3],xmm5[4,5,6,7]
9012 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
9013 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} ymm0 = <4,5,0,1,12,13,14,15,8,9,12,13,0,1,u,u,20,21,16,17,28,29,30,31,24,25,28,29,16,17,u,u>
9014 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm0, %ymm3, %ymm6
9015 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm0, %ymm9
9016 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm2 = ymm6[0,1,2],ymm2[3,4,5,6,7],ymm6[8,9,10],ymm2[11,12,13,14,15]
9017 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,6,5,4]
9018 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4],xmm6[5,6,7]
9019 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3],ymm2[4,5,6,7]
9020 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9021 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9022 ; AVX2-FAST-PERLANE-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
9023 ; AVX2-FAST-PERLANE-NEXT: # ymm1 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6],ymm0[7]
9024 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9025 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9026 ; AVX2-FAST-PERLANE-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
9027 ; AVX2-FAST-PERLANE-NEXT: # ymm2 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
9028 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm0 = xmm2[2,1,0,3]
9029 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9030 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm2, %xmm2
9031 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,1,2,1]
9032 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9033 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm8, %xmm0, %xmm2
9034 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm12 = xmm3[0,1,2,3,6,5,6,4]
9035 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm12 = xmm2[0,1,2,3],xmm12[4],xmm2[5,6],xmm12[7]
9036 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9037 ; AVX2-FAST-PERLANE-NEXT: vpblendd $219, (%rsp), %ymm0, %ymm2 # 32-byte Folded Reload
9038 ; AVX2-FAST-PERLANE-NEXT: # ymm2 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
9039 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm2, %xmm13
9040 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm0 = xmm13[0,3,2,1]
9041 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill
9042 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,1,2,3]
9043 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9044 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm10, %xmm0, %xmm13
9045 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm15 = xmm2[2,1,2,0,4,5,6,7]
9046 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm13 = xmm15[0],xmm13[1,2],xmm15[3],xmm13[4,5,6,7]
9047 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm12
9048 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm9, %ymm1, %ymm15
9049 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm12 = ymm15[0,1,2],ymm12[3,4,5,6,7],ymm15[8,9,10],ymm12[11,12,13,14,15]
9050 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,6,5,4]
9051 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0,1,2,3,4],xmm15[5,6,7]
9052 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3],ymm12[4,5,6,7]
9053 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9054 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9055 ; AVX2-FAST-PERLANE-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
9056 ; AVX2-FAST-PERLANE-NEXT: # ymm14 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6],ymm0[7]
9057 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9058 ; AVX2-FAST-PERLANE-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm13 # 32-byte Folded Reload
9059 ; AVX2-FAST-PERLANE-NEXT: # ymm13 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
9060 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm15 = xmm13[2,1,0,3]
9061 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm13, %xmm13
9062 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[0,1,2,1]
9063 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm8, %xmm15, %xmm0
9064 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm11 = xmm13[0,1,2,3,6,5,6,4]
9065 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1,2,3],xmm11[4],xmm0[5,6],xmm11[7]
9066 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9067 ; AVX2-FAST-PERLANE-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm11 # 32-byte Folded Reload
9068 ; AVX2-FAST-PERLANE-NEXT: # ymm11 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
9069 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm11, %xmm6
9070 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[0,3,2,1]
9071 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[2,1,2,3]
9072 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm10, %xmm6, %xmm0
9073 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm10, %xmm3
9074 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm5 = xmm11[2,1,2,0,4,5,6,7]
9075 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm5[0],xmm0[1,2],xmm5[3],xmm0[4,5,6,7]
9076 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
9077 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm9, %ymm14, %ymm5
9078 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm1 = ymm5[0,1,2],ymm1[3,4,5,6,7],ymm5[8,9,10],ymm1[11,12,13,14,15]
9079 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,6,5,4]
9080 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm5[5,6,7]
9081 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
9082 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9083 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9084 ; AVX2-FAST-PERLANE-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
9085 ; AVX2-FAST-PERLANE-NEXT: # ymm0 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
9086 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[2,1,0,3]
9087 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm8, %xmm12, %xmm1
9088 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm0, %xmm0
9089 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm8 = xmm0[0,1,2,1]
9090 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm0 = xmm8[0,1,2,3,6,5,6,4]
9091 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm2 = xmm1[0,1,2,3],xmm0[4],xmm1[5,6],xmm0[7]
9092 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9093 ; AVX2-FAST-PERLANE-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
9094 ; AVX2-FAST-PERLANE-NEXT: # ymm5 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6],ymm0[7]
9095 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9096 ; AVX2-FAST-PERLANE-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
9097 ; AVX2-FAST-PERLANE-NEXT: # ymm0 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
9098 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm0, %xmm10
9099 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[0,3,2,1]
9100 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm3, %xmm10, %xmm1
9101 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,1,2,3]
9102 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm9 = xmm3[2,1,2,0,4,5,6,7]
9103 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm1 = xmm9[0],xmm1[1,2],xmm9[3],xmm1[4,5,6,7]
9104 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
9105 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} ymm9 = ymm5[4,5,0,1,12,13,14,15,8,9,12,13,0,1,u,u,20,21,16,17,28,29,30,31,24,25,28,29,16,17,u,u]
9106 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm2 = ymm9[0,1,2],ymm2[3,4,5,6,7],ymm9[8,9,10],ymm2[11,12,13,14,15]
9107 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,6,5,4]
9108 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm9[5,6,7]
9109 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm2[4,5,6,7]
9110 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9111 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm2 = <2,3,2,3,2,3,2,3,u,u,10,11,14,15,u,u>
9112 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
9113 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm2, %xmm0, %xmm1
9114 ; AVX2-FAST-PERLANE-NEXT: vpshufhw $103, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload
9115 ; AVX2-FAST-PERLANE-NEXT: # xmm9 = mem[0,1,2,3,7,5,6,5]
9116 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm9[4],xmm1[5,6],xmm9[7]
9117 ; AVX2-FAST-PERLANE-NEXT: vpbroadcastq {{.*#+}} xmm9 = [14,15,2,3,6,7,0,0,14,15,2,3,6,7,0,0]
9118 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm9, %xmm7, %xmm7
9119 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[3,1,2,1,4,5,6,7]
9120 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm7[1,2],xmm4[3],xmm7[4,5,6,7]
9121 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} ymm7 = <6,7,2,3,14,15,14,15,14,15,10,11,u,u,2,3,22,23,18,19,30,31,30,31,30,31,26,27,u,u,18,19>
9122 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9123 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm7, %ymm0, %ymm0
9124 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
9125 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
9126 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,3,2]
9127 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm4[0,1,2,3,4],xmm0[5,6,7]
9128 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
9129 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9130 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm2, %xmm15, %xmm0
9131 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm1 = xmm13[0,1,2,3,7,5,6,5]
9132 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4],xmm0[5,6],xmm1[7]
9133 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm9, %xmm6, %xmm1
9134 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm6 = xmm11[3,1,2,1,4,5,6,7]
9135 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm1 = xmm6[0],xmm1[1,2],xmm6[3],xmm1[4,5,6,7]
9136 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm7, %ymm14, %ymm6
9137 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
9138 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm0 = ymm6[0,1,2],ymm0[3,4,5,6,7],ymm6[8,9,10],ymm0[11,12,13,14,15]
9139 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[0,1,3,2]
9140 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm6[5,6,7]
9141 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
9142 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9143 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
9144 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm2, %xmm0, %xmm0
9145 ; AVX2-FAST-PERLANE-NEXT: vpshufhw $103, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
9146 ; AVX2-FAST-PERLANE-NEXT: # xmm6 = mem[0,1,2,3,7,5,6,5]
9147 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm6[4],xmm0[5,6],xmm6[7]
9148 ; AVX2-FAST-PERLANE-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload
9149 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm9, %xmm1, %xmm6
9150 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $103, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
9151 ; AVX2-FAST-PERLANE-NEXT: # xmm11 = mem[3,1,2,1,4,5,6,7]
9152 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm6 = xmm11[0],xmm6[1,2],xmm11[3],xmm6[4,5,6,7]
9153 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
9154 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm7, %ymm1, %ymm11
9155 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
9156 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm0 = ymm11[0,1,2],ymm0[3,4,5,6,7],ymm11[8,9,10],ymm0[11,12,13,14,15]
9157 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[0,1,3,2]
9158 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm11[5,6,7]
9159 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm0[4,5,6,7]
9160 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm2, %xmm12, %xmm0
9161 ; AVX2-FAST-PERLANE-NEXT: vpshufhw {{.*#+}} xmm2 = xmm8[0,1,2,3,7,5,6,5]
9162 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4],xmm0[5,6],xmm2[7]
9163 ; AVX2-FAST-PERLANE-NEXT: vpshufb %ymm7, %ymm5, %ymm2
9164 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm9, %xmm10, %xmm5
9165 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[3,1,2,1,4,5,6,7]
9166 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm5[1,2],xmm3[3],xmm5[4,5,6,7]
9167 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
9168 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7],ymm2[8,9,10],ymm0[11,12,13,14,15]
9169 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,3,2]
9170 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3,4],xmm2[5,6,7]
9171 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm4 = ymm2[0,1,2,3],ymm0[4,5,6,7]
9172 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9173 ; AVX2-FAST-PERLANE-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
9174 ; AVX2-FAST-PERLANE-NEXT: # ymm0 = ymm0[0],mem[1],ymm0[2,3],mem[4],ymm0[5,6],mem[7]
9175 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm0, %xmm5
9176 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,3,2,1]
9177 ; AVX2-FAST-PERLANE-NEXT: vpbroadcastq {{.*#+}} xmm7 = [8,9,0,0,0,1,12,13,8,9,0,0,0,1,12,13]
9178 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm7, %xmm5, %xmm3
9179 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm8 = <0,1,2,3,0,1,4,5,u,u,12,13,u,u,u,u>
9180 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm8, %xmm2, %xmm9
9181 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm3 = xmm9[0,1,2,3],xmm3[4],xmm9[5],xmm3[6,7]
9182 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
9183 ; AVX2-FAST-PERLANE-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
9184 ; AVX2-FAST-PERLANE-NEXT: # ymm3 = mem[0,1,2,3,4],ymm3[5,6,7]
9185 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9186 ; AVX2-FAST-PERLANE-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm9 # 32-byte Folded Reload
9187 ; AVX2-FAST-PERLANE-NEXT: # ymm9 = ymm0[0],mem[1],ymm0[2,3],mem[4],ymm0[5,6],mem[7]
9188 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm9, %xmm10
9189 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[0,3,2,1]
9190 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm7, %xmm10, %xmm11
9191 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm8, %xmm9, %xmm12
9192 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm11 = xmm12[0,1,2,3],xmm11[4],xmm12[5],xmm11[6,7]
9193 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
9194 ; AVX2-FAST-PERLANE-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm11 # 32-byte Folded Reload
9195 ; AVX2-FAST-PERLANE-NEXT: # ymm11 = mem[0,1,2,3,4],ymm11[5,6,7]
9196 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9197 ; AVX2-FAST-PERLANE-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm12 # 32-byte Folded Reload
9198 ; AVX2-FAST-PERLANE-NEXT: # ymm12 = ymm0[0],mem[1],ymm0[2,3],mem[4],ymm0[5,6],mem[7]
9199 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm12, %xmm13
9200 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[0,3,2,1]
9201 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm7, %xmm13, %xmm14
9202 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm8, %xmm12, %xmm15
9203 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm14 = xmm15[0,1,2,3],xmm14[4],xmm15[5],xmm14[6,7]
9204 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
9205 ; AVX2-FAST-PERLANE-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload
9206 ; AVX2-FAST-PERLANE-NEXT: # ymm14 = mem[0,1,2,3,4],ymm14[5,6,7]
9207 ; AVX2-FAST-PERLANE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9208 ; AVX2-FAST-PERLANE-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
9209 ; AVX2-FAST-PERLANE-NEXT: # ymm15 = ymm0[0],mem[1],ymm0[2,3],mem[4],ymm0[5,6],mem[7]
9210 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm15, %xmm0
9211 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm7, %xmm0, %xmm7
9212 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm15 = xmm15[0,3,2,1]
9213 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm8, %xmm15, %xmm8
9214 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm7 = xmm8[0,1,2,3],xmm7[4],xmm8[5],xmm7[6,7]
9215 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
9216 ; AVX2-FAST-PERLANE-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
9217 ; AVX2-FAST-PERLANE-NEXT: # ymm7 = mem[0,1,2,3,4],ymm7[5,6,7]
9218 ; AVX2-FAST-PERLANE-NEXT: vpbroadcastq {{.*#+}} xmm8 = [10,11,0,0,2,3,14,15,10,11,0,0,2,3,14,15]
9219 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm8, %xmm10, %xmm10
9220 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm1 = <0,1,2,3,2,3,6,7,u,u,14,15,u,u,u,u>
9221 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm1, %xmm9, %xmm9
9222 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1,2,3],xmm10[4],xmm9[5],xmm10[6,7]
9223 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
9224 ; AVX2-FAST-PERLANE-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
9225 ; AVX2-FAST-PERLANE-NEXT: # ymm9 = mem[0,1,2,3,4],ymm9[5,6,7]
9226 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm8, %xmm5, %xmm5
9227 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm1, %xmm2, %xmm2
9228 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm5[4],xmm2[5],xmm5[6,7]
9229 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
9230 ; AVX2-FAST-PERLANE-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
9231 ; AVX2-FAST-PERLANE-NEXT: # ymm2 = mem[0,1,2,3,4],ymm2[5,6,7]
9232 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm8, %xmm13, %xmm5
9233 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm1, %xmm12, %xmm10
9234 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm5 = xmm10[0,1,2,3],xmm5[4],xmm10[5],xmm5[6,7]
9235 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
9236 ; AVX2-FAST-PERLANE-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
9237 ; AVX2-FAST-PERLANE-NEXT: # ymm5 = mem[0,1,2,3,4],ymm5[5,6,7]
9238 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm8, %xmm0, %xmm0
9239 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm1, %xmm15, %xmm1
9240 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4],xmm1[5],xmm0[6,7]
9241 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
9242 ; AVX2-FAST-PERLANE-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
9243 ; AVX2-FAST-PERLANE-NEXT: # ymm0 = mem[0,1,2,3,4],ymm0[5,6,7]
9244 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
9245 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm1, 96(%rsi)
9246 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
9247 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm1, 32(%rsi)
9248 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
9249 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm1, 64(%rsi)
9250 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
9251 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm1, (%rsi)
9252 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
9253 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm1, 96(%rdx)
9254 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
9255 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm1, 32(%rdx)
9256 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
9257 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm1, 64(%rdx)
9258 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
9259 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm1, (%rdx)
9260 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
9261 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm1, 32(%rcx)
9262 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
9263 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm1, 96(%rcx)
9264 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
9265 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm1, 64(%rcx)
9266 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
9267 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm1, (%rcx)
9268 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm6, 96(%r8)
9269 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
9270 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm1, 32(%r8)
9271 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
9272 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm1, 64(%r8)
9273 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm4, (%r8)
9274 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm7, 96(%r9)
9275 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm14, 32(%r9)
9276 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm11, (%r9)
9277 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm3, 64(%r9)
9278 ; AVX2-FAST-PERLANE-NEXT: movq {{[0-9]+}}(%rsp), %rax
9279 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm0, 96(%rax)
9280 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm5, 32(%rax)
9281 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm2, 64(%rax)
9282 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm9, (%rax)
9283 ; AVX2-FAST-PERLANE-NEXT: addq $1272, %rsp # imm = 0x4F8
9284 ; AVX2-FAST-PERLANE-NEXT: vzeroupper
9285 ; AVX2-FAST-PERLANE-NEXT: retq
9286 ;
9287 ; AVX512F-ONLY-SLOW-LABEL: load_i16_stride6_vf64:
9288 ; AVX512F-ONLY-SLOW: # %bb.0:
9289 ; AVX512F-ONLY-SLOW-NEXT: subq $1416, %rsp # imm = 0x588
9290 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} xmm9 = <0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15>
9291 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 608(%rdi), %ymm0
9292 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9293 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 576(%rdi), %ymm1
9294 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9295 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]
9296 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm9, %xmm1, %xmm0
9297 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm2
9298 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm1, %ymm20
9299 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[0,2,0,3]
9300 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm2, %xmm16
9301 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
9302 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3],xmm1[4,5],xmm0[6,7]
9303 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 544(%rdi), %ymm1
9304 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9305 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 512(%rdi), %ymm2
9306 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9307 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm12 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
9308 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm12, %xmm2
9309 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} xmm11 = <0,1,4,5,4,5,u,u,0,1,12,13,u,u,4,5>
9310 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm11, %xmm2, %xmm1
9311 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm2, %xmm21
9312 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm12[2,2,2,2,4,5,6,7]
9313 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,2,2]
9314 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3],xmm1[4,5],xmm2[6],xmm1[7]
9315 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
9316 ; AVX512F-ONLY-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
9317 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9318 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 448(%rdi), %ymm0
9319 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 416(%rdi), %ymm1
9320 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9321 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 384(%rdi), %ymm2
9322 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9323 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm10 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5,6],ymm1[7]
9324 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm9, %xmm10, %xmm1
9325 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm10, %xmm3
9326 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[0,2,0,3]
9327 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm3, %xmm22
9328 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
9329 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3],xmm2[4,5],xmm1[6,7]
9330 ; AVX512F-ONLY-SLOW-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],mem[2,3]
9331 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9332 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, 480(%rdi), %ymm0, %ymm0
9333 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9334 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0],ymm2[1],ymm0[2,3,4,5],ymm2[6],ymm0[7]
9335 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} ymm5 = [0,1,12,13,8,9,12,13,8,9,12,13,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
9336 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm5, %ymm2, %ymm0
9337 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm2, %ymm23
9338 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
9339 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9340 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 736(%rdi), %ymm0
9341 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9342 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 704(%rdi), %ymm1
9343 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9344 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm8 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
9345 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm8[2,2,2,2,4,5,6,7]
9346 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,2,2]
9347 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm8, %xmm2
9348 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm11, %xmm2, %xmm1
9349 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm2, %xmm28
9350 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3],xmm1[4,5],xmm0[6],xmm1[7]
9351 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 640(%rdi), %ymm1
9352 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
9353 ; AVX512F-ONLY-SLOW-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm1[2,3],mem[2,3]
9354 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9355 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, 672(%rdi), %ymm1, %ymm1
9356 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9357 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3,4,5],ymm2[6],ymm1[7]
9358 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} ymm3 = <0,1,12,13,8,9,12,13,8,9,12,13,4,5,u,u,16,17,28,29,24,25,28,29,24,25,28,29,20,21,u,u>
9359 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm3, %ymm2, %ymm1
9360 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm3, %ymm17
9361 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm2, %ymm29
9362 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
9363 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,6]
9364 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
9365 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9366 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 224(%rdi), %ymm0
9367 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9368 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 192(%rdi), %ymm1
9369 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9370 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm14 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]
9371 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm14, %xmm15
9372 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm15[0,2,0,3]
9373 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
9374 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm9, %xmm14, %xmm1
9375 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3],xmm0[4,5],xmm1[6,7]
9376 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 160(%rdi), %ymm1
9377 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9378 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 128(%rdi), %ymm2
9379 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
9380 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm2, %ymm30
9381 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm4[2,2,2,2,4,5,6,7]
9382 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,2]
9383 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm4, %xmm6
9384 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm11, %xmm6, %xmm3
9385 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3],xmm3[4,5],xmm1[6],xmm3[7]
9386 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
9387 ; AVX512F-ONLY-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
9388 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9389 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa (%rdi), %ymm0
9390 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9391 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 32(%rdi), %ymm1
9392 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm1, (%rsp) # 32-byte Spill
9393 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
9394 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm9, %xmm2, %xmm0
9395 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm9
9396 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm9[0,2,0,3]
9397 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
9398 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2],xmm0[3],xmm3[4,5],xmm0[6,7]
9399 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 64(%rdi), %ymm3
9400 ; AVX512F-ONLY-SLOW-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm3[2,3],mem[2,3]
9401 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, 96(%rdi), %ymm3, %ymm3
9402 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9403 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm7 = ymm3[0],ymm1[1],ymm3[2,3,4,5],ymm1[6],ymm3[7]
9404 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm1, %ymm31
9405 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm5, %ymm7, %ymm3
9406 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm3[3,4,5,6,7]
9407 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9408 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 352(%rdi), %ymm0
9409 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9410 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 320(%rdi), %ymm1
9411 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9412 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
9413 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm5, %xmm3
9414 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm11, %xmm3, %xmm11
9415 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm13 = xmm5[2,2,2,2,4,5,6,7]
9416 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[0,1,2,2]
9417 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm13 = xmm11[0,1,2],xmm13[3],xmm11[4,5],xmm13[6],xmm11[7]
9418 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa 256(%rdi), %ymm11
9419 ; AVX512F-ONLY-SLOW-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm11[2,3],mem[2,3]
9420 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9421 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, 288(%rdi), %ymm11, %ymm0
9422 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9423 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm11 = ymm0[0],ymm1[1],ymm0[2,3,4,5],ymm1[6],ymm0[7]
9424 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm17, %ymm0
9425 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm0, %ymm11, %ymm0
9426 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
9427 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} ymm13 = ymm0[0,1,2],ymm13[3,4,5,6,7],ymm0[8,9,10],ymm13[11,12,13,14,15]
9428 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,4,6]
9429 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm13[4,5,6,7]
9430 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9431 ; AVX512F-ONLY-SLOW-NEXT: vpbroadcastq {{.*#+}} xmm13 = [2,3,14,15,10,11,0,0,2,3,14,15,10,11,0,0]
9432 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm13, %xmm15, %xmm0
9433 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} xmm15 = <2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15>
9434 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm15, %xmm14, %xmm14
9435 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm14[0,1],xmm0[2],xmm14[3],xmm0[4,5],xmm14[6,7]
9436 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} xmm14 = <6,7,2,3,4,5,u,u,2,3,14,15,u,u,6,7>
9437 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm14, %xmm6, %xmm6
9438 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,5,5,5]
9439 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm4 = xmm6[0,1,2],xmm4[3],xmm6[4,5],xmm4[6],xmm6[7]
9440 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
9441 ; AVX512F-ONLY-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm4, %zmm0
9442 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9443 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm13, %xmm9, %xmm0
9444 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm15, %xmm2, %xmm2
9445 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3],xmm0[4,5],xmm2[6,7]
9446 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} ymm2 = [2,3,14,15,10,11,10,11,14,15,10,11,10,11,6,7,18,19,30,31,26,27,26,27,30,31,26,27,26,27,22,23]
9447 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm2, %ymm7, %ymm4
9448 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm4[3,4,5,6,7]
9449 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9450 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm14, %xmm3, %xmm0
9451 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm5[0,1,2,3,5,5,5,5]
9452 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]
9453 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} ymm1 = <2,3,14,15,10,11,10,11,14,15,10,11,u,u,6,7,18,19,30,31,26,27,26,27,30,31,26,27,u,u,22,23>
9454 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm1, %ymm11, %ymm3
9455 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
9456 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} ymm0 = ymm3[0,1,2],ymm0[3,4,5,6,7],ymm3[8,9,10],ymm0[11,12,13,14,15]
9457 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,5,7]
9458 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
9459 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm0, %ymm27
9460 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm16, %xmm0
9461 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm13, %xmm0, %xmm0
9462 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm20, %ymm3
9463 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm15, %xmm3, %xmm3
9464 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1],xmm0[2],xmm3[3],xmm0[4,5],xmm3[6,7]
9465 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm21, %xmm3
9466 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm14, %xmm3, %xmm3
9467 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm4 = xmm12[0,1,2,3,5,5,5,5]
9468 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[3],xmm3[4,5],xmm4[6],xmm3[7]
9469 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
9470 ; AVX512F-ONLY-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm3, %zmm0
9471 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9472 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm23, %ymm0
9473 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm2, %ymm0, %ymm0
9474 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm22, %xmm2
9475 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm13, %xmm2, %xmm2
9476 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm15, %xmm10, %xmm3
9477 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2],xmm3[3],xmm2[4,5],xmm3[6,7]
9478 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
9479 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9480 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm29, %ymm0
9481 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm1, %ymm0, %ymm0
9482 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm28, %xmm1
9483 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm14, %xmm1, %xmm1
9484 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm8[0,1,2,3,5,5,5,5]
9485 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3],xmm1[4,5],xmm2[6],xmm1[7]
9486 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
9487 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
9488 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,5,7]
9489 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
9490 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm0, %ymm26
9491 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9492 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
9493 ; AVX512F-ONLY-SLOW-NEXT: # ymm0 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
9494 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm1
9495 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,1,2,3]
9496 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,3,2,1]
9497 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm1[0,0,2,3,4,5,6,7]
9498 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm1, %xmm28
9499 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,6,6,6]
9500 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm2[2,1,2,0,4,5,6,7]
9501 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm2, %xmm21
9502 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5,6,7]
9503 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm30, %ymm1
9504 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
9505 ; AVX512F-ONLY-SLOW-NEXT: # ymm1 = mem[0,1],ymm1[2],mem[3,4],ymm1[5],mem[6,7]
9506 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm2
9507 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,1,0,3]
9508 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm3[0,0,0,0,4,5,6,7]
9509 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm3, %xmm20
9510 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,6,7]
9511 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,1,2,1]
9512 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm3[0,1,2,3,6,5,6,4]
9513 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm3, %xmm19
9514 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4],xmm1[5,6],xmm2[7]
9515 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
9516 ; AVX512F-ONLY-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm1
9517 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu (%rsp), %ymm0 # 32-byte Reload
9518 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
9519 ; AVX512F-ONLY-SLOW-NEXT: # ymm0 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
9520 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm2
9521 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,1,2,3]
9522 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,3,2,1]
9523 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm2[0,0,2,3,4,5,6,7]
9524 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm2, %xmm18
9525 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,3,3]
9526 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm3[2,1,2,0,4,5,6,7]
9527 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm3, %xmm16
9528 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm0[1,2],xmm2[3],xmm0[4,5,6,7]
9529 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm31, %ymm0
9530 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 32-byte Folded Reload
9531 ; AVX512F-ONLY-SLOW-NEXT: # ymm4 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6],ymm0[7]
9532 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} ymm0 = [4,5,0,1,12,13,14,15,8,9,0,1,12,13,8,9,20,21,16,17,28,29,30,31,24,25,16,17,28,29,24,25]
9533 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm0, %ymm4, %ymm3
9534 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm4, %ymm17
9535 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4],xmm3[5,6,7]
9536 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
9537 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
9538 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
9539 ; AVX512F-ONLY-SLOW-NEXT: # ymm3 = ymm3[0,1],mem[2],ymm3[3,4],mem[5],ymm3[6,7]
9540 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm3, %xmm4
9541 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm15 = xmm3[2,1,0,3]
9542 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm15[0,0,0,0,4,5,6,7]
9543 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,6,7]
9544 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm4[0,1,2,1]
9545 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm4 = xmm12[0,1,2,3,6,5,6,4]
9546 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm4[4],xmm3[5,6],xmm4[7]
9547 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
9548 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
9549 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm13 # 32-byte Folded Reload
9550 ; AVX512F-ONLY-SLOW-NEXT: # ymm13 = mem[0,1],ymm4[2],mem[3],ymm4[4],mem[5,6],ymm4[7]
9551 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} ymm5 = <4,5,0,1,12,13,14,15,8,9,12,13,0,1,u,u,20,21,16,17,28,29,30,31,24,25,28,29,16,17,u,u>
9552 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm5, %ymm13, %ymm4
9553 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm5, %ymm22
9554 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7],ymm4[8,9,10],ymm3[11,12,13,14,15]
9555 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,6,5,4]
9556 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
9557 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm3
9558 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm29 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
9559 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $226, %zmm1, %zmm29, %zmm2
9560 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0]
9561 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm2, %zmm1, %zmm3
9562 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm1, %zmm24
9563 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9564 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
9565 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
9566 ; AVX512F-ONLY-SLOW-NEXT: # ymm1 = ymm1[0,1],mem[2],ymm1[3,4],mem[5],ymm1[6,7]
9567 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm2
9568 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm1[2,1,2,3]
9569 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm2[0,3,2,1]
9570 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm11[0,0,2,3,4,5,6,7]
9571 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,6,6,6]
9572 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm10[2,1,2,0,4,5,6,7]
9573 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2],xmm2[3],xmm1[4,5,6,7]
9574 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
9575 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
9576 ; AVX512F-ONLY-SLOW-NEXT: # ymm2 = mem[0,1],ymm2[2],mem[3,4],ymm2[5],mem[6,7]
9577 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm3
9578 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm2[2,1,0,3]
9579 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm9[0,0,0,0,4,5,6,7]
9580 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,6,7]
9581 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm3[0,1,2,1]
9582 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm8[0,1,2,3,6,5,6,4]
9583 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4],xmm2[5,6],xmm3[7]
9584 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
9585 ; AVX512F-ONLY-SLOW-NEXT: vinserti32x4 $2, %xmm1, %zmm2, %zmm23
9586 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
9587 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
9588 ; AVX512F-ONLY-SLOW-NEXT: # ymm1 = ymm1[0,1],mem[2],ymm1[3,4],mem[5],ymm1[6,7]
9589 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm2
9590 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm1[2,1,2,3]
9591 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm2[0,3,2,1]
9592 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm7[0,0,2,3,4,5,6,7]
9593 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,3,3]
9594 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm6[2,1,2,0,4,5,6,7]
9595 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2],xmm2[3],xmm1[4,5,6,7]
9596 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
9597 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm5 # 32-byte Folded Reload
9598 ; AVX512F-ONLY-SLOW-NEXT: # ymm5 = mem[0,1],ymm2[2],mem[3],ymm2[4],mem[5,6],ymm2[7]
9599 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm0, %ymm5, %ymm0
9600 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm0[5,6,7]
9601 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5,6,7]
9602 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9603 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
9604 ; AVX512F-ONLY-SLOW-NEXT: # ymm1 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
9605 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm0
9606 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[2,1,0,3]
9607 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm4[0,0,0,0,4,5,6,7]
9608 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,6,7]
9609 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[0,1,2,1]
9610 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm14 = xmm3[0,1,2,3,6,5,6,4]
9611 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm14 = xmm1[0,1,2,3],xmm14[4],xmm1[5,6],xmm14[7]
9612 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9613 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
9614 ; AVX512F-ONLY-SLOW-NEXT: # ymm1 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6],ymm0[7]
9615 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm22, %ymm0
9616 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm0, %ymm1, %ymm0
9617 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
9618 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} ymm14 = ymm0[0,1,2],ymm14[3,4,5,6,7],ymm0[8,9,10],ymm14[11,12,13,14,15]
9619 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,5,4]
9620 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm14[4,5,6,7]
9621 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm25
9622 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $226, %zmm23, %zmm29, %zmm2
9623 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm2, %zmm24, %zmm25
9624 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm21, %xmm0
9625 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[3,1,2,1,4,5,6,7]
9626 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm28, %xmm2
9627 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,3,3,4,5,6,7]
9628 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,7,7,7]
9629 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2],xmm0[3],xmm2[4,5,6,7]
9630 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm19, %xmm2
9631 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,5]
9632 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm20, %xmm14
9633 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm14 = xmm14[1,1,1,1,4,5,6,7]
9634 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,4,5,7,7]
9635 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm14[0,1,2,3],xmm2[4],xmm14[5,6],xmm2[7]
9636 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
9637 ; AVX512F-ONLY-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm2, %zmm28
9638 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm16, %xmm0
9639 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[3,1,2,1,4,5,6,7]
9640 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm18, %xmm0
9641 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm14 = xmm0[0,1,3,3,4,5,6,7]
9642 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,7,7,7,7]
9643 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm14[1,2],xmm2[3],xmm14[4,5,6,7]
9644 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} ymm14 = [6,7,2,3,14,15,14,15,14,15,2,3,14,15,10,11,22,23,18,19,30,31,30,31,30,31,18,19,30,31,26,27]
9645 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm17, %ymm0
9646 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm14, %ymm0, %ymm0
9647 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4],xmm0[5,6,7]
9648 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
9649 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm12[0,1,2,3,7,5,6,5]
9650 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm12 = xmm15[1,1,1,1,4,5,6,7]
9651 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,5,7,7]
9652 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm12[0,1,2,3],xmm2[4],xmm12[5,6],xmm2[7]
9653 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} ymm12 = <6,7,2,3,14,15,14,15,14,15,10,11,u,u,2,3,22,23,18,19,30,31,30,31,30,31,26,27,u,u,18,19>
9654 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm12, %ymm13, %ymm13
9655 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
9656 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = ymm13[0,1,2],ymm2[3,4,5,6,7],ymm13[8,9,10],ymm2[11,12,13,14,15]
9657 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,4,7,4,5]
9658 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm13[0,1,2,3],ymm2[4,5,6,7]
9659 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm13 # 32-byte Folded Reload
9660 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9661 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm13 # 32-byte Folded Reload
9662 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9663 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm27, %zmm0, %zmm13
9664 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9665 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm26, %zmm0, %zmm13
9666 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9667 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm23
9668 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $226, %zmm28, %zmm29, %zmm0
9669 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm0, %zmm24, %zmm23
9670 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm10[3,1,2,1,4,5,6,7]
9671 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm11[0,1,3,3,4,5,6,7]
9672 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,7,7,7]
9673 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2],xmm0[3],xmm2[4,5,6,7]
9674 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm8[0,1,2,3,7,5,6,5]
9675 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm9[1,1,1,1,4,5,6,7]
9676 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5,7,7]
9677 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm8[0,1,2,3],xmm2[4],xmm8[5,6],xmm2[7]
9678 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
9679 ; AVX512F-ONLY-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm2, %zmm0
9680 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm6[3,1,2,1,4,5,6,7]
9681 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm7[0,1,3,3,4,5,6,7]
9682 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,7,7,7,7]
9683 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm6[1,2],xmm2[3],xmm6[4,5,6,7]
9684 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm14, %ymm5, %ymm5
9685 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4],xmm5[5,6,7]
9686 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm5[4,5,6,7]
9687 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm12, %ymm1, %ymm1
9688 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,5]
9689 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[1,1,1,1,4,5,6,7]
9690 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,7,7]
9691 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4],xmm4[5,6],xmm3[7]
9692 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
9693 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = ymm1[0,1,2],ymm3[3,4,5,6,7],ymm1[8,9,10],ymm3[11,12,13,14,15]
9694 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,4,5]
9695 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
9696 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm28
9697 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $226, %zmm0, %zmm29, %zmm2
9698 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm2, %zmm24, %zmm28
9699 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9700 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
9701 ; AVX512F-ONLY-SLOW-NEXT: # ymm2 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
9702 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} xmm1 = <8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15>
9703 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm1, %xmm2, %xmm0
9704 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm13
9705 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm13[2,2,2,2,4,5,6,7]
9706 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2,3],xmm3[4],xmm0[5,6,7]
9707 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm30, %ymm3
9708 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
9709 ; AVX512F-ONLY-SLOW-NEXT: # ymm3 = ymm3[0],mem[1],ymm3[2,3],mem[4],ymm3[5,6],mem[7]
9710 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm3, %xmm5
9711 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[0,3,2,1]
9712 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm4[0,1,0,2,4,5,6,7]
9713 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm4, %xmm22
9714 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,6,6,6]
9715 ; AVX512F-ONLY-SLOW-NEXT: vpbroadcastq {{.*#+}} xmm10 = [8,9,0,0,0,1,12,13,8,9,0,0,0,1,12,13]
9716 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm10, %xmm5, %xmm4
9717 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm5, %xmm18
9718 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm4[4],xmm3[5],xmm4[6,7]
9719 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
9720 ; AVX512F-ONLY-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm3, %zmm30
9721 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm31, %ymm0
9722 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $66, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
9723 ; AVX512F-ONLY-SLOW-NEXT: # ymm5 = ymm0[0],mem[1],ymm0[2,3,4,5],mem[6],ymm0[7]
9724 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu (%rsp), %ymm0 # 32-byte Reload
9725 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
9726 ; AVX512F-ONLY-SLOW-NEXT: # ymm8 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
9727 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm1, %xmm8, %xmm0
9728 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm8, %xmm4
9729 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm4[2,2,2,2,4,5,6,7]
9730 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm4, %xmm19
9731 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2,3],xmm3[4],xmm0[5,6,7]
9732 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 {{.*#+}} ymm29 = [65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
9733 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} ymm3 = [128,128,128,128,128,128,128,128,128,128,4,5,0,1,12,13,24,25,20,21,128,128,128,128,128,128,128,128,128,128,128,128]
9734 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm3, %ymm5, %ymm4
9735 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm5, %ymm20
9736 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $236, %ymm29, %ymm4, %ymm0
9737 ; AVX512F-ONLY-SLOW-NEXT: movw $31, %ax
9738 ; AVX512F-ONLY-SLOW-NEXT: kmovw %eax, %k1
9739 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa32 %zmm0, %zmm30 {%k1}
9740 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9741 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $66, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
9742 ; AVX512F-ONLY-SLOW-NEXT: # ymm5 = ymm0[0],mem[1],ymm0[2,3,4,5],mem[6],ymm0[7]
9743 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9744 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $109, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
9745 ; AVX512F-ONLY-SLOW-NEXT: # ymm0 = mem[0],ymm0[1],mem[2,3],ymm0[4],mem[5,6],ymm0[7]
9746 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm6
9747 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[0,3,2,1]
9748 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm4[0,1,0,2,4,5,6,7]
9749 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm4, %xmm21
9750 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,6,6,6]
9751 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm10, %xmm6, %xmm4
9752 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm6, %xmm27
9753 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm4[4],xmm0[5],xmm4[6,7]
9754 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
9755 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} ymm6 = [8,9,4,5,4,5,6,7,0,1,4,5,0,1,12,13,24,25,20,21,20,21,22,23,16,17,20,21,16,17,28,29]
9756 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm6, %ymm5, %ymm4
9757 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm6, %ymm16
9758 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm5, %ymm26
9759 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4],ymm0[5,6,7]
9760 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm0, %ymm24
9761 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9762 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $66, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
9763 ; AVX512F-ONLY-SLOW-NEXT: # ymm5 = ymm0[0],mem[1],ymm0[2,3,4,5],mem[6],ymm0[7]
9764 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9765 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
9766 ; AVX512F-ONLY-SLOW-NEXT: # ymm15 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
9767 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm1, %xmm15, %xmm0
9768 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm15, %xmm14
9769 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm14[2,2,2,2,4,5,6,7]
9770 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm4 = xmm0[0],xmm4[1],xmm0[2,3],xmm4[4],xmm0[5,6,7]
9771 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm3, %ymm5, %ymm0
9772 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm5, %ymm17
9773 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $236, %ymm29, %ymm0, %ymm4
9774 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9775 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm6 # 32-byte Folded Reload
9776 ; AVX512F-ONLY-SLOW-NEXT: # ymm6 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
9777 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm1, %xmm6, %xmm1
9778 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm6, %xmm12
9779 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm12[2,2,2,2,4,5,6,7]
9780 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm3[1],xmm1[2,3],xmm3[4],xmm1[5,6,7]
9781 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9782 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
9783 ; AVX512F-ONLY-SLOW-NEXT: # ymm3 = ymm0[0],mem[1],ymm0[2,3],mem[4],ymm0[5,6],mem[7]
9784 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm3, %xmm11
9785 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm3[0,3,2,1]
9786 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm9[0,1,0,2,4,5,6,7]
9787 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,6,6,6]
9788 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm10, %xmm11, %xmm5
9789 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm5[4],xmm3[5],xmm5[6,7]
9790 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
9791 ; AVX512F-ONLY-SLOW-NEXT: vinserti32x4 $2, %xmm1, %zmm3, %zmm31
9792 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa32 %zmm4, %zmm31 {%k1}
9793 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9794 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $66, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm7 # 32-byte Folded Reload
9795 ; AVX512F-ONLY-SLOW-NEXT: # ymm7 = ymm0[0],mem[1],ymm0[2,3,4,5],mem[6],ymm0[7]
9796 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
9797 ; AVX512F-ONLY-SLOW-NEXT: vpblendd $109, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
9798 ; AVX512F-ONLY-SLOW-NEXT: # ymm1 = mem[0],ymm0[1],mem[2,3],ymm0[4],mem[5,6],ymm0[7]
9799 ; AVX512F-ONLY-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm5
9800 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm10, %xmm5, %xmm10
9801 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[0,3,2,1]
9802 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm4[0,1,0,2,4,5,6,7]
9803 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,6,6,6]
9804 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm10[4],xmm1[5],xmm10[6,7]
9805 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm16, %ymm0
9806 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm0, %ymm7, %ymm10
9807 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
9808 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm10[0,1,2,3,4],ymm1[5,6,7]
9809 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm0, %ymm16
9810 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} xmm3 = <10,11,u,u,2,3,14,15,u,u,10,11,12,13,14,15>
9811 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm3, %xmm2, %xmm2
9812 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm13[1,1,2,3]
9813 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,5,5,5]
9814 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm10 = xmm2[0],xmm10[1],xmm2[2,3],xmm10[4],xmm2[5,6,7]
9815 ; AVX512F-ONLY-SLOW-NEXT: vpbroadcastq {{.*#+}} xmm2 = [10,11,0,0,2,3,14,15,10,11,0,0,2,3,14,15]
9816 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm18, %xmm0
9817 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm2, %xmm0, %xmm0
9818 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm22, %xmm1
9819 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm13 = xmm1[0,1,1,3,4,5,6,7]
9820 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[0,1,3,3]
9821 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm13[0,1,2,3],xmm0[4],xmm13[5],xmm0[6,7]
9822 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
9823 ; AVX512F-ONLY-SLOW-NEXT: vinserti32x4 $2, %xmm10, %zmm0, %zmm1
9824 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm3, %xmm8, %xmm8
9825 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm19[1,1,2,3]
9826 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,5,5,5]
9827 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0],xmm10[1],xmm8[2,3],xmm10[4],xmm8[5,6,7]
9828 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} ymm10 = [128,128,128,128,128,128,128,128,128,128,6,7,2,3,14,15,26,27,22,23,128,128,128,128,128,128,128,128,128,128,128,128]
9829 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm20, %ymm0
9830 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm10, %ymm0, %ymm13
9831 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $236, %ymm29, %ymm13, %ymm8
9832 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa32 %zmm8, %zmm1 {%k1}
9833 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm27, %xmm0
9834 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm2, %xmm0, %xmm8
9835 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %xmm21, %xmm0
9836 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm13 = xmm0[0,1,1,3,4,5,6,7]
9837 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[0,1,3,3]
9838 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm8 = xmm13[0,1,2,3],xmm8[4],xmm13[5],xmm8[6,7]
9839 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa {{.*#+}} ymm13 = [10,11,6,7,4,5,6,7,6,7,6,7,2,3,14,15,26,27,22,23,20,21,22,23,22,23,22,23,18,19,30,31]
9840 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm26, %ymm0
9841 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm13, %ymm0, %ymm0
9842 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
9843 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm8[5,6,7]
9844 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %ymm17, %ymm8
9845 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm10, %ymm8, %ymm8
9846 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm3, %xmm15, %xmm10
9847 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm14 = xmm14[1,1,2,3]
9848 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,5,5,5,5]
9849 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0],xmm14[1],xmm10[2,3],xmm14[4],xmm10[5,6,7]
9850 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $236, %ymm29, %ymm8, %ymm10
9851 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm3, %xmm6, %xmm3
9852 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm12[1,1,2,3]
9853 ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,5,5,5,5]
9854 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm6[1],xmm3[2,3],xmm6[4],xmm3[5,6,7]
9855 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm24, %zmm0, %zmm6
9856 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm16, %zmm0, %zmm8
9857 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0
9858 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm2, %xmm11, %xmm11
9859 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7]
9860 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[0,1,3,3]
9861 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1,2,3],xmm11[4],xmm9[5],xmm11[6,7]
9862 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
9863 ; AVX512F-ONLY-SLOW-NEXT: vinserti32x4 $2, %xmm3, %zmm9, %zmm3
9864 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa32 %zmm10, %zmm3 {%k1}
9865 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %ymm13, %ymm7, %ymm7
9866 ; AVX512F-ONLY-SLOW-NEXT: vpshufb %xmm2, %xmm5, %xmm2
9867 ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7]
9868 ; AVX512F-ONLY-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,1,3,3]
9869 ; AVX512F-ONLY-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3],xmm2[4],xmm4[5],xmm2[6,7]
9870 ; AVX512F-ONLY-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
9871 ; AVX512F-ONLY-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3,4],ymm2[5,6,7]
9872 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm2
9873 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm4 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
9874 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
9875 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm5 # 64-byte Folded Reload
9876 ; AVX512F-ONLY-SLOW-NEXT: movw $-2048, %ax # imm = 0xF800
9877 ; AVX512F-ONLY-SLOW-NEXT: kmovw %eax, %k1
9878 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
9879 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa32 %zmm7, %zmm5 {%k1}
9880 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm5, (%rsi)
9881 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
9882 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm5 # 64-byte Folded Reload
9883 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
9884 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa32 %zmm7, %zmm5 {%k1}
9885 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm5, 64(%rsi)
9886 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
9887 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm5 # 64-byte Folded Reload
9888 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
9889 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa32 %zmm7, %zmm5 {%k1}
9890 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm5, 64(%rdx)
9891 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
9892 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm5 # 64-byte Folded Reload
9893 ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
9894 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa32 %zmm4, %zmm5 {%k1}
9895 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm5, (%rdx)
9896 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm4 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0]
9897 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm30, %zmm4, %zmm6
9898 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm31, %zmm4, %zmm8
9899 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm1, %zmm4, %zmm0
9900 ; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm3, %zmm4, %zmm2
9901 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm25, 64(%rcx)
9902 ; AVX512F-ONLY-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
9903 ; AVX512F-ONLY-SLOW-NEXT: vmovaps %zmm1, (%rcx)
9904 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm28, 64(%r8)
9905 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm23, (%r8)
9906 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm8, 64(%r9)
9907 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm6, (%r9)
9908 ; AVX512F-ONLY-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
9909 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm2, 64(%rax)
9910 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm0, (%rax)
9911 ; AVX512F-ONLY-SLOW-NEXT: addq $1416, %rsp # imm = 0x588
9912 ; AVX512F-ONLY-SLOW-NEXT: vzeroupper
9913 ; AVX512F-ONLY-SLOW-NEXT: retq
9914 ;
9915 ; AVX512F-ONLY-FAST-LABEL: load_i16_stride6_vf64:
9916 ; AVX512F-ONLY-FAST: # %bb.0:
9917 ; AVX512F-ONLY-FAST-NEXT: subq $1512, %rsp # imm = 0x5E8
9918 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} xmm10 = <0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15>
9919 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 608(%rdi), %ymm0
9920 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9921 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 576(%rdi), %ymm1
9922 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9923 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]
9924 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm10, %xmm2, %xmm0
9925 ; AVX512F-ONLY-FAST-NEXT: vpbroadcastq {{.*#+}} xmm11 = [8,9,12,13,0,1,0,0,8,9,12,13,0,1,0,0]
9926 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm2, %xmm1
9927 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm2, %ymm20
9928 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,1,0,3]
9929 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm11, %xmm2, %xmm1
9930 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm2, %xmm21
9931 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3],xmm1[4,5],xmm0[6,7]
9932 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 544(%rdi), %ymm1
9933 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9934 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 512(%rdi), %ymm2
9935 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9936 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm15 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
9937 ; AVX512F-ONLY-FAST-NEXT: vpbroadcastd {{.*#+}} xmm9 = [8,9,4,5,8,9,4,5,8,9,4,5,8,9,4,5]
9938 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm9, %xmm15, %xmm1
9939 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm15, %xmm4
9940 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} xmm3 = <0,1,4,5,4,5,u,u,0,1,12,13,u,u,4,5>
9941 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm3, %xmm4, %xmm2
9942 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm4, %xmm22
9943 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3],xmm2[4,5],xmm1[6],xmm2[7]
9944 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
9945 ; AVX512F-ONLY-FAST-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
9946 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9947 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 416(%rdi), %ymm0
9948 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9949 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 384(%rdi), %ymm1
9950 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9951 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm14 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]
9952 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm10, %xmm14, %xmm0
9953 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm14, %xmm1
9954 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,1,0,3]
9955 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm11, %xmm2, %xmm1
9956 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm2, %xmm23
9957 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3],xmm1[4,5],xmm0[6,7]
9958 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 448(%rdi), %ymm1
9959 ; AVX512F-ONLY-FAST-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm1[2,3],mem[2,3]
9960 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9961 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, 480(%rdi), %ymm1, %ymm1
9962 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9963 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3,4,5],ymm2[6],ymm1[7]
9964 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} ymm8 = [0,1,12,13,8,9,12,13,8,9,12,13,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
9965 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm8, %ymm2, %ymm1
9966 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm2, %ymm24
9967 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
9968 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9969 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 736(%rdi), %ymm0
9970 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9971 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 704(%rdi), %ymm1
9972 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9973 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm12 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
9974 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm9, %xmm12, %xmm0
9975 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm12, %xmm2
9976 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm3, %xmm2, %xmm1
9977 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm2, %xmm25
9978 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3],xmm1[4,5],xmm0[6],xmm1[7]
9979 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 640(%rdi), %ymm1
9980 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
9981 ; AVX512F-ONLY-FAST-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm1[2,3],mem[2,3]
9982 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9983 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, 672(%rdi), %ymm1, %ymm1
9984 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9985 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3,4,5],ymm2[6],ymm1[7]
9986 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} ymm4 = <0,1,12,13,8,9,12,13,8,9,12,13,4,5,u,u,16,17,28,29,24,25,28,29,24,25,28,29,20,21,u,u>
9987 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm4, %ymm2, %ymm1
9988 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm4, %ymm16
9989 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm2, %ymm29
9990 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
9991 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,6]
9992 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
9993 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9994 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 224(%rdi), %ymm0
9995 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9996 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 192(%rdi), %ymm1
9997 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9998 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm13 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]
9999 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm13, %xmm0
10000 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm10, %xmm13, %xmm1
10001 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm7 = xmm0[2,1,0,3]
10002 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm11, %xmm7, %xmm0
10003 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3],xmm0[4,5],xmm1[6,7]
10004 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 160(%rdi), %ymm1
10005 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10006 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 128(%rdi), %ymm2
10007 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10008 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm5 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
10009 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm9, %xmm5, %xmm1
10010 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm5, %xmm6
10011 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm3, %xmm6, %xmm4
10012 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm4[0,1,2],xmm1[3],xmm4[4,5],xmm1[6],xmm4[7]
10013 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
10014 ; AVX512F-ONLY-FAST-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
10015 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
10016 ; AVX512F-ONLY-FAST-NEXT: vmovdqa (%rdi), %ymm0
10017 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10018 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 32(%rdi), %ymm1
10019 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10020 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
10021 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm10, %xmm2, %xmm0
10022 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm2, %xmm4
10023 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm10 = xmm4[2,1,0,3]
10024 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm11, %xmm10, %xmm4
10025 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm4[2],xmm0[3],xmm4[4,5],xmm0[6,7]
10026 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 64(%rdi), %ymm4
10027 ; AVX512F-ONLY-FAST-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm4[2,3],mem[2,3]
10028 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, 96(%rdi), %ymm4, %ymm4
10029 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10030 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm11 = ymm4[0],ymm1[1],ymm4[2,3,4,5],ymm1[6],ymm4[7]
10031 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm1, %ymm31
10032 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm8, %ymm11, %ymm4
10033 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm4[3,4,5,6,7]
10034 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
10035 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 352(%rdi), %ymm0
10036 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10037 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 320(%rdi), %ymm1
10038 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10039 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm4 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
10040 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm9, %xmm4, %xmm9
10041 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm4, %xmm8
10042 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm3, %xmm8, %xmm3
10043 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm9 = xmm3[0,1,2],xmm9[3],xmm3[4,5],xmm9[6],xmm3[7]
10044 ; AVX512F-ONLY-FAST-NEXT: vmovdqa 256(%rdi), %ymm3
10045 ; AVX512F-ONLY-FAST-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm3[2,3],mem[2,3]
10046 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10047 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, 288(%rdi), %ymm3, %ymm0
10048 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm0, (%rsp) # 32-byte Spill
10049 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm0[0],ymm1[1],ymm0[2,3,4,5],ymm1[6],ymm0[7]
10050 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm16, %ymm0
10051 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm0, %ymm3, %ymm0
10052 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
10053 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} ymm9 = ymm0[0,1,2],ymm9[3,4,5,6,7],ymm0[8,9,10],ymm9[11,12,13,14,15]
10054 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,4,6]
10055 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm9[4,5,6,7]
10056 ; AVX512F-ONLY-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10057 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} xmm9 = <2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15>
10058 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm9, %xmm13, %xmm0
10059 ; AVX512F-ONLY-FAST-NEXT: vpbroadcastq {{.*#+}} xmm13 = [10,11,14,15,2,3,0,0,10,11,14,15,2,3,0,0]
10060 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm13, %xmm7, %xmm7
10061 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm7 = xmm0[0,1],xmm7[2],xmm0[3],xmm7[4,5],xmm0[6,7]
10062 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} xmm0 = <6,7,2,3,4,5,u,u,2,3,14,15,u,u,6,7>
10063 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm0, %xmm6, %xmm6
10064 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,5,5]
10065 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3],xmm6[4,5],xmm5[6],xmm6[7]
10066 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
10067 ; AVX512F-ONLY-FAST-NEXT: vinserti32x4 $2, %xmm7, %zmm5, %zmm5
10068 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
10069 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm9, %xmm2, %xmm2
10070 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm13, %xmm10, %xmm5
10071 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm5[2],xmm2[3],xmm5[4,5],xmm2[6,7]
10072 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} ymm5 = [2,3,14,15,10,11,10,11,14,15,10,11,10,11,6,7,18,19,30,31,26,27,26,27,30,31,26,27,26,27,22,23]
10073 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm5, %ymm11, %ymm6
10074 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2],ymm6[3,4,5,6,7]
10075 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
10076 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm0, %xmm8, %xmm2
10077 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm1 = xmm4[0,1,2,3,5,5,5,5]
10078 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3],xmm2[4,5],xmm1[6],xmm2[7]
10079 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} ymm2 = <2,3,14,15,10,11,10,11,14,15,10,11,u,u,6,7,18,19,30,31,26,27,26,27,30,31,26,27,u,u,22,23>
10080 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm2, %ymm3, %ymm3
10081 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
10082 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3,4,5,6,7],ymm3[8,9,10],ymm1[11,12,13,14,15]
10083 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,5,7]
10084 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
10085 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm1, %ymm30
10086 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm20, %ymm1
10087 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm9, %xmm1, %xmm1
10088 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm21, %xmm3
10089 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm13, %xmm3, %xmm3
10090 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2],xmm1[3],xmm3[4,5],xmm1[6,7]
10091 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm22, %xmm3
10092 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm0, %xmm3, %xmm3
10093 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm4 = xmm15[0,1,2,3,5,5,5,5]
10094 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[3],xmm3[4,5],xmm4[6],xmm3[7]
10095 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
10096 ; AVX512F-ONLY-FAST-NEXT: vinserti32x4 $2, %xmm1, %zmm3, %zmm1
10097 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
10098 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm24, %ymm1
10099 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm5, %ymm1, %ymm1
10100 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm9, %xmm14, %xmm3
10101 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm23, %xmm4
10102 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm13, %xmm4, %xmm4
10103 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2],xmm3[3],xmm4[4,5],xmm3[6,7]
10104 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3,4,5,6,7]
10105 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
10106 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm29, %ymm1
10107 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm2, %ymm1, %ymm1
10108 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm25, %xmm2
10109 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm0, %xmm2, %xmm0
10110 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm2 = xmm12[0,1,2,3,5,5,5,5]
10111 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm2[3],xmm0[4,5],xmm2[6],xmm0[7]
10112 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
10113 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
10114 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,5,7]
10115 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
10116 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm0, %ymm28
10117 ; AVX512F-ONLY-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
10118 ; AVX512F-ONLY-FAST-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
10119 ; AVX512F-ONLY-FAST-NEXT: # ymm0 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
10120 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm0, %xmm1
10121 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm13 = xmm0[2,1,2,3]
10122 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} xmm0 = <u,u,0,1,4,5,u,u,12,13,12,13,12,13,12,13>
10123 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm14 = xmm1[0,3,2,1]
10124 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm0, %xmm14, %xmm1
10125 ; AVX512F-ONLY-FAST-NEXT: vpshuflw {{.*#+}} xmm2 = xmm13[2,1,2,0,4,5,6,7]
10126 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2],xmm2[3],xmm1[4,5,6,7]
10127 ; AVX512F-ONLY-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
10128 ; AVX512F-ONLY-FAST-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
10129 ; AVX512F-ONLY-FAST-NEXT: # ymm2 = mem[0,1],ymm2[2],mem[3,4],ymm2[5],mem[6,7]
10130 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm2, %xmm3
10131 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,1,0,3]
10132 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} xmm11 = <0,1,0,1,0,1,0,1,u,u,8,9,12,13,u,u>
10133 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm11, %xmm4, %xmm2
10134 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm4, %xmm24
10135 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[0,1,2,1]
10136 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm3 = xmm4[0,1,2,3,6,5,6,4]
10137 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm4, %xmm22
10138 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4],xmm2[5,6],xmm3[7]
10139 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
10140 ; AVX512F-ONLY-FAST-NEXT: vinserti32x4 $2, %xmm1, %zmm2, %zmm4
10141 ; AVX512F-ONLY-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
10142 ; AVX512F-ONLY-FAST-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
10143 ; AVX512F-ONLY-FAST-NEXT: # ymm1 = mem[0,1],ymm1[2],mem[3,4],ymm1[5],mem[6,7]
10144 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
10145 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,1,2,3]
10146 ; AVX512F-ONLY-FAST-NEXT: vpbroadcastq {{.*#+}} xmm3 = [12,13,0,1,4,5,0,0,12,13,0,1,4,5,0,0]
10147 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,3,2,1]
10148 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm3, %xmm2, %xmm1
10149 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm2, %xmm20
10150 ; AVX512F-ONLY-FAST-NEXT: vpshuflw {{.*#+}} xmm2 = xmm5[2,1,2,0,4,5,6,7]
10151 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm5, %xmm21
10152 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2],xmm2[3],xmm1[4,5,6,7]
10153 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm31, %ymm2
10154 ; AVX512F-ONLY-FAST-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm6 # 32-byte Folded Reload
10155 ; AVX512F-ONLY-FAST-NEXT: # ymm6 = mem[0,1],ymm2[2],mem[3],ymm2[4],mem[5,6],ymm2[7]
10156 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} ymm2 = [4,5,0,1,12,13,14,15,8,9,0,1,12,13,8,9,20,21,16,17,28,29,30,31,24,25,16,17,28,29,24,25]
10157 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm2, %ymm6, %ymm5
10158 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm6, %ymm19
10159 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm5[5,6,7]
10160 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm5 = ymm1[0,1,2,3],ymm5[4,5,6,7]
10161 ; AVX512F-ONLY-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
10162 ; AVX512F-ONLY-FAST-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
10163 ; AVX512F-ONLY-FAST-NEXT: # ymm1 = ymm1[0,1],mem[2],ymm1[3,4],mem[5],ymm1[6,7]
10164 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm1, %xmm6
10165 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm7 = xmm1[2,1,0,3]
10166 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm11, %xmm7, %xmm1
10167 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm7, %xmm18
10168 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[0,1,2,1]
10169 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm6 = xmm7[0,1,2,3,6,5,6,4]
10170 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm7, %xmm17
10171 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm6[4],xmm1[5,6],xmm6[7]
10172 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm6
10173 ; AVX512F-ONLY-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
10174 ; AVX512F-ONLY-FAST-NEXT: vpblendd $107, (%rsp), %ymm1, %ymm1 # 32-byte Folded Reload
10175 ; AVX512F-ONLY-FAST-NEXT: # ymm1 = mem[0,1],ymm1[2],mem[3],ymm1[4],mem[5,6],ymm1[7]
10176 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} ymm8 = <4,5,0,1,12,13,14,15,8,9,12,13,0,1,u,u,20,21,16,17,28,29,30,31,24,25,28,29,16,17,u,u>
10177 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm8, %ymm1, %ymm7
10178 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm8, %ymm26
10179 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm1, %ymm16
10180 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} ymm6 = ymm7[0,1,2],ymm6[3,4,5,6,7],ymm7[8,9,10],ymm6[11,12,13,14,15]
10181 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,6,5,4]
10182 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7]
10183 ; AVX512F-ONLY-FAST-NEXT: vinserti64x4 $1, %ymm6, %zmm0, %zmm1
10184 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 {{.*#+}} zmm29 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
10185 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $226, %zmm4, %zmm29, %zmm5
10186 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 {{.*#+}} zmm4 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0]
10187 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm5, %zmm4, %zmm1
10188 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm4, %zmm25
10189 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
10190 ; AVX512F-ONLY-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
10191 ; AVX512F-ONLY-FAST-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm4 # 32-byte Folded Reload
10192 ; AVX512F-ONLY-FAST-NEXT: # ymm4 = ymm1[0,1],mem[2],ymm1[3,4],mem[5],ymm1[6,7]
10193 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm4, %xmm5
10194 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm12 = xmm5[0,3,2,1]
10195 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm0, %xmm12, %xmm0
10196 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm10 = xmm4[2,1,2,3]
10197 ; AVX512F-ONLY-FAST-NEXT: vpshuflw {{.*#+}} xmm4 = xmm10[2,1,2,0,4,5,6,7]
10198 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm4[0],xmm0[1,2],xmm4[3],xmm0[4,5,6,7]
10199 ; AVX512F-ONLY-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
10200 ; AVX512F-ONLY-FAST-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm4 # 32-byte Folded Reload
10201 ; AVX512F-ONLY-FAST-NEXT: # ymm4 = mem[0,1],ymm1[2],mem[3,4],ymm1[5],mem[6,7]
10202 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm4, %xmm5
10203 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm8 = xmm4[2,1,0,3]
10204 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm11, %xmm8, %xmm4
10205 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm9 = xmm5[0,1,2,1]
10206 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm5 = xmm9[0,1,2,3,6,5,6,4]
10207 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm5[4],xmm4[5,6],xmm5[7]
10208 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
10209 ; AVX512F-ONLY-FAST-NEXT: vinserti32x4 $2, %xmm0, %zmm4, %zmm23
10210 ; AVX512F-ONLY-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
10211 ; AVX512F-ONLY-FAST-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
10212 ; AVX512F-ONLY-FAST-NEXT: # ymm0 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
10213 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm0, %xmm4
10214 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[0,3,2,1]
10215 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm3, %xmm7, %xmm3
10216 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm6 = xmm0[2,1,2,3]
10217 ; AVX512F-ONLY-FAST-NEXT: vpshuflw {{.*#+}} xmm0 = xmm6[2,1,2,0,4,5,6,7]
10218 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1,2],xmm0[3],xmm3[4,5,6,7]
10219 ; AVX512F-ONLY-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
10220 ; AVX512F-ONLY-FAST-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm5 # 32-byte Folded Reload
10221 ; AVX512F-ONLY-FAST-NEXT: # ymm5 = mem[0,1],ymm1[2],mem[3],ymm1[4],mem[5,6],ymm1[7]
10222 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm2, %ymm5, %ymm2
10223 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm2[5,6,7]
10224 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
10225 ; AVX512F-ONLY-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
10226 ; AVX512F-ONLY-FAST-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
10227 ; AVX512F-ONLY-FAST-NEXT: # ymm1 = ymm1[0,1],mem[2],ymm1[3,4],mem[5],ymm1[6,7]
10228 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm1, %xmm3
10229 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[2,1,0,3]
10230 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm11, %xmm4, %xmm1
10231 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,1,2,1]
10232 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm15 = xmm3[0,1,2,3,6,5,6,4]
10233 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm15 = xmm1[0,1,2,3],xmm15[4],xmm1[5,6],xmm15[7]
10234 ; AVX512F-ONLY-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
10235 ; AVX512F-ONLY-FAST-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
10236 ; AVX512F-ONLY-FAST-NEXT: # ymm1 = mem[0,1],ymm1[2],mem[3],ymm1[4],mem[5,6],ymm1[7]
10237 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm26, %ymm2
10238 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm2, %ymm1, %ymm11
10239 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
10240 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} ymm15 = ymm11[0,1,2],ymm15[3,4,5,6,7],ymm11[8,9,10],ymm15[11,12,13,14,15]
10241 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,4,6,5,4]
10242 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm15[4,5,6,7]
10243 ; AVX512F-ONLY-FAST-NEXT: vinserti64x4 $1, %ymm11, %zmm0, %zmm2
10244 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $226, %zmm23, %zmm29, %zmm0
10245 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm0, %zmm25, %zmm2
10246 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
10247 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} xmm2 = <u,u,2,3,6,7,u,u,14,15,14,15,14,15,14,15>
10248 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm2, %xmm14, %xmm0
10249 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm2, %xmm23
10250 ; AVX512F-ONLY-FAST-NEXT: vpshuflw {{.*#+}} xmm11 = xmm13[3,1,2,1,4,5,6,7]
10251 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm11 = xmm11[0],xmm0[1,2],xmm11[3],xmm0[4,5,6,7]
10252 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} xmm2 = <2,3,2,3,2,3,2,3,u,u,10,11,14,15,u,u>
10253 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm24, %xmm0
10254 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm2, %xmm0, %xmm15
10255 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm22, %xmm0
10256 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm14 = xmm0[0,1,2,3,7,5,6,5]
10257 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm14 = xmm15[0,1,2,3],xmm14[4],xmm15[5,6],xmm14[7]
10258 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
10259 ; AVX512F-ONLY-FAST-NEXT: vinserti32x4 $2, %xmm11, %zmm14, %zmm24
10260 ; AVX512F-ONLY-FAST-NEXT: vpbroadcastq {{.*#+}} xmm11 = [14,15,2,3,6,7,0,0,14,15,2,3,6,7,0,0]
10261 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm20, %xmm0
10262 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm11, %xmm0, %xmm15
10263 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm11, %xmm20
10264 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm21, %xmm0
10265 ; AVX512F-ONLY-FAST-NEXT: vpshuflw {{.*#+}} xmm13 = xmm0[3,1,2,1,4,5,6,7]
10266 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0],xmm15[1,2],xmm13[3],xmm15[4,5,6,7]
10267 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} ymm15 = [6,7,2,3,14,15,14,15,14,15,2,3,14,15,10,11,22,23,18,19,30,31,30,31,30,31,18,19,30,31,26,27]
10268 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm19, %ymm0
10269 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm15, %ymm0, %ymm0
10270 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0,1,2,3,4],xmm0[5,6,7]
10271 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3],ymm0[4,5,6,7]
10272 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm18, %xmm11
10273 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm2, %xmm11, %xmm13
10274 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm17, %xmm11
10275 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,7,5,6,5]
10276 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm11 = xmm13[0,1,2,3],xmm11[4],xmm13[5,6],xmm11[7]
10277 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} ymm13 = <6,7,2,3,14,15,14,15,14,15,10,11,u,u,2,3,22,23,18,19,30,31,30,31,30,31,26,27,u,u,18,19>
10278 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm16, %ymm14
10279 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm13, %ymm14, %ymm14
10280 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
10281 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} ymm11 = ymm14[0,1,2],ymm11[3,4,5,6,7],ymm14[8,9,10],ymm11[11,12,13,14,15]
10282 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,4,7,4,5]
10283 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm11 = ymm14[0,1,2,3],ymm11[4,5,6,7]
10284 ; AVX512F-ONLY-FAST-NEXT: vinserti64x4 $1, %ymm11, %zmm0, %zmm26
10285 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $226, %zmm24, %zmm29, %zmm0
10286 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm0, %zmm25, %zmm26
10287 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm23, %xmm0
10288 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm0, %xmm12, %xmm0
10289 ; AVX512F-ONLY-FAST-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm11 # 32-byte Folded Reload
10290 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
10291 ; AVX512F-ONLY-FAST-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm11 # 32-byte Folded Reload
10292 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
10293 ; AVX512F-ONLY-FAST-NEXT: vinserti64x4 $1, %ymm30, %zmm0, %zmm11
10294 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
10295 ; AVX512F-ONLY-FAST-NEXT: vinserti64x4 $1, %ymm28, %zmm0, %zmm11
10296 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
10297 ; AVX512F-ONLY-FAST-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[3,1,2,1,4,5,6,7]
10298 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm10[0],xmm0[1,2],xmm10[3],xmm0[4,5,6,7]
10299 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm2, %xmm8, %xmm8
10300 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,7,5,6,5]
10301 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2,3],xmm9[4],xmm8[5,6],xmm9[7]
10302 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
10303 ; AVX512F-ONLY-FAST-NEXT: vinserti32x4 $2, %xmm0, %zmm8, %zmm0
10304 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm20, %xmm8
10305 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm8, %xmm7, %xmm7
10306 ; AVX512F-ONLY-FAST-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[3,1,2,1,4,5,6,7]
10307 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0],xmm7[1,2],xmm6[3],xmm7[4,5,6,7]
10308 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm15, %ymm5, %ymm5
10309 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm5[5,6,7]
10310 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3],ymm5[4,5,6,7]
10311 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm13, %ymm1, %ymm1
10312 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm2, %xmm4, %xmm2
10313 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,5]
10314 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4],xmm2[5,6],xmm3[7]
10315 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
10316 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0,1,2],ymm2[3,4,5,6,7],ymm1[8,9,10],ymm2[11,12,13,14,15]
10317 ; AVX512F-ONLY-FAST-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,4,5]
10318 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
10319 ; AVX512F-ONLY-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm28
10320 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $226, %zmm0, %zmm29, %zmm5
10321 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm5, %zmm25, %zmm28
10322 ; AVX512F-ONLY-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
10323 ; AVX512F-ONLY-FAST-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm10 # 32-byte Folded Reload
10324 ; AVX512F-ONLY-FAST-NEXT: # ymm10 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
10325 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} xmm1 = <8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15>
10326 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm1, %xmm10, %xmm0
10327 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm10, %xmm2
10328 ; AVX512F-ONLY-FAST-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
10329 ; AVX512F-ONLY-FAST-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,2,2,2,4,5,6,7]
10330 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3],xmm2[4],xmm0[5,6,7]
10331 ; AVX512F-ONLY-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
10332 ; AVX512F-ONLY-FAST-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
10333 ; AVX512F-ONLY-FAST-NEXT: # ymm2 = ymm2[0],mem[1],ymm2[2,3],mem[4],ymm2[5,6],mem[7]
10334 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm2, %xmm3
10335 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[0,3,2,1]
10336 ; AVX512F-ONLY-FAST-NEXT: vpbroadcastq {{.*#+}} xmm8 = [8,9,0,0,0,1,12,13,8,9,0,0,0,1,12,13]
10337 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm8, %xmm3, %xmm2
10338 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm3, %xmm25
10339 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} xmm15 = <0,1,2,3,0,1,4,5,u,u,12,13,u,u,u,u>
10340 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm15, %xmm4, %xmm3
10341 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm4, %xmm27
10342 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4],xmm3[5],xmm2[6,7]
10343 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
10344 ; AVX512F-ONLY-FAST-NEXT: vinserti32x4 $2, %xmm0, %zmm2, %zmm30
10345 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm31, %ymm0
10346 ; AVX512F-ONLY-FAST-NEXT: vpblendd $66, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 32-byte Folded Reload
10347 ; AVX512F-ONLY-FAST-NEXT: # ymm4 = ymm0[0],mem[1],ymm0[2,3,4,5],mem[6],ymm0[7]
10348 ; AVX512F-ONLY-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
10349 ; AVX512F-ONLY-FAST-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm11 # 32-byte Folded Reload
10350 ; AVX512F-ONLY-FAST-NEXT: # ymm11 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
10351 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm1, %xmm11, %xmm0
10352 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm11, %xmm12
10353 ; AVX512F-ONLY-FAST-NEXT: vpshuflw {{.*#+}} xmm2 = xmm12[2,2,2,2,4,5,6,7]
10354 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3],xmm2[4],xmm0[5,6,7]
10355 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 {{.*#+}} ymm29 = [65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
10356 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} ymm2 = [128,128,128,128,128,128,128,128,128,128,4,5,0,1,12,13,24,25,20,21,128,128,128,128,128,128,128,128,128,128,128,128]
10357 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm2, %ymm4, %ymm3
10358 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm4, %ymm20
10359 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $236, %ymm29, %ymm3, %ymm0
10360 ; AVX512F-ONLY-FAST-NEXT: movw $31, %ax
10361 ; AVX512F-ONLY-FAST-NEXT: kmovw %eax, %k1
10362 ; AVX512F-ONLY-FAST-NEXT: vmovdqa32 %zmm0, %zmm30 {%k1}
10363 ; AVX512F-ONLY-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
10364 ; AVX512F-ONLY-FAST-NEXT: vpblendd $66, (%rsp), %ymm0, %ymm4 # 32-byte Folded Reload
10365 ; AVX512F-ONLY-FAST-NEXT: # ymm4 = ymm0[0],mem[1],ymm0[2,3,4,5],mem[6],ymm0[7]
10366 ; AVX512F-ONLY-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
10367 ; AVX512F-ONLY-FAST-NEXT: vpblendd $109, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
10368 ; AVX512F-ONLY-FAST-NEXT: # ymm0 = mem[0],ymm0[1],mem[2,3],ymm0[4],mem[5,6],ymm0[7]
10369 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
10370 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[0,3,2,1]
10371 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm8, %xmm3, %xmm0
10372 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm3, %xmm21
10373 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm15, %xmm5, %xmm3
10374 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm5, %xmm23
10375 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4],xmm3[5],xmm0[6,7]
10376 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
10377 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} ymm5 = [8,9,4,5,4,5,6,7,0,1,4,5,0,1,12,13,24,25,20,21,20,21,22,23,16,17,20,21,16,17,28,29]
10378 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm5, %ymm4, %ymm3
10379 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm5, %ymm16
10380 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm4, %ymm22
10381 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4],ymm0[5,6,7]
10382 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm0, %ymm24
10383 ; AVX512F-ONLY-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
10384 ; AVX512F-ONLY-FAST-NEXT: vpblendd $66, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 32-byte Folded Reload
10385 ; AVX512F-ONLY-FAST-NEXT: # ymm4 = ymm0[0],mem[1],ymm0[2,3,4,5],mem[6],ymm0[7]
10386 ; AVX512F-ONLY-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
10387 ; AVX512F-ONLY-FAST-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
10388 ; AVX512F-ONLY-FAST-NEXT: # ymm14 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
10389 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm1, %xmm14, %xmm0
10390 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm14, %xmm5
10391 ; AVX512F-ONLY-FAST-NEXT: vpshuflw {{.*#+}} xmm3 = xmm5[2,2,2,2,4,5,6,7]
10392 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm5, %xmm19
10393 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0],xmm3[1],xmm0[2,3],xmm3[4],xmm0[5,6,7]
10394 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm2, %ymm4, %ymm0
10395 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm4, %ymm18
10396 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $236, %ymm29, %ymm0, %ymm3
10397 ; AVX512F-ONLY-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
10398 ; AVX512F-ONLY-FAST-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
10399 ; AVX512F-ONLY-FAST-NEXT: # ymm5 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
10400 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm1, %xmm5, %xmm1
10401 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm5, %xmm0
10402 ; AVX512F-ONLY-FAST-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[2,2,2,2,4,5,6,7]
10403 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm0, %xmm17
10404 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2,3],xmm2[4],xmm1[5,6,7]
10405 ; AVX512F-ONLY-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
10406 ; AVX512F-ONLY-FAST-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
10407 ; AVX512F-ONLY-FAST-NEXT: # ymm2 = ymm0[0],mem[1],ymm0[2,3],mem[4],ymm0[5,6],mem[7]
10408 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm2, %xmm13
10409 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm9 = xmm2[0,3,2,1]
10410 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm8, %xmm13, %xmm2
10411 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm15, %xmm9, %xmm4
10412 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3],xmm2[4],xmm4[5],xmm2[6,7]
10413 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
10414 ; AVX512F-ONLY-FAST-NEXT: vinserti32x4 $2, %xmm1, %zmm2, %zmm31
10415 ; AVX512F-ONLY-FAST-NEXT: vmovdqa32 %zmm3, %zmm31 {%k1}
10416 ; AVX512F-ONLY-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
10417 ; AVX512F-ONLY-FAST-NEXT: vpblendd $66, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm7 # 32-byte Folded Reload
10418 ; AVX512F-ONLY-FAST-NEXT: # ymm7 = ymm0[0],mem[1],ymm0[2,3,4,5],mem[6],ymm0[7]
10419 ; AVX512F-ONLY-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
10420 ; AVX512F-ONLY-FAST-NEXT: vpblendd $109, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
10421 ; AVX512F-ONLY-FAST-NEXT: # ymm1 = mem[0],ymm0[1],mem[2,3],ymm0[4],mem[5,6],ymm0[7]
10422 ; AVX512F-ONLY-FAST-NEXT: vextracti128 $1, %ymm1, %xmm6
10423 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm8, %xmm6, %xmm8
10424 ; AVX512F-ONLY-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[0,3,2,1]
10425 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm15, %xmm4, %xmm1
10426 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm8[4],xmm1[5],xmm8[6,7]
10427 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm16, %ymm0
10428 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm0, %ymm7, %ymm8
10429 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
10430 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3,4],ymm1[5,6,7]
10431 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm0, %ymm16
10432 ; AVX512F-ONLY-FAST-NEXT: vpbroadcastd {{.*#+}} xmm3 = [10,11,6,7,10,11,6,7,10,11,6,7,10,11,6,7]
10433 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm3, %xmm12, %xmm8
10434 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} xmm12 = <10,11,u,u,2,3,14,15,u,u,10,11,12,13,14,15>
10435 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm12, %xmm11, %xmm11
10436 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm8 = xmm11[0],xmm8[1],xmm11[2,3],xmm8[4],xmm11[5,6,7]
10437 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} ymm1 = [128,128,128,128,128,128,128,128,128,128,6,7,2,3,14,15,26,27,22,23,128,128,128,128,128,128,128,128,128,128,128,128]
10438 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm20, %ymm0
10439 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm1, %ymm0, %ymm11
10440 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm1, %ymm20
10441 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $236, %ymm29, %ymm11, %ymm8
10442 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
10443 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm3, %xmm0, %xmm11
10444 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm12, %xmm10, %xmm10
10445 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm10[0],xmm11[1],xmm10[2,3],xmm11[4],xmm10[5,6,7]
10446 ; AVX512F-ONLY-FAST-NEXT: vpbroadcastq {{.*#+}} xmm10 = [10,11,0,0,2,3,14,15,10,11,0,0,2,3,14,15]
10447 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm25, %xmm0
10448 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm10, %xmm0, %xmm0
10449 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} xmm11 = <0,1,2,3,2,3,6,7,u,u,14,15,u,u,u,u>
10450 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm27, %xmm2
10451 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm11, %xmm2, %xmm15
10452 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm15[0,1,2,3],xmm0[4],xmm15[5],xmm0[6,7]
10453 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
10454 ; AVX512F-ONLY-FAST-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm2
10455 ; AVX512F-ONLY-FAST-NEXT: vmovdqa32 %zmm8, %zmm2 {%k1}
10456 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm21, %xmm0
10457 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm10, %xmm0, %xmm1
10458 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm23, %xmm0
10459 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm11, %xmm0, %xmm8
10460 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm8[0,1,2,3],xmm1[4],xmm8[5],xmm1[6,7]
10461 ; AVX512F-ONLY-FAST-NEXT: vmovdqa {{.*#+}} ymm8 = [10,11,6,7,4,5,6,7,6,7,6,7,2,3,14,15,26,27,22,23,20,21,22,23,22,23,22,23,18,19,30,31]
10462 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm22, %ymm0
10463 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm8, %ymm0, %ymm15
10464 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
10465 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm15[0,1,2,3,4],ymm1[5,6,7]
10466 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm18, %ymm0
10467 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %ymm20, %ymm15
10468 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm15, %ymm0, %ymm15
10469 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm19, %xmm0
10470 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm3, %xmm0, %xmm0
10471 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm12, %xmm14, %xmm14
10472 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm14[0],xmm0[1],xmm14[2,3],xmm0[4],xmm14[5,6,7]
10473 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $236, %ymm29, %ymm15, %ymm0
10474 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %xmm17, %xmm14
10475 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm3, %xmm14, %xmm3
10476 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm12, %xmm5, %xmm5
10477 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm5[0],xmm3[1],xmm5[2,3],xmm3[4],xmm5[5,6,7]
10478 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm10, %xmm13, %xmm5
10479 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm11, %xmm9, %xmm9
10480 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm5 = xmm9[0,1,2,3],xmm5[4],xmm9[5],xmm5[6,7]
10481 ; AVX512F-ONLY-FAST-NEXT: vinserti64x4 $1, %ymm24, %zmm0, %zmm9
10482 ; AVX512F-ONLY-FAST-NEXT: vinserti64x4 $1, %ymm16, %zmm0, %zmm12
10483 ; AVX512F-ONLY-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm1
10484 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
10485 ; AVX512F-ONLY-FAST-NEXT: vinserti32x4 $2, %xmm3, %zmm5, %zmm3
10486 ; AVX512F-ONLY-FAST-NEXT: vmovdqa32 %zmm0, %zmm3 {%k1}
10487 ; AVX512F-ONLY-FAST-NEXT: vpshufb %ymm8, %ymm7, %ymm0
10488 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm10, %xmm6, %xmm5
10489 ; AVX512F-ONLY-FAST-NEXT: vpshufb %xmm11, %xmm4, %xmm4
10490 ; AVX512F-ONLY-FAST-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm5[4],xmm4[5],xmm5[6,7]
10491 ; AVX512F-ONLY-FAST-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
10492 ; AVX512F-ONLY-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5,6,7]
10493 ; AVX512F-ONLY-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0
10494 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 {{.*#+}} zmm4 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
10495 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
10496 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm5 # 64-byte Folded Reload
10497 ; AVX512F-ONLY-FAST-NEXT: movw $-2048, %ax # imm = 0xF800
10498 ; AVX512F-ONLY-FAST-NEXT: kmovw %eax, %k1
10499 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
10500 ; AVX512F-ONLY-FAST-NEXT: vmovdqa32 %zmm6, %zmm5 {%k1}
10501 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm5, (%rsi)
10502 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
10503 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm5 # 64-byte Folded Reload
10504 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
10505 ; AVX512F-ONLY-FAST-NEXT: vmovdqa32 %zmm6, %zmm5 {%k1}
10506 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm5, 64(%rsi)
10507 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
10508 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm5 # 64-byte Folded Reload
10509 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
10510 ; AVX512F-ONLY-FAST-NEXT: vmovdqa32 %zmm6, %zmm5 {%k1}
10511 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm5, 64(%rdx)
10512 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
10513 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm5 # 64-byte Folded Reload
10514 ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
10515 ; AVX512F-ONLY-FAST-NEXT: vmovdqa32 %zmm4, %zmm5 {%k1}
10516 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm5, (%rdx)
10517 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 {{.*#+}} zmm4 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0]
10518 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm30, %zmm4, %zmm9
10519 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm31, %zmm4, %zmm12
10520 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm2, %zmm4, %zmm1
10521 ; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm3, %zmm4, %zmm0
10522 ; AVX512F-ONLY-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
10523 ; AVX512F-ONLY-FAST-NEXT: vmovaps %zmm2, 64(%rcx)
10524 ; AVX512F-ONLY-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
10525 ; AVX512F-ONLY-FAST-NEXT: vmovaps %zmm2, (%rcx)
10526 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm28, 64(%r8)
10527 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm26, (%r8)
10528 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm12, 64(%r9)
10529 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm9, (%r9)
10530 ; AVX512F-ONLY-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
10531 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm0, 64(%rax)
10532 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm1, (%rax)
10533 ; AVX512F-ONLY-FAST-NEXT: addq $1512, %rsp # imm = 0x5E8
10534 ; AVX512F-ONLY-FAST-NEXT: vzeroupper
10535 ; AVX512F-ONLY-FAST-NEXT: retq
10536 ;
10537 ; AVX512DQ-SLOW-LABEL: load_i16_stride6_vf64:
10538 ; AVX512DQ-SLOW: # %bb.0:
10539 ; AVX512DQ-SLOW-NEXT: subq $840, %rsp # imm = 0x348
10540 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} xmm9 = <0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15>
10541 ; AVX512DQ-SLOW-NEXT: vmovdqa 608(%rdi), %ymm0
10542 ; AVX512DQ-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10543 ; AVX512DQ-SLOW-NEXT: vmovdqa 576(%rdi), %ymm1
10544 ; AVX512DQ-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10545 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]
10546 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm9, %xmm1, %xmm0
10547 ; AVX512DQ-SLOW-NEXT: vextracti32x4 $1, %ymm1, %xmm24
10548 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm1, %ymm25
10549 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm24[0,2,0,3]
10550 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
10551 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3],xmm1[4,5],xmm0[6,7]
10552 ; AVX512DQ-SLOW-NEXT: vmovdqa 544(%rdi), %ymm1
10553 ; AVX512DQ-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10554 ; AVX512DQ-SLOW-NEXT: vmovdqa 512(%rdi), %ymm2
10555 ; AVX512DQ-SLOW-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10556 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm3 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
10557 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm3, %xmm2
10558 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} xmm10 = <0,1,4,5,4,5,u,u,0,1,12,13,u,u,4,5>
10559 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm10, %xmm2, %xmm1
10560 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm2, %xmm23
10561 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm3[2,2,2,2,4,5,6,7]
10562 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm3, %ymm22
10563 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,2,2]
10564 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3],xmm1[4,5],xmm2[6],xmm1[7]
10565 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
10566 ; AVX512DQ-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
10567 ; AVX512DQ-SLOW-NEXT: vmovdqa 448(%rdi), %ymm1
10568 ; AVX512DQ-SLOW-NEXT: vmovdqa 416(%rdi), %ymm2
10569 ; AVX512DQ-SLOW-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10570 ; AVX512DQ-SLOW-NEXT: vmovdqa 384(%rdi), %ymm3
10571 ; AVX512DQ-SLOW-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10572 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm15 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6],ymm2[7]
10573 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm9, %xmm15, %xmm2
10574 ; AVX512DQ-SLOW-NEXT: vextracti32x4 $1, %ymm15, %xmm21
10575 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm21[0,2,0,3]
10576 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
10577 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2],xmm2[3],xmm3[4,5],xmm2[6,7]
10578 ; AVX512DQ-SLOW-NEXT: vperm2i128 {{.*#+}} ymm3 = ymm1[2,3],mem[2,3]
10579 ; AVX512DQ-SLOW-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10580 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, 480(%rdi), %ymm1, %ymm1
10581 ; AVX512DQ-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10582 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm3 = ymm1[0],ymm3[1],ymm1[2,3,4,5],ymm3[6],ymm1[7]
10583 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} ymm6 = [0,1,12,13,8,9,12,13,8,9,12,13,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
10584 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm6, %ymm3, %ymm1
10585 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm3, %ymm20
10586 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm3 = ymm2[0,1,2],ymm1[3,4,5,6,7]
10587 ; AVX512DQ-SLOW-NEXT: vmovdqa 736(%rdi), %ymm1
10588 ; AVX512DQ-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10589 ; AVX512DQ-SLOW-NEXT: vmovdqa 704(%rdi), %ymm2
10590 ; AVX512DQ-SLOW-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10591 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm11 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
10592 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm11[2,2,2,2,4,5,6,7]
10593 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,2]
10594 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm11, %xmm4
10595 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm10, %xmm4, %xmm2
10596 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm4, %xmm19
10597 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3],xmm2[4,5],xmm1[6],xmm2[7]
10598 ; AVX512DQ-SLOW-NEXT: vmovdqa 640(%rdi), %ymm2
10599 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
10600 ; AVX512DQ-SLOW-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm2[2,3],mem[2,3]
10601 ; AVX512DQ-SLOW-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10602 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, 672(%rdi), %ymm2, %ymm2
10603 ; AVX512DQ-SLOW-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10604 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm2[0],ymm4[1],ymm2[2,3,4,5],ymm4[6],ymm2[7]
10605 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} ymm5 = <0,1,12,13,8,9,12,13,8,9,12,13,4,5,u,u,16,17,28,29,24,25,28,29,24,25,28,29,20,21,u,u>
10606 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm5, %ymm4, %ymm2
10607 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm5, %ymm27
10608 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm4, %ymm18
10609 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7],ymm2[8,9,10],ymm1[11,12,13,14,15]
10610 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,6]
10611 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
10612 ; AVX512DQ-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
10613 ; AVX512DQ-SLOW-NEXT: vpternlogq $226, %zmm0, %zmm16, %zmm3
10614 ; AVX512DQ-SLOW-NEXT: movw $-2048, %ax # imm = 0xF800
10615 ; AVX512DQ-SLOW-NEXT: kmovw %eax, %k1
10616 ; AVX512DQ-SLOW-NEXT: vinserti32x8 $1, %ymm1, %zmm0, %zmm3 {%k1}
10617 ; AVX512DQ-SLOW-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
10618 ; AVX512DQ-SLOW-NEXT: vmovdqa 224(%rdi), %ymm0
10619 ; AVX512DQ-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10620 ; AVX512DQ-SLOW-NEXT: vmovdqa 192(%rdi), %ymm1
10621 ; AVX512DQ-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10622 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm13 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]
10623 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm13, %xmm14
10624 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm14[0,2,0,3]
10625 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
10626 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm9, %xmm13, %xmm1
10627 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3],xmm0[4,5],xmm1[6,7]
10628 ; AVX512DQ-SLOW-NEXT: vmovdqa 160(%rdi), %ymm1
10629 ; AVX512DQ-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10630 ; AVX512DQ-SLOW-NEXT: vmovdqa 128(%rdi), %ymm2
10631 ; AVX512DQ-SLOW-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10632 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
10633 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm4[2,2,2,2,4,5,6,7]
10634 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,2]
10635 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm4, %xmm8
10636 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm10, %xmm8, %xmm3
10637 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3],xmm3[4,5],xmm1[6],xmm3[7]
10638 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
10639 ; AVX512DQ-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm17
10640 ; AVX512DQ-SLOW-NEXT: vmovdqa (%rdi), %ymm0
10641 ; AVX512DQ-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10642 ; AVX512DQ-SLOW-NEXT: vmovdqa 32(%rdi), %ymm1
10643 ; AVX512DQ-SLOW-NEXT: vmovdqu %ymm1, (%rsp) # 32-byte Spill
10644 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
10645 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm9, %xmm2, %xmm0
10646 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm9
10647 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm9[0,2,0,3]
10648 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
10649 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2],xmm0[3],xmm3[4,5],xmm0[6,7]
10650 ; AVX512DQ-SLOW-NEXT: vmovdqa 64(%rdi), %ymm3
10651 ; AVX512DQ-SLOW-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm3[2,3],mem[2,3]
10652 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, 96(%rdi), %ymm3, %ymm3
10653 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm3[0],ymm1[1],ymm3[2,3,4,5],ymm1[6],ymm3[7]
10654 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm3, %ymm31
10655 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm1, %ymm30
10656 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm6, %ymm5, %ymm3
10657 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm6 = ymm0[0,1,2],ymm3[3,4,5,6,7]
10658 ; AVX512DQ-SLOW-NEXT: vmovdqa 352(%rdi), %ymm0
10659 ; AVX512DQ-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10660 ; AVX512DQ-SLOW-NEXT: vmovdqa 320(%rdi), %ymm1
10661 ; AVX512DQ-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
10662 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm7 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
10663 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm7, %xmm3
10664 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm10, %xmm3, %xmm10
10665 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm12 = xmm7[2,2,2,2,4,5,6,7]
10666 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[0,1,2,2]
10667 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm12 = xmm10[0,1,2],xmm12[3],xmm10[4,5],xmm12[6],xmm10[7]
10668 ; AVX512DQ-SLOW-NEXT: vmovdqa 256(%rdi), %ymm10
10669 ; AVX512DQ-SLOW-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm10[2,3],mem[2,3]
10670 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, 288(%rdi), %ymm10, %ymm0
10671 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm10 = ymm0[0],ymm1[1],ymm0[2,3,4,5],ymm1[6],ymm0[7]
10672 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm0, %ymm29
10673 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm1, %ymm26
10674 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm27, %ymm0
10675 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm0, %ymm10, %ymm0
10676 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm12
10677 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} ymm12 = ymm0[0,1,2],ymm12[3,4,5,6,7],ymm0[8,9,10],ymm12[11,12,13,14,15]
10678 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,4,6]
10679 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm12[4,5,6,7]
10680 ; AVX512DQ-SLOW-NEXT: vpternlogq $226, %zmm17, %zmm16, %zmm6
10681 ; AVX512DQ-SLOW-NEXT: vinserti32x8 $1, %ymm0, %zmm0, %zmm6 {%k1}
10682 ; AVX512DQ-SLOW-NEXT: vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
10683 ; AVX512DQ-SLOW-NEXT: vpbroadcastq {{.*#+}} xmm12 = [2,3,14,15,10,11,0,0,2,3,14,15,10,11,0,0]
10684 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm12, %xmm14, %xmm0
10685 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} xmm14 = <2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15>
10686 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm14, %xmm13, %xmm13
10687 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm13[0,1],xmm0[2],xmm13[3],xmm0[4,5],xmm13[6,7]
10688 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} xmm13 = <6,7,2,3,4,5,u,u,2,3,14,15,u,u,6,7>
10689 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm13, %xmm8, %xmm6
10690 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,5,5,5]
10691 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm4 = xmm6[0,1,2],xmm4[3],xmm6[4,5],xmm4[6],xmm6[7]
10692 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
10693 ; AVX512DQ-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm4, %zmm0
10694 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm12, %xmm9, %xmm4
10695 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm14, %xmm2, %xmm2
10696 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm4[2],xmm2[3],xmm4[4,5],xmm2[6,7]
10697 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} ymm4 = [2,3,14,15,10,11,10,11,14,15,10,11,10,11,6,7,18,19,30,31,26,27,26,27,30,31,26,27,26,27,22,23]
10698 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm4, %ymm5, %ymm5
10699 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm2[0,1,2],ymm5[3,4,5,6,7]
10700 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm13, %xmm3, %xmm2
10701 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm7[0,1,2,3,5,5,5,5]
10702 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3],xmm2[4,5],xmm1[6],xmm2[7]
10703 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} ymm2 = <2,3,14,15,10,11,10,11,14,15,10,11,u,u,6,7,18,19,30,31,26,27,26,27,30,31,26,27,u,u,22,23>
10704 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm2, %ymm10, %ymm3
10705 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
10706 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3,4,5,6,7],ymm3[8,9,10],ymm1[11,12,13,14,15]
10707 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,5,7]
10708 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
10709 ; AVX512DQ-SLOW-NEXT: vpternlogq $226, %zmm0, %zmm16, %zmm5
10710 ; AVX512DQ-SLOW-NEXT: vinserti32x8 $1, %ymm1, %zmm0, %zmm5 {%k1}
10711 ; AVX512DQ-SLOW-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
10712 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm24, %xmm0
10713 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm12, %xmm0, %xmm0
10714 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm25, %ymm1
10715 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm14, %xmm1, %xmm1
10716 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3],xmm0[4,5],xmm1[6,7]
10717 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm23, %xmm1
10718 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm13, %xmm1, %xmm1
10719 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm22, %ymm3
10720 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,5,5]
10721 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm3[3],xmm1[4,5],xmm3[6],xmm1[7]
10722 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
10723 ; AVX512DQ-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
10724 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm20, %ymm1
10725 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm4, %ymm1, %ymm1
10726 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm21, %xmm3
10727 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm12, %xmm3, %xmm3
10728 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm14, %xmm15, %xmm4
10729 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0,1],xmm3[2],xmm4[3],xmm3[4,5],xmm4[6,7]
10730 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2],ymm1[3,4,5,6,7]
10731 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm18, %ymm1
10732 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm2, %ymm1, %ymm1
10733 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm19, %xmm2
10734 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm13, %xmm2, %xmm2
10735 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm11[0,1,2,3,5,5,5,5]
10736 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[3],xmm2[4,5],xmm3[6],xmm2[7]
10737 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
10738 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0,1,2],ymm2[3,4,5,6,7],ymm1[8,9,10],ymm2[11,12,13,14,15]
10739 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,5,7]
10740 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
10741 ; AVX512DQ-SLOW-NEXT: vpternlogq $226, %zmm0, %zmm16, %zmm4
10742 ; AVX512DQ-SLOW-NEXT: vinserti32x8 $1, %ymm1, %zmm0, %zmm4 {%k1}
10743 ; AVX512DQ-SLOW-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
10744 ; AVX512DQ-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
10745 ; AVX512DQ-SLOW-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
10746 ; AVX512DQ-SLOW-NEXT: # ymm0 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
10747 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm1
10748 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,1,2,3]
10749 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,3,2,1]
10750 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm1[0,0,2,3,4,5,6,7]
10751 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm1, %xmm20
10752 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,6,6,6]
10753 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm2[2,1,2,0,4,5,6,7]
10754 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm2, %xmm22
10755 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5,6,7]
10756 ; AVX512DQ-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
10757 ; AVX512DQ-SLOW-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
10758 ; AVX512DQ-SLOW-NEXT: # ymm1 = ymm1[0,1],mem[2],ymm1[3,4],mem[5],ymm1[6,7]
10759 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm2
10760 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,1,0,3]
10761 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm3[0,0,0,0,4,5,6,7]
10762 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm3, %xmm23
10763 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,6,7]
10764 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,1,2,1]
10765 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm3[0,1,2,3,6,5,6,4]
10766 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm3, %xmm24
10767 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4],xmm1[5,6],xmm2[7]
10768 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
10769 ; AVX512DQ-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm1
10770 ; AVX512DQ-SLOW-NEXT: vmovdqu (%rsp), %ymm0 # 32-byte Reload
10771 ; AVX512DQ-SLOW-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
10772 ; AVX512DQ-SLOW-NEXT: # ymm0 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
10773 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm2
10774 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,1,2,3]
10775 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,3,2,1]
10776 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm2[0,0,2,3,4,5,6,7]
10777 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm2, %xmm25
10778 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,3,3]
10779 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm3[2,1,2,0,4,5,6,7]
10780 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm3, %xmm16
10781 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm0[1,2],xmm2[3],xmm0[4,5,6,7]
10782 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm30, %ymm0
10783 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm31, %ymm3
10784 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1],ymm0[2],ymm3[3],ymm0[4],ymm3[5,6],ymm0[7]
10785 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} ymm0 = [4,5,0,1,12,13,14,15,8,9,0,1,12,13,8,9,20,21,16,17,28,29,30,31,24,25,16,17,28,29,24,25]
10786 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm0, %ymm4, %ymm3
10787 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm4, %ymm17
10788 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4],xmm3[5,6,7]
10789 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
10790 ; AVX512DQ-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
10791 ; AVX512DQ-SLOW-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
10792 ; AVX512DQ-SLOW-NEXT: # ymm3 = mem[0,1],ymm3[2],mem[3,4],ymm3[5],mem[6,7]
10793 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm3, %xmm4
10794 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm15 = xmm3[2,1,0,3]
10795 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm15[0,0,0,0,4,5,6,7]
10796 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,6,7]
10797 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm4[0,1,2,1]
10798 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm4 = xmm12[0,1,2,3,6,5,6,4]
10799 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm4[4],xmm3[5,6],xmm4[7]
10800 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
10801 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm26, %ymm4
10802 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm29, %ymm5
10803 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm13 = ymm5[0,1],ymm4[2],ymm5[3],ymm4[4],ymm5[5,6],ymm4[7]
10804 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} ymm5 = <4,5,0,1,12,13,14,15,8,9,12,13,0,1,u,u,20,21,16,17,28,29,30,31,24,25,28,29,16,17,u,u>
10805 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm5, %ymm13, %ymm4
10806 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm5, %ymm27
10807 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7],ymm4[8,9,10],ymm3[11,12,13,14,15]
10808 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,6,5,4]
10809 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
10810 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm3
10811 ; AVX512DQ-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm21 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
10812 ; AVX512DQ-SLOW-NEXT: vpternlogq $226, %zmm1, %zmm21, %zmm2
10813 ; AVX512DQ-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0]
10814 ; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm2, %zmm1, %zmm3
10815 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm1, %zmm18
10816 ; AVX512DQ-SLOW-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
10817 ; AVX512DQ-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
10818 ; AVX512DQ-SLOW-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
10819 ; AVX512DQ-SLOW-NEXT: # ymm1 = mem[0,1],ymm1[2],mem[3,4],ymm1[5],mem[6,7]
10820 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm2
10821 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm1[2,1,2,3]
10822 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm2[0,3,2,1]
10823 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm11[0,0,2,3,4,5,6,7]
10824 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,6,6,6]
10825 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm10[2,1,2,0,4,5,6,7]
10826 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2],xmm2[3],xmm1[4,5,6,7]
10827 ; AVX512DQ-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
10828 ; AVX512DQ-SLOW-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
10829 ; AVX512DQ-SLOW-NEXT: # ymm2 = mem[0,1],ymm2[2],mem[3,4],ymm2[5],mem[6,7]
10830 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm3
10831 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm2[2,1,0,3]
10832 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm9[0,0,0,0,4,5,6,7]
10833 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,6,7]
10834 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm3[0,1,2,1]
10835 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm8[0,1,2,3,6,5,6,4]
10836 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4],xmm2[5,6],xmm3[7]
10837 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
10838 ; AVX512DQ-SLOW-NEXT: vinserti32x4 $2, %xmm1, %zmm2, %zmm19
10839 ; AVX512DQ-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
10840 ; AVX512DQ-SLOW-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
10841 ; AVX512DQ-SLOW-NEXT: # ymm1 = mem[0,1],ymm1[2],mem[3,4],ymm1[5],mem[6,7]
10842 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm2
10843 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm1[2,1,2,3]
10844 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm2[0,3,2,1]
10845 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm7[0,0,2,3,4,5,6,7]
10846 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,3,3]
10847 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm6[2,1,2,0,4,5,6,7]
10848 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2],xmm2[3],xmm1[4,5,6,7]
10849 ; AVX512DQ-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
10850 ; AVX512DQ-SLOW-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm5 # 32-byte Folded Reload
10851 ; AVX512DQ-SLOW-NEXT: # ymm5 = mem[0,1],ymm2[2],mem[3],ymm2[4],mem[5,6],ymm2[7]
10852 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm0, %ymm5, %ymm0
10853 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm0[5,6,7]
10854 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5,6,7]
10855 ; AVX512DQ-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
10856 ; AVX512DQ-SLOW-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
10857 ; AVX512DQ-SLOW-NEXT: # ymm1 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
10858 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm0
10859 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[2,1,0,3]
10860 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm4[0,0,0,0,4,5,6,7]
10861 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,6,7]
10862 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[0,1,2,1]
10863 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm14 = xmm3[0,1,2,3,6,5,6,4]
10864 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm14 = xmm1[0,1,2,3],xmm14[4],xmm1[5,6],xmm14[7]
10865 ; AVX512DQ-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
10866 ; AVX512DQ-SLOW-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
10867 ; AVX512DQ-SLOW-NEXT: # ymm1 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6],ymm0[7]
10868 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm27, %ymm0
10869 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm0, %ymm1, %ymm0
10870 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
10871 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} ymm14 = ymm0[0,1,2],ymm14[3,4,5,6,7],ymm0[8,9,10],ymm14[11,12,13,14,15]
10872 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,5,4]
10873 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm14[4,5,6,7]
10874 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm28
10875 ; AVX512DQ-SLOW-NEXT: vpternlogq $226, %zmm19, %zmm21, %zmm2
10876 ; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm2, %zmm18, %zmm28
10877 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm22, %xmm0
10878 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[3,1,2,1,4,5,6,7]
10879 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm20, %xmm2
10880 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,3,3,4,5,6,7]
10881 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,7,7,7]
10882 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2],xmm0[3],xmm2[4,5,6,7]
10883 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm24, %xmm2
10884 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,5]
10885 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm23, %xmm14
10886 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm14 = xmm14[1,1,1,1,4,5,6,7]
10887 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,4,5,7,7]
10888 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm14[0,1,2,3],xmm2[4],xmm14[5,6],xmm2[7]
10889 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
10890 ; AVX512DQ-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm2, %zmm20
10891 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm16, %xmm0
10892 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[3,1,2,1,4,5,6,7]
10893 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm25, %xmm0
10894 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm14 = xmm0[0,1,3,3,4,5,6,7]
10895 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,7,7,7,7]
10896 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm14[1,2],xmm2[3],xmm14[4,5,6,7]
10897 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} ymm14 = [6,7,2,3,14,15,14,15,14,15,2,3,14,15,10,11,22,23,18,19,30,31,30,31,30,31,18,19,30,31,26,27]
10898 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm17, %ymm0
10899 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm14, %ymm0, %ymm0
10900 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4],xmm0[5,6,7]
10901 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
10902 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm12[0,1,2,3,7,5,6,5]
10903 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm12 = xmm15[1,1,1,1,4,5,6,7]
10904 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,5,7,7]
10905 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm12[0,1,2,3],xmm2[4],xmm12[5,6],xmm2[7]
10906 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} ymm12 = <6,7,2,3,14,15,14,15,14,15,10,11,u,u,2,3,22,23,18,19,30,31,30,31,30,31,26,27,u,u,18,19>
10907 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm12, %ymm13, %ymm13
10908 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
10909 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = ymm13[0,1,2],ymm2[3,4,5,6,7],ymm13[8,9,10],ymm2[11,12,13,14,15]
10910 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,4,7,4,5]
10911 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm13[0,1,2,3],ymm2[4,5,6,7]
10912 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm27
10913 ; AVX512DQ-SLOW-NEXT: vpternlogq $226, %zmm20, %zmm21, %zmm0
10914 ; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm0, %zmm18, %zmm27
10915 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm10[3,1,2,1,4,5,6,7]
10916 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm11[0,1,3,3,4,5,6,7]
10917 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,7,7,7]
10918 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2],xmm0[3],xmm2[4,5,6,7]
10919 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm8[0,1,2,3,7,5,6,5]
10920 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm9[1,1,1,1,4,5,6,7]
10921 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5,7,7]
10922 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm8[0,1,2,3],xmm2[4],xmm8[5,6],xmm2[7]
10923 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
10924 ; AVX512DQ-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm2, %zmm0
10925 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm6[3,1,2,1,4,5,6,7]
10926 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm7[0,1,3,3,4,5,6,7]
10927 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,7,7,7,7]
10928 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm6[1,2],xmm2[3],xmm6[4,5,6,7]
10929 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm14, %ymm5, %ymm5
10930 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4],xmm5[5,6,7]
10931 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm5[4,5,6,7]
10932 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm12, %ymm1, %ymm1
10933 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,5]
10934 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[1,1,1,1,4,5,6,7]
10935 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,7,7]
10936 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4],xmm4[5,6],xmm3[7]
10937 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
10938 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = ymm1[0,1,2],ymm3[3,4,5,6,7],ymm1[8,9,10],ymm3[11,12,13,14,15]
10939 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,4,5]
10940 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
10941 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm20
10942 ; AVX512DQ-SLOW-NEXT: vpternlogq $226, %zmm0, %zmm21, %zmm2
10943 ; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm2, %zmm18, %zmm20
10944 ; AVX512DQ-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
10945 ; AVX512DQ-SLOW-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
10946 ; AVX512DQ-SLOW-NEXT: # ymm3 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
10947 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} xmm1 = <8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15>
10948 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm1, %xmm3, %xmm0
10949 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm3, %xmm13
10950 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm13[2,2,2,2,4,5,6,7]
10951 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3],xmm2[4],xmm0[5,6,7]
10952 ; AVX512DQ-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
10953 ; AVX512DQ-SLOW-NEXT: vpblendd $109, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
10954 ; AVX512DQ-SLOW-NEXT: # ymm2 = mem[0],ymm2[1],mem[2,3],ymm2[4],mem[5,6],ymm2[7]
10955 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm5
10956 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[0,3,2,1]
10957 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm4[0,1,0,2,4,5,6,7]
10958 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm4, %xmm19
10959 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,6,6,6]
10960 ; AVX512DQ-SLOW-NEXT: vpbroadcastq {{.*#+}} xmm11 = [8,9,0,0,0,1,12,13,8,9,0,0,0,1,12,13]
10961 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm11, %xmm5, %xmm4
10962 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm5, %xmm18
10963 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm4[4],xmm2[5],xmm4[6,7]
10964 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
10965 ; AVX512DQ-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm2, %zmm22
10966 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm30, %ymm0
10967 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm31, %ymm2
10968 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm0[0],ymm2[1],ymm0[2,3,4,5],ymm2[6],ymm0[7]
10969 ; AVX512DQ-SLOW-NEXT: vmovdqu (%rsp), %ymm0 # 32-byte Reload
10970 ; AVX512DQ-SLOW-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
10971 ; AVX512DQ-SLOW-NEXT: # ymm8 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
10972 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm1, %xmm8, %xmm0
10973 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm8, %xmm4
10974 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm4[2,2,2,2,4,5,6,7]
10975 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm4, %xmm30
10976 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3],xmm2[4],xmm0[5,6,7]
10977 ; AVX512DQ-SLOW-NEXT: vmovdqa64 {{.*#+}} ymm21 = [65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
10978 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} ymm2 = [128,128,128,128,128,128,128,128,128,128,4,5,0,1,12,13,24,25,20,21,128,128,128,128,128,128,128,128,128,128,128,128]
10979 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm2, %ymm5, %ymm4
10980 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm5, %ymm31
10981 ; AVX512DQ-SLOW-NEXT: vpternlogq $236, %ymm21, %ymm4, %ymm0
10982 ; AVX512DQ-SLOW-NEXT: movw $31, %ax
10983 ; AVX512DQ-SLOW-NEXT: kmovw %eax, %k1
10984 ; AVX512DQ-SLOW-NEXT: vinserti32x8 $0, %ymm0, %zmm0, %zmm22 {%k1}
10985 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm26, %ymm0
10986 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm29, %ymm4
10987 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm0[0],ymm4[1],ymm0[2,3,4,5],ymm4[6],ymm0[7]
10988 ; AVX512DQ-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
10989 ; AVX512DQ-SLOW-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
10990 ; AVX512DQ-SLOW-NEXT: # ymm0 = ymm0[0],mem[1],ymm0[2,3],mem[4],ymm0[5,6],mem[7]
10991 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm6
10992 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[0,3,2,1]
10993 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm4[0,1,0,2,4,5,6,7]
10994 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm4, %xmm29
10995 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,6,6,6]
10996 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm11, %xmm6, %xmm4
10997 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm6, %xmm16
10998 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm4[4],xmm0[5],xmm4[6,7]
10999 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
11000 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} ymm6 = [8,9,4,5,4,5,6,7,0,1,4,5,0,1,12,13,24,25,20,21,20,21,22,23,16,17,20,21,16,17,28,29]
11001 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm6, %ymm5, %ymm4
11002 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm6, %ymm23
11003 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm5, %ymm17
11004 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4],ymm0[5,6,7]
11005 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm24
11006 ; AVX512DQ-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
11007 ; AVX512DQ-SLOW-NEXT: vpblendd $66, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
11008 ; AVX512DQ-SLOW-NEXT: # ymm14 = ymm0[0],mem[1],ymm0[2,3,4,5],mem[6],ymm0[7]
11009 ; AVX512DQ-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
11010 ; AVX512DQ-SLOW-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
11011 ; AVX512DQ-SLOW-NEXT: # ymm15 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
11012 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm1, %xmm15, %xmm0
11013 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm15, %xmm12
11014 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm12[2,2,2,2,4,5,6,7]
11015 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm4 = xmm0[0],xmm4[1],xmm0[2,3],xmm4[4],xmm0[5,6,7]
11016 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm2, %ymm14, %ymm0
11017 ; AVX512DQ-SLOW-NEXT: vpternlogq $236, %ymm21, %ymm0, %ymm4
11018 ; AVX512DQ-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
11019 ; AVX512DQ-SLOW-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm6 # 32-byte Folded Reload
11020 ; AVX512DQ-SLOW-NEXT: # ymm6 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
11021 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm1, %xmm6, %xmm1
11022 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm6, %xmm10
11023 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm10[2,2,2,2,4,5,6,7]
11024 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2,3],xmm2[4],xmm1[5,6,7]
11025 ; AVX512DQ-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
11026 ; AVX512DQ-SLOW-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
11027 ; AVX512DQ-SLOW-NEXT: # ymm2 = ymm0[0],mem[1],ymm0[2,3],mem[4],ymm0[5,6],mem[7]
11028 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm9
11029 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm2[0,3,2,1]
11030 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm7[0,1,0,2,4,5,6,7]
11031 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,6,6,6]
11032 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm11, %xmm9, %xmm5
11033 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm5[4],xmm2[5],xmm5[6,7]
11034 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
11035 ; AVX512DQ-SLOW-NEXT: vinserti32x4 $2, %xmm1, %zmm2, %zmm25
11036 ; AVX512DQ-SLOW-NEXT: vinserti32x8 $0, %ymm4, %zmm0, %zmm25 {%k1}
11037 ; AVX512DQ-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
11038 ; AVX512DQ-SLOW-NEXT: vpblendd $66, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
11039 ; AVX512DQ-SLOW-NEXT: # ymm5 = ymm0[0],mem[1],ymm0[2,3,4,5],mem[6],ymm0[7]
11040 ; AVX512DQ-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
11041 ; AVX512DQ-SLOW-NEXT: vpblendd $109, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
11042 ; AVX512DQ-SLOW-NEXT: # ymm1 = mem[0],ymm0[1],mem[2,3],ymm0[4],mem[5,6],ymm0[7]
11043 ; AVX512DQ-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm4
11044 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm11, %xmm4, %xmm11
11045 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[0,3,2,1]
11046 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm2[0,1,0,2,4,5,6,7]
11047 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,6,6,6]
11048 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm11[4],xmm1[5],xmm11[6,7]
11049 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm23, %ymm0
11050 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm0, %ymm5, %ymm11
11051 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
11052 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3,4],ymm1[5,6,7]
11053 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm23
11054 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} xmm1 = <10,11,u,u,2,3,14,15,u,u,10,11,12,13,14,15>
11055 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm1, %xmm3, %xmm3
11056 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm13[1,1,2,3]
11057 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,5,5,5,5]
11058 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm11 = xmm3[0],xmm11[1],xmm3[2,3],xmm11[4],xmm3[5,6,7]
11059 ; AVX512DQ-SLOW-NEXT: vpbroadcastq {{.*#+}} xmm3 = [10,11,0,0,2,3,14,15,10,11,0,0,2,3,14,15]
11060 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm18, %xmm0
11061 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm3, %xmm0, %xmm0
11062 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm19, %xmm13
11063 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm13 = xmm13[0,1,1,3,4,5,6,7]
11064 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[0,1,3,3]
11065 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm13[0,1,2,3],xmm0[4],xmm13[5],xmm0[6,7]
11066 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
11067 ; AVX512DQ-SLOW-NEXT: vinserti32x4 $2, %xmm11, %zmm0, %zmm26
11068 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm1, %xmm8, %xmm8
11069 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm30[1,1,2,3]
11070 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,5,5,5,5]
11071 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0],xmm11[1],xmm8[2,3],xmm11[4],xmm8[5,6,7]
11072 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} ymm11 = [128,128,128,128,128,128,128,128,128,128,6,7,2,3,14,15,26,27,22,23,128,128,128,128,128,128,128,128,128,128,128,128]
11073 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm31, %ymm0
11074 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm11, %ymm0, %ymm13
11075 ; AVX512DQ-SLOW-NEXT: vpternlogq $236, %ymm21, %ymm13, %ymm8
11076 ; AVX512DQ-SLOW-NEXT: vinserti32x8 $0, %ymm8, %zmm0, %zmm26 {%k1}
11077 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm16, %xmm0
11078 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm3, %xmm0, %xmm8
11079 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %xmm29, %xmm0
11080 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm13 = xmm0[0,1,1,3,4,5,6,7]
11081 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[0,1,3,3]
11082 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm8 = xmm13[0,1,2,3],xmm8[4],xmm13[5],xmm8[6,7]
11083 ; AVX512DQ-SLOW-NEXT: vmovdqa {{.*#+}} ymm13 = [10,11,6,7,4,5,6,7,6,7,6,7,2,3,14,15,26,27,22,23,20,21,22,23,22,23,22,23,18,19,30,31]
11084 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm17, %ymm0
11085 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm13, %ymm0, %ymm0
11086 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
11087 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm8[5,6,7]
11088 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0
11089 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm11, %ymm14, %ymm8
11090 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm1, %xmm15, %xmm11
11091 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[1,1,2,3]
11092 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,5,5,5,5]
11093 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm11 = xmm11[0],xmm12[1],xmm11[2,3],xmm12[4],xmm11[5,6,7]
11094 ; AVX512DQ-SLOW-NEXT: vpternlogq $236, %ymm21, %ymm8, %ymm11
11095 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm1, %xmm6, %xmm1
11096 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm10[1,1,2,3]
11097 ; AVX512DQ-SLOW-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,5,5,5,5]
11098 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm6[1],xmm1[2,3],xmm6[4],xmm1[5,6,7]
11099 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm3, %xmm9, %xmm6
11100 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,1,3,4,5,6,7]
11101 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[0,1,3,3]
11102 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0,1,2,3],xmm6[4],xmm7[5],xmm6[6,7]
11103 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
11104 ; AVX512DQ-SLOW-NEXT: vinserti32x4 $2, %xmm1, %zmm6, %zmm1
11105 ; AVX512DQ-SLOW-NEXT: vinserti32x8 $0, %ymm11, %zmm0, %zmm1 {%k1}
11106 ; AVX512DQ-SLOW-NEXT: vpshufb %ymm13, %ymm5, %ymm5
11107 ; AVX512DQ-SLOW-NEXT: vpshufb %xmm3, %xmm4, %xmm3
11108 ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,1,3,4,5,6,7]
11109 ; AVX512DQ-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,3,3]
11110 ; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4],xmm2[5],xmm3[6,7]
11111 ; AVX512DQ-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
11112 ; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3,4],ymm2[5,6,7]
11113 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm2
11114 ; AVX512DQ-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
11115 ; AVX512DQ-SLOW-NEXT: vmovaps %zmm3, (%rsi)
11116 ; AVX512DQ-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
11117 ; AVX512DQ-SLOW-NEXT: vmovaps %zmm3, 64(%rsi)
11118 ; AVX512DQ-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
11119 ; AVX512DQ-SLOW-NEXT: vmovaps %zmm3, 64(%rdx)
11120 ; AVX512DQ-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
11121 ; AVX512DQ-SLOW-NEXT: vmovaps %zmm3, (%rdx)
11122 ; AVX512DQ-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0]
11123 ; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm22, %zmm3, %zmm24
11124 ; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm25, %zmm3, %zmm23
11125 ; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm26, %zmm3, %zmm0
11126 ; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm1, %zmm3, %zmm2
11127 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm28, 64(%rcx)
11128 ; AVX512DQ-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
11129 ; AVX512DQ-SLOW-NEXT: vmovaps %zmm1, (%rcx)
11130 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm20, 64(%r8)
11131 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm27, (%r8)
11132 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm23, 64(%r9)
11133 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm24, (%r9)
11134 ; AVX512DQ-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
11135 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm2, 64(%rax)
11136 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm0, (%rax)
11137 ; AVX512DQ-SLOW-NEXT: addq $840, %rsp # imm = 0x348
11138 ; AVX512DQ-SLOW-NEXT: vzeroupper
11139 ; AVX512DQ-SLOW-NEXT: retq
11140 ;
11141 ; AVX512DQ-FAST-LABEL: load_i16_stride6_vf64:
11142 ; AVX512DQ-FAST: # %bb.0:
11143 ; AVX512DQ-FAST-NEXT: subq $936, %rsp # imm = 0x3A8
11144 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} xmm9 = <0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15>
11145 ; AVX512DQ-FAST-NEXT: vmovdqa 608(%rdi), %ymm0
11146 ; AVX512DQ-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11147 ; AVX512DQ-FAST-NEXT: vmovdqa 576(%rdi), %ymm1
11148 ; AVX512DQ-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11149 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]
11150 ; AVX512DQ-FAST-NEXT: vpshufb %xmm9, %xmm2, %xmm0
11151 ; AVX512DQ-FAST-NEXT: vpbroadcastq {{.*#+}} xmm11 = [8,9,12,13,0,1,0,0,8,9,12,13,0,1,0,0]
11152 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm2, %xmm1
11153 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm2, %ymm25
11154 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,1,0,3]
11155 ; AVX512DQ-FAST-NEXT: vpshufb %xmm11, %xmm2, %xmm1
11156 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm2, %xmm24
11157 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3],xmm1[4,5],xmm0[6,7]
11158 ; AVX512DQ-FAST-NEXT: vmovdqa 544(%rdi), %ymm1
11159 ; AVX512DQ-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11160 ; AVX512DQ-FAST-NEXT: vmovdqa 512(%rdi), %ymm2
11161 ; AVX512DQ-FAST-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11162 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
11163 ; AVX512DQ-FAST-NEXT: vpbroadcastd {{.*#+}} xmm13 = [8,9,4,5,8,9,4,5,8,9,4,5,8,9,4,5]
11164 ; AVX512DQ-FAST-NEXT: vpshufb %xmm13, %xmm2, %xmm1
11165 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm2, %xmm4
11166 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm2, %ymm23
11167 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} xmm3 = <0,1,4,5,4,5,u,u,0,1,12,13,u,u,4,5>
11168 ; AVX512DQ-FAST-NEXT: vpshufb %xmm3, %xmm4, %xmm2
11169 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm4, %xmm22
11170 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3],xmm2[4,5],xmm1[6],xmm2[7]
11171 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
11172 ; AVX512DQ-FAST-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
11173 ; AVX512DQ-FAST-NEXT: vmovdqa 416(%rdi), %ymm1
11174 ; AVX512DQ-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11175 ; AVX512DQ-FAST-NEXT: vmovdqa 384(%rdi), %ymm2
11176 ; AVX512DQ-FAST-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11177 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm15 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5,6],ymm1[7]
11178 ; AVX512DQ-FAST-NEXT: vpshufb %xmm9, %xmm15, %xmm1
11179 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm15, %xmm2
11180 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,1,0,3]
11181 ; AVX512DQ-FAST-NEXT: vpshufb %xmm11, %xmm4, %xmm2
11182 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm4, %xmm21
11183 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3],xmm2[4,5],xmm1[6,7]
11184 ; AVX512DQ-FAST-NEXT: vmovdqa 448(%rdi), %ymm2
11185 ; AVX512DQ-FAST-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm2[2,3],mem[2,3]
11186 ; AVX512DQ-FAST-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11187 ; AVX512DQ-FAST-NEXT: vinserti128 $1, 480(%rdi), %ymm2, %ymm2
11188 ; AVX512DQ-FAST-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11189 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm4 = ymm2[0],ymm4[1],ymm2[2,3,4,5],ymm4[6],ymm2[7]
11190 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} ymm6 = [0,1,12,13,8,9,12,13,8,9,12,13,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
11191 ; AVX512DQ-FAST-NEXT: vpshufb %ymm6, %ymm4, %ymm2
11192 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm4, %ymm20
11193 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm4 = ymm1[0,1,2],ymm2[3,4,5,6,7]
11194 ; AVX512DQ-FAST-NEXT: vmovdqa 736(%rdi), %ymm1
11195 ; AVX512DQ-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11196 ; AVX512DQ-FAST-NEXT: vmovdqa 704(%rdi), %ymm2
11197 ; AVX512DQ-FAST-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11198 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm14 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
11199 ; AVX512DQ-FAST-NEXT: vpshufb %xmm13, %xmm14, %xmm1
11200 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm14, %xmm5
11201 ; AVX512DQ-FAST-NEXT: vpshufb %xmm3, %xmm5, %xmm2
11202 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm5, %xmm19
11203 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3],xmm2[4,5],xmm1[6],xmm2[7]
11204 ; AVX512DQ-FAST-NEXT: vmovdqa 640(%rdi), %ymm2
11205 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
11206 ; AVX512DQ-FAST-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm2[2,3],mem[2,3]
11207 ; AVX512DQ-FAST-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11208 ; AVX512DQ-FAST-NEXT: vinserti128 $1, 672(%rdi), %ymm2, %ymm2
11209 ; AVX512DQ-FAST-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11210 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm5 = ymm2[0],ymm5[1],ymm2[2,3,4,5],ymm5[6],ymm2[7]
11211 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} ymm7 = <0,1,12,13,8,9,12,13,8,9,12,13,4,5,u,u,16,17,28,29,24,25,28,29,24,25,28,29,20,21,u,u>
11212 ; AVX512DQ-FAST-NEXT: vpshufb %ymm7, %ymm5, %ymm2
11213 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm7, %ymm27
11214 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm5, %ymm18
11215 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7],ymm2[8,9,10],ymm1[11,12,13,14,15]
11216 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,6]
11217 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
11218 ; AVX512DQ-FAST-NEXT: vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
11219 ; AVX512DQ-FAST-NEXT: vpternlogq $226, %zmm0, %zmm16, %zmm4
11220 ; AVX512DQ-FAST-NEXT: movw $-2048, %ax # imm = 0xF800
11221 ; AVX512DQ-FAST-NEXT: kmovw %eax, %k1
11222 ; AVX512DQ-FAST-NEXT: vinserti32x8 $1, %ymm1, %zmm0, %zmm4 {%k1}
11223 ; AVX512DQ-FAST-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
11224 ; AVX512DQ-FAST-NEXT: vmovdqa 224(%rdi), %ymm0
11225 ; AVX512DQ-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11226 ; AVX512DQ-FAST-NEXT: vmovdqa 192(%rdi), %ymm1
11227 ; AVX512DQ-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11228 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm12 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]
11229 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm12, %xmm0
11230 ; AVX512DQ-FAST-NEXT: vpshufb %xmm9, %xmm12, %xmm1
11231 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm7 = xmm0[2,1,0,3]
11232 ; AVX512DQ-FAST-NEXT: vpshufb %xmm11, %xmm7, %xmm0
11233 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3],xmm0[4,5],xmm1[6,7]
11234 ; AVX512DQ-FAST-NEXT: vmovdqa 160(%rdi), %ymm1
11235 ; AVX512DQ-FAST-NEXT: vmovdqa 128(%rdi), %ymm2
11236 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm5 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
11237 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm2, %ymm30
11238 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm1, %ymm31
11239 ; AVX512DQ-FAST-NEXT: vpshufb %xmm13, %xmm5, %xmm1
11240 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm5, %xmm10
11241 ; AVX512DQ-FAST-NEXT: vpshufb %xmm3, %xmm10, %xmm4
11242 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm4[0,1,2],xmm1[3],xmm4[4,5],xmm1[6],xmm4[7]
11243 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
11244 ; AVX512DQ-FAST-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm17
11245 ; AVX512DQ-FAST-NEXT: vmovdqa (%rdi), %ymm0
11246 ; AVX512DQ-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11247 ; AVX512DQ-FAST-NEXT: vmovdqa 32(%rdi), %ymm1
11248 ; AVX512DQ-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11249 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
11250 ; AVX512DQ-FAST-NEXT: vpshufb %xmm9, %xmm2, %xmm0
11251 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm2, %xmm4
11252 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm9 = xmm4[2,1,0,3]
11253 ; AVX512DQ-FAST-NEXT: vpshufb %xmm11, %xmm9, %xmm4
11254 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm4[2],xmm0[3],xmm4[4,5],xmm0[6,7]
11255 ; AVX512DQ-FAST-NEXT: vmovdqa 64(%rdi), %ymm4
11256 ; AVX512DQ-FAST-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm4[2,3],mem[2,3]
11257 ; AVX512DQ-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11258 ; AVX512DQ-FAST-NEXT: vinserti128 $1, 96(%rdi), %ymm4, %ymm4
11259 ; AVX512DQ-FAST-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11260 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm11 = ymm4[0],ymm1[1],ymm4[2,3,4,5],ymm1[6],ymm4[7]
11261 ; AVX512DQ-FAST-NEXT: vpshufb %ymm6, %ymm11, %ymm4
11262 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm6 = ymm0[0,1,2],ymm4[3,4,5,6,7]
11263 ; AVX512DQ-FAST-NEXT: vmovdqa 352(%rdi), %ymm0
11264 ; AVX512DQ-FAST-NEXT: vmovdqu %ymm0, (%rsp) # 32-byte Spill
11265 ; AVX512DQ-FAST-NEXT: vmovdqa 320(%rdi), %ymm1
11266 ; AVX512DQ-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11267 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm8 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
11268 ; AVX512DQ-FAST-NEXT: vpshufb %xmm13, %xmm8, %xmm13
11269 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm8, %xmm4
11270 ; AVX512DQ-FAST-NEXT: vpshufb %xmm3, %xmm4, %xmm3
11271 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm13 = xmm3[0,1,2],xmm13[3],xmm3[4,5],xmm13[6],xmm3[7]
11272 ; AVX512DQ-FAST-NEXT: vmovdqa 256(%rdi), %ymm3
11273 ; AVX512DQ-FAST-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm3[2,3],mem[2,3]
11274 ; AVX512DQ-FAST-NEXT: vinserti128 $1, 288(%rdi), %ymm3, %ymm0
11275 ; AVX512DQ-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
11276 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm0[0],ymm1[1],ymm0[2,3,4,5],ymm1[6],ymm0[7]
11277 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm1, %ymm26
11278 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm27, %ymm0
11279 ; AVX512DQ-FAST-NEXT: vpshufb %ymm0, %ymm3, %ymm0
11280 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
11281 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} ymm13 = ymm0[0,1,2],ymm13[3,4,5,6,7],ymm0[8,9,10],ymm13[11,12,13,14,15]
11282 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,4,6]
11283 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm13[4,5,6,7]
11284 ; AVX512DQ-FAST-NEXT: vpternlogq $226, %zmm17, %zmm16, %zmm6
11285 ; AVX512DQ-FAST-NEXT: vinserti32x8 $1, %ymm0, %zmm0, %zmm6 {%k1}
11286 ; AVX512DQ-FAST-NEXT: vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
11287 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} xmm13 = <2,3,14,15,u,u,6,7,u,u,u,u,12,13,14,15>
11288 ; AVX512DQ-FAST-NEXT: vpshufb %xmm13, %xmm12, %xmm0
11289 ; AVX512DQ-FAST-NEXT: vpbroadcastq {{.*#+}} xmm12 = [10,11,14,15,2,3,0,0,10,11,14,15,2,3,0,0]
11290 ; AVX512DQ-FAST-NEXT: vpshufb %xmm12, %xmm7, %xmm7
11291 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm7 = xmm0[0,1],xmm7[2],xmm0[3],xmm7[4,5],xmm0[6,7]
11292 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} xmm0 = <6,7,2,3,4,5,u,u,2,3,14,15,u,u,6,7>
11293 ; AVX512DQ-FAST-NEXT: vpshufb %xmm0, %xmm10, %xmm6
11294 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,5,5]
11295 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3],xmm6[4,5],xmm5[6],xmm6[7]
11296 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
11297 ; AVX512DQ-FAST-NEXT: vinserti32x4 $2, %xmm7, %zmm5, %zmm5
11298 ; AVX512DQ-FAST-NEXT: vpshufb %xmm13, %xmm2, %xmm2
11299 ; AVX512DQ-FAST-NEXT: vpshufb %xmm12, %xmm9, %xmm6
11300 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm6[2],xmm2[3],xmm6[4,5],xmm2[6,7]
11301 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} ymm6 = [2,3,14,15,10,11,10,11,14,15,10,11,10,11,6,7,18,19,30,31,26,27,26,27,30,31,26,27,26,27,22,23]
11302 ; AVX512DQ-FAST-NEXT: vpshufb %ymm6, %ymm11, %ymm7
11303 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm7 = ymm2[0,1,2],ymm7[3,4,5,6,7]
11304 ; AVX512DQ-FAST-NEXT: vpshufb %xmm0, %xmm4, %xmm2
11305 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm1 = xmm8[0,1,2,3,5,5,5,5]
11306 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3],xmm2[4,5],xmm1[6],xmm2[7]
11307 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} ymm2 = <2,3,14,15,10,11,10,11,14,15,10,11,u,u,6,7,18,19,30,31,26,27,26,27,30,31,26,27,u,u,22,23>
11308 ; AVX512DQ-FAST-NEXT: vpshufb %ymm2, %ymm3, %ymm3
11309 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
11310 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3,4,5,6,7],ymm3[8,9,10],ymm1[11,12,13,14,15]
11311 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,5,7]
11312 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
11313 ; AVX512DQ-FAST-NEXT: vpternlogq $226, %zmm5, %zmm16, %zmm7
11314 ; AVX512DQ-FAST-NEXT: vinserti32x8 $1, %ymm1, %zmm0, %zmm7 {%k1}
11315 ; AVX512DQ-FAST-NEXT: vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
11316 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm25, %ymm1
11317 ; AVX512DQ-FAST-NEXT: vpshufb %xmm13, %xmm1, %xmm1
11318 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm24, %xmm3
11319 ; AVX512DQ-FAST-NEXT: vpshufb %xmm12, %xmm3, %xmm3
11320 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2],xmm1[3],xmm3[4,5],xmm1[6,7]
11321 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm22, %xmm3
11322 ; AVX512DQ-FAST-NEXT: vpshufb %xmm0, %xmm3, %xmm3
11323 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm23, %ymm4
11324 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,5,5,5]
11325 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[3],xmm3[4,5],xmm4[6],xmm3[7]
11326 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
11327 ; AVX512DQ-FAST-NEXT: vinserti32x4 $2, %xmm1, %zmm3, %zmm1
11328 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm20, %ymm3
11329 ; AVX512DQ-FAST-NEXT: vpshufb %ymm6, %ymm3, %ymm3
11330 ; AVX512DQ-FAST-NEXT: vpshufb %xmm13, %xmm15, %xmm4
11331 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm21, %xmm5
11332 ; AVX512DQ-FAST-NEXT: vpshufb %xmm12, %xmm5, %xmm5
11333 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2],xmm4[3],xmm5[4,5],xmm4[6,7]
11334 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2],ymm3[3,4,5,6,7]
11335 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm18, %ymm3
11336 ; AVX512DQ-FAST-NEXT: vpshufb %ymm2, %ymm3, %ymm2
11337 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm19, %xmm3
11338 ; AVX512DQ-FAST-NEXT: vpshufb %xmm0, %xmm3, %xmm0
11339 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm3 = xmm14[0,1,2,3,5,5,5,5]
11340 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[3],xmm0[4,5],xmm3[6],xmm0[7]
11341 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
11342 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7],ymm2[8,9,10],ymm0[11,12,13,14,15]
11343 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,5,7]
11344 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
11345 ; AVX512DQ-FAST-NEXT: vpternlogq $226, %zmm1, %zmm16, %zmm4
11346 ; AVX512DQ-FAST-NEXT: vinserti32x8 $1, %ymm0, %zmm0, %zmm4 {%k1}
11347 ; AVX512DQ-FAST-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
11348 ; AVX512DQ-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
11349 ; AVX512DQ-FAST-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
11350 ; AVX512DQ-FAST-NEXT: # ymm0 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
11351 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm0, %xmm1
11352 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm13 = xmm0[2,1,2,3]
11353 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} xmm0 = <u,u,0,1,4,5,u,u,12,13,12,13,12,13,12,13>
11354 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm14 = xmm1[0,3,2,1]
11355 ; AVX512DQ-FAST-NEXT: vpshufb %xmm0, %xmm14, %xmm1
11356 ; AVX512DQ-FAST-NEXT: vpshuflw {{.*#+}} xmm2 = xmm13[2,1,2,0,4,5,6,7]
11357 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2],xmm2[3],xmm1[4,5,6,7]
11358 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm30, %ymm2
11359 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm31, %ymm3
11360 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1],ymm2[2],ymm3[3,4],ymm2[5],ymm3[6,7]
11361 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm2, %xmm3
11362 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,1,0,3]
11363 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} xmm11 = <0,1,0,1,0,1,0,1,u,u,8,9,12,13,u,u>
11364 ; AVX512DQ-FAST-NEXT: vpshufb %xmm11, %xmm4, %xmm2
11365 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm4, %xmm20
11366 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[0,1,2,1]
11367 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm3 = xmm4[0,1,2,3,6,5,6,4]
11368 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm4, %xmm22
11369 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4],xmm2[5,6],xmm3[7]
11370 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
11371 ; AVX512DQ-FAST-NEXT: vinserti32x4 $2, %xmm1, %zmm2, %zmm4
11372 ; AVX512DQ-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
11373 ; AVX512DQ-FAST-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
11374 ; AVX512DQ-FAST-NEXT: # ymm1 = mem[0,1],ymm1[2],mem[3,4],ymm1[5],mem[6,7]
11375 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
11376 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,1,2,3]
11377 ; AVX512DQ-FAST-NEXT: vpbroadcastq {{.*#+}} xmm3 = [12,13,0,1,4,5,0,0,12,13,0,1,4,5,0,0]
11378 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,3,2,1]
11379 ; AVX512DQ-FAST-NEXT: vpshufb %xmm3, %xmm2, %xmm1
11380 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm2, %xmm24
11381 ; AVX512DQ-FAST-NEXT: vpshuflw {{.*#+}} xmm2 = xmm5[2,1,2,0,4,5,6,7]
11382 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm5, %xmm23
11383 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2],xmm2[3],xmm1[4,5,6,7]
11384 ; AVX512DQ-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
11385 ; AVX512DQ-FAST-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm6 # 32-byte Folded Reload
11386 ; AVX512DQ-FAST-NEXT: # ymm6 = mem[0,1],ymm2[2],mem[3],ymm2[4],mem[5,6],ymm2[7]
11387 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} ymm2 = [4,5,0,1,12,13,14,15,8,9,0,1,12,13,8,9,20,21,16,17,28,29,30,31,24,25,16,17,28,29,24,25]
11388 ; AVX512DQ-FAST-NEXT: vpshufb %ymm2, %ymm6, %ymm5
11389 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm6, %ymm25
11390 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm5[5,6,7]
11391 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm5 = ymm1[0,1,2,3],ymm5[4,5,6,7]
11392 ; AVX512DQ-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
11393 ; AVX512DQ-FAST-NEXT: vpblendd $219, (%rsp), %ymm1, %ymm1 # 32-byte Folded Reload
11394 ; AVX512DQ-FAST-NEXT: # ymm1 = mem[0,1],ymm1[2],mem[3,4],ymm1[5],mem[6,7]
11395 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm1, %xmm6
11396 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm7 = xmm1[2,1,0,3]
11397 ; AVX512DQ-FAST-NEXT: vpshufb %xmm11, %xmm7, %xmm1
11398 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm7, %xmm18
11399 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[0,1,2,1]
11400 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm6 = xmm7[0,1,2,3,6,5,6,4]
11401 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm7, %xmm17
11402 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm6[4],xmm1[5,6],xmm6[7]
11403 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm6
11404 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm26, %ymm1
11405 ; AVX512DQ-FAST-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
11406 ; AVX512DQ-FAST-NEXT: # ymm1 = mem[0,1],ymm1[2],mem[3],ymm1[4],mem[5,6],ymm1[7]
11407 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} ymm8 = <4,5,0,1,12,13,14,15,8,9,12,13,0,1,u,u,20,21,16,17,28,29,30,31,24,25,28,29,16,17,u,u>
11408 ; AVX512DQ-FAST-NEXT: vpshufb %ymm8, %ymm1, %ymm7
11409 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm8, %ymm28
11410 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm1, %ymm16
11411 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} ymm6 = ymm7[0,1,2],ymm6[3,4,5,6,7],ymm7[8,9,10],ymm6[11,12,13,14,15]
11412 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,6,5,4]
11413 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7]
11414 ; AVX512DQ-FAST-NEXT: vinserti64x4 $1, %ymm6, %zmm0, %zmm1
11415 ; AVX512DQ-FAST-NEXT: vmovdqa64 {{.*#+}} zmm21 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
11416 ; AVX512DQ-FAST-NEXT: vpternlogq $226, %zmm4, %zmm21, %zmm5
11417 ; AVX512DQ-FAST-NEXT: vmovdqa64 {{.*#+}} zmm4 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0]
11418 ; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm5, %zmm4, %zmm1
11419 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm4, %zmm27
11420 ; AVX512DQ-FAST-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
11421 ; AVX512DQ-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
11422 ; AVX512DQ-FAST-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm4 # 32-byte Folded Reload
11423 ; AVX512DQ-FAST-NEXT: # ymm4 = ymm1[0,1],mem[2],ymm1[3,4],mem[5],ymm1[6,7]
11424 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm4, %xmm5
11425 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm12 = xmm5[0,3,2,1]
11426 ; AVX512DQ-FAST-NEXT: vpshufb %xmm0, %xmm12, %xmm0
11427 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm10 = xmm4[2,1,2,3]
11428 ; AVX512DQ-FAST-NEXT: vpshuflw {{.*#+}} xmm4 = xmm10[2,1,2,0,4,5,6,7]
11429 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm4[0],xmm0[1,2],xmm4[3],xmm0[4,5,6,7]
11430 ; AVX512DQ-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
11431 ; AVX512DQ-FAST-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm4 # 32-byte Folded Reload
11432 ; AVX512DQ-FAST-NEXT: # ymm4 = ymm1[0,1],mem[2],ymm1[3,4],mem[5],ymm1[6,7]
11433 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm4, %xmm5
11434 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm8 = xmm4[2,1,0,3]
11435 ; AVX512DQ-FAST-NEXT: vpshufb %xmm11, %xmm8, %xmm4
11436 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm9 = xmm5[0,1,2,1]
11437 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm5 = xmm9[0,1,2,3,6,5,6,4]
11438 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm5[4],xmm4[5,6],xmm5[7]
11439 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
11440 ; AVX512DQ-FAST-NEXT: vinserti32x4 $2, %xmm0, %zmm4, %zmm19
11441 ; AVX512DQ-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
11442 ; AVX512DQ-FAST-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
11443 ; AVX512DQ-FAST-NEXT: # ymm0 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
11444 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm0, %xmm4
11445 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[0,3,2,1]
11446 ; AVX512DQ-FAST-NEXT: vpshufb %xmm3, %xmm7, %xmm3
11447 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm6 = xmm0[2,1,2,3]
11448 ; AVX512DQ-FAST-NEXT: vpshuflw {{.*#+}} xmm0 = xmm6[2,1,2,0,4,5,6,7]
11449 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1,2],xmm0[3],xmm3[4,5,6,7]
11450 ; AVX512DQ-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
11451 ; AVX512DQ-FAST-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm5 # 32-byte Folded Reload
11452 ; AVX512DQ-FAST-NEXT: # ymm5 = mem[0,1],ymm1[2],mem[3],ymm1[4],mem[5,6],ymm1[7]
11453 ; AVX512DQ-FAST-NEXT: vpshufb %ymm2, %ymm5, %ymm2
11454 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm2[5,6,7]
11455 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
11456 ; AVX512DQ-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
11457 ; AVX512DQ-FAST-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
11458 ; AVX512DQ-FAST-NEXT: # ymm1 = ymm1[0,1],mem[2],ymm1[3,4],mem[5],ymm1[6,7]
11459 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm1, %xmm3
11460 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[2,1,0,3]
11461 ; AVX512DQ-FAST-NEXT: vpshufb %xmm11, %xmm4, %xmm1
11462 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,1,2,1]
11463 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm15 = xmm3[0,1,2,3,6,5,6,4]
11464 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm15 = xmm1[0,1,2,3],xmm15[4],xmm1[5,6],xmm15[7]
11465 ; AVX512DQ-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
11466 ; AVX512DQ-FAST-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
11467 ; AVX512DQ-FAST-NEXT: # ymm1 = mem[0,1],ymm1[2],mem[3],ymm1[4],mem[5,6],ymm1[7]
11468 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm28, %ymm2
11469 ; AVX512DQ-FAST-NEXT: vpshufb %ymm2, %ymm1, %ymm11
11470 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
11471 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} ymm15 = ymm11[0,1,2],ymm15[3,4,5,6,7],ymm11[8,9,10],ymm15[11,12,13,14,15]
11472 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,4,6,5,4]
11473 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm15[4,5,6,7]
11474 ; AVX512DQ-FAST-NEXT: vinserti64x4 $1, %ymm11, %zmm0, %zmm2
11475 ; AVX512DQ-FAST-NEXT: vpternlogq $226, %zmm19, %zmm21, %zmm0
11476 ; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm0, %zmm27, %zmm2
11477 ; AVX512DQ-FAST-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
11478 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} xmm2 = <u,u,2,3,6,7,u,u,14,15,14,15,14,15,14,15>
11479 ; AVX512DQ-FAST-NEXT: vpshufb %xmm2, %xmm14, %xmm0
11480 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm2, %xmm19
11481 ; AVX512DQ-FAST-NEXT: vpshuflw {{.*#+}} xmm11 = xmm13[3,1,2,1,4,5,6,7]
11482 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm11 = xmm11[0],xmm0[1,2],xmm11[3],xmm0[4,5,6,7]
11483 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} xmm2 = <2,3,2,3,2,3,2,3,u,u,10,11,14,15,u,u>
11484 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm20, %xmm0
11485 ; AVX512DQ-FAST-NEXT: vpshufb %xmm2, %xmm0, %xmm15
11486 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm22, %xmm0
11487 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm14 = xmm0[0,1,2,3,7,5,6,5]
11488 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm14 = xmm15[0,1,2,3],xmm14[4],xmm15[5,6],xmm14[7]
11489 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
11490 ; AVX512DQ-FAST-NEXT: vinserti32x4 $2, %xmm11, %zmm14, %zmm20
11491 ; AVX512DQ-FAST-NEXT: vpbroadcastq {{.*#+}} xmm11 = [14,15,2,3,6,7,0,0,14,15,2,3,6,7,0,0]
11492 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm24, %xmm0
11493 ; AVX512DQ-FAST-NEXT: vpshufb %xmm11, %xmm0, %xmm15
11494 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm11, %xmm22
11495 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm23, %xmm0
11496 ; AVX512DQ-FAST-NEXT: vpshuflw {{.*#+}} xmm13 = xmm0[3,1,2,1,4,5,6,7]
11497 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0],xmm15[1,2],xmm13[3],xmm15[4,5,6,7]
11498 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} ymm15 = [6,7,2,3,14,15,14,15,14,15,2,3,14,15,10,11,22,23,18,19,30,31,30,31,30,31,18,19,30,31,26,27]
11499 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm25, %ymm0
11500 ; AVX512DQ-FAST-NEXT: vpshufb %ymm15, %ymm0, %ymm0
11501 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0,1,2,3,4],xmm0[5,6,7]
11502 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3],ymm0[4,5,6,7]
11503 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm18, %xmm11
11504 ; AVX512DQ-FAST-NEXT: vpshufb %xmm2, %xmm11, %xmm13
11505 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm17, %xmm11
11506 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,7,5,6,5]
11507 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm11 = xmm13[0,1,2,3],xmm11[4],xmm13[5,6],xmm11[7]
11508 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} ymm13 = <6,7,2,3,14,15,14,15,14,15,10,11,u,u,2,3,22,23,18,19,30,31,30,31,30,31,26,27,u,u,18,19>
11509 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm16, %ymm14
11510 ; AVX512DQ-FAST-NEXT: vpshufb %ymm13, %ymm14, %ymm14
11511 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
11512 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} ymm11 = ymm14[0,1,2],ymm11[3,4,5,6,7],ymm14[8,9,10],ymm11[11,12,13,14,15]
11513 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,4,7,4,5]
11514 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm11 = ymm14[0,1,2,3],ymm11[4,5,6,7]
11515 ; AVX512DQ-FAST-NEXT: vinserti64x4 $1, %ymm11, %zmm0, %zmm29
11516 ; AVX512DQ-FAST-NEXT: vpternlogq $226, %zmm20, %zmm21, %zmm0
11517 ; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm0, %zmm27, %zmm29
11518 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm19, %xmm0
11519 ; AVX512DQ-FAST-NEXT: vpshufb %xmm0, %xmm12, %xmm0
11520 ; AVX512DQ-FAST-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[3,1,2,1,4,5,6,7]
11521 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm10[0],xmm0[1,2],xmm10[3],xmm0[4,5,6,7]
11522 ; AVX512DQ-FAST-NEXT: vpshufb %xmm2, %xmm8, %xmm8
11523 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,7,5,6,5]
11524 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2,3],xmm9[4],xmm8[5,6],xmm9[7]
11525 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
11526 ; AVX512DQ-FAST-NEXT: vinserti32x4 $2, %xmm0, %zmm8, %zmm0
11527 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm22, %xmm8
11528 ; AVX512DQ-FAST-NEXT: vpshufb %xmm8, %xmm7, %xmm7
11529 ; AVX512DQ-FAST-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[3,1,2,1,4,5,6,7]
11530 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0],xmm7[1,2],xmm6[3],xmm7[4,5,6,7]
11531 ; AVX512DQ-FAST-NEXT: vpshufb %ymm15, %ymm5, %ymm5
11532 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm5[5,6,7]
11533 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3],ymm5[4,5,6,7]
11534 ; AVX512DQ-FAST-NEXT: vpshufb %ymm13, %ymm1, %ymm1
11535 ; AVX512DQ-FAST-NEXT: vpshufb %xmm2, %xmm4, %xmm2
11536 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,5]
11537 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4],xmm2[5,6],xmm3[7]
11538 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
11539 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0,1,2],ymm2[3,4,5,6,7],ymm1[8,9,10],ymm2[11,12,13,14,15]
11540 ; AVX512DQ-FAST-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,4,5]
11541 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
11542 ; AVX512DQ-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm20
11543 ; AVX512DQ-FAST-NEXT: vpternlogq $226, %zmm0, %zmm21, %zmm5
11544 ; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm5, %zmm27, %zmm20
11545 ; AVX512DQ-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
11546 ; AVX512DQ-FAST-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm10 # 32-byte Folded Reload
11547 ; AVX512DQ-FAST-NEXT: # ymm10 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
11548 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} xmm1 = <8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15>
11549 ; AVX512DQ-FAST-NEXT: vpshufb %xmm1, %xmm10, %xmm0
11550 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm10, %xmm15
11551 ; AVX512DQ-FAST-NEXT: vpshuflw {{.*#+}} xmm2 = xmm15[2,2,2,2,4,5,6,7]
11552 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3],xmm2[4],xmm0[5,6,7]
11553 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm30, %ymm2
11554 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm31, %ymm3
11555 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2,3],ymm3[4],ymm2[5,6],ymm3[7]
11556 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm2, %xmm3
11557 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[0,3,2,1]
11558 ; AVX512DQ-FAST-NEXT: vpbroadcastq {{.*#+}} xmm4 = [8,9,0,0,0,1,12,13,8,9,0,0,0,1,12,13]
11559 ; AVX512DQ-FAST-NEXT: vpshufb %xmm4, %xmm3, %xmm2
11560 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm3, %xmm31
11561 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} xmm9 = <0,1,2,3,0,1,4,5,u,u,12,13,u,u,u,u>
11562 ; AVX512DQ-FAST-NEXT: vpshufb %xmm9, %xmm5, %xmm3
11563 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm5, %xmm30
11564 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4],xmm3[5],xmm2[6,7]
11565 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
11566 ; AVX512DQ-FAST-NEXT: vinserti32x4 $2, %xmm0, %zmm2, %zmm22
11567 ; AVX512DQ-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
11568 ; AVX512DQ-FAST-NEXT: vpblendd $66, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
11569 ; AVX512DQ-FAST-NEXT: # ymm5 = ymm0[0],mem[1],ymm0[2,3,4,5],mem[6],ymm0[7]
11570 ; AVX512DQ-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
11571 ; AVX512DQ-FAST-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm11 # 32-byte Folded Reload
11572 ; AVX512DQ-FAST-NEXT: # ymm11 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
11573 ; AVX512DQ-FAST-NEXT: vpshufb %xmm1, %xmm11, %xmm0
11574 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm11, %xmm13
11575 ; AVX512DQ-FAST-NEXT: vpshuflw {{.*#+}} xmm2 = xmm13[2,2,2,2,4,5,6,7]
11576 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3],xmm2[4],xmm0[5,6,7]
11577 ; AVX512DQ-FAST-NEXT: vmovdqa64 {{.*#+}} ymm28 = [65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
11578 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} ymm2 = [128,128,128,128,128,128,128,128,128,128,4,5,0,1,12,13,24,25,20,21,128,128,128,128,128,128,128,128,128,128,128,128]
11579 ; AVX512DQ-FAST-NEXT: vpshufb %ymm2, %ymm5, %ymm3
11580 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm5, %ymm18
11581 ; AVX512DQ-FAST-NEXT: vpternlogq $236, %ymm28, %ymm3, %ymm0
11582 ; AVX512DQ-FAST-NEXT: movw $31, %ax
11583 ; AVX512DQ-FAST-NEXT: kmovw %eax, %k1
11584 ; AVX512DQ-FAST-NEXT: vinserti32x8 $0, %ymm0, %zmm0, %zmm22 {%k1}
11585 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm26, %ymm0
11586 ; AVX512DQ-FAST-NEXT: vpblendd $66, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
11587 ; AVX512DQ-FAST-NEXT: # ymm5 = ymm0[0],mem[1],ymm0[2,3,4,5],mem[6],ymm0[7]
11588 ; AVX512DQ-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
11589 ; AVX512DQ-FAST-NEXT: vpblendd $146, (%rsp), %ymm0, %ymm0 # 32-byte Folded Reload
11590 ; AVX512DQ-FAST-NEXT: # ymm0 = ymm0[0],mem[1],ymm0[2,3],mem[4],ymm0[5,6],mem[7]
11591 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
11592 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm6 = xmm0[0,3,2,1]
11593 ; AVX512DQ-FAST-NEXT: vpshufb %xmm4, %xmm3, %xmm0
11594 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm3, %xmm19
11595 ; AVX512DQ-FAST-NEXT: vpshufb %xmm9, %xmm6, %xmm3
11596 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm6, %xmm27
11597 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4],xmm3[5],xmm0[6,7]
11598 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
11599 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} ymm6 = [8,9,4,5,4,5,6,7,0,1,4,5,0,1,12,13,24,25,20,21,20,21,22,23,16,17,20,21,16,17,28,29]
11600 ; AVX512DQ-FAST-NEXT: vpshufb %ymm6, %ymm5, %ymm3
11601 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm6, %ymm25
11602 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm5, %ymm21
11603 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4],ymm0[5,6,7]
11604 ; AVX512DQ-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm23
11605 ; AVX512DQ-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
11606 ; AVX512DQ-FAST-NEXT: vpblendd $66, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
11607 ; AVX512DQ-FAST-NEXT: # ymm5 = ymm0[0],mem[1],ymm0[2,3,4,5],mem[6],ymm0[7]
11608 ; AVX512DQ-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
11609 ; AVX512DQ-FAST-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
11610 ; AVX512DQ-FAST-NEXT: # ymm14 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
11611 ; AVX512DQ-FAST-NEXT: vpshufb %xmm1, %xmm14, %xmm0
11612 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm14, %xmm6
11613 ; AVX512DQ-FAST-NEXT: vpshuflw {{.*#+}} xmm3 = xmm6[2,2,2,2,4,5,6,7]
11614 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm6, %xmm17
11615 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0],xmm3[1],xmm0[2,3],xmm3[4],xmm0[5,6,7]
11616 ; AVX512DQ-FAST-NEXT: vpshufb %ymm2, %ymm5, %ymm0
11617 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm5, %ymm16
11618 ; AVX512DQ-FAST-NEXT: vpternlogq $236, %ymm28, %ymm0, %ymm3
11619 ; AVX512DQ-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
11620 ; AVX512DQ-FAST-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm6 # 32-byte Folded Reload
11621 ; AVX512DQ-FAST-NEXT: # ymm6 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
11622 ; AVX512DQ-FAST-NEXT: vpshufb %xmm1, %xmm6, %xmm1
11623 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm6, %xmm12
11624 ; AVX512DQ-FAST-NEXT: vpshuflw {{.*#+}} xmm2 = xmm12[2,2,2,2,4,5,6,7]
11625 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2,3],xmm2[4],xmm1[5,6,7]
11626 ; AVX512DQ-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
11627 ; AVX512DQ-FAST-NEXT: vpblendd $109, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
11628 ; AVX512DQ-FAST-NEXT: # ymm2 = mem[0],ymm0[1],mem[2,3],ymm0[4],mem[5,6],ymm0[7]
11629 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm2, %xmm8
11630 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm7 = xmm2[0,3,2,1]
11631 ; AVX512DQ-FAST-NEXT: vpshufb %xmm4, %xmm8, %xmm2
11632 ; AVX512DQ-FAST-NEXT: vpshufb %xmm9, %xmm7, %xmm5
11633 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm5[0,1,2,3],xmm2[4],xmm5[5],xmm2[6,7]
11634 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
11635 ; AVX512DQ-FAST-NEXT: vinserti32x4 $2, %xmm1, %zmm2, %zmm24
11636 ; AVX512DQ-FAST-NEXT: vinserti32x8 $0, %ymm3, %zmm0, %zmm24 {%k1}
11637 ; AVX512DQ-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
11638 ; AVX512DQ-FAST-NEXT: vpblendd $66, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
11639 ; AVX512DQ-FAST-NEXT: # ymm5 = ymm0[0],mem[1],ymm0[2,3,4,5],mem[6],ymm0[7]
11640 ; AVX512DQ-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
11641 ; AVX512DQ-FAST-NEXT: vpblendd $109, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
11642 ; AVX512DQ-FAST-NEXT: # ymm1 = mem[0],ymm0[1],mem[2,3],ymm0[4],mem[5,6],ymm0[7]
11643 ; AVX512DQ-FAST-NEXT: vextracti128 $1, %ymm1, %xmm3
11644 ; AVX512DQ-FAST-NEXT: vpshufb %xmm4, %xmm3, %xmm4
11645 ; AVX512DQ-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[0,3,2,1]
11646 ; AVX512DQ-FAST-NEXT: vpshufb %xmm9, %xmm2, %xmm1
11647 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm4[4],xmm1[5],xmm4[6,7]
11648 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm25, %ymm0
11649 ; AVX512DQ-FAST-NEXT: vpshufb %ymm0, %ymm5, %ymm4
11650 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
11651 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4],ymm1[5,6,7]
11652 ; AVX512DQ-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm25
11653 ; AVX512DQ-FAST-NEXT: vpbroadcastd {{.*#+}} xmm1 = [10,11,6,7,10,11,6,7,10,11,6,7,10,11,6,7]
11654 ; AVX512DQ-FAST-NEXT: vpshufb %xmm1, %xmm13, %xmm9
11655 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} xmm13 = <10,11,u,u,2,3,14,15,u,u,10,11,12,13,14,15>
11656 ; AVX512DQ-FAST-NEXT: vpshufb %xmm13, %xmm11, %xmm11
11657 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm9 = xmm11[0],xmm9[1],xmm11[2,3],xmm9[4],xmm11[5,6,7]
11658 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} ymm4 = [128,128,128,128,128,128,128,128,128,128,6,7,2,3,14,15,26,27,22,23,128,128,128,128,128,128,128,128,128,128,128,128]
11659 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm18, %ymm0
11660 ; AVX512DQ-FAST-NEXT: vpshufb %ymm4, %ymm0, %ymm11
11661 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm4, %ymm18
11662 ; AVX512DQ-FAST-NEXT: vpternlogq $236, %ymm28, %ymm11, %ymm9
11663 ; AVX512DQ-FAST-NEXT: vpshufb %xmm1, %xmm15, %xmm11
11664 ; AVX512DQ-FAST-NEXT: vpshufb %xmm13, %xmm10, %xmm10
11665 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm10[0],xmm11[1],xmm10[2,3],xmm11[4],xmm10[5,6,7]
11666 ; AVX512DQ-FAST-NEXT: vpbroadcastq {{.*#+}} xmm10 = [10,11,0,0,2,3,14,15,10,11,0,0,2,3,14,15]
11667 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm31, %xmm4
11668 ; AVX512DQ-FAST-NEXT: vpshufb %xmm10, %xmm4, %xmm4
11669 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} xmm11 = <0,1,2,3,2,3,6,7,u,u,14,15,u,u,u,u>
11670 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm30, %xmm15
11671 ; AVX512DQ-FAST-NEXT: vpshufb %xmm11, %xmm15, %xmm15
11672 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm4 = xmm15[0,1,2,3],xmm4[4],xmm15[5],xmm4[6,7]
11673 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
11674 ; AVX512DQ-FAST-NEXT: vinserti32x4 $2, %xmm0, %zmm4, %zmm26
11675 ; AVX512DQ-FAST-NEXT: vinserti32x8 $0, %ymm9, %zmm0, %zmm26 {%k1}
11676 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm19, %xmm0
11677 ; AVX512DQ-FAST-NEXT: vpshufb %xmm10, %xmm0, %xmm4
11678 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm27, %xmm0
11679 ; AVX512DQ-FAST-NEXT: vpshufb %xmm11, %xmm0, %xmm9
11680 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm4 = xmm9[0,1,2,3],xmm4[4],xmm9[5],xmm4[6,7]
11681 ; AVX512DQ-FAST-NEXT: vmovdqa {{.*#+}} ymm9 = [10,11,6,7,4,5,6,7,6,7,6,7,2,3,14,15,26,27,22,23,20,21,22,23,22,23,22,23,18,19,30,31]
11682 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm21, %ymm0
11683 ; AVX512DQ-FAST-NEXT: vpshufb %ymm9, %ymm0, %ymm15
11684 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
11685 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm4 = ymm15[0,1,2,3,4],ymm4[5,6,7]
11686 ; AVX512DQ-FAST-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm4
11687 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm16, %ymm0
11688 ; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm18, %ymm15
11689 ; AVX512DQ-FAST-NEXT: vpshufb %ymm15, %ymm0, %ymm15
11690 ; AVX512DQ-FAST-NEXT: vmovdqa64 %xmm17, %xmm0
11691 ; AVX512DQ-FAST-NEXT: vpshufb %xmm1, %xmm0, %xmm0
11692 ; AVX512DQ-FAST-NEXT: vpshufb %xmm13, %xmm14, %xmm14
11693 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm14[0],xmm0[1],xmm14[2,3],xmm0[4],xmm14[5,6,7]
11694 ; AVX512DQ-FAST-NEXT: vpternlogq $236, %ymm28, %ymm15, %ymm0
11695 ; AVX512DQ-FAST-NEXT: vpshufb %xmm1, %xmm12, %xmm1
11696 ; AVX512DQ-FAST-NEXT: vpshufb %xmm13, %xmm6, %xmm6
11697 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm6[0],xmm1[1],xmm6[2,3],xmm1[4],xmm6[5,6,7]
11698 ; AVX512DQ-FAST-NEXT: vpshufb %xmm10, %xmm8, %xmm6
11699 ; AVX512DQ-FAST-NEXT: vpshufb %xmm11, %xmm7, %xmm7
11700 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0,1,2,3],xmm6[4],xmm7[5],xmm6[6,7]
11701 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
11702 ; AVX512DQ-FAST-NEXT: vinserti32x4 $2, %xmm1, %zmm6, %zmm1
11703 ; AVX512DQ-FAST-NEXT: vinserti32x8 $0, %ymm0, %zmm0, %zmm1 {%k1}
11704 ; AVX512DQ-FAST-NEXT: vpshufb %ymm9, %ymm5, %ymm0
11705 ; AVX512DQ-FAST-NEXT: vpshufb %xmm10, %xmm3, %xmm3
11706 ; AVX512DQ-FAST-NEXT: vpshufb %xmm11, %xmm2, %xmm2
11707 ; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4],xmm2[5],xmm3[6,7]
11708 ; AVX512DQ-FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
11709 ; AVX512DQ-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
11710 ; AVX512DQ-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0
11711 ; AVX512DQ-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
11712 ; AVX512DQ-FAST-NEXT: vmovaps %zmm2, (%rsi)
11713 ; AVX512DQ-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
11714 ; AVX512DQ-FAST-NEXT: vmovaps %zmm2, 64(%rsi)
11715 ; AVX512DQ-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
11716 ; AVX512DQ-FAST-NEXT: vmovaps %zmm2, 64(%rdx)
11717 ; AVX512DQ-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
11718 ; AVX512DQ-FAST-NEXT: vmovaps %zmm2, (%rdx)
11719 ; AVX512DQ-FAST-NEXT: vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0]
11720 ; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm22, %zmm2, %zmm23
11721 ; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm24, %zmm2, %zmm25
11722 ; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm26, %zmm2, %zmm4
11723 ; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm1, %zmm2, %zmm0
11724 ; AVX512DQ-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
11725 ; AVX512DQ-FAST-NEXT: vmovaps %zmm1, 64(%rcx)
11726 ; AVX512DQ-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
11727 ; AVX512DQ-FAST-NEXT: vmovaps %zmm1, (%rcx)
11728 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm20, 64(%r8)
11729 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm29, (%r8)
11730 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm25, 64(%r9)
11731 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm23, (%r9)
11732 ; AVX512DQ-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
11733 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm0, 64(%rax)
11734 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm4, (%rax)
11735 ; AVX512DQ-FAST-NEXT: addq $936, %rsp # imm = 0x3A8
11736 ; AVX512DQ-FAST-NEXT: vzeroupper
11737 ; AVX512DQ-FAST-NEXT: retq
11738 ;
11739 ; AVX512BW-LABEL: load_i16_stride6_vf64:
11740 ; AVX512BW: # %bb.0:
11741 ; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
11742 ; AVX512BW-NEXT: vmovdqa64 448(%rdi), %zmm3
11743 ; AVX512BW-NEXT: vmovdqa64 384(%rdi), %zmm2
11744 ; AVX512BW-NEXT: vmovdqa64 512(%rdi), %zmm0
11745 ; AVX512BW-NEXT: vmovdqa64 576(%rdi), %zmm5
11746 ; AVX512BW-NEXT: vmovdqa64 704(%rdi), %zmm4
11747 ; AVX512BW-NEXT: vmovdqa64 640(%rdi), %zmm6
11748 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm10
11749 ; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm13
11750 ; AVX512BW-NEXT: vmovdqa64 128(%rdi), %zmm1
11751 ; AVX512BW-NEXT: vmovdqa64 192(%rdi), %zmm12
11752 ; AVX512BW-NEXT: vmovdqa64 320(%rdi), %zmm9
11753 ; AVX512BW-NEXT: vmovdqa64 256(%rdi), %zmm11
11754 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm14 = [0,0,0,0,0,0,4,10,16,22,28,34,40,46,52,58,0,0,0,0,0,0,4,10,16,22,28,34,40,46,52,58]
11755 ; AVX512BW-NEXT: # zmm14 = mem[0,1,2,3,0,1,2,3]
11756 ; AVX512BW-NEXT: vmovdqa64 %zmm11, %zmm15
11757 ; AVX512BW-NEXT: vpermt2w %zmm9, %zmm14, %zmm15
11758 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm16 = [0,6,12,18,24,30,0,0,0,0,0,34,40,46,52,58,0,6,12,18,24,30,0,0,0,0,0,34,40,46,52,58]
11759 ; AVX512BW-NEXT: # zmm16 = mem[0,1,2,3,0,1,2,3]
11760 ; AVX512BW-NEXT: vmovdqa64 %zmm12, %zmm17
11761 ; AVX512BW-NEXT: vpermt2w %zmm1, %zmm16, %zmm17
11762 ; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm7 = <0,6,12,18,24,30,36,42,48,54,60,u,u,u,u,u>
11763 ; AVX512BW-NEXT: vmovdqa64 %zmm10, %zmm8
11764 ; AVX512BW-NEXT: vpermt2w %zmm13, %zmm7, %zmm8
11765 ; AVX512BW-NEXT: movl $4192256, %edi # imm = 0x3FF800
11766 ; AVX512BW-NEXT: kmovd %edi, %k1
11767 ; AVX512BW-NEXT: vmovdqu16 %zmm17, %zmm8 {%k1}
11768 ; AVX512BW-NEXT: movw $-2048, %di # imm = 0xF800
11769 ; AVX512BW-NEXT: kmovd %edi, %k2
11770 ; AVX512BW-NEXT: vmovdqa32 %zmm15, %zmm8 {%k2}
11771 ; AVX512BW-NEXT: vpermi2w %zmm4, %zmm6, %zmm14
11772 ; AVX512BW-NEXT: vpermi2w %zmm0, %zmm5, %zmm16
11773 ; AVX512BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm7
11774 ; AVX512BW-NEXT: vmovdqu16 %zmm16, %zmm7 {%k1}
11775 ; AVX512BW-NEXT: vmovdqa32 %zmm14, %zmm7 {%k2}
11776 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm16 = [0,0,0,0,0,0,5,11,17,23,29,35,41,47,53,59,0,0,0,0,0,0,5,11,17,23,29,35,41,47,53,59]
11777 ; AVX512BW-NEXT: # zmm16 = mem[0,1,2,3,0,1,2,3]
11778 ; AVX512BW-NEXT: vmovdqa64 %zmm11, %zmm17
11779 ; AVX512BW-NEXT: vpermt2w %zmm9, %zmm16, %zmm17
11780 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm18 = [1,7,13,19,25,31,0,0,0,0,0,35,41,47,53,59,1,7,13,19,25,31,0,0,0,0,0,35,41,47,53,59]
11781 ; AVX512BW-NEXT: # zmm18 = mem[0,1,2,3,0,1,2,3]
11782 ; AVX512BW-NEXT: vmovdqa64 %zmm12, %zmm19
11783 ; AVX512BW-NEXT: vpermt2w %zmm1, %zmm18, %zmm19
11784 ; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm14 = <1,7,13,19,25,31,37,43,49,55,61,u,u,u,u,u>
11785 ; AVX512BW-NEXT: vmovdqa64 %zmm10, %zmm15
11786 ; AVX512BW-NEXT: vpermt2w %zmm13, %zmm14, %zmm15
11787 ; AVX512BW-NEXT: vmovdqu16 %zmm19, %zmm15 {%k1}
11788 ; AVX512BW-NEXT: vmovdqa32 %zmm17, %zmm15 {%k2}
11789 ; AVX512BW-NEXT: vpermi2w %zmm4, %zmm6, %zmm16
11790 ; AVX512BW-NEXT: vpermi2w %zmm0, %zmm5, %zmm18
11791 ; AVX512BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm14
11792 ; AVX512BW-NEXT: vmovdqu16 %zmm18, %zmm14 {%k1}
11793 ; AVX512BW-NEXT: vmovdqa32 %zmm16, %zmm14 {%k2}
11794 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm18 = [0,0,0,0,0,0,6,12,18,24,30,36,42,48,54,60,0,0,0,0,0,0,6,12,18,24,30,36,42,48,54,60]
11795 ; AVX512BW-NEXT: # zmm18 = mem[0,1,2,3,0,1,2,3]
11796 ; AVX512BW-NEXT: vmovdqa64 %zmm11, %zmm19
11797 ; AVX512BW-NEXT: vpermt2w %zmm9, %zmm18, %zmm19
11798 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm20 = [34,40,46,52,58,0,0,0,0,0,0,4,10,16,22,28,34,40,46,52,58,0,0,0,0,0,0,4,10,16,22,28]
11799 ; AVX512BW-NEXT: # zmm20 = mem[0,1,2,3,0,1,2,3]
11800 ; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm21
11801 ; AVX512BW-NEXT: vpermt2w %zmm12, %zmm20, %zmm21
11802 ; AVX512BW-NEXT: vmovdqa64 {{.*#+}} ymm16 = <34,40,46,52,58,0,6,12,18,24,30,u,u,u,u,u>
11803 ; AVX512BW-NEXT: vmovdqa64 %zmm13, %zmm17
11804 ; AVX512BW-NEXT: vpermt2w %zmm10, %zmm16, %zmm17
11805 ; AVX512BW-NEXT: movl $2095104, %edi # imm = 0x1FF800
11806 ; AVX512BW-NEXT: kmovd %edi, %k2
11807 ; AVX512BW-NEXT: vmovdqu16 %zmm21, %zmm17 {%k2}
11808 ; AVX512BW-NEXT: movl $-2097152, %edi # imm = 0xFFE00000
11809 ; AVX512BW-NEXT: kmovd %edi, %k1
11810 ; AVX512BW-NEXT: vmovdqu16 %zmm19, %zmm17 {%k1}
11811 ; AVX512BW-NEXT: vpermi2w %zmm4, %zmm6, %zmm18
11812 ; AVX512BW-NEXT: vpermi2w %zmm5, %zmm0, %zmm20
11813 ; AVX512BW-NEXT: vpermi2w %zmm2, %zmm3, %zmm16
11814 ; AVX512BW-NEXT: vmovdqu16 %zmm20, %zmm16 {%k2}
11815 ; AVX512BW-NEXT: vmovdqu16 %zmm18, %zmm16 {%k1}
11816 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm20 = [0,0,0,0,0,1,7,13,19,25,31,37,43,49,55,61,0,0,0,0,0,1,7,13,19,25,31,37,43,49,55,61]
11817 ; AVX512BW-NEXT: # zmm20 = mem[0,1,2,3,0,1,2,3]
11818 ; AVX512BW-NEXT: vmovdqa64 %zmm11, %zmm21
11819 ; AVX512BW-NEXT: vpermt2w %zmm9, %zmm20, %zmm21
11820 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm22 = [35,41,47,53,59,0,0,0,0,0,0,5,11,17,23,29,35,41,47,53,59,0,0,0,0,0,0,5,11,17,23,29]
11821 ; AVX512BW-NEXT: # zmm22 = mem[0,1,2,3,0,1,2,3]
11822 ; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm23
11823 ; AVX512BW-NEXT: vpermt2w %zmm12, %zmm22, %zmm23
11824 ; AVX512BW-NEXT: vmovdqa64 {{.*#+}} ymm18 = <35,41,47,53,59,1,7,13,19,25,31,u,u,u,u,u>
11825 ; AVX512BW-NEXT: vmovdqa64 %zmm13, %zmm19
11826 ; AVX512BW-NEXT: vpermt2w %zmm10, %zmm18, %zmm19
11827 ; AVX512BW-NEXT: vmovdqu16 %zmm23, %zmm19 {%k2}
11828 ; AVX512BW-NEXT: vmovdqu16 %zmm21, %zmm19 {%k1}
11829 ; AVX512BW-NEXT: vpermi2w %zmm4, %zmm6, %zmm20
11830 ; AVX512BW-NEXT: vpermi2w %zmm5, %zmm0, %zmm22
11831 ; AVX512BW-NEXT: vpermi2w %zmm2, %zmm3, %zmm18
11832 ; AVX512BW-NEXT: vmovdqu16 %zmm22, %zmm18 {%k2}
11833 ; AVX512BW-NEXT: vmovdqu16 %zmm20, %zmm18 {%k1}
11834 ; AVX512BW-NEXT: vmovdqa64 {{.*#+}} ymm20 = <4,10,16,22,28,34,40,46,52,58,u,u,u,u,u,u>
11835 ; AVX512BW-NEXT: vmovdqa64 %zmm10, %zmm21
11836 ; AVX512BW-NEXT: vpermt2w %zmm13, %zmm20, %zmm21
11837 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm22 = [36,42,48,54,60,0,0,0,0,0,0,6,12,18,24,30,36,42,48,54,60,0,0,0,0,0,0,6,12,18,24,30]
11838 ; AVX512BW-NEXT: # zmm22 = mem[0,1,2,3,0,1,2,3]
11839 ; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm23
11840 ; AVX512BW-NEXT: vpermt2w %zmm12, %zmm22, %zmm23
11841 ; AVX512BW-NEXT: movw $31, %di
11842 ; AVX512BW-NEXT: kmovd %edi, %k2
11843 ; AVX512BW-NEXT: vmovdqa32 %zmm21, %zmm23 {%k2}
11844 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm21 = [0,0,0,0,0,34,40,46,52,58,0,6,12,18,24,30,0,0,0,0,0,34,40,46,52,58,0,6,12,18,24,30]
11845 ; AVX512BW-NEXT: # zmm21 = mem[0,1,2,3,0,1,2,3]
11846 ; AVX512BW-NEXT: vmovdqa64 %zmm9, %zmm24
11847 ; AVX512BW-NEXT: vpermt2w %zmm11, %zmm21, %zmm24
11848 ; AVX512BW-NEXT: vmovdqu16 %zmm24, %zmm23 {%k1}
11849 ; AVX512BW-NEXT: vpermi2w %zmm6, %zmm4, %zmm21
11850 ; AVX512BW-NEXT: vpermi2w %zmm5, %zmm0, %zmm22
11851 ; AVX512BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm20
11852 ; AVX512BW-NEXT: vmovdqa32 %zmm20, %zmm22 {%k2}
11853 ; AVX512BW-NEXT: vmovdqu16 %zmm21, %zmm22 {%k1}
11854 ; AVX512BW-NEXT: vmovdqa64 {{.*#+}} ymm20 = <5,11,17,23,29,35,41,47,53,59,u,u,u,u,u,u>
11855 ; AVX512BW-NEXT: vpermt2w %zmm13, %zmm20, %zmm10
11856 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm13 = [37,43,49,55,61,0,0,0,0,0,1,7,13,19,25,31,37,43,49,55,61,0,0,0,0,0,1,7,13,19,25,31]
11857 ; AVX512BW-NEXT: # zmm13 = mem[0,1,2,3,0,1,2,3]
11858 ; AVX512BW-NEXT: vpermt2w %zmm12, %zmm13, %zmm1
11859 ; AVX512BW-NEXT: vmovdqa32 %zmm10, %zmm1 {%k2}
11860 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm10 = [0,0,0,0,0,35,41,47,53,59,1,7,13,19,25,31,0,0,0,0,0,35,41,47,53,59,1,7,13,19,25,31]
11861 ; AVX512BW-NEXT: # zmm10 = mem[0,1,2,3,0,1,2,3]
11862 ; AVX512BW-NEXT: vpermt2w %zmm11, %zmm10, %zmm9
11863 ; AVX512BW-NEXT: vmovdqu16 %zmm9, %zmm1 {%k1}
11864 ; AVX512BW-NEXT: vpermt2w %zmm6, %zmm10, %zmm4
11865 ; AVX512BW-NEXT: vpermt2w %zmm5, %zmm13, %zmm0
11866 ; AVX512BW-NEXT: vpermt2w %zmm3, %zmm20, %zmm2
11867 ; AVX512BW-NEXT: vmovdqa32 %zmm2, %zmm0 {%k2}
11868 ; AVX512BW-NEXT: vmovdqu16 %zmm4, %zmm0 {%k1}
11869 ; AVX512BW-NEXT: vmovdqa64 %zmm7, 64(%rsi)
11870 ; AVX512BW-NEXT: vmovdqa64 %zmm8, (%rsi)
11871 ; AVX512BW-NEXT: vmovdqa64 %zmm14, 64(%rdx)
11872 ; AVX512BW-NEXT: vmovdqa64 %zmm15, (%rdx)
11873 ; AVX512BW-NEXT: vmovdqa64 %zmm16, 64(%rcx)
11874 ; AVX512BW-NEXT: vmovdqa64 %zmm17, (%rcx)
11875 ; AVX512BW-NEXT: vmovdqa64 %zmm18, 64(%r8)
11876 ; AVX512BW-NEXT: vmovdqa64 %zmm19, (%r8)
11877 ; AVX512BW-NEXT: vmovdqa64 %zmm22, 64(%r9)
11878 ; AVX512BW-NEXT: vmovdqa64 %zmm23, (%r9)
11879 ; AVX512BW-NEXT: vmovdqa64 %zmm0, 64(%rax)
11880 ; AVX512BW-NEXT: vmovdqa64 %zmm1, (%rax)
11881 ; AVX512BW-NEXT: vzeroupper
11882 ; AVX512BW-NEXT: retq
11883 %wide.vec = load <384 x i16>, ptr %in.vec, align 64
11884 %strided.vec0 = shufflevector <384 x i16> %wide.vec, <384 x i16> poison, <64 x i32> <i32 0, i32 6, i32 12, i32 18, i32 24, i32 30, i32 36, i32 42, i32 48, i32 54, i32 60, i32 66, i32 72, i32 78, i32 84, i32 90, i32 96, i32 102, i32 108, i32 114, i32 120, i32 126, i32 132, i32 138, i32 144, i32 150, i32 156, i32 162, i32 168, i32 174, i32 180, i32 186, i32 192, i32 198, i32 204, i32 210, i32 216, i32 222, i32 228, i32 234, i32 240, i32 246, i32 252, i32 258, i32 264, i32 270, i32 276, i32 282, i32 288, i32 294, i32 300, i32 306, i32 312, i32 318, i32 324, i32 330, i32 336, i32 342, i32 348, i32 354, i32 360, i32 366, i32 372, i32 378>
11885 %strided.vec1 = shufflevector <384 x i16> %wide.vec, <384 x i16> poison, <64 x i32> <i32 1, i32 7, i32 13, i32 19, i32 25, i32 31, i32 37, i32 43, i32 49, i32 55, i32 61, i32 67, i32 73, i32 79, i32 85, i32 91, i32 97, i32 103, i32 109, i32 115, i32 121, i32 127, i32 133, i32 139, i32 145, i32 151, i32 157, i32 163, i32 169, i32 175, i32 181, i32 187, i32 193, i32 199, i32 205, i32 211, i32 217, i32 223, i32 229, i32 235, i32 241, i32 247, i32 253, i32 259, i32 265, i32 271, i32 277, i32 283, i32 289, i32 295, i32 301, i32 307, i32 313, i32 319, i32 325, i32 331, i32 337, i32 343, i32 349, i32 355, i32 361, i32 367, i32 373, i32 379>
11886 %strided.vec2 = shufflevector <384 x i16> %wide.vec, <384 x i16> poison, <64 x i32> <i32 2, i32 8, i32 14, i32 20, i32 26, i32 32, i32 38, i32 44, i32 50, i32 56, i32 62, i32 68, i32 74, i32 80, i32 86, i32 92, i32 98, i32 104, i32 110, i32 116, i32 122, i32 128, i32 134, i32 140, i32 146, i32 152, i32 158, i32 164, i32 170, i32 176, i32 182, i32 188, i32 194, i32 200, i32 206, i32 212, i32 218, i32 224, i32 230, i32 236, i32 242, i32 248, i32 254, i32 260, i32 266, i32 272, i32 278, i32 284, i32 290, i32 296, i32 302, i32 308, i32 314, i32 320, i32 326, i32 332, i32 338, i32 344, i32 350, i32 356, i32 362, i32 368, i32 374, i32 380>
11887 %strided.vec3 = shufflevector <384 x i16> %wide.vec, <384 x i16> poison, <64 x i32> <i32 3, i32 9, i32 15, i32 21, i32 27, i32 33, i32 39, i32 45, i32 51, i32 57, i32 63, i32 69, i32 75, i32 81, i32 87, i32 93, i32 99, i32 105, i32 111, i32 117, i32 123, i32 129, i32 135, i32 141, i32 147, i32 153, i32 159, i32 165, i32 171, i32 177, i32 183, i32 189, i32 195, i32 201, i32 207, i32 213, i32 219, i32 225, i32 231, i32 237, i32 243, i32 249, i32 255, i32 261, i32 267, i32 273, i32 279, i32 285, i32 291, i32 297, i32 303, i32 309, i32 315, i32 321, i32 327, i32 333, i32 339, i32 345, i32 351, i32 357, i32 363, i32 369, i32 375, i32 381>
11888 %strided.vec4 = shufflevector <384 x i16> %wide.vec, <384 x i16> poison, <64 x i32> <i32 4, i32 10, i32 16, i32 22, i32 28, i32 34, i32 40, i32 46, i32 52, i32 58, i32 64, i32 70, i32 76, i32 82, i32 88, i32 94, i32 100, i32 106, i32 112, i32 118, i32 124, i32 130, i32 136, i32 142, i32 148, i32 154, i32 160, i32 166, i32 172, i32 178, i32 184, i32 190, i32 196, i32 202, i32 208, i32 214, i32 220, i32 226, i32 232, i32 238, i32 244, i32 250, i32 256, i32 262, i32 268, i32 274, i32 280, i32 286, i32 292, i32 298, i32 304, i32 310, i32 316, i32 322, i32 328, i32 334, i32 340, i32 346, i32 352, i32 358, i32 364, i32 370, i32 376, i32 382>
11889 %strided.vec5 = shufflevector <384 x i16> %wide.vec, <384 x i16> poison, <64 x i32> <i32 5, i32 11, i32 17, i32 23, i32 29, i32 35, i32 41, i32 47, i32 53, i32 59, i32 65, i32 71, i32 77, i32 83, i32 89, i32 95, i32 101, i32 107, i32 113, i32 119, i32 125, i32 131, i32 137, i32 143, i32 149, i32 155, i32 161, i32 167, i32 173, i32 179, i32 185, i32 191, i32 197, i32 203, i32 209, i32 215, i32 221, i32 227, i32 233, i32 239, i32 245, i32 251, i32 257, i32 263, i32 269, i32 275, i32 281, i32 287, i32 293, i32 299, i32 305, i32 311, i32 317, i32 323, i32 329, i32 335, i32 341, i32 347, i32 353, i32 359, i32 365, i32 371, i32 377, i32 383>
11890 store <64 x i16> %strided.vec0, ptr %out.vec0, align 64
11891 store <64 x i16> %strided.vec1, ptr %out.vec1, align 64
11892 store <64 x i16> %strided.vec2, ptr %out.vec2, align 64
11893 store <64 x i16> %strided.vec3, ptr %out.vec3, align 64
11894 store <64 x i16> %strided.vec4, ptr %out.vec4, align 64
11895 store <64 x i16> %strided.vec5, ptr %out.vec5, align 64
11896 ret void
11897 }
11898 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
11899 ; AVX: {{.*}}
11900 ; AVX1: {{.*}}
11901 ; AVX2: {{.*}}
11902 ; AVX2-ONLY: {{.*}}
11903 ; AVX512: {{.*}}
11904 ; AVX512BW-ONLY-FAST: {{.*}}
11905 ; AVX512BW-ONLY-SLOW: {{.*}}
11906 ; AVX512DQBW-FAST: {{.*}}
11907 ; AVX512DQBW-SLOW: {{.*}}
11908 ; AVX512F: {{.*}}
11909 ; FALLBACK0: {{.*}}
11910 ; FALLBACK1: {{.*}}
11911 ; FALLBACK10: {{.*}}
11912 ; FALLBACK11: {{.*}}
11913 ; FALLBACK12: {{.*}}
11914 ; FALLBACK2: {{.*}}
11915 ; FALLBACK3: {{.*}}
11916 ; FALLBACK4: {{.*}}
11917 ; FALLBACK5: {{.*}}
11918 ; FALLBACK6: {{.*}}
11919 ; FALLBACK7: {{.*}}
11920 ; FALLBACK8: {{.*}}
11921 ; FALLBACK9: {{.*}}