| blob | a31aebcab24e542c767d663597deed7b06ac8766 |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc < %s -x86-experimental-vector-widening-legalization -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
3 ; RUN: llc < %s -x86-experimental-vector-widening-legalization -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
4 ; RUN: llc < %s -x86-experimental-vector-widening-legalization -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
5 ; RUN: llc < %s -x86-experimental-vector-widening-legalization -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
6 ; RUN: llc < %s -x86-experimental-vector-widening-legalization -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512BW
7 ; RUN: llc < %s -x86-experimental-vector-widening-legalization -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512VL --check-prefix=AVX512BWVL
8 ; RUN: llc < %s -x86-experimental-vector-widening-legalization -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512DQ
9 ; RUN: llc < %s -x86-experimental-vector-widening-legalization -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512dq,+avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512VL --check-prefix=AVX512DQVL
11 ;
12 ; vXi64
13 ;
15 define i64 @test_v2i64(<2 x i64> %a0) {
16 ; SSE-LABEL: test_v2i64:
17 ; SSE: # %bb.0:
18 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
19 ; SSE-NEXT: movdqa %xmm0, %xmm2
20 ; SSE-NEXT: psrlq $32, %xmm2
21 ; SSE-NEXT: pmuludq %xmm1, %xmm2
22 ; SSE-NEXT: movdqa %xmm0, %xmm3
23 ; SSE-NEXT: psrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
24 ; SSE-NEXT: pmuludq %xmm0, %xmm3
25 ; SSE-NEXT: paddq %xmm2, %xmm3
26 ; SSE-NEXT: psllq $32, %xmm3
27 ; SSE-NEXT: pmuludq %xmm1, %xmm0
28 ; SSE-NEXT: paddq %xmm3, %xmm0
29 ; SSE-NEXT: movq %xmm0, %rax
30 ; SSE-NEXT: retq
31 ;
32 ; AVX-LABEL: test_v2i64:
33 ; AVX: # %bb.0:
34 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
35 ; AVX-NEXT: vpsrlq $32, %xmm0, %xmm2
36 ; AVX-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
37 ; AVX-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
38 ; AVX-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
39 ; AVX-NEXT: vpaddq %xmm2, %xmm3, %xmm2
40 ; AVX-NEXT: vpsllq $32, %xmm2, %xmm2
41 ; AVX-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
42 ; AVX-NEXT: vpaddq %xmm2, %xmm0, %xmm0
43 ; AVX-NEXT: vmovq %xmm0, %rax
44 ; AVX-NEXT: retq
45 ;
46 ; AVX512BW-LABEL: test_v2i64:
47 ; AVX512BW: # %bb.0:
48 ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
49 ; AVX512BW-NEXT: vpsrlq $32, %xmm0, %xmm2
50 ; AVX512BW-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
51 ; AVX512BW-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
52 ; AVX512BW-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
53 ; AVX512BW-NEXT: vpaddq %xmm2, %xmm3, %xmm2
54 ; AVX512BW-NEXT: vpsllq $32, %xmm2, %xmm2
55 ; AVX512BW-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
56 ; AVX512BW-NEXT: vpaddq %xmm2, %xmm0, %xmm0
57 ; AVX512BW-NEXT: vmovq %xmm0, %rax
58 ; AVX512BW-NEXT: retq
59 ;
60 ; AVX512BWVL-LABEL: test_v2i64:
61 ; AVX512BWVL: # %bb.0:
62 ; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
63 ; AVX512BWVL-NEXT: vpsrlq $32, %xmm0, %xmm2
64 ; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
65 ; AVX512BWVL-NEXT: vpsrlq $32, %xmm1, %xmm3
66 ; AVX512BWVL-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
67 ; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm3, %xmm2
68 ; AVX512BWVL-NEXT: vpsllq $32, %xmm2, %xmm2
69 ; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
70 ; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm0, %xmm0
71 ; AVX512BWVL-NEXT: vmovq %xmm0, %rax
72 ; AVX512BWVL-NEXT: retq
73 ;
74 ; AVX512DQ-LABEL: test_v2i64:
75 ; AVX512DQ: # %bb.0:
76 ; AVX512DQ-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
77 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
78 ; AVX512DQ-NEXT: vpmullq %zmm1, %zmm0, %zmm0
79 ; AVX512DQ-NEXT: vmovq %xmm0, %rax
80 ; AVX512DQ-NEXT: vzeroupper
81 ; AVX512DQ-NEXT: retq
82 ;
83 ; AVX512DQVL-LABEL: test_v2i64:
84 ; AVX512DQVL: # %bb.0:
85 ; AVX512DQVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
86 ; AVX512DQVL-NEXT: vpmullq %xmm1, %xmm0, %xmm0
87 ; AVX512DQVL-NEXT: vmovq %xmm0, %rax
88 ; AVX512DQVL-NEXT: retq
89 %1 = call i64 @llvm.experimental.vector.reduce.mul.v2i64(<2 x i64> %a0)
90 ret i64 %1
91 }
93 define i64 @test_v4i64(<4 x i64> %a0) {
94 ; SSE-LABEL: test_v4i64:
95 ; SSE: # %bb.0:
96 ; SSE-NEXT: movdqa %xmm0, %xmm2
97 ; SSE-NEXT: psrlq $32, %xmm2
98 ; SSE-NEXT: pmuludq %xmm1, %xmm2
99 ; SSE-NEXT: movdqa %xmm1, %xmm3
100 ; SSE-NEXT: psrlq $32, %xmm3
101 ; SSE-NEXT: pmuludq %xmm0, %xmm3
102 ; SSE-NEXT: paddq %xmm2, %xmm3
103 ; SSE-NEXT: psllq $32, %xmm3
104 ; SSE-NEXT: pmuludq %xmm1, %xmm0
105 ; SSE-NEXT: paddq %xmm3, %xmm0
106 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
107 ; SSE-NEXT: movdqa %xmm0, %xmm2
108 ; SSE-NEXT: psrlq $32, %xmm2
109 ; SSE-NEXT: pmuludq %xmm1, %xmm2
110 ; SSE-NEXT: movdqa %xmm0, %xmm3
111 ; SSE-NEXT: psrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
112 ; SSE-NEXT: pmuludq %xmm0, %xmm3
113 ; SSE-NEXT: paddq %xmm2, %xmm3
114 ; SSE-NEXT: psllq $32, %xmm3
115 ; SSE-NEXT: pmuludq %xmm1, %xmm0
116 ; SSE-NEXT: paddq %xmm3, %xmm0
117 ; SSE-NEXT: movq %xmm0, %rax
118 ; SSE-NEXT: retq
119 ;
120 ; AVX1-LABEL: test_v4i64:
121 ; AVX1: # %bb.0:
122 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
123 ; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm2
124 ; AVX1-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
125 ; AVX1-NEXT: vpsrlq $32, %xmm1, %xmm3
126 ; AVX1-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
127 ; AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2
128 ; AVX1-NEXT: vpsllq $32, %xmm2, %xmm2
129 ; AVX1-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
130 ; AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
131 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
132 ; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm2
133 ; AVX1-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
134 ; AVX1-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
135 ; AVX1-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
136 ; AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2
137 ; AVX1-NEXT: vpsllq $32, %xmm2, %xmm2
138 ; AVX1-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
139 ; AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
140 ; AVX1-NEXT: vmovq %xmm0, %rax
141 ; AVX1-NEXT: vzeroupper
142 ; AVX1-NEXT: retq
143 ;
144 ; AVX2-LABEL: test_v4i64:
145 ; AVX2: # %bb.0:
146 ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
147 ; AVX2-NEXT: vpsrlq $32, %xmm0, %xmm2
148 ; AVX2-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
149 ; AVX2-NEXT: vpsrlq $32, %xmm1, %xmm3
150 ; AVX2-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
151 ; AVX2-NEXT: vpaddq %xmm2, %xmm3, %xmm2
152 ; AVX2-NEXT: vpsllq $32, %xmm2, %xmm2
153 ; AVX2-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
154 ; AVX2-NEXT: vpaddq %xmm2, %xmm0, %xmm0
155 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
156 ; AVX2-NEXT: vpsrlq $32, %xmm0, %xmm2
157 ; AVX2-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
158 ; AVX2-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
159 ; AVX2-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
160 ; AVX2-NEXT: vpaddq %xmm2, %xmm3, %xmm2
161 ; AVX2-NEXT: vpsllq $32, %xmm2, %xmm2
162 ; AVX2-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
163 ; AVX2-NEXT: vpaddq %xmm2, %xmm0, %xmm0
164 ; AVX2-NEXT: vmovq %xmm0, %rax
165 ; AVX2-NEXT: vzeroupper
166 ; AVX2-NEXT: retq
167 ;
168 ; AVX512BW-LABEL: test_v4i64:
169 ; AVX512BW: # %bb.0:
170 ; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
171 ; AVX512BW-NEXT: vpsrlq $32, %xmm0, %xmm2
172 ; AVX512BW-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
173 ; AVX512BW-NEXT: vpsrlq $32, %xmm1, %xmm3
174 ; AVX512BW-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
175 ; AVX512BW-NEXT: vpaddq %xmm2, %xmm3, %xmm2
176 ; AVX512BW-NEXT: vpsllq $32, %xmm2, %xmm2
177 ; AVX512BW-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
178 ; AVX512BW-NEXT: vpaddq %xmm2, %xmm0, %xmm0
179 ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
180 ; AVX512BW-NEXT: vpsrlq $32, %xmm0, %xmm2
181 ; AVX512BW-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
182 ; AVX512BW-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
183 ; AVX512BW-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
184 ; AVX512BW-NEXT: vpaddq %xmm2, %xmm3, %xmm2
185 ; AVX512BW-NEXT: vpsllq $32, %xmm2, %xmm2
186 ; AVX512BW-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
187 ; AVX512BW-NEXT: vpaddq %xmm2, %xmm0, %xmm0
188 ; AVX512BW-NEXT: vmovq %xmm0, %rax
189 ; AVX512BW-NEXT: vzeroupper
190 ; AVX512BW-NEXT: retq
191 ;
192 ; AVX512BWVL-LABEL: test_v4i64:
193 ; AVX512BWVL: # %bb.0:
194 ; AVX512BWVL-NEXT: vextracti128 $1, %ymm0, %xmm1
195 ; AVX512BWVL-NEXT: vpsrlq $32, %xmm0, %xmm2
196 ; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
197 ; AVX512BWVL-NEXT: vpsrlq $32, %xmm1, %xmm3
198 ; AVX512BWVL-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
199 ; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm3, %xmm2
200 ; AVX512BWVL-NEXT: vpsllq $32, %xmm2, %xmm2
201 ; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
202 ; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm0, %xmm0
203 ; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
204 ; AVX512BWVL-NEXT: vpsrlq $32, %xmm0, %xmm2
205 ; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
206 ; AVX512BWVL-NEXT: vpsrlq $32, %xmm1, %xmm3
207 ; AVX512BWVL-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
208 ; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm3, %xmm2
209 ; AVX512BWVL-NEXT: vpsllq $32, %xmm2, %xmm2
210 ; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
211 ; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm0, %xmm0
212 ; AVX512BWVL-NEXT: vmovq %xmm0, %rax
213 ; AVX512BWVL-NEXT: vzeroupper
214 ; AVX512BWVL-NEXT: retq
215 ;
216 ; AVX512DQ-LABEL: test_v4i64:
217 ; AVX512DQ: # %bb.0:
218 ; AVX512DQ-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
219 ; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm1
220 ; AVX512DQ-NEXT: vpmullq %zmm1, %zmm0, %zmm0
221 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
222 ; AVX512DQ-NEXT: vpmullq %zmm1, %zmm0, %zmm0
223 ; AVX512DQ-NEXT: vmovq %xmm0, %rax
224 ; AVX512DQ-NEXT: vzeroupper
225 ; AVX512DQ-NEXT: retq
226 ;
227 ; AVX512DQVL-LABEL: test_v4i64:
228 ; AVX512DQVL: # %bb.0:
229 ; AVX512DQVL-NEXT: vextracti128 $1, %ymm0, %xmm1
230 ; AVX512DQVL-NEXT: vpmullq %xmm1, %xmm0, %xmm0
231 ; AVX512DQVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
232 ; AVX512DQVL-NEXT: vpmullq %xmm1, %xmm0, %xmm0
233 ; AVX512DQVL-NEXT: vmovq %xmm0, %rax
234 ; AVX512DQVL-NEXT: vzeroupper
235 ; AVX512DQVL-NEXT: retq
236 %1 = call i64 @llvm.experimental.vector.reduce.mul.v4i64(<4 x i64> %a0)
237 ret i64 %1
238 }
240 define i64 @test_v8i64(<8 x i64> %a0) {
241 ; SSE-LABEL: test_v8i64:
242 ; SSE: # %bb.0:
243 ; SSE-NEXT: movdqa %xmm1, %xmm4
244 ; SSE-NEXT: psrlq $32, %xmm4
245 ; SSE-NEXT: pmuludq %xmm3, %xmm4
246 ; SSE-NEXT: movdqa %xmm3, %xmm5
247 ; SSE-NEXT: psrlq $32, %xmm5
248 ; SSE-NEXT: pmuludq %xmm1, %xmm5
249 ; SSE-NEXT: paddq %xmm4, %xmm5
250 ; SSE-NEXT: psllq $32, %xmm5
251 ; SSE-NEXT: pmuludq %xmm3, %xmm1
252 ; SSE-NEXT: paddq %xmm5, %xmm1
253 ; SSE-NEXT: movdqa %xmm0, %xmm3
254 ; SSE-NEXT: psrlq $32, %xmm3
255 ; SSE-NEXT: pmuludq %xmm2, %xmm3
256 ; SSE-NEXT: movdqa %xmm2, %xmm4
257 ; SSE-NEXT: psrlq $32, %xmm4
258 ; SSE-NEXT: pmuludq %xmm0, %xmm4
259 ; SSE-NEXT: paddq %xmm3, %xmm4
260 ; SSE-NEXT: psllq $32, %xmm4
261 ; SSE-NEXT: pmuludq %xmm2, %xmm0
262 ; SSE-NEXT: paddq %xmm4, %xmm0
263 ; SSE-NEXT: movdqa %xmm0, %xmm2
264 ; SSE-NEXT: psrlq $32, %xmm2
265 ; SSE-NEXT: pmuludq %xmm1, %xmm2
266 ; SSE-NEXT: movdqa %xmm1, %xmm3
267 ; SSE-NEXT: psrlq $32, %xmm3
268 ; SSE-NEXT: pmuludq %xmm0, %xmm3
269 ; SSE-NEXT: paddq %xmm2, %xmm3
270 ; SSE-NEXT: psllq $32, %xmm3
271 ; SSE-NEXT: pmuludq %xmm1, %xmm0
272 ; SSE-NEXT: paddq %xmm3, %xmm0
273 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
274 ; SSE-NEXT: movdqa %xmm0, %xmm2
275 ; SSE-NEXT: psrlq $32, %xmm2
276 ; SSE-NEXT: pmuludq %xmm1, %xmm2
277 ; SSE-NEXT: movdqa %xmm0, %xmm3
278 ; SSE-NEXT: psrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
279 ; SSE-NEXT: pmuludq %xmm0, %xmm3
280 ; SSE-NEXT: paddq %xmm2, %xmm3
281 ; SSE-NEXT: psllq $32, %xmm3
282 ; SSE-NEXT: pmuludq %xmm1, %xmm0
283 ; SSE-NEXT: paddq %xmm3, %xmm0
284 ; SSE-NEXT: movq %xmm0, %rax
285 ; SSE-NEXT: retq
286 ;
287 ; AVX1-LABEL: test_v8i64:
288 ; AVX1: # %bb.0:
289 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
290 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
291 ; AVX1-NEXT: vpsrlq $32, %xmm3, %xmm4
292 ; AVX1-NEXT: vpmuludq %xmm2, %xmm4, %xmm4
293 ; AVX1-NEXT: vpsrlq $32, %xmm2, %xmm5
294 ; AVX1-NEXT: vpmuludq %xmm5, %xmm3, %xmm5
295 ; AVX1-NEXT: vpaddq %xmm4, %xmm5, %xmm4
296 ; AVX1-NEXT: vpsllq $32, %xmm4, %xmm4
297 ; AVX1-NEXT: vpmuludq %xmm2, %xmm3, %xmm2
298 ; AVX1-NEXT: vpaddq %xmm4, %xmm2, %xmm2
299 ; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm3
300 ; AVX1-NEXT: vpmuludq %xmm1, %xmm3, %xmm3
301 ; AVX1-NEXT: vpsrlq $32, %xmm1, %xmm4
302 ; AVX1-NEXT: vpmuludq %xmm4, %xmm0, %xmm4
303 ; AVX1-NEXT: vpaddq %xmm3, %xmm4, %xmm3
304 ; AVX1-NEXT: vpsllq $32, %xmm3, %xmm3
305 ; AVX1-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
306 ; AVX1-NEXT: vpaddq %xmm3, %xmm0, %xmm0
307 ; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm1
308 ; AVX1-NEXT: vpmuludq %xmm2, %xmm1, %xmm1
309 ; AVX1-NEXT: vpsrlq $32, %xmm2, %xmm3
310 ; AVX1-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
311 ; AVX1-NEXT: vpaddq %xmm1, %xmm3, %xmm1
312 ; AVX1-NEXT: vpsllq $32, %xmm1, %xmm1
313 ; AVX1-NEXT: vpmuludq %xmm2, %xmm0, %xmm0
314 ; AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
315 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
316 ; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm2
317 ; AVX1-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
318 ; AVX1-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
319 ; AVX1-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
320 ; AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2
321 ; AVX1-NEXT: vpsllq $32, %xmm2, %xmm2
322 ; AVX1-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
323 ; AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
324 ; AVX1-NEXT: vmovq %xmm0, %rax
325 ; AVX1-NEXT: vzeroupper
326 ; AVX1-NEXT: retq
327 ;
328 ; AVX2-LABEL: test_v8i64:
329 ; AVX2: # %bb.0:
330 ; AVX2-NEXT: vpsrlq $32, %ymm0, %ymm2
331 ; AVX2-NEXT: vpmuludq %ymm1, %ymm2, %ymm2
332 ; AVX2-NEXT: vpsrlq $32, %ymm1, %ymm3
333 ; AVX2-NEXT: vpmuludq %ymm3, %ymm0, %ymm3
334 ; AVX2-NEXT: vpaddq %ymm2, %ymm3, %ymm2
335 ; AVX2-NEXT: vpsllq $32, %ymm2, %ymm2
336 ; AVX2-NEXT: vpmuludq %ymm1, %ymm0, %ymm0
337 ; AVX2-NEXT: vpaddq %ymm2, %ymm0, %ymm0
338 ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
339 ; AVX2-NEXT: vpsrlq $32, %xmm0, %xmm2
340 ; AVX2-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
341 ; AVX2-NEXT: vpsrlq $32, %xmm1, %xmm3
342 ; AVX2-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
343 ; AVX2-NEXT: vpaddq %xmm2, %xmm3, %xmm2
344 ; AVX2-NEXT: vpsllq $32, %xmm2, %xmm2
345 ; AVX2-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
346 ; AVX2-NEXT: vpaddq %xmm2, %xmm0, %xmm0
347 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
348 ; AVX2-NEXT: vpsrlq $32, %xmm0, %xmm2
349 ; AVX2-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
350 ; AVX2-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
351 ; AVX2-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
352 ; AVX2-NEXT: vpaddq %xmm2, %xmm3, %xmm2
353 ; AVX2-NEXT: vpsllq $32, %xmm2, %xmm2
354 ; AVX2-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
355 ; AVX2-NEXT: vpaddq %xmm2, %xmm0, %xmm0
356 ; AVX2-NEXT: vmovq %xmm0, %rax
357 ; AVX2-NEXT: vzeroupper
358 ; AVX2-NEXT: retq
359 ;
360 ; AVX512BW-LABEL: test_v8i64:
361 ; AVX512BW: # %bb.0:
362 ; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
363 ; AVX512BW-NEXT: vpsrlq $32, %zmm0, %zmm2
364 ; AVX512BW-NEXT: vpmuludq %zmm1, %zmm2, %zmm2
365 ; AVX512BW-NEXT: vpsrlq $32, %zmm1, %zmm3
366 ; AVX512BW-NEXT: vpmuludq %zmm3, %zmm0, %zmm3
367 ; AVX512BW-NEXT: vpaddq %zmm2, %zmm3, %zmm2
368 ; AVX512BW-NEXT: vpsllq $32, %zmm2, %zmm2
369 ; AVX512BW-NEXT: vpmuludq %zmm1, %zmm0, %zmm0
370 ; AVX512BW-NEXT: vpaddq %zmm2, %zmm0, %zmm0
371 ; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
372 ; AVX512BW-NEXT: vpsrlq $32, %xmm0, %xmm2
373 ; AVX512BW-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
374 ; AVX512BW-NEXT: vpsrlq $32, %xmm1, %xmm3
375 ; AVX512BW-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
376 ; AVX512BW-NEXT: vpaddq %xmm2, %xmm3, %xmm2
377 ; AVX512BW-NEXT: vpsllq $32, %xmm2, %xmm2
378 ; AVX512BW-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
379 ; AVX512BW-NEXT: vpaddq %xmm2, %xmm0, %xmm0
380 ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
381 ; AVX512BW-NEXT: vpsrlq $32, %xmm0, %xmm2
382 ; AVX512BW-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
383 ; AVX512BW-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
384 ; AVX512BW-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
385 ; AVX512BW-NEXT: vpaddq %xmm2, %xmm3, %xmm2
386 ; AVX512BW-NEXT: vpsllq $32, %xmm2, %xmm2
387 ; AVX512BW-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
388 ; AVX512BW-NEXT: vpaddq %xmm2, %xmm0, %xmm0
389 ; AVX512BW-NEXT: vmovq %xmm0, %rax
390 ; AVX512BW-NEXT: vzeroupper
391 ; AVX512BW-NEXT: retq
392 ;
393 ; AVX512BWVL-LABEL: test_v8i64:
394 ; AVX512BWVL: # %bb.0:
395 ; AVX512BWVL-NEXT: vextracti64x4 $1, %zmm0, %ymm1
396 ; AVX512BWVL-NEXT: vpsrlq $32, %zmm0, %zmm2
397 ; AVX512BWVL-NEXT: vpmuludq %zmm1, %zmm2, %zmm2
398 ; AVX512BWVL-NEXT: vpsrlq $32, %zmm1, %zmm3
399 ; AVX512BWVL-NEXT: vpmuludq %zmm3, %zmm0, %zmm3
400 ; AVX512BWVL-NEXT: vpaddq %zmm2, %zmm3, %zmm2
401 ; AVX512BWVL-NEXT: vpsllq $32, %zmm2, %zmm2
402 ; AVX512BWVL-NEXT: vpmuludq %zmm1, %zmm0, %zmm0
403 ; AVX512BWVL-NEXT: vpaddq %zmm2, %zmm0, %zmm0
404 ; AVX512BWVL-NEXT: vextracti128 $1, %ymm0, %xmm1
405 ; AVX512BWVL-NEXT: vpsrlq $32, %xmm0, %xmm2
406 ; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
407 ; AVX512BWVL-NEXT: vpsrlq $32, %xmm1, %xmm3
408 ; AVX512BWVL-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
409 ; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm3, %xmm2
410 ; AVX512BWVL-NEXT: vpsllq $32, %xmm2, %xmm2
411 ; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
412 ; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm0, %xmm0
413 ; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
414 ; AVX512BWVL-NEXT: vpsrlq $32, %xmm0, %xmm2
415 ; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
416 ; AVX512BWVL-NEXT: vpsrlq $32, %xmm1, %xmm3
417 ; AVX512BWVL-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
418 ; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm3, %xmm2
419 ; AVX512BWVL-NEXT: vpsllq $32, %xmm2, %xmm2
420 ; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
421 ; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm0, %xmm0
422 ; AVX512BWVL-NEXT: vmovq %xmm0, %rax
423 ; AVX512BWVL-NEXT: vzeroupper
424 ; AVX512BWVL-NEXT: retq
425 ;
426 ; AVX512DQ-LABEL: test_v8i64:
427 ; AVX512DQ: # %bb.0:
428 ; AVX512DQ-NEXT: vextracti64x4 $1, %zmm0, %ymm1
429 ; AVX512DQ-NEXT: vpmullq %zmm1, %zmm0, %zmm0
430 ; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm1
431 ; AVX512DQ-NEXT: vpmullq %zmm1, %zmm0, %zmm0
432 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
433 ; AVX512DQ-NEXT: vpmullq %zmm1, %zmm0, %zmm0
434 ; AVX512DQ-NEXT: vmovq %xmm0, %rax
435 ; AVX512DQ-NEXT: vzeroupper
436 ; AVX512DQ-NEXT: retq
437 ;
438 ; AVX512DQVL-LABEL: test_v8i64:
439 ; AVX512DQVL: # %bb.0:
440 ; AVX512DQVL-NEXT: vextracti64x4 $1, %zmm0, %ymm1
441 ; AVX512DQVL-NEXT: vpmullq %zmm1, %zmm0, %zmm0
442 ; AVX512DQVL-NEXT: vextracti128 $1, %ymm0, %xmm1
443 ; AVX512DQVL-NEXT: vpmullq %xmm1, %xmm0, %xmm0
444 ; AVX512DQVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
445 ; AVX512DQVL-NEXT: vpmullq %xmm1, %xmm0, %xmm0
446 ; AVX512DQVL-NEXT: vmovq %xmm0, %rax
447 ; AVX512DQVL-NEXT: vzeroupper
448 ; AVX512DQVL-NEXT: retq
449 %1 = call i64 @llvm.experimental.vector.reduce.mul.v8i64(<8 x i64> %a0)
450 ret i64 %1
451 }
453 define i64 @test_v16i64(<16 x i64> %a0) {
454 ; SSE-LABEL: test_v16i64:
455 ; SSE: # %bb.0:
456 ; SSE-NEXT: movdqa %xmm2, %xmm8
457 ; SSE-NEXT: psrlq $32, %xmm8
458 ; SSE-NEXT: pmuludq %xmm6, %xmm8
459 ; SSE-NEXT: movdqa %xmm6, %xmm9
460 ; SSE-NEXT: psrlq $32, %xmm9
461 ; SSE-NEXT: pmuludq %xmm2, %xmm9
462 ; SSE-NEXT: paddq %xmm8, %xmm9
463 ; SSE-NEXT: psllq $32, %xmm9
464 ; SSE-NEXT: pmuludq %xmm6, %xmm2
465 ; SSE-NEXT: paddq %xmm9, %xmm2
466 ; SSE-NEXT: movdqa %xmm0, %xmm8
467 ; SSE-NEXT: psrlq $32, %xmm8
468 ; SSE-NEXT: pmuludq %xmm4, %xmm8
469 ; SSE-NEXT: movdqa %xmm4, %xmm6
470 ; SSE-NEXT: psrlq $32, %xmm6
471 ; SSE-NEXT: pmuludq %xmm0, %xmm6
472 ; SSE-NEXT: paddq %xmm8, %xmm6
473 ; SSE-NEXT: psllq $32, %xmm6
474 ; SSE-NEXT: pmuludq %xmm4, %xmm0
475 ; SSE-NEXT: paddq %xmm6, %xmm0
476 ; SSE-NEXT: movdqa %xmm3, %xmm4
477 ; SSE-NEXT: psrlq $32, %xmm4
478 ; SSE-NEXT: pmuludq %xmm7, %xmm4
479 ; SSE-NEXT: movdqa %xmm7, %xmm6
480 ; SSE-NEXT: psrlq $32, %xmm6
481 ; SSE-NEXT: pmuludq %xmm3, %xmm6
482 ; SSE-NEXT: paddq %xmm4, %xmm6
483 ; SSE-NEXT: psllq $32, %xmm6
484 ; SSE-NEXT: pmuludq %xmm7, %xmm3
485 ; SSE-NEXT: paddq %xmm6, %xmm3
486 ; SSE-NEXT: movdqa %xmm1, %xmm4
487 ; SSE-NEXT: psrlq $32, %xmm4
488 ; SSE-NEXT: pmuludq %xmm5, %xmm4
489 ; SSE-NEXT: movdqa %xmm5, %xmm6
490 ; SSE-NEXT: psrlq $32, %xmm6
491 ; SSE-NEXT: pmuludq %xmm1, %xmm6
492 ; SSE-NEXT: paddq %xmm4, %xmm6
493 ; SSE-NEXT: psllq $32, %xmm6
494 ; SSE-NEXT: pmuludq %xmm5, %xmm1
495 ; SSE-NEXT: paddq %xmm6, %xmm1
496 ; SSE-NEXT: movdqa %xmm1, %xmm4
497 ; SSE-NEXT: psrlq $32, %xmm4
498 ; SSE-NEXT: pmuludq %xmm3, %xmm4
499 ; SSE-NEXT: movdqa %xmm3, %xmm5
500 ; SSE-NEXT: psrlq $32, %xmm5
501 ; SSE-NEXT: pmuludq %xmm1, %xmm5
502 ; SSE-NEXT: paddq %xmm4, %xmm5
503 ; SSE-NEXT: psllq $32, %xmm5
504 ; SSE-NEXT: pmuludq %xmm3, %xmm1
505 ; SSE-NEXT: paddq %xmm5, %xmm1
506 ; SSE-NEXT: movdqa %xmm0, %xmm3
507 ; SSE-NEXT: psrlq $32, %xmm3
508 ; SSE-NEXT: pmuludq %xmm2, %xmm3
509 ; SSE-NEXT: movdqa %xmm2, %xmm4
510 ; SSE-NEXT: psrlq $32, %xmm4
511 ; SSE-NEXT: pmuludq %xmm0, %xmm4
512 ; SSE-NEXT: paddq %xmm3, %xmm4
513 ; SSE-NEXT: psllq $32, %xmm4
514 ; SSE-NEXT: pmuludq %xmm2, %xmm0
515 ; SSE-NEXT: paddq %xmm4, %xmm0
516 ; SSE-NEXT: movdqa %xmm0, %xmm2
517 ; SSE-NEXT: psrlq $32, %xmm2
518 ; SSE-NEXT: pmuludq %xmm1, %xmm2
519 ; SSE-NEXT: movdqa %xmm1, %xmm3
520 ; SSE-NEXT: psrlq $32, %xmm3
521 ; SSE-NEXT: pmuludq %xmm0, %xmm3
522 ; SSE-NEXT: paddq %xmm2, %xmm3
523 ; SSE-NEXT: psllq $32, %xmm3
524 ; SSE-NEXT: pmuludq %xmm1, %xmm0
525 ; SSE-NEXT: paddq %xmm3, %xmm0
526 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
527 ; SSE-NEXT: movdqa %xmm0, %xmm2
528 ; SSE-NEXT: psrlq $32, %xmm2
529 ; SSE-NEXT: pmuludq %xmm1, %xmm2
530 ; SSE-NEXT: movdqa %xmm0, %xmm3
531 ; SSE-NEXT: psrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
532 ; SSE-NEXT: pmuludq %xmm0, %xmm3
533 ; SSE-NEXT: paddq %xmm2, %xmm3
534 ; SSE-NEXT: psllq $32, %xmm3
535 ; SSE-NEXT: pmuludq %xmm1, %xmm0
536 ; SSE-NEXT: paddq %xmm3, %xmm0
537 ; SSE-NEXT: movq %xmm0, %rax
538 ; SSE-NEXT: retq
539 ;
540 ; AVX1-LABEL: test_v16i64:
541 ; AVX1: # %bb.0:
542 ; AVX1-NEXT: vpsrlq $32, %xmm1, %xmm4
543 ; AVX1-NEXT: vpmuludq %xmm3, %xmm4, %xmm4
544 ; AVX1-NEXT: vpsrlq $32, %xmm3, %xmm5
545 ; AVX1-NEXT: vpmuludq %xmm5, %xmm1, %xmm5
546 ; AVX1-NEXT: vpaddq %xmm4, %xmm5, %xmm4
547 ; AVX1-NEXT: vpsllq $32, %xmm4, %xmm4
548 ; AVX1-NEXT: vpmuludq %xmm3, %xmm1, %xmm5
549 ; AVX1-NEXT: vpaddq %xmm4, %xmm5, %xmm4
550 ; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm5
551 ; AVX1-NEXT: vpmuludq %xmm2, %xmm5, %xmm5
552 ; AVX1-NEXT: vpsrlq $32, %xmm2, %xmm6
553 ; AVX1-NEXT: vpmuludq %xmm6, %xmm0, %xmm6
554 ; AVX1-NEXT: vpaddq %xmm5, %xmm6, %xmm5
555 ; AVX1-NEXT: vpsllq $32, %xmm5, %xmm5
556 ; AVX1-NEXT: vpmuludq %xmm2, %xmm0, %xmm6
557 ; AVX1-NEXT: vpaddq %xmm5, %xmm6, %xmm5
558 ; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm3
559 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
560 ; AVX1-NEXT: vpsrlq $32, %xmm1, %xmm6
561 ; AVX1-NEXT: vpmuludq %xmm3, %xmm6, %xmm6
562 ; AVX1-NEXT: vpsrlq $32, %xmm3, %xmm7
563 ; AVX1-NEXT: vpmuludq %xmm7, %xmm1, %xmm7
564 ; AVX1-NEXT: vpaddq %xmm6, %xmm7, %xmm6
565 ; AVX1-NEXT: vpsllq $32, %xmm6, %xmm6
566 ; AVX1-NEXT: vpmuludq %xmm3, %xmm1, %xmm1
567 ; AVX1-NEXT: vpaddq %xmm6, %xmm1, %xmm1
568 ; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
569 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
570 ; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm3
571 ; AVX1-NEXT: vpmuludq %xmm2, %xmm3, %xmm3
572 ; AVX1-NEXT: vpsrlq $32, %xmm2, %xmm6
573 ; AVX1-NEXT: vpmuludq %xmm6, %xmm0, %xmm6
574 ; AVX1-NEXT: vpaddq %xmm3, %xmm6, %xmm3
575 ; AVX1-NEXT: vpsllq $32, %xmm3, %xmm3
576 ; AVX1-NEXT: vpmuludq %xmm2, %xmm0, %xmm0
577 ; AVX1-NEXT: vpaddq %xmm3, %xmm0, %xmm0
578 ; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm2
579 ; AVX1-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
580 ; AVX1-NEXT: vpsrlq $32, %xmm1, %xmm3
581 ; AVX1-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
582 ; AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2
583 ; AVX1-NEXT: vpsllq $32, %xmm2, %xmm2
584 ; AVX1-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
585 ; AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
586 ; AVX1-NEXT: vpsrlq $32, %xmm5, %xmm1
587 ; AVX1-NEXT: vpmuludq %xmm4, %xmm1, %xmm1
588 ; AVX1-NEXT: vpsrlq $32, %xmm4, %xmm2
589 ; AVX1-NEXT: vpmuludq %xmm2, %xmm5, %xmm2
590 ; AVX1-NEXT: vpaddq %xmm1, %xmm2, %xmm1
591 ; AVX1-NEXT: vpsllq $32, %xmm1, %xmm1
592 ; AVX1-NEXT: vpmuludq %xmm4, %xmm5, %xmm2
593 ; AVX1-NEXT: vpaddq %xmm1, %xmm2, %xmm1
594 ; AVX1-NEXT: vpsrlq $32, %xmm1, %xmm2
595 ; AVX1-NEXT: vpmuludq %xmm0, %xmm2, %xmm2
596 ; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm3
597 ; AVX1-NEXT: vpmuludq %xmm3, %xmm1, %xmm3
598 ; AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2
599 ; AVX1-NEXT: vpsllq $32, %xmm2, %xmm2
600 ; AVX1-NEXT: vpmuludq %xmm0, %xmm1, %xmm0
601 ; AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
602 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
603 ; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm2
604 ; AVX1-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
605 ; AVX1-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
606 ; AVX1-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
607 ; AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2
608 ; AVX1-NEXT: vpsllq $32, %xmm2, %xmm2
609 ; AVX1-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
610 ; AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
611 ; AVX1-NEXT: vmovq %xmm0, %rax
612 ; AVX1-NEXT: vzeroupper
613 ; AVX1-NEXT: retq
614 ;
615 ; AVX2-LABEL: test_v16i64:
616 ; AVX2: # %bb.0:
617 ; AVX2-NEXT: vpsrlq $32, %ymm1, %ymm4
618 ; AVX2-NEXT: vpmuludq %ymm3, %ymm4, %ymm4
619 ; AVX2-NEXT: vpsrlq $32, %ymm3, %ymm5
620 ; AVX2-NEXT: vpmuludq %ymm5, %ymm1, %ymm5
621 ; AVX2-NEXT: vpaddq %ymm4, %ymm5, %ymm4
622 ; AVX2-NEXT: vpsllq $32, %ymm4, %ymm4
623 ; AVX2-NEXT: vpmuludq %ymm3, %ymm1, %ymm1
624 ; AVX2-NEXT: vpaddq %ymm4, %ymm1, %ymm1
625 ; AVX2-NEXT: vpsrlq $32, %ymm0, %ymm3
626 ; AVX2-NEXT: vpmuludq %ymm2, %ymm3, %ymm3
627 ; AVX2-NEXT: vpsrlq $32, %ymm2, %ymm4
628 ; AVX2-NEXT: vpmuludq %ymm4, %ymm0, %ymm4
629 ; AVX2-NEXT: vpaddq %ymm3, %ymm4, %ymm3
630 ; AVX2-NEXT: vpsllq $32, %ymm3, %ymm3
631 ; AVX2-NEXT: vpmuludq %ymm2, %ymm0, %ymm0
632 ; AVX2-NEXT: vpaddq %ymm3, %ymm0, %ymm0
633 ; AVX2-NEXT: vpsrlq $32, %ymm0, %ymm2
634 ; AVX2-NEXT: vpmuludq %ymm1, %ymm2, %ymm2
635 ; AVX2-NEXT: vpsrlq $32, %ymm1, %ymm3
636 ; AVX2-NEXT: vpmuludq %ymm3, %ymm0, %ymm3
637 ; AVX2-NEXT: vpaddq %ymm2, %ymm3, %ymm2
638 ; AVX2-NEXT: vpsllq $32, %ymm2, %ymm2
639 ; AVX2-NEXT: vpmuludq %ymm1, %ymm0, %ymm0
640 ; AVX2-NEXT: vpaddq %ymm2, %ymm0, %ymm0
641 ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
642 ; AVX2-NEXT: vpsrlq $32, %xmm0, %xmm2
643 ; AVX2-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
644 ; AVX2-NEXT: vpsrlq $32, %xmm1, %xmm3
645 ; AVX2-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
646 ; AVX2-NEXT: vpaddq %xmm2, %xmm3, %xmm2
647 ; AVX2-NEXT: vpsllq $32, %xmm2, %xmm2
648 ; AVX2-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
649 ; AVX2-NEXT: vpaddq %xmm2, %xmm0, %xmm0
650 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
651 ; AVX2-NEXT: vpsrlq $32, %xmm0, %xmm2
652 ; AVX2-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
653 ; AVX2-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
654 ; AVX2-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
655 ; AVX2-NEXT: vpaddq %xmm2, %xmm3, %xmm2
656 ; AVX2-NEXT: vpsllq $32, %xmm2, %xmm2
657 ; AVX2-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
658 ; AVX2-NEXT: vpaddq %xmm2, %xmm0, %xmm0
659 ; AVX2-NEXT: vmovq %xmm0, %rax
660 ; AVX2-NEXT: vzeroupper
661 ; AVX2-NEXT: retq
662 ;
663 ; AVX512BW-LABEL: test_v16i64:
664 ; AVX512BW: # %bb.0:
665 ; AVX512BW-NEXT: vpsrlq $32, %zmm0, %zmm2
666 ; AVX512BW-NEXT: vpmuludq %zmm1, %zmm2, %zmm2
667 ; AVX512BW-NEXT: vpsrlq $32, %zmm1, %zmm3
668 ; AVX512BW-NEXT: vpmuludq %zmm3, %zmm0, %zmm3
669 ; AVX512BW-NEXT: vpaddq %zmm2, %zmm3, %zmm2
670 ; AVX512BW-NEXT: vpsllq $32, %zmm2, %zmm2
671 ; AVX512BW-NEXT: vpmuludq %zmm1, %zmm0, %zmm0
672 ; AVX512BW-NEXT: vpaddq %zmm2, %zmm0, %zmm0
673 ; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
674 ; AVX512BW-NEXT: vpsrlq $32, %zmm0, %zmm2
675 ; AVX512BW-NEXT: vpmuludq %zmm1, %zmm2, %zmm2
676 ; AVX512BW-NEXT: vpsrlq $32, %zmm1, %zmm3
677 ; AVX512BW-NEXT: vpmuludq %zmm3, %zmm0, %zmm3
678 ; AVX512BW-NEXT: vpaddq %zmm2, %zmm3, %zmm2
679 ; AVX512BW-NEXT: vpsllq $32, %zmm2, %zmm2
680 ; AVX512BW-NEXT: vpmuludq %zmm1, %zmm0, %zmm0
681 ; AVX512BW-NEXT: vpaddq %zmm2, %zmm0, %zmm0
682 ; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
683 ; AVX512BW-NEXT: vpsrlq $32, %xmm0, %xmm2
684 ; AVX512BW-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
685 ; AVX512BW-NEXT: vpsrlq $32, %xmm1, %xmm3
686 ; AVX512BW-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
687 ; AVX512BW-NEXT: vpaddq %xmm2, %xmm3, %xmm2
688 ; AVX512BW-NEXT: vpsllq $32, %xmm2, %xmm2
689 ; AVX512BW-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
690 ; AVX512BW-NEXT: vpaddq %xmm2, %xmm0, %xmm0
691 ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
692 ; AVX512BW-NEXT: vpsrlq $32, %xmm0, %xmm2
693 ; AVX512BW-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
694 ; AVX512BW-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
695 ; AVX512BW-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
696 ; AVX512BW-NEXT: vpaddq %xmm2, %xmm3, %xmm2
697 ; AVX512BW-NEXT: vpsllq $32, %xmm2, %xmm2
698 ; AVX512BW-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
699 ; AVX512BW-NEXT: vpaddq %xmm2, %xmm0, %xmm0
700 ; AVX512BW-NEXT: vmovq %xmm0, %rax
701 ; AVX512BW-NEXT: vzeroupper
702 ; AVX512BW-NEXT: retq
703 ;
704 ; AVX512BWVL-LABEL: test_v16i64:
705 ; AVX512BWVL: # %bb.0:
706 ; AVX512BWVL-NEXT: vpsrlq $32, %zmm0, %zmm2
707 ; AVX512BWVL-NEXT: vpmuludq %zmm1, %zmm2, %zmm2
708 ; AVX512BWVL-NEXT: vpsrlq $32, %zmm1, %zmm3
709 ; AVX512BWVL-NEXT: vpmuludq %zmm3, %zmm0, %zmm3
710 ; AVX512BWVL-NEXT: vpaddq %zmm2, %zmm3, %zmm2
711 ; AVX512BWVL-NEXT: vpsllq $32, %zmm2, %zmm2
712 ; AVX512BWVL-NEXT: vpmuludq %zmm1, %zmm0, %zmm0
713 ; AVX512BWVL-NEXT: vpaddq %zmm2, %zmm0, %zmm0
714 ; AVX512BWVL-NEXT: vextracti64x4 $1, %zmm0, %ymm1
715 ; AVX512BWVL-NEXT: vpsrlq $32, %zmm0, %zmm2
716 ; AVX512BWVL-NEXT: vpmuludq %zmm1, %zmm2, %zmm2
717 ; AVX512BWVL-NEXT: vpsrlq $32, %zmm1, %zmm3
718 ; AVX512BWVL-NEXT: vpmuludq %zmm3, %zmm0, %zmm3
719 ; AVX512BWVL-NEXT: vpaddq %zmm2, %zmm3, %zmm2
720 ; AVX512BWVL-NEXT: vpsllq $32, %zmm2, %zmm2
721 ; AVX512BWVL-NEXT: vpmuludq %zmm1, %zmm0, %zmm0
722 ; AVX512BWVL-NEXT: vpaddq %zmm2, %zmm0, %zmm0
723 ; AVX512BWVL-NEXT: vextracti128 $1, %ymm0, %xmm1
724 ; AVX512BWVL-NEXT: vpsrlq $32, %xmm0, %xmm2
725 ; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
726 ; AVX512BWVL-NEXT: vpsrlq $32, %xmm1, %xmm3
727 ; AVX512BWVL-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
728 ; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm3, %xmm2
729 ; AVX512BWVL-NEXT: vpsllq $32, %xmm2, %xmm2
730 ; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
731 ; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm0, %xmm0
732 ; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
733 ; AVX512BWVL-NEXT: vpsrlq $32, %xmm0, %xmm2
734 ; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
735 ; AVX512BWVL-NEXT: vpsrlq $32, %xmm1, %xmm3
736 ; AVX512BWVL-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
737 ; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm3, %xmm2
738 ; AVX512BWVL-NEXT: vpsllq $32, %xmm2, %xmm2
739 ; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
740 ; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm0, %xmm0
741 ; AVX512BWVL-NEXT: vmovq %xmm0, %rax
742 ; AVX512BWVL-NEXT: vzeroupper
743 ; AVX512BWVL-NEXT: retq
744 ;
745 ; AVX512DQ-LABEL: test_v16i64:
746 ; AVX512DQ: # %bb.0:
747 ; AVX512DQ-NEXT: vpmullq %zmm1, %zmm0, %zmm0
748 ; AVX512DQ-NEXT: vextracti64x4 $1, %zmm0, %ymm1
749 ; AVX512DQ-NEXT: vpmullq %zmm1, %zmm0, %zmm0
750 ; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm1
751 ; AVX512DQ-NEXT: vpmullq %zmm1, %zmm0, %zmm0
752 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
753 ; AVX512DQ-NEXT: vpmullq %zmm1, %zmm0, %zmm0
754 ; AVX512DQ-NEXT: vmovq %xmm0, %rax
755 ; AVX512DQ-NEXT: vzeroupper
756 ; AVX512DQ-NEXT: retq
757 ;
758 ; AVX512DQVL-LABEL: test_v16i64:
759 ; AVX512DQVL: # %bb.0:
760 ; AVX512DQVL-NEXT: vpmullq %zmm1, %zmm0, %zmm0
761 ; AVX512DQVL-NEXT: vextracti64x4 $1, %zmm0, %ymm1
762 ; AVX512DQVL-NEXT: vpmullq %zmm1, %zmm0, %zmm0
763 ; AVX512DQVL-NEXT: vextracti128 $1, %ymm0, %xmm1
764 ; AVX512DQVL-NEXT: vpmullq %xmm1, %xmm0, %xmm0
765 ; AVX512DQVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
766 ; AVX512DQVL-NEXT: vpmullq %xmm1, %xmm0, %xmm0
767 ; AVX512DQVL-NEXT: vmovq %xmm0, %rax
768 ; AVX512DQVL-NEXT: vzeroupper
769 ; AVX512DQVL-NEXT: retq
770 %1 = call i64 @llvm.experimental.vector.reduce.mul.v16i64(<16 x i64> %a0)
771 ret i64 %1
772 }
774 ;
775 ; vXi32
776 ;
778 define i32 @test_v2i32(<2 x i32> %a0) {
779 ; SSE2-LABEL: test_v2i32:
780 ; SSE2: # %bb.0:
781 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
782 ; SSE2-NEXT: pmuludq %xmm0, %xmm1
783 ; SSE2-NEXT: movd %xmm1, %eax
784 ; SSE2-NEXT: retq
785 ;
786 ; SSE41-LABEL: test_v2i32:
787 ; SSE41: # %bb.0:
788 ; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
789 ; SSE41-NEXT: pmulld %xmm0, %xmm1
790 ; SSE41-NEXT: movd %xmm1, %eax
791 ; SSE41-NEXT: retq
792 ;
793 ; AVX-LABEL: test_v2i32:
794 ; AVX: # %bb.0:
795 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
796 ; AVX-NEXT: vpmulld %xmm1, %xmm0, %xmm0
797 ; AVX-NEXT: vmovd %xmm0, %eax
798 ; AVX-NEXT: retq
799 ;
800 ; AVX512-LABEL: test_v2i32:
801 ; AVX512: # %bb.0:
802 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
803 ; AVX512-NEXT: vpmulld %xmm1, %xmm0, %xmm0
804 ; AVX512-NEXT: vmovd %xmm0, %eax
805 ; AVX512-NEXT: retq
806 %1 = call i32 @llvm.experimental.vector.reduce.mul.v2i32(<2 x i32> %a0)
807 ret i32 %1
808 }
810 define i32 @test_v4i32(<4 x i32> %a0) {
811 ; SSE2-LABEL: test_v4i32:
812 ; SSE2: # %bb.0:
813 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
814 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,1,1]
815 ; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
816 ; SSE2-NEXT: pmuludq %xmm2, %xmm3
817 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
818 ; SSE2-NEXT: pmuludq %xmm0, %xmm1
819 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
820 ; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
821 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
822 ; SSE2-NEXT: pmuludq %xmm0, %xmm1
823 ; SSE2-NEXT: movd %xmm1, %eax
824 ; SSE2-NEXT: retq
825 ;
826 ; SSE41-LABEL: test_v4i32:
827 ; SSE41: # %bb.0:
828 ; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
829 ; SSE41-NEXT: pmulld %xmm0, %xmm1
830 ; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
831 ; SSE41-NEXT: pmulld %xmm1, %xmm0
832 ; SSE41-NEXT: movd %xmm0, %eax
833 ; SSE41-NEXT: retq
834 ;
835 ; AVX-LABEL: test_v4i32:
836 ; AVX: # %bb.0:
837 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
838 ; AVX-NEXT: vpmulld %xmm1, %xmm0, %xmm0
839 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
840 ; AVX-NEXT: vpmulld %xmm1, %xmm0, %xmm0
841 ; AVX-NEXT: vmovd %xmm0, %eax
842 ; AVX-NEXT: retq
843 ;
844 ; AVX512-LABEL: test_v4i32:
845 ; AVX512: # %bb.0:
846 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
847 ; AVX512-NEXT: vpmulld %xmm1, %xmm0, %xmm0
848 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
849 ; AVX512-NEXT: vpmulld %xmm1, %xmm0, %xmm0
850 ; AVX512-NEXT: vmovd %xmm0, %eax
851 ; AVX512-NEXT: retq
852 %1 = call i32 @llvm.experimental.vector.reduce.mul.v4i32(<4 x i32> %a0)
853 ret i32 %1
854 }
856 define i32 @test_v8i32(<8 x i32> %a0) {
857 ; SSE2-LABEL: test_v8i32:
858 ; SSE2: # %bb.0:
859 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
860 ; SSE2-NEXT: pmuludq %xmm1, %xmm0
861 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
862 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
863 ; SSE2-NEXT: pmuludq %xmm2, %xmm1
864 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3]
865 ; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
866 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
867 ; SSE2-NEXT: pmuludq %xmm0, %xmm2
868 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
869 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,2,0,0]
870 ; SSE2-NEXT: pmuludq %xmm1, %xmm2
871 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
872 ; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
873 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
874 ; SSE2-NEXT: pmuludq %xmm0, %xmm1
875 ; SSE2-NEXT: movd %xmm1, %eax
876 ; SSE2-NEXT: retq
877 ;
878 ; SSE41-LABEL: test_v8i32:
879 ; SSE41: # %bb.0:
880 ; SSE41-NEXT: pmulld %xmm1, %xmm0
881 ; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
882 ; SSE41-NEXT: pmulld %xmm0, %xmm1
883 ; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
884 ; SSE41-NEXT: pmulld %xmm1, %xmm0
885 ; SSE41-NEXT: movd %xmm0, %eax
886 ; SSE41-NEXT: retq
887 ;
888 ; AVX1-LABEL: test_v8i32:
889 ; AVX1: # %bb.0:
890 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
891 ; AVX1-NEXT: vpmulld %xmm1, %xmm0, %xmm0
892 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
893 ; AVX1-NEXT: vpmulld %xmm1, %xmm0, %xmm0
894 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
895 ; AVX1-NEXT: vpmulld %xmm1, %xmm0, %xmm0
896 ; AVX1-NEXT: vmovd %xmm0, %eax
897 ; AVX1-NEXT: vzeroupper
898 ; AVX1-NEXT: retq
899 ;
900 ; AVX2-LABEL: test_v8i32:
901 ; AVX2: # %bb.0:
902 ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
903 ; AVX2-NEXT: vpmulld %xmm1, %xmm0, %xmm0
904 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
905 ; AVX2-NEXT: vpmulld %xmm1, %xmm0, %xmm0
906 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
907 ; AVX2-NEXT: vpmulld %xmm1, %xmm0, %xmm0
908 ; AVX2-NEXT: vmovd %xmm0, %eax
909 ; AVX2-NEXT: vzeroupper
910 ; AVX2-NEXT: retq
911 ;
912 ; AVX512-LABEL: test_v8i32:
913 ; AVX512: # %bb.0:
914 ; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
915 ; AVX512-NEXT: vpmulld %xmm1, %xmm0, %xmm0
916 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
917 ; AVX512-NEXT: vpmulld %xmm1, %xmm0, %xmm0
918 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
919 ; AVX512-NEXT: vpmulld %xmm1, %xmm0, %xmm0
920 ; AVX512-NEXT: vmovd %xmm0, %eax
921 ; AVX512-NEXT: vzeroupper
922 ; AVX512-NEXT: retq
923 %1 = call i32 @llvm.experimental.vector.reduce.mul.v8i32(<8 x i32> %a0)
924 ret i32 %1
925 }
927 define i32 @test_v16i32(<16 x i32> %a0) {
928 ; SSE2-LABEL: test_v16i32:
929 ; SSE2: # %bb.0:
930 ; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
931 ; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
932 ; SSE2-NEXT: pmuludq %xmm4, %xmm5
933 ; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
934 ; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
935 ; SSE2-NEXT: pmuludq %xmm4, %xmm6
936 ; SSE2-NEXT: pmuludq %xmm5, %xmm6
937 ; SSE2-NEXT: pmuludq %xmm3, %xmm1
938 ; SSE2-NEXT: pmuludq %xmm2, %xmm0
939 ; SSE2-NEXT: pmuludq %xmm1, %xmm0
940 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
941 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm6[0,2,2,3]
942 ; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
943 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
944 ; SSE2-NEXT: pmuludq %xmm0, %xmm1
945 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
946 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm6[2,2,0,0]
947 ; SSE2-NEXT: pmuludq %xmm6, %xmm1
948 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
949 ; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
950 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
951 ; SSE2-NEXT: pmuludq %xmm0, %xmm1
952 ; SSE2-NEXT: movd %xmm1, %eax
953 ; SSE2-NEXT: retq
954 ;
955 ; SSE41-LABEL: test_v16i32:
956 ; SSE41: # %bb.0:
957 ; SSE41-NEXT: pmulld %xmm3, %xmm1
958 ; SSE41-NEXT: pmulld %xmm2, %xmm1
959 ; SSE41-NEXT: pmulld %xmm0, %xmm1
960 ; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
961 ; SSE41-NEXT: pmulld %xmm1, %xmm0
962 ; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
963 ; SSE41-NEXT: pmulld %xmm0, %xmm1
964 ; SSE41-NEXT: movd %xmm1, %eax
965 ; SSE41-NEXT: retq
966 ;
967 ; AVX1-LABEL: test_v16i32:
968 ; AVX1: # %bb.0:
969 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
970 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
971 ; AVX1-NEXT: vpmulld %xmm2, %xmm3, %xmm2
972 ; AVX1-NEXT: vpmulld %xmm2, %xmm1, %xmm1
973 ; AVX1-NEXT: vpmulld %xmm1, %xmm0, %xmm0
974 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
975 ; AVX1-NEXT: vpmulld %xmm1, %xmm0, %xmm0
976 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
977 ; AVX1-NEXT: vpmulld %xmm1, %xmm0, %xmm0
978 ; AVX1-NEXT: vmovd %xmm0, %eax
979 ; AVX1-NEXT: vzeroupper
980 ; AVX1-NEXT: retq
981 ;
982 ; AVX2-LABEL: test_v16i32:
983 ; AVX2: # %bb.0:
984 ; AVX2-NEXT: vpmulld %ymm1, %ymm0, %ymm0
985 ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
986 ; AVX2-NEXT: vpmulld %xmm1, %xmm0, %xmm0
987 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
988 ; AVX2-NEXT: vpmulld %xmm1, %xmm0, %xmm0
989 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
990 ; AVX2-NEXT: vpmulld %xmm1, %xmm0, %xmm0
991 ; AVX2-NEXT: vmovd %xmm0, %eax
992 ; AVX2-NEXT: vzeroupper
993 ; AVX2-NEXT: retq
994 ;
995 ; AVX512-LABEL: test_v16i32:
996 ; AVX512: # %bb.0:
997 ; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
998 ; AVX512-NEXT: vpmulld %zmm1, %zmm0, %zmm0
999 ; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
1000 ; AVX512-NEXT: vpmulld %xmm1, %xmm0, %xmm0
1001 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1002 ; AVX512-NEXT: vpmulld %xmm1, %xmm0, %xmm0
1003 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1004 ; AVX512-NEXT: vpmulld %xmm1, %xmm0, %xmm0
1005 ; AVX512-NEXT: vmovd %xmm0, %eax
1006 ; AVX512-NEXT: vzeroupper
1007 ; AVX512-NEXT: retq
1008 %1 = call i32 @llvm.experimental.vector.reduce.mul.v16i32(<16 x i32> %a0)
1009 ret i32 %1
1010 }
1012 define i32 @test_v32i32(<32 x i32> %a0) {
1013 ; SSE2-LABEL: test_v32i32:
1014 ; SSE2: # %bb.0:
1015 ; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm2[1,1,3,3]
1016 ; SSE2-NEXT: pmuludq %xmm6, %xmm2
1017 ; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm0[1,1,3,3]
1018 ; SSE2-NEXT: pmuludq %xmm4, %xmm0
1019 ; SSE2-NEXT: pmuludq %xmm2, %xmm0
1020 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
1021 ; SSE2-NEXT: pmuludq %xmm7, %xmm3
1022 ; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm1[1,1,3,3]
1023 ; SSE2-NEXT: pmuludq %xmm5, %xmm1
1024 ; SSE2-NEXT: pmuludq %xmm3, %xmm1
1025 ; SSE2-NEXT: pmuludq %xmm0, %xmm1
1026 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
1027 ; SSE2-NEXT: pmuludq %xmm8, %xmm0
1028 ; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
1029 ; SSE2-NEXT: pmuludq %xmm9, %xmm3
1030 ; SSE2-NEXT: pmuludq %xmm0, %xmm3
1031 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3]
1032 ; SSE2-NEXT: pmuludq %xmm2, %xmm0
1033 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm5[1,1,3,3]
1034 ; SSE2-NEXT: pmuludq %xmm10, %xmm2
1035 ; SSE2-NEXT: pmuludq %xmm0, %xmm2
1036 ; SSE2-NEXT: pmuludq %xmm3, %xmm2
1037 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
1038 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
1039 ; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1040 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1041 ; SSE2-NEXT: pmuludq %xmm0, %xmm1
1042 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
1043 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,2,0,0]
1044 ; SSE2-NEXT: pmuludq %xmm2, %xmm1
1045 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
1046 ; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1047 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1048 ; SSE2-NEXT: pmuludq %xmm0, %xmm1
1049 ; SSE2-NEXT: movd %xmm1, %eax
1050 ; SSE2-NEXT: retq
1051 ;
1052 ; SSE41-LABEL: test_v32i32:
1053 ; SSE41: # %bb.0:
1054 ; SSE41-NEXT: pmulld %xmm6, %xmm2
1055 ; SSE41-NEXT: pmulld %xmm7, %xmm3
1056 ; SSE41-NEXT: pmulld %xmm5, %xmm3
1057 ; SSE41-NEXT: pmulld %xmm1, %xmm3
1058 ; SSE41-NEXT: pmulld %xmm4, %xmm2
1059 ; SSE41-NEXT: pmulld %xmm3, %xmm2
1060 ; SSE41-NEXT: pmulld %xmm0, %xmm2
1061 ; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
1062 ; SSE41-NEXT: pmulld %xmm2, %xmm0
1063 ; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1064 ; SSE41-NEXT: pmulld %xmm0, %xmm1
1065 ; SSE41-NEXT: movd %xmm1, %eax
1066 ; SSE41-NEXT: retq
1067 ;
1068 ; AVX1-LABEL: test_v32i32:
1069 ; AVX1: # %bb.0:
1070 ; AVX1-NEXT: vpmulld %xmm3, %xmm1, %xmm4
1071 ; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm3
1072 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
1073 ; AVX1-NEXT: vpmulld %xmm3, %xmm1, %xmm1
1074 ; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
1075 ; AVX1-NEXT: vpmulld %xmm1, %xmm3, %xmm1
1076 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
1077 ; AVX1-NEXT: vpmulld %xmm1, %xmm3, %xmm1
1078 ; AVX1-NEXT: vpmulld %xmm4, %xmm2, %xmm2
1079 ; AVX1-NEXT: vpmulld %xmm1, %xmm2, %xmm1
1080 ; AVX1-NEXT: vpmulld %xmm1, %xmm0, %xmm0
1081 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1082 ; AVX1-NEXT: vpmulld %xmm1, %xmm0, %xmm0
1083 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1084 ; AVX1-NEXT: vpmulld %xmm1, %xmm0, %xmm0
1085 ; AVX1-NEXT: vmovd %xmm0, %eax
1086 ; AVX1-NEXT: vzeroupper
1087 ; AVX1-NEXT: retq
1088 ;
1089 ; AVX2-LABEL: test_v32i32:
1090 ; AVX2: # %bb.0:
1091 ; AVX2-NEXT: vpmulld %ymm3, %ymm1, %ymm1
1092 ; AVX2-NEXT: vpmulld %ymm1, %ymm2, %ymm1
1093 ; AVX2-NEXT: vpmulld %ymm1, %ymm0, %ymm0
1094 ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
1095 ; AVX2-NEXT: vpmulld %xmm1, %xmm0, %xmm0
1096 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1097 ; AVX2-NEXT: vpmulld %xmm1, %xmm0, %xmm0
1098 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1099 ; AVX2-NEXT: vpmulld %xmm1, %xmm0, %xmm0
1100 ; AVX2-NEXT: vmovd %xmm0, %eax
1101 ; AVX2-NEXT: vzeroupper
1102 ; AVX2-NEXT: retq
1103 ;
1104 ; AVX512-LABEL: test_v32i32:
1105 ; AVX512: # %bb.0:
1106 ; AVX512-NEXT: vpmulld %zmm1, %zmm0, %zmm0
1107 ; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
1108 ; AVX512-NEXT: vpmulld %zmm1, %zmm0, %zmm0
1109 ; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
1110 ; AVX512-NEXT: vpmulld %xmm1, %xmm0, %xmm0
1111 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1112 ; AVX512-NEXT: vpmulld %xmm1, %xmm0, %xmm0
1113 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1114 ; AVX512-NEXT: vpmulld %xmm1, %xmm0, %xmm0
1115 ; AVX512-NEXT: vmovd %xmm0, %eax
1116 ; AVX512-NEXT: vzeroupper
1117 ; AVX512-NEXT: retq
1118 %1 = call i32 @llvm.experimental.vector.reduce.mul.v32i32(<32 x i32> %a0)
1119 ret i32 %1
1120 }
1122 ;
1123 ; vXi16
1124 ;
1126 define i16 @test_v2i16(<2 x i16> %a0) {
1127 ; SSE-LABEL: test_v2i16:
1128 ; SSE: # %bb.0:
1129 ; SSE-NEXT: movdqa %xmm0, %xmm1
1130 ; SSE-NEXT: psrld $16, %xmm1
1131 ; SSE-NEXT: pmullw %xmm0, %xmm1
1132 ; SSE-NEXT: movd %xmm1, %eax
1133 ; SSE-NEXT: # kill: def $ax killed $ax killed $eax
1134 ; SSE-NEXT: retq
1135 ;
1136 ; AVX-LABEL: test_v2i16:
1137 ; AVX: # %bb.0:
1138 ; AVX-NEXT: vpsrld $16, %xmm0, %xmm1
1139 ; AVX-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1140 ; AVX-NEXT: vmovd %xmm0, %eax
1141 ; AVX-NEXT: # kill: def $ax killed $ax killed $eax
1142 ; AVX-NEXT: retq
1143 ;
1144 ; AVX512-LABEL: test_v2i16:
1145 ; AVX512: # %bb.0:
1146 ; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1
1147 ; AVX512-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1148 ; AVX512-NEXT: vmovd %xmm0, %eax
1149 ; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
1150 ; AVX512-NEXT: retq
1151 %1 = call i16 @llvm.experimental.vector.reduce.mul.v2i16(<2 x i16> %a0)
1152 ret i16 %1
1153 }
1155 define i16 @test_v4i16(<4 x i16> %a0) {
1156 ; SSE-LABEL: test_v4i16:
1157 ; SSE: # %bb.0:
1158 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1159 ; SSE-NEXT: pmullw %xmm0, %xmm1
1160 ; SSE-NEXT: movdqa %xmm1, %xmm0
1161 ; SSE-NEXT: psrld $16, %xmm0
1162 ; SSE-NEXT: pmullw %xmm1, %xmm0
1163 ; SSE-NEXT: movd %xmm0, %eax
1164 ; SSE-NEXT: # kill: def $ax killed $ax killed $eax
1165 ; SSE-NEXT: retq
1166 ;
1167 ; AVX-LABEL: test_v4i16:
1168 ; AVX: # %bb.0:
1169 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1170 ; AVX-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1171 ; AVX-NEXT: vpsrld $16, %xmm0, %xmm1
1172 ; AVX-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1173 ; AVX-NEXT: vmovd %xmm0, %eax
1174 ; AVX-NEXT: # kill: def $ax killed $ax killed $eax
1175 ; AVX-NEXT: retq
1176 ;
1177 ; AVX512-LABEL: test_v4i16:
1178 ; AVX512: # %bb.0:
1179 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1180 ; AVX512-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1181 ; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1
1182 ; AVX512-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1183 ; AVX512-NEXT: vmovd %xmm0, %eax
1184 ; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
1185 ; AVX512-NEXT: retq
1186 %1 = call i16 @llvm.experimental.vector.reduce.mul.v4i16(<4 x i16> %a0)
1187 ret i16 %1
1188 }
1190 define i16 @test_v8i16(<8 x i16> %a0) {
1191 ; SSE-LABEL: test_v8i16:
1192 ; SSE: # %bb.0:
1193 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1194 ; SSE-NEXT: pmullw %xmm0, %xmm1
1195 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
1196 ; SSE-NEXT: pmullw %xmm1, %xmm0
1197 ; SSE-NEXT: movdqa %xmm0, %xmm1
1198 ; SSE-NEXT: psrld $16, %xmm1
1199 ; SSE-NEXT: pmullw %xmm0, %xmm1
1200 ; SSE-NEXT: movd %xmm1, %eax
1201 ; SSE-NEXT: # kill: def $ax killed $ax killed $eax
1202 ; SSE-NEXT: retq
1203 ;
1204 ; AVX-LABEL: test_v8i16:
1205 ; AVX: # %bb.0:
1206 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1207 ; AVX-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1208 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1209 ; AVX-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1210 ; AVX-NEXT: vpsrld $16, %xmm0, %xmm1
1211 ; AVX-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1212 ; AVX-NEXT: vmovd %xmm0, %eax
1213 ; AVX-NEXT: # kill: def $ax killed $ax killed $eax
1214 ; AVX-NEXT: retq
1215 ;
1216 ; AVX512-LABEL: test_v8i16:
1217 ; AVX512: # %bb.0:
1218 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1219 ; AVX512-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1220 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1221 ; AVX512-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1222 ; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1
1223 ; AVX512-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1224 ; AVX512-NEXT: vmovd %xmm0, %eax
1225 ; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
1226 ; AVX512-NEXT: retq
1227 %1 = call i16 @llvm.experimental.vector.reduce.mul.v8i16(<8 x i16> %a0)
1228 ret i16 %1
1229 }
1231 define i16 @test_v16i16(<16 x i16> %a0) {
1232 ; SSE-LABEL: test_v16i16:
1233 ; SSE: # %bb.0:
1234 ; SSE-NEXT: pmullw %xmm1, %xmm0
1235 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1236 ; SSE-NEXT: pmullw %xmm0, %xmm1
1237 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
1238 ; SSE-NEXT: pmullw %xmm1, %xmm0
1239 ; SSE-NEXT: movdqa %xmm0, %xmm1
1240 ; SSE-NEXT: psrld $16, %xmm1
1241 ; SSE-NEXT: pmullw %xmm0, %xmm1
1242 ; SSE-NEXT: movd %xmm1, %eax
1243 ; SSE-NEXT: # kill: def $ax killed $ax killed $eax
1244 ; SSE-NEXT: retq
1245 ;
1246 ; AVX1-LABEL: test_v16i16:
1247 ; AVX1: # %bb.0:
1248 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
1249 ; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1250 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1251 ; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1252 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1253 ; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1254 ; AVX1-NEXT: vpsrld $16, %xmm0, %xmm1
1255 ; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1256 ; AVX1-NEXT: vmovd %xmm0, %eax
1257 ; AVX1-NEXT: # kill: def $ax killed $ax killed $eax
1258 ; AVX1-NEXT: vzeroupper
1259 ; AVX1-NEXT: retq
1260 ;
1261 ; AVX2-LABEL: test_v16i16:
1262 ; AVX2: # %bb.0:
1263 ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
1264 ; AVX2-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1265 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1266 ; AVX2-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1267 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1268 ; AVX2-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1269 ; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
1270 ; AVX2-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1271 ; AVX2-NEXT: vmovd %xmm0, %eax
1272 ; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
1273 ; AVX2-NEXT: vzeroupper
1274 ; AVX2-NEXT: retq
1275 ;
1276 ; AVX512-LABEL: test_v16i16:
1277 ; AVX512: # %bb.0:
1278 ; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
1279 ; AVX512-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1280 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1281 ; AVX512-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1282 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1283 ; AVX512-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1284 ; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1
1285 ; AVX512-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1286 ; AVX512-NEXT: vmovd %xmm0, %eax
1287 ; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
1288 ; AVX512-NEXT: vzeroupper
1289 ; AVX512-NEXT: retq
1290 %1 = call i16 @llvm.experimental.vector.reduce.mul.v16i16(<16 x i16> %a0)
1291 ret i16 %1
1292 }
1294 define i16 @test_v32i16(<32 x i16> %a0) {
1295 ; SSE-LABEL: test_v32i16:
1296 ; SSE: # %bb.0:
1297 ; SSE-NEXT: pmullw %xmm3, %xmm1
1298 ; SSE-NEXT: pmullw %xmm2, %xmm1
1299 ; SSE-NEXT: pmullw %xmm0, %xmm1
1300 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
1301 ; SSE-NEXT: pmullw %xmm1, %xmm0
1302 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1303 ; SSE-NEXT: pmullw %xmm0, %xmm1
1304 ; SSE-NEXT: movdqa %xmm1, %xmm0
1305 ; SSE-NEXT: psrld $16, %xmm0
1306 ; SSE-NEXT: pmullw %xmm1, %xmm0
1307 ; SSE-NEXT: movd %xmm0, %eax
1308 ; SSE-NEXT: # kill: def $ax killed $ax killed $eax
1309 ; SSE-NEXT: retq
1310 ;
1311 ; AVX1-LABEL: test_v32i16:
1312 ; AVX1: # %bb.0:
1313 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
1314 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
1315 ; AVX1-NEXT: vpmullw %xmm2, %xmm3, %xmm2
1316 ; AVX1-NEXT: vpmullw %xmm2, %xmm1, %xmm1
1317 ; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1318 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1319 ; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1320 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1321 ; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1322 ; AVX1-NEXT: vpsrld $16, %xmm0, %xmm1
1323 ; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1324 ; AVX1-NEXT: vmovd %xmm0, %eax
1325 ; AVX1-NEXT: # kill: def $ax killed $ax killed $eax
1326 ; AVX1-NEXT: vzeroupper
1327 ; AVX1-NEXT: retq
1328 ;
1329 ; AVX2-LABEL: test_v32i16:
1330 ; AVX2: # %bb.0:
1331 ; AVX2-NEXT: vpmullw %ymm1, %ymm0, %ymm0
1332 ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
1333 ; AVX2-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1334 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1335 ; AVX2-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1336 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1337 ; AVX2-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1338 ; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
1339 ; AVX2-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1340 ; AVX2-NEXT: vmovd %xmm0, %eax
1341 ; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
1342 ; AVX2-NEXT: vzeroupper
1343 ; AVX2-NEXT: retq
1344 ;
1345 ; AVX512BW-LABEL: test_v32i16:
1346 ; AVX512BW: # %bb.0:
1347 ; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
1348 ; AVX512BW-NEXT: vpmullw %zmm1, %zmm0, %zmm0
1349 ; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
1350 ; AVX512BW-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1351 ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1352 ; AVX512BW-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1353 ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1354 ; AVX512BW-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1355 ; AVX512BW-NEXT: vpsrld $16, %xmm0, %xmm1
1356 ; AVX512BW-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1357 ; AVX512BW-NEXT: vmovd %xmm0, %eax
1358 ; AVX512BW-NEXT: # kill: def $ax killed $ax killed $eax
1359 ; AVX512BW-NEXT: vzeroupper
1360 ; AVX512BW-NEXT: retq
1361 ;
1362 ; AVX512BWVL-LABEL: test_v32i16:
1363 ; AVX512BWVL: # %bb.0:
1364 ; AVX512BWVL-NEXT: vextracti64x4 $1, %zmm0, %ymm1
1365 ; AVX512BWVL-NEXT: vpmullw %zmm1, %zmm0, %zmm0
1366 ; AVX512BWVL-NEXT: vextracti128 $1, %ymm0, %xmm1
1367 ; AVX512BWVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1368 ; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1369 ; AVX512BWVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1370 ; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1371 ; AVX512BWVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1372 ; AVX512BWVL-NEXT: vpsrld $16, %xmm0, %xmm1
1373 ; AVX512BWVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1374 ; AVX512BWVL-NEXT: vmovd %xmm0, %eax
1375 ; AVX512BWVL-NEXT: # kill: def $ax killed $ax killed $eax
1376 ; AVX512BWVL-NEXT: vzeroupper
1377 ; AVX512BWVL-NEXT: retq
1378 ;
1379 ; AVX512DQ-LABEL: test_v32i16:
1380 ; AVX512DQ: # %bb.0:
1381 ; AVX512DQ-NEXT: vpmullw %ymm1, %ymm0, %ymm0
1382 ; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm1
1383 ; AVX512DQ-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1384 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1385 ; AVX512DQ-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1386 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1387 ; AVX512DQ-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1388 ; AVX512DQ-NEXT: vpsrld $16, %xmm0, %xmm1
1389 ; AVX512DQ-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1390 ; AVX512DQ-NEXT: vmovd %xmm0, %eax
1391 ; AVX512DQ-NEXT: # kill: def $ax killed $ax killed $eax
1392 ; AVX512DQ-NEXT: vzeroupper
1393 ; AVX512DQ-NEXT: retq
1394 ;
1395 ; AVX512DQVL-LABEL: test_v32i16:
1396 ; AVX512DQVL: # %bb.0:
1397 ; AVX512DQVL-NEXT: vpmullw %ymm1, %ymm0, %ymm0
1398 ; AVX512DQVL-NEXT: vextracti128 $1, %ymm0, %xmm1
1399 ; AVX512DQVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1400 ; AVX512DQVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1401 ; AVX512DQVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1402 ; AVX512DQVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1403 ; AVX512DQVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1404 ; AVX512DQVL-NEXT: vpsrld $16, %xmm0, %xmm1
1405 ; AVX512DQVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1406 ; AVX512DQVL-NEXT: vmovd %xmm0, %eax
1407 ; AVX512DQVL-NEXT: # kill: def $ax killed $ax killed $eax
1408 ; AVX512DQVL-NEXT: vzeroupper
1409 ; AVX512DQVL-NEXT: retq
1410 %1 = call i16 @llvm.experimental.vector.reduce.mul.v32i16(<32 x i16> %a0)
1411 ret i16 %1
1412 }
1414 define i16 @test_v64i16(<64 x i16> %a0) {
1415 ; SSE-LABEL: test_v64i16:
1416 ; SSE: # %bb.0:
1417 ; SSE-NEXT: pmullw %xmm6, %xmm2
1418 ; SSE-NEXT: pmullw %xmm7, %xmm3
1419 ; SSE-NEXT: pmullw %xmm5, %xmm3
1420 ; SSE-NEXT: pmullw %xmm1, %xmm3
1421 ; SSE-NEXT: pmullw %xmm4, %xmm2
1422 ; SSE-NEXT: pmullw %xmm3, %xmm2
1423 ; SSE-NEXT: pmullw %xmm0, %xmm2
1424 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
1425 ; SSE-NEXT: pmullw %xmm2, %xmm0
1426 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1427 ; SSE-NEXT: pmullw %xmm0, %xmm1
1428 ; SSE-NEXT: movdqa %xmm1, %xmm0
1429 ; SSE-NEXT: psrld $16, %xmm0
1430 ; SSE-NEXT: pmullw %xmm1, %xmm0
1431 ; SSE-NEXT: movd %xmm0, %eax
1432 ; SSE-NEXT: # kill: def $ax killed $ax killed $eax
1433 ; SSE-NEXT: retq
1434 ;
1435 ; AVX1-LABEL: test_v64i16:
1436 ; AVX1: # %bb.0:
1437 ; AVX1-NEXT: vpmullw %xmm3, %xmm1, %xmm4
1438 ; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm3
1439 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
1440 ; AVX1-NEXT: vpmullw %xmm3, %xmm1, %xmm1
1441 ; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
1442 ; AVX1-NEXT: vpmullw %xmm1, %xmm3, %xmm1
1443 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
1444 ; AVX1-NEXT: vpmullw %xmm1, %xmm3, %xmm1
1445 ; AVX1-NEXT: vpmullw %xmm4, %xmm2, %xmm2
1446 ; AVX1-NEXT: vpmullw %xmm1, %xmm2, %xmm1
1447 ; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1448 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1449 ; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1450 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1451 ; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1452 ; AVX1-NEXT: vpsrld $16, %xmm0, %xmm1
1453 ; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1454 ; AVX1-NEXT: vmovd %xmm0, %eax
1455 ; AVX1-NEXT: # kill: def $ax killed $ax killed $eax
1456 ; AVX1-NEXT: vzeroupper
1457 ; AVX1-NEXT: retq
1458 ;
1459 ; AVX2-LABEL: test_v64i16:
1460 ; AVX2: # %bb.0:
1461 ; AVX2-NEXT: vpmullw %ymm3, %ymm1, %ymm1
1462 ; AVX2-NEXT: vpmullw %ymm1, %ymm2, %ymm1
1463 ; AVX2-NEXT: vpmullw %ymm1, %ymm0, %ymm0
1464 ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
1465 ; AVX2-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1466 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1467 ; AVX2-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1468 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1469 ; AVX2-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1470 ; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
1471 ; AVX2-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1472 ; AVX2-NEXT: vmovd %xmm0, %eax
1473 ; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
1474 ; AVX2-NEXT: vzeroupper
1475 ; AVX2-NEXT: retq
1476 ;
1477 ; AVX512BW-LABEL: test_v64i16:
1478 ; AVX512BW: # %bb.0:
1479 ; AVX512BW-NEXT: vpmullw %zmm1, %zmm0, %zmm0
1480 ; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
1481 ; AVX512BW-NEXT: vpmullw %zmm1, %zmm0, %zmm0
1482 ; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
1483 ; AVX512BW-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1484 ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1485 ; AVX512BW-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1486 ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1487 ; AVX512BW-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1488 ; AVX512BW-NEXT: vpsrld $16, %xmm0, %xmm1
1489 ; AVX512BW-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1490 ; AVX512BW-NEXT: vmovd %xmm0, %eax
1491 ; AVX512BW-NEXT: # kill: def $ax killed $ax killed $eax
1492 ; AVX512BW-NEXT: vzeroupper
1493 ; AVX512BW-NEXT: retq
1494 ;
1495 ; AVX512BWVL-LABEL: test_v64i16:
1496 ; AVX512BWVL: # %bb.0:
1497 ; AVX512BWVL-NEXT: vpmullw %zmm1, %zmm0, %zmm0
1498 ; AVX512BWVL-NEXT: vextracti64x4 $1, %zmm0, %ymm1
1499 ; AVX512BWVL-NEXT: vpmullw %zmm1, %zmm0, %zmm0
1500 ; AVX512BWVL-NEXT: vextracti128 $1, %ymm0, %xmm1
1501 ; AVX512BWVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1502 ; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1503 ; AVX512BWVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1504 ; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1505 ; AVX512BWVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1506 ; AVX512BWVL-NEXT: vpsrld $16, %xmm0, %xmm1
1507 ; AVX512BWVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1508 ; AVX512BWVL-NEXT: vmovd %xmm0, %eax
1509 ; AVX512BWVL-NEXT: # kill: def $ax killed $ax killed $eax
1510 ; AVX512BWVL-NEXT: vzeroupper
1511 ; AVX512BWVL-NEXT: retq
1512 ;
1513 ; AVX512DQ-LABEL: test_v64i16:
1514 ; AVX512DQ: # %bb.0:
1515 ; AVX512DQ-NEXT: vpmullw %ymm3, %ymm1, %ymm1
1516 ; AVX512DQ-NEXT: vpmullw %ymm1, %ymm2, %ymm1
1517 ; AVX512DQ-NEXT: vpmullw %ymm1, %ymm0, %ymm0
1518 ; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm1
1519 ; AVX512DQ-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1520 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1521 ; AVX512DQ-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1522 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1523 ; AVX512DQ-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1524 ; AVX512DQ-NEXT: vpsrld $16, %xmm0, %xmm1
1525 ; AVX512DQ-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1526 ; AVX512DQ-NEXT: vmovd %xmm0, %eax
1527 ; AVX512DQ-NEXT: # kill: def $ax killed $ax killed $eax
1528 ; AVX512DQ-NEXT: vzeroupper
1529 ; AVX512DQ-NEXT: retq
1530 ;
1531 ; AVX512DQVL-LABEL: test_v64i16:
1532 ; AVX512DQVL: # %bb.0:
1533 ; AVX512DQVL-NEXT: vpmullw %ymm3, %ymm1, %ymm1
1534 ; AVX512DQVL-NEXT: vpmullw %ymm1, %ymm2, %ymm1
1535 ; AVX512DQVL-NEXT: vpmullw %ymm1, %ymm0, %ymm0
1536 ; AVX512DQVL-NEXT: vextracti128 $1, %ymm0, %xmm1
1537 ; AVX512DQVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1538 ; AVX512DQVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1539 ; AVX512DQVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1540 ; AVX512DQVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1541 ; AVX512DQVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1542 ; AVX512DQVL-NEXT: vpsrld $16, %xmm0, %xmm1
1543 ; AVX512DQVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1544 ; AVX512DQVL-NEXT: vmovd %xmm0, %eax
1545 ; AVX512DQVL-NEXT: # kill: def $ax killed $ax killed $eax
1546 ; AVX512DQVL-NEXT: vzeroupper
1547 ; AVX512DQVL-NEXT: retq
1548 %1 = call i16 @llvm.experimental.vector.reduce.mul.v64i16(<64 x i16> %a0)
1549 ret i16 %1
1550 }
1552 ;
1553 ; vXi8
1554 ;
1556 define i8 @test_v2i8(<2 x i8> %a0) {
1557 ; SSE2-LABEL: test_v2i8:
1558 ; SSE2: # %bb.0:
1559 ; SSE2-NEXT: movdqa %xmm0, %xmm1
1560 ; SSE2-NEXT: psrlw $8, %xmm1
1561 ; SSE2-NEXT: pmullw %xmm0, %xmm1
1562 ; SSE2-NEXT: movd %xmm1, %eax
1563 ; SSE2-NEXT: # kill: def $al killed $al killed $eax
1564 ; SSE2-NEXT: retq
1565 ;
1566 ; SSE41-LABEL: test_v2i8:
1567 ; SSE41: # %bb.0:
1568 ; SSE41-NEXT: movdqa %xmm0, %xmm1
1569 ; SSE41-NEXT: psrlw $8, %xmm1
1570 ; SSE41-NEXT: pmullw %xmm0, %xmm1
1571 ; SSE41-NEXT: pextrb $0, %xmm1, %eax
1572 ; SSE41-NEXT: # kill: def $al killed $al killed $eax
1573 ; SSE41-NEXT: retq
1574 ;
1575 ; AVX-LABEL: test_v2i8:
1576 ; AVX: # %bb.0:
1577 ; AVX-NEXT: vpsrlw $8, %xmm0, %xmm1
1578 ; AVX-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1579 ; AVX-NEXT: vpextrb $0, %xmm0, %eax
1580 ; AVX-NEXT: # kill: def $al killed $al killed $eax
1581 ; AVX-NEXT: retq
1582 ;
1583 ; AVX512-LABEL: test_v2i8:
1584 ; AVX512: # %bb.0:
1585 ; AVX512-NEXT: vpsrlw $8, %xmm0, %xmm1
1586 ; AVX512-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1587 ; AVX512-NEXT: vpextrb $0, %xmm0, %eax
1588 ; AVX512-NEXT: # kill: def $al killed $al killed $eax
1589 ; AVX512-NEXT: retq
1590 %1 = call i8 @llvm.experimental.vector.reduce.mul.v2i8(<2 x i8> %a0)
1591 ret i8 %1
1592 }
1594 define i8 @test_v4i8(<4 x i8> %a0) {
1595 ; SSE2-LABEL: test_v4i8:
1596 ; SSE2: # %bb.0:
1597 ; SSE2-NEXT: movdqa %xmm0, %xmm1
1598 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1599 ; SSE2-NEXT: psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
1600 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1601 ; SSE2-NEXT: pmullw %xmm1, %xmm0
1602 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm0
1603 ; SSE2-NEXT: packuswb %xmm0, %xmm0
1604 ; SSE2-NEXT: movdqa %xmm0, %xmm1
1605 ; SSE2-NEXT: psrlw $8, %xmm1
1606 ; SSE2-NEXT: pmullw %xmm0, %xmm1
1607 ; SSE2-NEXT: movd %xmm1, %eax
1608 ; SSE2-NEXT: # kill: def $al killed $al killed $eax
1609 ; SSE2-NEXT: retq
1610 ;
1611 ; SSE41-LABEL: test_v4i8:
1612 ; SSE41: # %bb.0:
1613 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1614 ; SSE41-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1615 ; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
1616 ; SSE41-NEXT: pmullw %xmm1, %xmm0
1617 ; SSE41-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,u,u,u,u,u,u,u,u,u,u,u,u]
1618 ; SSE41-NEXT: movdqa %xmm0, %xmm1
1619 ; SSE41-NEXT: psrlw $8, %xmm1
1620 ; SSE41-NEXT: pmullw %xmm0, %xmm1
1621 ; SSE41-NEXT: pextrb $0, %xmm1, %eax
1622 ; SSE41-NEXT: # kill: def $al killed $al killed $eax
1623 ; SSE41-NEXT: retq
1624 ;
1625 ; AVX-LABEL: test_v4i8:
1626 ; AVX: # %bb.0:
1627 ; AVX-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1628 ; AVX-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1629 ; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
1630 ; AVX-NEXT: vpmullw %xmm0, %xmm1, %xmm0
1631 ; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,u,u,u,u,u,u,u,u,u,u,u,u]
1632 ; AVX-NEXT: vpsrlw $8, %xmm0, %xmm1
1633 ; AVX-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1634 ; AVX-NEXT: vpextrb $0, %xmm0, %eax
1635 ; AVX-NEXT: # kill: def $al killed $al killed $eax
1636 ; AVX-NEXT: retq
1637 ;
1638 ; AVX512-LABEL: test_v4i8:
1639 ; AVX512: # %bb.0:
1640 ; AVX512-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1641 ; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1642 ; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
1643 ; AVX512-NEXT: vpmullw %xmm0, %xmm1, %xmm0
1644 ; AVX512-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,u,u,u,u,u,u,u,u,u,u,u,u]
1645 ; AVX512-NEXT: vpsrlw $8, %xmm0, %xmm1
1646 ; AVX512-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1647 ; AVX512-NEXT: vpextrb $0, %xmm0, %eax
1648 ; AVX512-NEXT: # kill: def $al killed $al killed $eax
1649 ; AVX512-NEXT: retq
1650 %1 = call i8 @llvm.experimental.vector.reduce.mul.v4i8(<4 x i8> %a0)
1651 ret i8 %1
1652 }
1654 define i8 @test_v8i8(<8 x i8> %a0) {
1655 ; SSE2-LABEL: test_v8i8:
1656 ; SSE2: # %bb.0:
1657 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,2,3,3]
1658 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1659 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1660 ; SSE2-NEXT: pmullw %xmm0, %xmm1
1661 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,2,3,0]
1662 ; SSE2-NEXT: pmullw %xmm1, %xmm0
1663 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm0
1664 ; SSE2-NEXT: packuswb %xmm0, %xmm0
1665 ; SSE2-NEXT: movdqa %xmm0, %xmm1
1666 ; SSE2-NEXT: psrlw $8, %xmm1
1667 ; SSE2-NEXT: pmullw %xmm0, %xmm1
1668 ; SSE2-NEXT: movd %xmm1, %eax
1669 ; SSE2-NEXT: # kill: def $al killed $al killed $eax
1670 ; SSE2-NEXT: retq
1671 ;
1672 ; SSE41-LABEL: test_v8i8:
1673 ; SSE41: # %bb.0:
1674 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1675 ; SSE41-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1676 ; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1677 ; SSE41-NEXT: pmullw %xmm1, %xmm0
1678 ; SSE41-NEXT: movdqa {{.*#+}} xmm1 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
1679 ; SSE41-NEXT: pshufb %xmm1, %xmm0
1680 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1681 ; SSE41-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1682 ; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
1683 ; SSE41-NEXT: pmullw %xmm2, %xmm0
1684 ; SSE41-NEXT: pshufb %xmm1, %xmm0
1685 ; SSE41-NEXT: movdqa %xmm0, %xmm1
1686 ; SSE41-NEXT: psrlw $8, %xmm1
1687 ; SSE41-NEXT: pmullw %xmm0, %xmm1
1688 ; SSE41-NEXT: pextrb $0, %xmm1, %eax
1689 ; SSE41-NEXT: # kill: def $al killed $al killed $eax
1690 ; SSE41-NEXT: retq
1691 ;
1692 ; AVX-LABEL: test_v8i8:
1693 ; AVX: # %bb.0:
1694 ; AVX-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1695 ; AVX-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1696 ; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1697 ; AVX-NEXT: vpmullw %xmm0, %xmm1, %xmm0
1698 ; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
1699 ; AVX-NEXT: vpshufb %xmm1, %xmm0, %xmm0
1700 ; AVX-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1701 ; AVX-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1702 ; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
1703 ; AVX-NEXT: vpmullw %xmm0, %xmm2, %xmm0
1704 ; AVX-NEXT: vpshufb %xmm1, %xmm0, %xmm0
1705 ; AVX-NEXT: vpsrlw $8, %xmm0, %xmm1
1706 ; AVX-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1707 ; AVX-NEXT: vpextrb $0, %xmm0, %eax
1708 ; AVX-NEXT: # kill: def $al killed $al killed $eax
1709 ; AVX-NEXT: retq
1710 ;
1711 ; AVX512-LABEL: test_v8i8:
1712 ; AVX512: # %bb.0:
1713 ; AVX512-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1714 ; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1715 ; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1716 ; AVX512-NEXT: vpmullw %xmm0, %xmm1, %xmm0
1717 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm1 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
1718 ; AVX512-NEXT: vpshufb %xmm1, %xmm0, %xmm0
1719 ; AVX512-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1720 ; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1721 ; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
1722 ; AVX512-NEXT: vpmullw %xmm0, %xmm2, %xmm0
1723 ; AVX512-NEXT: vpshufb %xmm1, %xmm0, %xmm0
1724 ; AVX512-NEXT: vpsrlw $8, %xmm0, %xmm1
1725 ; AVX512-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1726 ; AVX512-NEXT: vpextrb $0, %xmm0, %eax
1727 ; AVX512-NEXT: # kill: def $al killed $al killed $eax
1728 ; AVX512-NEXT: retq
1729 %1 = call i8 @llvm.experimental.vector.reduce.mul.v8i8(<8 x i8> %a0)
1730 ret i8 %1
1731 }
1733 define i8 @test_v16i8(<16 x i8> %a0) {
1734 ; SSE2-LABEL: test_v16i8:
1735 ; SSE2: # %bb.0:
1736 ; SSE2-NEXT: movdqa %xmm0, %xmm1
1737 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
1738 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1739 ; SSE2-NEXT: pmullw %xmm1, %xmm0
1740 ; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
1741 ; SSE2-NEXT: movdqa %xmm0, %xmm2
1742 ; SSE2-NEXT: pand %xmm1, %xmm2
1743 ; SSE2-NEXT: pxor %xmm3, %xmm3
1744 ; SSE2-NEXT: packuswb %xmm3, %xmm2
1745 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,2,3,3]
1746 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
1747 ; SSE2-NEXT: pmullw %xmm0, %xmm2
1748 ; SSE2-NEXT: movdqa %xmm2, %xmm0
1749 ; SSE2-NEXT: pand %xmm1, %xmm0
1750 ; SSE2-NEXT: packuswb %xmm3, %xmm0
1751 ; SSE2-NEXT: psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
1752 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1753 ; SSE2-NEXT: pmullw %xmm2, %xmm0
1754 ; SSE2-NEXT: movdqa %xmm0, %xmm2
1755 ; SSE2-NEXT: pand %xmm1, %xmm2
1756 ; SSE2-NEXT: packuswb %xmm3, %xmm2
1757 ; SSE2-NEXT: psrlw $8, %xmm2
1758 ; SSE2-NEXT: pmullw %xmm0, %xmm2
1759 ; SSE2-NEXT: pand %xmm1, %xmm2
1760 ; SSE2-NEXT: packuswb %xmm3, %xmm2
1761 ; SSE2-NEXT: movd %xmm2, %eax
1762 ; SSE2-NEXT: # kill: def $al killed $al killed $eax
1763 ; SSE2-NEXT: retq
1764 ;
1765 ; SSE41-LABEL: test_v16i8:
1766 ; SSE41: # %bb.0:
1767 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1768 ; SSE41-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1769 ; SSE41-NEXT: pmullw %xmm1, %xmm0
1770 ; SSE41-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
1771 ; SSE41-NEXT: pand %xmm1, %xmm0
1772 ; SSE41-NEXT: pxor %xmm2, %xmm2
1773 ; SSE41-NEXT: packuswb %xmm2, %xmm0
1774 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1775 ; SSE41-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1776 ; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1777 ; SSE41-NEXT: pmullw %xmm3, %xmm0
1778 ; SSE41-NEXT: pand %xmm1, %xmm0
1779 ; SSE41-NEXT: packuswb %xmm2, %xmm0
1780 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1781 ; SSE41-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1782 ; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
1783 ; SSE41-NEXT: pmullw %xmm3, %xmm0
1784 ; SSE41-NEXT: pand %xmm1, %xmm0
1785 ; SSE41-NEXT: packuswb %xmm2, %xmm0
1786 ; SSE41-NEXT: movdqa %xmm0, %xmm1
1787 ; SSE41-NEXT: psrlw $8, %xmm1
1788 ; SSE41-NEXT: pmullw %xmm0, %xmm1
1789 ; SSE41-NEXT: pextrb $0, %xmm1, %eax
1790 ; SSE41-NEXT: # kill: def $al killed $al killed $eax
1791 ; SSE41-NEXT: retq
1792 ;
1793 ; AVX1-LABEL: test_v16i8:
1794 ; AVX1: # %bb.0:
1795 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1796 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1797 ; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1798 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
1799 ; AVX1-NEXT: vpand %xmm1, %xmm0, %xmm0
1800 ; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
1801 ; AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
1802 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1803 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1804 ; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
1805 ; AVX1-NEXT: vpmullw %xmm0, %xmm3, %xmm0
1806 ; AVX1-NEXT: vpand %xmm1, %xmm0, %xmm0
1807 ; AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
1808 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1809 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1810 ; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
1811 ; AVX1-NEXT: vpmullw %xmm0, %xmm3, %xmm0
1812 ; AVX1-NEXT: vpand %xmm1, %xmm0, %xmm0
1813 ; AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
1814 ; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm1
1815 ; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1816 ; AVX1-NEXT: vpextrb $0, %xmm0, %eax
1817 ; AVX1-NEXT: # kill: def $al killed $al killed $eax
1818 ; AVX1-NEXT: retq
1819 ;
1820 ; AVX2-LABEL: test_v16i8:
1821 ; AVX2: # %bb.0:
1822 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1823 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1824 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1825 ; AVX2-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1826 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
1827 ; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
1828 ; AVX2-NEXT: vpackuswb %xmm0, %xmm0, %xmm2
1829 ; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
1830 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
1831 ; AVX2-NEXT: vpmullw %xmm2, %xmm0, %xmm0
1832 ; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
1833 ; AVX2-NEXT: vpackuswb %xmm0, %xmm0, %xmm2
1834 ; AVX2-NEXT: vpsrld $16, %xmm2, %xmm2
1835 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
1836 ; AVX2-NEXT: vpmullw %xmm2, %xmm0, %xmm0
1837 ; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
1838 ; AVX2-NEXT: vpackuswb %xmm0, %xmm0, %xmm1
1839 ; AVX2-NEXT: vpsrlw $8, %xmm1, %xmm1
1840 ; AVX2-NEXT: vpmullw %xmm1, %xmm0, %xmm0
1841 ; AVX2-NEXT: vpextrb $0, %xmm0, %eax
1842 ; AVX2-NEXT: # kill: def $al killed $al killed $eax
1843 ; AVX2-NEXT: retq
1844 ;
1845 ; AVX512BW-LABEL: test_v16i8:
1846 ; AVX512BW: # %bb.0:
1847 ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1848 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1849 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1850 ; AVX512BW-NEXT: vpmullw %ymm1, %ymm0, %ymm0
1851 ; AVX512BW-NEXT: vpmovwb %zmm0, %ymm1
1852 ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
1853 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1854 ; AVX512BW-NEXT: vpmullw %ymm1, %ymm0, %ymm0
1855 ; AVX512BW-NEXT: vpmovwb %zmm0, %ymm1
1856 ; AVX512BW-NEXT: vpsrld $16, %xmm1, %xmm1
1857 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1858 ; AVX512BW-NEXT: vpmullw %ymm1, %ymm0, %ymm0
1859 ; AVX512BW-NEXT: vpmovwb %zmm0, %ymm1
1860 ; AVX512BW-NEXT: vpsrlw $8, %xmm1, %xmm1
1861 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1862 ; AVX512BW-NEXT: vpmullw %ymm1, %ymm0, %ymm0
1863 ; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0
1864 ; AVX512BW-NEXT: vpextrb $0, %xmm0, %eax
1865 ; AVX512BW-NEXT: # kill: def $al killed $al killed $eax
1866 ; AVX512BW-NEXT: vzeroupper
1867 ; AVX512BW-NEXT: retq
1868 ;
1869 ; AVX512BWVL-LABEL: test_v16i8:
1870 ; AVX512BWVL: # %bb.0:
1871 ; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1872 ; AVX512BWVL-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1873 ; AVX512BWVL-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1874 ; AVX512BWVL-NEXT: vpmullw %ymm1, %ymm0, %ymm0
1875 ; AVX512BWVL-NEXT: vpmovwb %ymm0, %xmm1
1876 ; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
1877 ; AVX512BWVL-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1878 ; AVX512BWVL-NEXT: vpmullw %ymm1, %ymm0, %ymm0
1879 ; AVX512BWVL-NEXT: vpmovwb %ymm0, %xmm1
1880 ; AVX512BWVL-NEXT: vpsrld $16, %xmm1, %xmm1
1881 ; AVX512BWVL-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1882 ; AVX512BWVL-NEXT: vpmullw %ymm1, %ymm0, %ymm0
1883 ; AVX512BWVL-NEXT: vpmovwb %ymm0, %xmm1
1884 ; AVX512BWVL-NEXT: vpsrlw $8, %xmm1, %xmm1
1885 ; AVX512BWVL-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1886 ; AVX512BWVL-NEXT: vpmullw %ymm1, %ymm0, %ymm0
1887 ; AVX512BWVL-NEXT: vpmovwb %ymm0, %xmm0
1888 ; AVX512BWVL-NEXT: vpextrb $0, %xmm0, %eax
1889 ; AVX512BWVL-NEXT: # kill: def $al killed $al killed $eax
1890 ; AVX512BWVL-NEXT: vzeroupper
1891 ; AVX512BWVL-NEXT: retq
1892 ;
1893 ; AVX512DQ-LABEL: test_v16i8:
1894 ; AVX512DQ: # %bb.0:
1895 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1896 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1897 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1898 ; AVX512DQ-NEXT: vpmullw %ymm1, %ymm0, %ymm0
1899 ; AVX512DQ-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
1900 ; AVX512DQ-NEXT: vpmovdb %zmm1, %xmm1
1901 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
1902 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1903 ; AVX512DQ-NEXT: vpmullw %ymm1, %ymm0, %ymm0
1904 ; AVX512DQ-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
1905 ; AVX512DQ-NEXT: vpmovdb %zmm1, %xmm1
1906 ; AVX512DQ-NEXT: vpsrld $16, %xmm1, %xmm1
1907 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1908 ; AVX512DQ-NEXT: vpmullw %ymm1, %ymm0, %ymm0
1909 ; AVX512DQ-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
1910 ; AVX512DQ-NEXT: vpmovdb %zmm1, %xmm1
1911 ; AVX512DQ-NEXT: vpsrlw $8, %xmm1, %xmm1
1912 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1913 ; AVX512DQ-NEXT: vpmullw %ymm1, %ymm0, %ymm0
1914 ; AVX512DQ-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
1915 ; AVX512DQ-NEXT: vpmovdb %zmm0, %xmm0
1916 ; AVX512DQ-NEXT: vpextrb $0, %xmm0, %eax
1917 ; AVX512DQ-NEXT: # kill: def $al killed $al killed $eax
1918 ; AVX512DQ-NEXT: vzeroupper
1919 ; AVX512DQ-NEXT: retq
1920 ;
1921 ; AVX512DQVL-LABEL: test_v16i8:
1922 ; AVX512DQVL: # %bb.0:
1923 ; AVX512DQVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1924 ; AVX512DQVL-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1925 ; AVX512DQVL-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1926 ; AVX512DQVL-NEXT: vpmullw %ymm1, %ymm0, %ymm0
1927 ; AVX512DQVL-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
1928 ; AVX512DQVL-NEXT: vpmovdb %zmm1, %xmm1
1929 ; AVX512DQVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
1930 ; AVX512DQVL-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1931 ; AVX512DQVL-NEXT: vpmullw %ymm1, %ymm0, %ymm0
1932 ; AVX512DQVL-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
1933 ; AVX512DQVL-NEXT: vpmovdb %zmm1, %xmm1
1934 ; AVX512DQVL-NEXT: vpsrld $16, %xmm1, %xmm1
1935 ; AVX512DQVL-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1936 ; AVX512DQVL-NEXT: vpmullw %ymm1, %ymm0, %ymm0
1937 ; AVX512DQVL-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
1938 ; AVX512DQVL-NEXT: vpmovdb %zmm1, %xmm1
1939 ; AVX512DQVL-NEXT: vpsrlw $8, %xmm1, %xmm1
1940 ; AVX512DQVL-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1941 ; AVX512DQVL-NEXT: vpmullw %ymm1, %ymm0, %ymm0
1942 ; AVX512DQVL-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
1943 ; AVX512DQVL-NEXT: vpmovdb %zmm0, %xmm0
1944 ; AVX512DQVL-NEXT: vpextrb $0, %xmm0, %eax
1945 ; AVX512DQVL-NEXT: # kill: def $al killed $al killed $eax
1946 ; AVX512DQVL-NEXT: vzeroupper
1947 ; AVX512DQVL-NEXT: retq
1948 %1 = call i8 @llvm.experimental.vector.reduce.mul.v16i8(<16 x i8> %a0)
1949 ret i8 %1
1950 }
1952 define i8 @test_v32i8(<32 x i8> %a0) {
1953 ; SSE2-LABEL: test_v32i8:
1954 ; SSE2: # %bb.0:
1955 ; SSE2-NEXT: movdqa %xmm1, %xmm2
1956 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
1957 ; SSE2-NEXT: movdqa %xmm0, %xmm3
1958 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
1959 ; SSE2-NEXT: pmullw %xmm2, %xmm3
1960 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1961 ; SSE2-NEXT: pmullw %xmm3, %xmm1
1962 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1963 ; SSE2-NEXT: pmullw %xmm1, %xmm0
1964 ; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
1965 ; SSE2-NEXT: movdqa %xmm0, %xmm2
1966 ; SSE2-NEXT: pand %xmm1, %xmm2
1967 ; SSE2-NEXT: pxor %xmm3, %xmm3
1968 ; SSE2-NEXT: packuswb %xmm3, %xmm2
1969 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,2,3,3]
1970 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
1971 ; SSE2-NEXT: pmullw %xmm0, %xmm2
1972 ; SSE2-NEXT: movdqa %xmm2, %xmm0
1973 ; SSE2-NEXT: pand %xmm1, %xmm0
1974 ; SSE2-NEXT: packuswb %xmm3, %xmm0
1975 ; SSE2-NEXT: psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
1976 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1977 ; SSE2-NEXT: pmullw %xmm2, %xmm0
1978 ; SSE2-NEXT: movdqa %xmm0, %xmm2
1979 ; SSE2-NEXT: pand %xmm1, %xmm2
1980 ; SSE2-NEXT: packuswb %xmm3, %xmm2
1981 ; SSE2-NEXT: psrlw $8, %xmm2
1982 ; SSE2-NEXT: pmullw %xmm0, %xmm2
1983 ; SSE2-NEXT: pand %xmm1, %xmm2
1984 ; SSE2-NEXT: packuswb %xmm3, %xmm2
1985 ; SSE2-NEXT: movd %xmm2, %eax
1986 ; SSE2-NEXT: # kill: def $al killed $al killed $eax
1987 ; SSE2-NEXT: retq
1988 ;
1989 ; SSE41-LABEL: test_v32i8:
1990 ; SSE41: # %bb.0:
1991 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
1992 ; SSE41-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
1993 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
1994 ; SSE41-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
1995 ; SSE41-NEXT: pmullw %xmm2, %xmm3
1996 ; SSE41-NEXT: pshufb {{.*#+}} xmm3 = xmm3[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
1997 ; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
1998 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
1999 ; SSE41-NEXT: pmullw %xmm1, %xmm3
2000 ; SSE41-NEXT: pmullw %xmm0, %xmm3
2001 ; SSE41-NEXT: pand %xmm2, %xmm3
2002 ; SSE41-NEXT: pxor %xmm0, %xmm0
2003 ; SSE41-NEXT: packuswb %xmm0, %xmm3
2004 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm1 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
2005 ; SSE41-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2006 ; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
2007 ; SSE41-NEXT: pmullw %xmm1, %xmm3
2008 ; SSE41-NEXT: pand %xmm2, %xmm3
2009 ; SSE41-NEXT: packuswb %xmm0, %xmm3
2010 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm1 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
2011 ; SSE41-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2012 ; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
2013 ; SSE41-NEXT: pmullw %xmm1, %xmm3
2014 ; SSE41-NEXT: pand %xmm2, %xmm3
2015 ; SSE41-NEXT: packuswb %xmm0, %xmm3
2016 ; SSE41-NEXT: movdqa %xmm3, %xmm0
2017 ; SSE41-NEXT: psrlw $8, %xmm0
2018 ; SSE41-NEXT: pmullw %xmm3, %xmm0
2019 ; SSE41-NEXT: pextrb $0, %xmm0, %eax
2020 ; SSE41-NEXT: # kill: def $al killed $al killed $eax
2021 ; SSE41-NEXT: retq
2022 ;
2023 ; AVX1-LABEL: test_v32i8:
2024 ; AVX1: # %bb.0:
2025 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
2026 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2027 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2028 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2029 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2030 ; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2031 ; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
2032 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
2033 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2034 ; AVX1-NEXT: vpmullw %xmm0, %xmm2, %xmm0
2035 ; AVX1-NEXT: vpmullw %xmm0, %xmm3, %xmm0
2036 ; AVX1-NEXT: vpand %xmm1, %xmm0, %xmm0
2037 ; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
2038 ; AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
2039 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2040 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2041 ; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
2042 ; AVX1-NEXT: vpmullw %xmm0, %xmm3, %xmm0
2043 ; AVX1-NEXT: vpand %xmm1, %xmm0, %xmm0
2044 ; AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
2045 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2046 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2047 ; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
2048 ; AVX1-NEXT: vpmullw %xmm0, %xmm3, %xmm0
2049 ; AVX1-NEXT: vpand %xmm1, %xmm0, %xmm0
2050 ; AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
2051 ; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm1
2052 ; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2053 ; AVX1-NEXT: vpextrb $0, %xmm0, %eax
2054 ; AVX1-NEXT: # kill: def $al killed $al killed $eax
2055 ; AVX1-NEXT: vzeroupper
2056 ; AVX1-NEXT: retq
2057 ;
2058 ; AVX2-LABEL: test_v32i8:
2059 ; AVX2: # %bb.0:
2060 ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
2061 ; AVX2-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2062 ; AVX2-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2063 ; AVX2-NEXT: vpmullw %xmm2, %xmm3, %xmm2
2064 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
2065 ; AVX2-NEXT: vpand %xmm3, %xmm2, %xmm4
2066 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2067 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2068 ; AVX2-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2069 ; AVX2-NEXT: vpand %xmm3, %xmm0, %xmm0
2070 ; AVX2-NEXT: vpackuswb %xmm4, %xmm0, %xmm0
2071 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2072 ; AVX2-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2073 ; AVX2-NEXT: vpand %xmm3, %xmm0, %xmm0
2074 ; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
2075 ; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2076 ; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
2077 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2078 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2079 ; AVX2-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2080 ; AVX2-NEXT: vpand %xmm3, %xmm0, %xmm0
2081 ; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2082 ; AVX2-NEXT: vpsrld $16, %xmm0, %xmm2
2083 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2084 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2085 ; AVX2-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2086 ; AVX2-NEXT: vpand %xmm3, %xmm0, %xmm0
2087 ; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2088 ; AVX2-NEXT: vpsrlw $8, %xmm0, %xmm1
2089 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2090 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2091 ; AVX2-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2092 ; AVX2-NEXT: vpextrb $0, %xmm0, %eax
2093 ; AVX2-NEXT: # kill: def $al killed $al killed $eax
2094 ; AVX2-NEXT: vzeroupper
2095 ; AVX2-NEXT: retq
2096 ;
2097 ; AVX512BW-LABEL: test_v32i8:
2098 ; AVX512BW: # %bb.0:
2099 ; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
2100 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
2101 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
2102 ; AVX512BW-NEXT: vpmullw %zmm1, %zmm0, %zmm0
2103 ; AVX512BW-NEXT: vpmovwb %zmm0, %ymm1
2104 ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
2105 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
2106 ; AVX512BW-NEXT: vpmullw %zmm1, %zmm0, %zmm0
2107 ; AVX512BW-NEXT: vpmovwb %zmm0, %ymm1
2108 ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
2109 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
2110 ; AVX512BW-NEXT: vpmullw %zmm1, %zmm0, %zmm0
2111 ; AVX512BW-NEXT: vpmovwb %zmm0, %ymm1
2112 ; AVX512BW-NEXT: vpsrld $16, %xmm1, %xmm1
2113 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
2114 ; AVX512BW-NEXT: vpmullw %zmm1, %zmm0, %zmm0
2115 ; AVX512BW-NEXT: vpmovwb %zmm0, %ymm1
2116 ; AVX512BW-NEXT: vpsrlw $8, %xmm1, %xmm1
2117 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
2118 ; AVX512BW-NEXT: vpmullw %zmm1, %zmm0, %zmm0
2119 ; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0
2120 ; AVX512BW-NEXT: vpextrb $0, %xmm0, %eax
2121 ; AVX512BW-NEXT: # kill: def $al killed $al killed $eax
2122 ; AVX512BW-NEXT: vzeroupper
2123 ; AVX512BW-NEXT: retq
2124 ;
2125 ; AVX512BWVL-LABEL: test_v32i8:
2126 ; AVX512BWVL: # %bb.0:
2127 ; AVX512BWVL-NEXT: vextracti128 $1, %ymm0, %xmm1
2128 ; AVX512BWVL-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
2129 ; AVX512BWVL-NEXT: vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
2130 ; AVX512BWVL-NEXT: vpmullw %zmm1, %zmm0, %zmm0
2131 ; AVX512BWVL-NEXT: vpmovwb %zmm0, %ymm1
2132 ; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
2133 ; AVX512BWVL-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
2134 ; AVX512BWVL-NEXT: vpmullw %zmm1, %zmm0, %zmm0
2135 ; AVX512BWVL-NEXT: vpmovwb %zmm0, %ymm1
2136 ; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
2137 ; AVX512BWVL-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
2138 ; AVX512BWVL-NEXT: vpmullw %zmm1, %zmm0, %zmm0
2139 ; AVX512BWVL-NEXT: vpmovwb %zmm0, %ymm1
2140 ; AVX512BWVL-NEXT: vpsrld $16, %xmm1, %xmm1
2141 ; AVX512BWVL-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
2142 ; AVX512BWVL-NEXT: vpmullw %zmm1, %zmm0, %zmm0
2143 ; AVX512BWVL-NEXT: vpmovwb %zmm0, %ymm1
2144 ; AVX512BWVL-NEXT: vpsrlw $8, %xmm1, %xmm1
2145 ; AVX512BWVL-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
2146 ; AVX512BWVL-NEXT: vpmullw %zmm1, %zmm0, %zmm0
2147 ; AVX512BWVL-NEXT: vpmovwb %zmm0, %ymm0
2148 ; AVX512BWVL-NEXT: vpextrb $0, %xmm0, %eax
2149 ; AVX512BWVL-NEXT: # kill: def $al killed $al killed $eax
2150 ; AVX512BWVL-NEXT: vzeroupper
2151 ; AVX512BWVL-NEXT: retq
2152 ;
2153 ; AVX512DQ-LABEL: test_v32i8:
2154 ; AVX512DQ: # %bb.0:
2155 ; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm1
2156 ; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2157 ; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2158 ; AVX512DQ-NEXT: vpmullw %xmm2, %xmm3, %xmm2
2159 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
2160 ; AVX512DQ-NEXT: vpand %xmm3, %xmm2, %xmm4
2161 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2162 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2163 ; AVX512DQ-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2164 ; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0
2165 ; AVX512DQ-NEXT: vpackuswb %xmm4, %xmm0, %xmm0
2166 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2167 ; AVX512DQ-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2168 ; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0
2169 ; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
2170 ; AVX512DQ-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2171 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
2172 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2173 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2174 ; AVX512DQ-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2175 ; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0
2176 ; AVX512DQ-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2177 ; AVX512DQ-NEXT: vpsrld $16, %xmm0, %xmm2
2178 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2179 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2180 ; AVX512DQ-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2181 ; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0
2182 ; AVX512DQ-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2183 ; AVX512DQ-NEXT: vpsrlw $8, %xmm0, %xmm1
2184 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2185 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2186 ; AVX512DQ-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2187 ; AVX512DQ-NEXT: vpextrb $0, %xmm0, %eax
2188 ; AVX512DQ-NEXT: # kill: def $al killed $al killed $eax
2189 ; AVX512DQ-NEXT: vzeroupper
2190 ; AVX512DQ-NEXT: retq
2191 ;
2192 ; AVX512DQVL-LABEL: test_v32i8:
2193 ; AVX512DQVL: # %bb.0:
2194 ; AVX512DQVL-NEXT: vextracti128 $1, %ymm0, %xmm1
2195 ; AVX512DQVL-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2196 ; AVX512DQVL-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2197 ; AVX512DQVL-NEXT: vpmullw %xmm2, %xmm3, %xmm2
2198 ; AVX512DQVL-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2199 ; AVX512DQVL-NEXT: vpmullw %xmm2, %xmm1, %xmm1
2200 ; AVX512DQVL-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2201 ; AVX512DQVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2202 ; AVX512DQVL-NEXT: vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
2203 ; AVX512DQVL-NEXT: vpand %xmm1, %xmm0, %xmm2
2204 ; AVX512DQVL-NEXT: vpxor %xmm3, %xmm3, %xmm3
2205 ; AVX512DQVL-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
2206 ; AVX512DQVL-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
2207 ; AVX512DQVL-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
2208 ; AVX512DQVL-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2209 ; AVX512DQVL-NEXT: vpand %xmm1, %xmm0, %xmm2
2210 ; AVX512DQVL-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
2211 ; AVX512DQVL-NEXT: vpsrld $16, %xmm2, %xmm2
2212 ; AVX512DQVL-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
2213 ; AVX512DQVL-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2214 ; AVX512DQVL-NEXT: vpand %xmm1, %xmm0, %xmm1
2215 ; AVX512DQVL-NEXT: vpackuswb %xmm3, %xmm1, %xmm1
2216 ; AVX512DQVL-NEXT: vpsrlw $8, %xmm1, %xmm1
2217 ; AVX512DQVL-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2218 ; AVX512DQVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2219 ; AVX512DQVL-NEXT: vpextrb $0, %xmm0, %eax
2220 ; AVX512DQVL-NEXT: # kill: def $al killed $al killed $eax
2221 ; AVX512DQVL-NEXT: vzeroupper
2222 ; AVX512DQVL-NEXT: retq
2223 %1 = call i8 @llvm.experimental.vector.reduce.mul.v32i8(<32 x i8> %a0)
2224 ret i8 %1
2225 }
2227 define i8 @test_v64i8(<64 x i8> %a0) {
2228 ; SSE2-LABEL: test_v64i8:
2229 ; SSE2: # %bb.0:
2230 ; SSE2-NEXT: movdqa %xmm3, %xmm4
2231 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
2232 ; SSE2-NEXT: movdqa %xmm1, %xmm5
2233 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
2234 ; SSE2-NEXT: pmullw %xmm4, %xmm5
2235 ; SSE2-NEXT: movdqa %xmm2, %xmm4
2236 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
2237 ; SSE2-NEXT: pmullw %xmm5, %xmm4
2238 ; SSE2-NEXT: movdqa %xmm0, %xmm5
2239 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
2240 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
2241 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2242 ; SSE2-NEXT: pmullw %xmm3, %xmm1
2243 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
2244 ; SSE2-NEXT: pmullw %xmm1, %xmm2
2245 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2246 ; SSE2-NEXT: pmullw %xmm2, %xmm0
2247 ; SSE2-NEXT: pmullw %xmm4, %xmm0
2248 ; SSE2-NEXT: pmullw %xmm5, %xmm0
2249 ; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
2250 ; SSE2-NEXT: movdqa %xmm0, %xmm2
2251 ; SSE2-NEXT: pand %xmm1, %xmm2
2252 ; SSE2-NEXT: pxor %xmm3, %xmm3
2253 ; SSE2-NEXT: packuswb %xmm3, %xmm2
2254 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,2,3,3]
2255 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
2256 ; SSE2-NEXT: pmullw %xmm0, %xmm2
2257 ; SSE2-NEXT: movdqa %xmm2, %xmm0
2258 ; SSE2-NEXT: pand %xmm1, %xmm0
2259 ; SSE2-NEXT: packuswb %xmm3, %xmm0
2260 ; SSE2-NEXT: psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
2261 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2262 ; SSE2-NEXT: pmullw %xmm2, %xmm0
2263 ; SSE2-NEXT: movdqa %xmm0, %xmm2
2264 ; SSE2-NEXT: pand %xmm1, %xmm2
2265 ; SSE2-NEXT: packuswb %xmm3, %xmm2
2266 ; SSE2-NEXT: psrlw $8, %xmm2
2267 ; SSE2-NEXT: pmullw %xmm0, %xmm2
2268 ; SSE2-NEXT: pand %xmm1, %xmm2
2269 ; SSE2-NEXT: packuswb %xmm3, %xmm2
2270 ; SSE2-NEXT: movd %xmm2, %eax
2271 ; SSE2-NEXT: # kill: def $al killed $al killed $eax
2272 ; SSE2-NEXT: retq
2273 ;
2274 ; SSE41-LABEL: test_v64i8:
2275 ; SSE41: # %bb.0:
2276 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm4 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
2277 ; SSE41-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
2278 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm5 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2279 ; SSE41-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2280 ; SSE41-NEXT: pmullw %xmm3, %xmm1
2281 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2282 ; SSE41-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
2283 ; SSE41-NEXT: pmullw %xmm1, %xmm2
2284 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm6 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2285 ; SSE41-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2286 ; SSE41-NEXT: pmullw %xmm3, %xmm6
2287 ; SSE41-NEXT: movdqa {{.*#+}} xmm3 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
2288 ; SSE41-NEXT: pshufb %xmm3, %xmm6
2289 ; SSE41-NEXT: pmullw %xmm4, %xmm5
2290 ; SSE41-NEXT: pshufb %xmm3, %xmm5
2291 ; SSE41-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
2292 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm4 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2293 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm5 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
2294 ; SSE41-NEXT: pmullw %xmm4, %xmm5
2295 ; SSE41-NEXT: pshufb %xmm3, %xmm5
2296 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm3 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2297 ; SSE41-NEXT: pmullw %xmm2, %xmm3
2298 ; SSE41-NEXT: pmullw %xmm0, %xmm3
2299 ; SSE41-NEXT: pand %xmm1, %xmm3
2300 ; SSE41-NEXT: pxor %xmm0, %xmm0
2301 ; SSE41-NEXT: packuswb %xmm0, %xmm3
2302 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
2303 ; SSE41-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2304 ; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
2305 ; SSE41-NEXT: pmullw %xmm2, %xmm3
2306 ; SSE41-NEXT: pand %xmm1, %xmm3
2307 ; SSE41-NEXT: packuswb %xmm0, %xmm3
2308 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
2309 ; SSE41-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2310 ; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
2311 ; SSE41-NEXT: pmullw %xmm2, %xmm3
2312 ; SSE41-NEXT: pand %xmm1, %xmm3
2313 ; SSE41-NEXT: packuswb %xmm0, %xmm3
2314 ; SSE41-NEXT: movdqa %xmm3, %xmm0
2315 ; SSE41-NEXT: psrlw $8, %xmm0
2316 ; SSE41-NEXT: pmullw %xmm3, %xmm0
2317 ; SSE41-NEXT: pextrb $0, %xmm0, %eax
2318 ; SSE41-NEXT: # kill: def $al killed $al killed $eax
2319 ; SSE41-NEXT: retq
2320 ;
2321 ; AVX1-LABEL: test_v64i8:
2322 ; AVX1: # %bb.0:
2323 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
2324 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
2325 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
2326 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm5 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
2327 ; AVX1-NEXT: vpmullw %xmm3, %xmm5, %xmm3
2328 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm5 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2329 ; AVX1-NEXT: vpmullw %xmm3, %xmm5, %xmm3
2330 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm5 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2331 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2332 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2333 ; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2334 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm1 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
2335 ; AVX1-NEXT: vpshufb %xmm1, %xmm0, %xmm0
2336 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2337 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
2338 ; AVX1-NEXT: vpmullw %xmm2, %xmm4, %xmm2
2339 ; AVX1-NEXT: vpshufb %xmm1, %xmm2, %xmm2
2340 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
2341 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2342 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2343 ; AVX1-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2344 ; AVX1-NEXT: vpshufb %xmm1, %xmm0, %xmm0
2345 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2346 ; AVX1-NEXT: vpmullw %xmm0, %xmm3, %xmm0
2347 ; AVX1-NEXT: vpmullw %xmm0, %xmm5, %xmm0
2348 ; AVX1-NEXT: vpand %xmm4, %xmm0, %xmm0
2349 ; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
2350 ; AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2351 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2352 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2353 ; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
2354 ; AVX1-NEXT: vpmullw %xmm0, %xmm2, %xmm0
2355 ; AVX1-NEXT: vpand %xmm4, %xmm0, %xmm0
2356 ; AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2357 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2358 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2359 ; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
2360 ; AVX1-NEXT: vpmullw %xmm0, %xmm2, %xmm0
2361 ; AVX1-NEXT: vpand %xmm4, %xmm0, %xmm0
2362 ; AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2363 ; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm1
2364 ; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2365 ; AVX1-NEXT: vpextrb $0, %xmm0, %eax
2366 ; AVX1-NEXT: # kill: def $al killed $al killed $eax
2367 ; AVX1-NEXT: vzeroupper
2368 ; AVX1-NEXT: retq
2369 ;
2370 ; AVX2-LABEL: test_v64i8:
2371 ; AVX2: # %bb.0:
2372 ; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
2373 ; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2374 ; AVX2-NEXT: vpmullw %ymm2, %ymm3, %ymm2
2375 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2376 ; AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2
2377 ; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
2378 ; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2379 ; AVX2-NEXT: vpmullw %ymm1, %ymm0, %ymm0
2380 ; AVX2-NEXT: vpand %ymm3, %ymm0, %ymm0
2381 ; AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0
2382 ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
2383 ; AVX2-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2384 ; AVX2-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2385 ; AVX2-NEXT: vpmullw %xmm2, %xmm3, %xmm2
2386 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
2387 ; AVX2-NEXT: vpand %xmm3, %xmm2, %xmm4
2388 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2389 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2390 ; AVX2-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2391 ; AVX2-NEXT: vpand %xmm3, %xmm0, %xmm0
2392 ; AVX2-NEXT: vpackuswb %xmm4, %xmm0, %xmm0
2393 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2394 ; AVX2-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2395 ; AVX2-NEXT: vpand %xmm3, %xmm0, %xmm0
2396 ; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
2397 ; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2398 ; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
2399 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2400 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2401 ; AVX2-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2402 ; AVX2-NEXT: vpand %xmm3, %xmm0, %xmm0
2403 ; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2404 ; AVX2-NEXT: vpsrld $16, %xmm0, %xmm2
2405 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2406 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2407 ; AVX2-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2408 ; AVX2-NEXT: vpand %xmm3, %xmm0, %xmm0
2409 ; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2410 ; AVX2-NEXT: vpsrlw $8, %xmm0, %xmm1
2411 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2412 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2413 ; AVX2-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2414 ; AVX2-NEXT: vpextrb $0, %xmm0, %eax
2415 ; AVX2-NEXT: # kill: def $al killed $al killed $eax
2416 ; AVX2-NEXT: vzeroupper
2417 ; AVX2-NEXT: retq
2418 ;
2419 ; AVX512BW-LABEL: test_v64i8:
2420 ; AVX512BW: # %bb.0:
2421 ; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
2422 ; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2423 ; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
2424 ; AVX512BW-NEXT: vpmullw %zmm3, %zmm2, %zmm2
2425 ; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2426 ; AVX512BW-NEXT: vpandq %zmm3, %zmm2, %zmm2
2427 ; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2428 ; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
2429 ; AVX512BW-NEXT: vpmullw %zmm1, %zmm0, %zmm0
2430 ; AVX512BW-NEXT: vpandq %zmm3, %zmm0, %zmm0
2431 ; AVX512BW-NEXT: vpackuswb %zmm2, %zmm0, %zmm0
2432 ; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
2433 ; AVX512BW-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2434 ; AVX512BW-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2435 ; AVX512BW-NEXT: vpmullw %xmm3, %xmm2, %xmm2
2436 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
2437 ; AVX512BW-NEXT: vpand %xmm3, %xmm2, %xmm4
2438 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2439 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2440 ; AVX512BW-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2441 ; AVX512BW-NEXT: vpand %xmm3, %xmm0, %xmm0
2442 ; AVX512BW-NEXT: vpackuswb %xmm4, %xmm0, %xmm0
2443 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2444 ; AVX512BW-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2445 ; AVX512BW-NEXT: vpand %xmm3, %xmm0, %xmm0
2446 ; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
2447 ; AVX512BW-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2448 ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
2449 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2450 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2451 ; AVX512BW-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2452 ; AVX512BW-NEXT: vpand %xmm3, %xmm0, %xmm0
2453 ; AVX512BW-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2454 ; AVX512BW-NEXT: vpsrld $16, %xmm0, %xmm2
2455 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2456 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2457 ; AVX512BW-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2458 ; AVX512BW-NEXT: vpand %xmm3, %xmm0, %xmm0
2459 ; AVX512BW-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2460 ; AVX512BW-NEXT: vpsrlw $8, %xmm0, %xmm1
2461 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2462 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2463 ; AVX512BW-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2464 ; AVX512BW-NEXT: vpextrb $0, %xmm0, %eax
2465 ; AVX512BW-NEXT: # kill: def $al killed $al killed $eax
2466 ; AVX512BW-NEXT: vzeroupper
2467 ; AVX512BW-NEXT: retq
2468 ;
2469 ; AVX512BWVL-LABEL: test_v64i8:
2470 ; AVX512BWVL: # %bb.0:
2471 ; AVX512BWVL-NEXT: vextracti64x4 $1, %zmm0, %ymm1
2472 ; AVX512BWVL-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2473 ; AVX512BWVL-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
2474 ; AVX512BWVL-NEXT: vpmullw %zmm3, %zmm2, %zmm2
2475 ; AVX512BWVL-NEXT: vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2476 ; AVX512BWVL-NEXT: vpandq %zmm3, %zmm2, %zmm2
2477 ; AVX512BWVL-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2478 ; AVX512BWVL-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
2479 ; AVX512BWVL-NEXT: vpmullw %zmm1, %zmm0, %zmm0
2480 ; AVX512BWVL-NEXT: vpandq %zmm3, %zmm0, %zmm0
2481 ; AVX512BWVL-NEXT: vpackuswb %zmm2, %zmm0, %zmm0
2482 ; AVX512BWVL-NEXT: vextracti128 $1, %ymm0, %xmm1
2483 ; AVX512BWVL-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2484 ; AVX512BWVL-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2485 ; AVX512BWVL-NEXT: vpmullw %xmm3, %xmm2, %xmm2
2486 ; AVX512BWVL-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2487 ; AVX512BWVL-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2488 ; AVX512BWVL-NEXT: vpmullw %xmm2, %xmm1, %xmm1
2489 ; AVX512BWVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2490 ; AVX512BWVL-NEXT: vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
2491 ; AVX512BWVL-NEXT: vpand %xmm1, %xmm0, %xmm2
2492 ; AVX512BWVL-NEXT: vpxor %xmm3, %xmm3, %xmm3
2493 ; AVX512BWVL-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
2494 ; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
2495 ; AVX512BWVL-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
2496 ; AVX512BWVL-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2497 ; AVX512BWVL-NEXT: vpand %xmm1, %xmm0, %xmm2
2498 ; AVX512BWVL-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
2499 ; AVX512BWVL-NEXT: vpsrld $16, %xmm2, %xmm2
2500 ; AVX512BWVL-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
2501 ; AVX512BWVL-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2502 ; AVX512BWVL-NEXT: vpand %xmm1, %xmm0, %xmm1
2503 ; AVX512BWVL-NEXT: vpackuswb %xmm3, %xmm1, %xmm1
2504 ; AVX512BWVL-NEXT: vpsrlw $8, %xmm1, %xmm1
2505 ; AVX512BWVL-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2506 ; AVX512BWVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2507 ; AVX512BWVL-NEXT: vpextrb $0, %xmm0, %eax
2508 ; AVX512BWVL-NEXT: # kill: def $al killed $al killed $eax
2509 ; AVX512BWVL-NEXT: vzeroupper
2510 ; AVX512BWVL-NEXT: retq
2511 ;
2512 ; AVX512DQ-LABEL: test_v64i8:
2513 ; AVX512DQ: # %bb.0:
2514 ; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
2515 ; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2516 ; AVX512DQ-NEXT: vpmullw %ymm2, %ymm3, %ymm2
2517 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2518 ; AVX512DQ-NEXT: vpand %ymm3, %ymm2, %ymm2
2519 ; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
2520 ; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2521 ; AVX512DQ-NEXT: vpmullw %ymm1, %ymm0, %ymm0
2522 ; AVX512DQ-NEXT: vpand %ymm3, %ymm0, %ymm0
2523 ; AVX512DQ-NEXT: vpackuswb %ymm2, %ymm0, %ymm0
2524 ; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm1
2525 ; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2526 ; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2527 ; AVX512DQ-NEXT: vpmullw %xmm2, %xmm3, %xmm2
2528 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
2529 ; AVX512DQ-NEXT: vpand %xmm3, %xmm2, %xmm4
2530 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2531 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2532 ; AVX512DQ-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2533 ; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0
2534 ; AVX512DQ-NEXT: vpackuswb %xmm4, %xmm0, %xmm0
2535 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2536 ; AVX512DQ-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2537 ; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0
2538 ; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
2539 ; AVX512DQ-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2540 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
2541 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2542 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2543 ; AVX512DQ-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2544 ; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0
2545 ; AVX512DQ-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2546 ; AVX512DQ-NEXT: vpsrld $16, %xmm0, %xmm2
2547 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2548 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2549 ; AVX512DQ-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2550 ; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0
2551 ; AVX512DQ-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2552 ; AVX512DQ-NEXT: vpsrlw $8, %xmm0, %xmm1
2553 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2554 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2555 ; AVX512DQ-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2556 ; AVX512DQ-NEXT: vpextrb $0, %xmm0, %eax
2557 ; AVX512DQ-NEXT: # kill: def $al killed $al killed $eax
2558 ; AVX512DQ-NEXT: vzeroupper
2559 ; AVX512DQ-NEXT: retq
2560 ;
2561 ; AVX512DQVL-LABEL: test_v64i8:
2562 ; AVX512DQVL: # %bb.0:
2563 ; AVX512DQVL-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
2564 ; AVX512DQVL-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2565 ; AVX512DQVL-NEXT: vpmullw %ymm2, %ymm3, %ymm2
2566 ; AVX512DQVL-NEXT: vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2567 ; AVX512DQVL-NEXT: vpand %ymm3, %ymm2, %ymm2
2568 ; AVX512DQVL-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
2569 ; AVX512DQVL-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2570 ; AVX512DQVL-NEXT: vpmullw %ymm1, %ymm0, %ymm0
2571 ; AVX512DQVL-NEXT: vpand %ymm3, %ymm0, %ymm0
2572 ; AVX512DQVL-NEXT: vpackuswb %ymm2, %ymm0, %ymm0
2573 ; AVX512DQVL-NEXT: vextracti128 $1, %ymm0, %xmm1
2574 ; AVX512DQVL-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2575 ; AVX512DQVL-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2576 ; AVX512DQVL-NEXT: vpmullw %xmm2, %xmm3, %xmm2
2577 ; AVX512DQVL-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2578 ; AVX512DQVL-NEXT: vpmullw %xmm2, %xmm1, %xmm1
2579 ; AVX512DQVL-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2580 ; AVX512DQVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2581 ; AVX512DQVL-NEXT: vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
2582 ; AVX512DQVL-NEXT: vpand %xmm1, %xmm0, %xmm2
2583 ; AVX512DQVL-NEXT: vpxor %xmm3, %xmm3, %xmm3
2584 ; AVX512DQVL-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
2585 ; AVX512DQVL-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
2586 ; AVX512DQVL-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
2587 ; AVX512DQVL-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2588 ; AVX512DQVL-NEXT: vpand %xmm1, %xmm0, %xmm2
2589 ; AVX512DQVL-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
2590 ; AVX512DQVL-NEXT: vpsrld $16, %xmm2, %xmm2
2591 ; AVX512DQVL-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
2592 ; AVX512DQVL-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2593 ; AVX512DQVL-NEXT: vpand %xmm1, %xmm0, %xmm1
2594 ; AVX512DQVL-NEXT: vpackuswb %xmm3, %xmm1, %xmm1
2595 ; AVX512DQVL-NEXT: vpsrlw $8, %xmm1, %xmm1
2596 ; AVX512DQVL-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2597 ; AVX512DQVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2598 ; AVX512DQVL-NEXT: vpextrb $0, %xmm0, %eax
2599 ; AVX512DQVL-NEXT: # kill: def $al killed $al killed $eax
2600 ; AVX512DQVL-NEXT: vzeroupper
2601 ; AVX512DQVL-NEXT: retq
2602 %1 = call i8 @llvm.experimental.vector.reduce.mul.v64i8(<64 x i8> %a0)
2603 ret i8 %1
2604 }
2606 define i8 @test_v128i8(<128 x i8> %a0) {
2607 ; SSE2-LABEL: test_v128i8:
2608 ; SSE2: # %bb.0:
2609 ; SSE2-NEXT: movdqa %xmm6, %xmm8
2610 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm0[8],xmm8[9],xmm0[9],xmm8[10],xmm0[10],xmm8[11],xmm0[11],xmm8[12],xmm0[12],xmm8[13],xmm0[13],xmm8[14],xmm0[14],xmm8[15],xmm0[15]
2611 ; SSE2-NEXT: movdqa %xmm2, %xmm9
2612 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm0[8],xmm9[9],xmm0[9],xmm9[10],xmm0[10],xmm9[11],xmm0[11],xmm9[12],xmm0[12],xmm9[13],xmm0[13],xmm9[14],xmm0[14],xmm9[15],xmm0[15]
2613 ; SSE2-NEXT: pmullw %xmm8, %xmm9
2614 ; SSE2-NEXT: movdqa %xmm4, %xmm10
2615 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm0[8],xmm10[9],xmm0[9],xmm10[10],xmm0[10],xmm10[11],xmm0[11],xmm10[12],xmm0[12],xmm10[13],xmm0[13],xmm10[14],xmm0[14],xmm10[15],xmm0[15]
2616 ; SSE2-NEXT: pmullw %xmm9, %xmm10
2617 ; SSE2-NEXT: movdqa %xmm0, %xmm9
2618 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm0[8],xmm9[9],xmm0[9],xmm9[10],xmm0[10],xmm9[11],xmm0[11],xmm9[12],xmm0[12],xmm9[13],xmm0[13],xmm9[14],xmm0[14],xmm9[15],xmm0[15]
2619 ; SSE2-NEXT: movdqa %xmm7, %xmm8
2620 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm0[8],xmm8[9],xmm0[9],xmm8[10],xmm0[10],xmm8[11],xmm0[11],xmm8[12],xmm0[12],xmm8[13],xmm0[13],xmm8[14],xmm0[14],xmm8[15],xmm0[15]
2621 ; SSE2-NEXT: movdqa %xmm3, %xmm11
2622 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm0[8],xmm11[9],xmm0[9],xmm11[10],xmm0[10],xmm11[11],xmm0[11],xmm11[12],xmm0[12],xmm11[13],xmm0[13],xmm11[14],xmm0[14],xmm11[15],xmm0[15]
2623 ; SSE2-NEXT: pmullw %xmm8, %xmm11
2624 ; SSE2-NEXT: movdqa %xmm5, %xmm12
2625 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm12 = xmm12[8],xmm0[8],xmm12[9],xmm0[9],xmm12[10],xmm0[10],xmm12[11],xmm0[11],xmm12[12],xmm0[12],xmm12[13],xmm0[13],xmm12[14],xmm0[14],xmm12[15],xmm0[15]
2626 ; SSE2-NEXT: pmullw %xmm11, %xmm12
2627 ; SSE2-NEXT: movdqa %xmm1, %xmm8
2628 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm0[8],xmm8[9],xmm0[9],xmm8[10],xmm0[10],xmm8[11],xmm0[11],xmm8[12],xmm0[12],xmm8[13],xmm0[13],xmm8[14],xmm0[14],xmm8[15],xmm0[15]
2629 ; SSE2-NEXT: pmullw %xmm12, %xmm8
2630 ; SSE2-NEXT: pmullw %xmm10, %xmm8
2631 ; SSE2-NEXT: pmullw %xmm9, %xmm8
2632 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3],xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7]
2633 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
2634 ; SSE2-NEXT: pmullw %xmm6, %xmm2
2635 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
2636 ; SSE2-NEXT: pmullw %xmm2, %xmm4
2637 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2638 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7]
2639 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
2640 ; SSE2-NEXT: pmullw %xmm7, %xmm3
2641 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
2642 ; SSE2-NEXT: pmullw %xmm3, %xmm5
2643 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2644 ; SSE2-NEXT: pmullw %xmm5, %xmm1
2645 ; SSE2-NEXT: pmullw %xmm4, %xmm1
2646 ; SSE2-NEXT: pmullw %xmm8, %xmm1
2647 ; SSE2-NEXT: pmullw %xmm0, %xmm1
2648 ; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
2649 ; SSE2-NEXT: movdqa %xmm1, %xmm2
2650 ; SSE2-NEXT: pand %xmm0, %xmm2
2651 ; SSE2-NEXT: pxor %xmm3, %xmm3
2652 ; SSE2-NEXT: packuswb %xmm3, %xmm2
2653 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,2,3,3]
2654 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
2655 ; SSE2-NEXT: pmullw %xmm1, %xmm2
2656 ; SSE2-NEXT: movdqa %xmm2, %xmm1
2657 ; SSE2-NEXT: pand %xmm0, %xmm1
2658 ; SSE2-NEXT: packuswb %xmm3, %xmm1
2659 ; SSE2-NEXT: psrldq {{.*#+}} xmm1 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
2660 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2661 ; SSE2-NEXT: pmullw %xmm2, %xmm1
2662 ; SSE2-NEXT: movdqa %xmm1, %xmm2
2663 ; SSE2-NEXT: pand %xmm0, %xmm2
2664 ; SSE2-NEXT: packuswb %xmm3, %xmm2
2665 ; SSE2-NEXT: psrlw $8, %xmm2
2666 ; SSE2-NEXT: pmullw %xmm1, %xmm2
2667 ; SSE2-NEXT: pand %xmm0, %xmm2
2668 ; SSE2-NEXT: packuswb %xmm3, %xmm2
2669 ; SSE2-NEXT: movd %xmm2, %eax
2670 ; SSE2-NEXT: # kill: def $al killed $al killed $eax
2671 ; SSE2-NEXT: retq
2672 ;
2673 ; SSE41-LABEL: test_v128i8:
2674 ; SSE41: # %bb.0:
2675 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm8 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
2676 ; SSE41-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
2677 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm9 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2678 ; SSE41-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
2679 ; SSE41-NEXT: pmullw %xmm6, %xmm2
2680 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm10 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
2681 ; SSE41-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
2682 ; SSE41-NEXT: pmullw %xmm2, %xmm4
2683 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2684 ; SSE41-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2685 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm11 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
2686 ; SSE41-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15]
2687 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm6 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
2688 ; SSE41-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
2689 ; SSE41-NEXT: pmullw %xmm7, %xmm3
2690 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm7 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2691 ; SSE41-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
2692 ; SSE41-NEXT: pmullw %xmm3, %xmm5
2693 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2694 ; SSE41-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2695 ; SSE41-NEXT: pmullw %xmm5, %xmm1
2696 ; SSE41-NEXT: pmullw %xmm4, %xmm1
2697 ; SSE41-NEXT: pmullw %xmm7, %xmm3
2698 ; SSE41-NEXT: movdqa {{.*#+}} xmm5 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
2699 ; SSE41-NEXT: pshufb %xmm5, %xmm3
2700 ; SSE41-NEXT: pmullw %xmm11, %xmm6
2701 ; SSE41-NEXT: pshufb %xmm5, %xmm6
2702 ; SSE41-NEXT: pmullw %xmm10, %xmm2
2703 ; SSE41-NEXT: pshufb %xmm5, %xmm2
2704 ; SSE41-NEXT: pmullw %xmm8, %xmm9
2705 ; SSE41-NEXT: pshufb %xmm5, %xmm9
2706 ; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
2707 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm7 = xmm9[0],zero,xmm9[1],zero,xmm9[2],zero,xmm9[3],zero,xmm9[4],zero,xmm9[5],zero,xmm9[6],zero,xmm9[7],zero
2708 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2709 ; SSE41-NEXT: pmullw %xmm7, %xmm2
2710 ; SSE41-NEXT: pshufb %xmm5, %xmm2
2711 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
2712 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
2713 ; SSE41-NEXT: pmullw %xmm6, %xmm3
2714 ; SSE41-NEXT: pshufb %xmm5, %xmm3
2715 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
2716 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2717 ; SSE41-NEXT: pmullw %xmm3, %xmm2
2718 ; SSE41-NEXT: pshufb %xmm5, %xmm2
2719 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2720 ; SSE41-NEXT: pmullw %xmm1, %xmm2
2721 ; SSE41-NEXT: pmullw %xmm0, %xmm2
2722 ; SSE41-NEXT: pand %xmm4, %xmm2
2723 ; SSE41-NEXT: pxor %xmm0, %xmm0
2724 ; SSE41-NEXT: packuswb %xmm0, %xmm2
2725 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm1 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2726 ; SSE41-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2727 ; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
2728 ; SSE41-NEXT: pmullw %xmm1, %xmm2
2729 ; SSE41-NEXT: pand %xmm4, %xmm2
2730 ; SSE41-NEXT: packuswb %xmm0, %xmm2
2731 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm1 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2732 ; SSE41-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2733 ; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
2734 ; SSE41-NEXT: pmullw %xmm1, %xmm2
2735 ; SSE41-NEXT: pand %xmm4, %xmm2
2736 ; SSE41-NEXT: packuswb %xmm0, %xmm2
2737 ; SSE41-NEXT: movdqa %xmm2, %xmm0
2738 ; SSE41-NEXT: psrlw $8, %xmm0
2739 ; SSE41-NEXT: pmullw %xmm2, %xmm0
2740 ; SSE41-NEXT: pextrb $0, %xmm0, %eax
2741 ; SSE41-NEXT: # kill: def $al killed $al killed $eax
2742 ; SSE41-NEXT: retq
2743 ;
2744 ; AVX1-LABEL: test_v128i8:
2745 ; AVX1: # %bb.0:
2746 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
2747 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm5 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2748 ; AVX1-NEXT: vpmullw %xmm4, %xmm5, %xmm4
2749 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm5 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
2750 ; AVX1-NEXT: vpmullw %xmm4, %xmm5, %xmm9
2751 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm8 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2752 ; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm11
2753 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm11[8],xmm0[8],xmm11[9],xmm0[9],xmm11[10],xmm0[10],xmm11[11],xmm0[11],xmm11[12],xmm0[12],xmm11[13],xmm0[13],xmm11[14],xmm0[14],xmm11[15],xmm0[15]
2754 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4
2755 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm5 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
2756 ; AVX1-NEXT: vpmullw %xmm7, %xmm5, %xmm10
2757 ; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm7
2758 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm5 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15]
2759 ; AVX1-NEXT: vpmullw %xmm10, %xmm5, %xmm10
2760 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5
2761 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
2762 ; AVX1-NEXT: vpmullw %xmm10, %xmm6, %xmm6
2763 ; AVX1-NEXT: vpmullw %xmm6, %xmm9, %xmm9
2764 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm7 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
2765 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2766 ; AVX1-NEXT: vpmullw %xmm7, %xmm5, %xmm5
2767 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm7 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
2768 ; AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm5
2769 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm6 = xmm11[0],zero,xmm11[1],zero,xmm11[2],zero,xmm11[3],zero,xmm11[4],zero,xmm11[5],zero,xmm11[6],zero,xmm11[7],zero
2770 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
2771 ; AVX1-NEXT: vpmullw %xmm6, %xmm4, %xmm4
2772 ; AVX1-NEXT: vpshufb %xmm7, %xmm4, %xmm4
2773 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2774 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2775 ; AVX1-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2776 ; AVX1-NEXT: vpshufb %xmm7, %xmm0, %xmm0
2777 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
2778 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2779 ; AVX1-NEXT: vpmullw %xmm2, %xmm1, %xmm1
2780 ; AVX1-NEXT: vpshufb %xmm7, %xmm1, %xmm1
2781 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
2782 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2783 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2784 ; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2785 ; AVX1-NEXT: vpshufb %xmm7, %xmm0, %xmm0
2786 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
2787 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm3 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2788 ; AVX1-NEXT: vpmullw %xmm1, %xmm3, %xmm1
2789 ; AVX1-NEXT: vpshufb %xmm7, %xmm1, %xmm1
2790 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2791 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2792 ; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2793 ; AVX1-NEXT: vpshufb %xmm7, %xmm0, %xmm0
2794 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2795 ; AVX1-NEXT: vpmullw %xmm0, %xmm9, %xmm0
2796 ; AVX1-NEXT: vpmullw %xmm0, %xmm8, %xmm0
2797 ; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0
2798 ; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
2799 ; AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2800 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2801 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2802 ; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
2803 ; AVX1-NEXT: vpmullw %xmm0, %xmm3, %xmm0
2804 ; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0
2805 ; AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2806 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2807 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2808 ; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
2809 ; AVX1-NEXT: vpmullw %xmm0, %xmm3, %xmm0
2810 ; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0
2811 ; AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2812 ; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm1
2813 ; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2814 ; AVX1-NEXT: vpextrb $0, %xmm0, %eax
2815 ; AVX1-NEXT: # kill: def $al killed $al killed $eax
2816 ; AVX1-NEXT: vzeroupper
2817 ; AVX1-NEXT: retq
2818 ;
2819 ; AVX2-LABEL: test_v128i8:
2820 ; AVX2: # %bb.0:
2821 ; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm4 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
2822 ; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm5 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
2823 ; AVX2-NEXT: vpmullw %ymm4, %ymm5, %ymm4
2824 ; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm5 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
2825 ; AVX2-NEXT: vpmullw %ymm4, %ymm5, %ymm4
2826 ; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm5 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2827 ; AVX2-NEXT: vpmullw %ymm4, %ymm5, %ymm4
2828 ; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
2829 ; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm1 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
2830 ; AVX2-NEXT: vpmullw %ymm3, %ymm1, %ymm1
2831 ; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
2832 ; AVX2-NEXT: vpmullw %ymm1, %ymm2, %ymm1
2833 ; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm0 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2834 ; AVX2-NEXT: vpmullw %ymm1, %ymm0, %ymm0
2835 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2836 ; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm0
2837 ; AVX2-NEXT: vpand %ymm1, %ymm4, %ymm1
2838 ; AVX2-NEXT: vpackuswb %ymm0, %ymm1, %ymm0
2839 ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
2840 ; AVX2-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2841 ; AVX2-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2842 ; AVX2-NEXT: vpmullw %xmm2, %xmm3, %xmm2
2843 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
2844 ; AVX2-NEXT: vpand %xmm3, %xmm2, %xmm4
2845 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2846 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2847 ; AVX2-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2848 ; AVX2-NEXT: vpand %xmm3, %xmm0, %xmm0
2849 ; AVX2-NEXT: vpackuswb %xmm4, %xmm0, %xmm0
2850 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2851 ; AVX2-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2852 ; AVX2-NEXT: vpand %xmm3, %xmm0, %xmm0
2853 ; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
2854 ; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2855 ; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
2856 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2857 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2858 ; AVX2-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2859 ; AVX2-NEXT: vpand %xmm3, %xmm0, %xmm0
2860 ; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2861 ; AVX2-NEXT: vpsrld $16, %xmm0, %xmm2
2862 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2863 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2864 ; AVX2-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2865 ; AVX2-NEXT: vpand %xmm3, %xmm0, %xmm0
2866 ; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2867 ; AVX2-NEXT: vpsrlw $8, %xmm0, %xmm1
2868 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2869 ; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2870 ; AVX2-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2871 ; AVX2-NEXT: vpextrb $0, %xmm0, %eax
2872 ; AVX2-NEXT: # kill: def $al killed $al killed $eax
2873 ; AVX2-NEXT: vzeroupper
2874 ; AVX2-NEXT: retq
2875 ;
2876 ; AVX512BW-LABEL: test_v128i8:
2877 ; AVX512BW: # %bb.0:
2878 ; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
2879 ; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2880 ; AVX512BW-NEXT: vpmullw %zmm2, %zmm3, %zmm2
2881 ; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2882 ; AVX512BW-NEXT: vpandq %zmm3, %zmm2, %zmm4
2883 ; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
2884 ; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2885 ; AVX512BW-NEXT: vpmullw %zmm1, %zmm0, %zmm0
2886 ; AVX512BW-NEXT: vpandq %zmm3, %zmm0, %zmm1
2887 ; AVX512BW-NEXT: vpackuswb %zmm4, %zmm1, %zmm1
2888 ; AVX512BW-NEXT: vextracti64x4 $1, %zmm1, %ymm1
2889 ; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm4 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
2890 ; AVX512BW-NEXT: vpmullw %zmm4, %zmm2, %zmm2
2891 ; AVX512BW-NEXT: vpandq %zmm3, %zmm2, %zmm2
2892 ; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
2893 ; AVX512BW-NEXT: vpmullw %zmm1, %zmm0, %zmm0
2894 ; AVX512BW-NEXT: vpandq %zmm3, %zmm0, %zmm0
2895 ; AVX512BW-NEXT: vpackuswb %zmm2, %zmm0, %zmm0
2896 ; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
2897 ; AVX512BW-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2898 ; AVX512BW-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2899 ; AVX512BW-NEXT: vpmullw %xmm3, %xmm2, %xmm2
2900 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
2901 ; AVX512BW-NEXT: vpand %xmm3, %xmm2, %xmm4
2902 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2903 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2904 ; AVX512BW-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2905 ; AVX512BW-NEXT: vpand %xmm3, %xmm0, %xmm0
2906 ; AVX512BW-NEXT: vpackuswb %xmm4, %xmm0, %xmm0
2907 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2908 ; AVX512BW-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2909 ; AVX512BW-NEXT: vpand %xmm3, %xmm0, %xmm0
2910 ; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
2911 ; AVX512BW-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2912 ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
2913 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2914 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2915 ; AVX512BW-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2916 ; AVX512BW-NEXT: vpand %xmm3, %xmm0, %xmm0
2917 ; AVX512BW-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2918 ; AVX512BW-NEXT: vpsrld $16, %xmm0, %xmm2
2919 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2920 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2921 ; AVX512BW-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2922 ; AVX512BW-NEXT: vpand %xmm3, %xmm0, %xmm0
2923 ; AVX512BW-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
2924 ; AVX512BW-NEXT: vpsrlw $8, %xmm0, %xmm1
2925 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2926 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2927 ; AVX512BW-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2928 ; AVX512BW-NEXT: vpextrb $0, %xmm0, %eax
2929 ; AVX512BW-NEXT: # kill: def $al killed $al killed $eax
2930 ; AVX512BW-NEXT: vzeroupper
2931 ; AVX512BW-NEXT: retq
2932 ;
2933 ; AVX512BWVL-LABEL: test_v128i8:
2934 ; AVX512BWVL: # %bb.0:
2935 ; AVX512BWVL-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
2936 ; AVX512BWVL-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2937 ; AVX512BWVL-NEXT: vpmullw %zmm2, %zmm3, %zmm2
2938 ; AVX512BWVL-NEXT: vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2939 ; AVX512BWVL-NEXT: vpandq %zmm3, %zmm2, %zmm4
2940 ; AVX512BWVL-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
2941 ; AVX512BWVL-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2942 ; AVX512BWVL-NEXT: vpmullw %zmm1, %zmm0, %zmm0
2943 ; AVX512BWVL-NEXT: vpandq %zmm3, %zmm0, %zmm1
2944 ; AVX512BWVL-NEXT: vpackuswb %zmm4, %zmm1, %zmm1
2945 ; AVX512BWVL-NEXT: vextracti64x4 $1, %zmm1, %ymm1
2946 ; AVX512BWVL-NEXT: vpunpckhbw {{.*#+}} zmm4 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
2947 ; AVX512BWVL-NEXT: vpmullw %zmm4, %zmm2, %zmm2
2948 ; AVX512BWVL-NEXT: vpandq %zmm3, %zmm2, %zmm2
2949 ; AVX512BWVL-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
2950 ; AVX512BWVL-NEXT: vpmullw %zmm1, %zmm0, %zmm0
2951 ; AVX512BWVL-NEXT: vpandq %zmm3, %zmm0, %zmm0
2952 ; AVX512BWVL-NEXT: vpackuswb %zmm2, %zmm0, %zmm0
2953 ; AVX512BWVL-NEXT: vextracti128 $1, %ymm0, %xmm1
2954 ; AVX512BWVL-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2955 ; AVX512BWVL-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2956 ; AVX512BWVL-NEXT: vpmullw %xmm3, %xmm2, %xmm2
2957 ; AVX512BWVL-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2958 ; AVX512BWVL-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2959 ; AVX512BWVL-NEXT: vpmullw %xmm2, %xmm1, %xmm1
2960 ; AVX512BWVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2961 ; AVX512BWVL-NEXT: vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
2962 ; AVX512BWVL-NEXT: vpand %xmm1, %xmm0, %xmm2
2963 ; AVX512BWVL-NEXT: vpxor %xmm3, %xmm3, %xmm3
2964 ; AVX512BWVL-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
2965 ; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
2966 ; AVX512BWVL-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
2967 ; AVX512BWVL-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2968 ; AVX512BWVL-NEXT: vpand %xmm1, %xmm0, %xmm2
2969 ; AVX512BWVL-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
2970 ; AVX512BWVL-NEXT: vpsrld $16, %xmm2, %xmm2
2971 ; AVX512BWVL-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
2972 ; AVX512BWVL-NEXT: vpmullw %xmm2, %xmm0, %xmm0
2973 ; AVX512BWVL-NEXT: vpand %xmm1, %xmm0, %xmm1
2974 ; AVX512BWVL-NEXT: vpackuswb %xmm3, %xmm1, %xmm1
2975 ; AVX512BWVL-NEXT: vpsrlw $8, %xmm1, %xmm1
2976 ; AVX512BWVL-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
2977 ; AVX512BWVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
2978 ; AVX512BWVL-NEXT: vpextrb $0, %xmm0, %eax
2979 ; AVX512BWVL-NEXT: # kill: def $al killed $al killed $eax
2980 ; AVX512BWVL-NEXT: vzeroupper
2981 ; AVX512BWVL-NEXT: retq
2982 ;
2983 ; AVX512DQ-LABEL: test_v128i8:
2984 ; AVX512DQ: # %bb.0:
2985 ; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} ymm4 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
2986 ; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} ymm5 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
2987 ; AVX512DQ-NEXT: vpmullw %ymm4, %ymm5, %ymm4
2988 ; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} ymm5 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
2989 ; AVX512DQ-NEXT: vpmullw %ymm4, %ymm5, %ymm4
2990 ; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} ymm5 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2991 ; AVX512DQ-NEXT: vpmullw %ymm4, %ymm5, %ymm4
2992 ; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
2993 ; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} ymm1 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
2994 ; AVX512DQ-NEXT: vpmullw %ymm3, %ymm1, %ymm1
2995 ; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
2996 ; AVX512DQ-NEXT: vpmullw %ymm1, %ymm2, %ymm1
2997 ; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} ymm0 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2998 ; AVX512DQ-NEXT: vpmullw %ymm1, %ymm0, %ymm0
2999 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
3000 ; AVX512DQ-NEXT: vpand %ymm1, %ymm0, %ymm0
3001 ; AVX512DQ-NEXT: vpand %ymm1, %ymm4, %ymm1
3002 ; AVX512DQ-NEXT: vpackuswb %ymm0, %ymm1, %ymm0
3003 ; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm1
3004 ; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
3005 ; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3006 ; AVX512DQ-NEXT: vpmullw %xmm2, %xmm3, %xmm2
3007 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
3008 ; AVX512DQ-NEXT: vpand %xmm3, %xmm2, %xmm4
3009 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
3010 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
3011 ; AVX512DQ-NEXT: vpmullw %xmm1, %xmm0, %xmm0
3012 ; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0
3013 ; AVX512DQ-NEXT: vpackuswb %xmm4, %xmm0, %xmm0
3014 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
3015 ; AVX512DQ-NEXT: vpmullw %xmm2, %xmm0, %xmm0
3016 ; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0
3017 ; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
3018 ; AVX512DQ-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
3019 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
3020 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
3021 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
3022 ; AVX512DQ-NEXT: vpmullw %xmm2, %xmm0, %xmm0
3023 ; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0
3024 ; AVX512DQ-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
3025 ; AVX512DQ-NEXT: vpsrld $16, %xmm0, %xmm2
3026 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
3027 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
3028 ; AVX512DQ-NEXT: vpmullw %xmm2, %xmm0, %xmm0
3029 ; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0
3030 ; AVX512DQ-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
3031 ; AVX512DQ-NEXT: vpsrlw $8, %xmm0, %xmm1
3032 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
3033 ; AVX512DQ-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
3034 ; AVX512DQ-NEXT: vpmullw %xmm1, %xmm0, %xmm0
3035 ; AVX512DQ-NEXT: vpextrb $0, %xmm0, %eax
3036 ; AVX512DQ-NEXT: # kill: def $al killed $al killed $eax
3037 ; AVX512DQ-NEXT: vzeroupper
3038 ; AVX512DQ-NEXT: retq
3039 ;
3040 ; AVX512DQVL-LABEL: test_v128i8:
3041 ; AVX512DQVL: # %bb.0:
3042 ; AVX512DQVL-NEXT: vpunpcklbw {{.*#+}} ymm4 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
3043 ; AVX512DQVL-NEXT: vpunpcklbw {{.*#+}} ymm5 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
3044 ; AVX512DQVL-NEXT: vpmullw %ymm4, %ymm5, %ymm4
3045 ; AVX512DQVL-NEXT: vpunpcklbw {{.*#+}} ymm5 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
3046 ; AVX512DQVL-NEXT: vpmullw %ymm4, %ymm5, %ymm4
3047 ; AVX512DQVL-NEXT: vpunpcklbw {{.*#+}} ymm5 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
3048 ; AVX512DQVL-NEXT: vpmullw %ymm4, %ymm5, %ymm4
3049 ; AVX512DQVL-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
3050 ; AVX512DQVL-NEXT: vpunpckhbw {{.*#+}} ymm1 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
3051 ; AVX512DQVL-NEXT: vpmullw %ymm3, %ymm1, %ymm1
3052 ; AVX512DQVL-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
3053 ; AVX512DQVL-NEXT: vpmullw %ymm1, %ymm2, %ymm1
3054 ; AVX512DQVL-NEXT: vpunpckhbw {{.*#+}} ymm0 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
3055 ; AVX512DQVL-NEXT: vpmullw %ymm1, %ymm0, %ymm0
3056 ; AVX512DQVL-NEXT: vmovdqa {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
3057 ; AVX512DQVL-NEXT: vpand %ymm1, %ymm0, %ymm0
3058 ; AVX512DQVL-NEXT: vpand %ymm1, %ymm4, %ymm1
3059 ; AVX512DQVL-NEXT: vpackuswb %ymm0, %ymm1, %ymm0
3060 ; AVX512DQVL-NEXT: vextracti128 $1, %ymm0, %xmm1
3061 ; AVX512DQVL-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
3062 ; AVX512DQVL-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3063 ; AVX512DQVL-NEXT: vpmullw %xmm2, %xmm3, %xmm2
3064 ; AVX512DQVL-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
3065 ; AVX512DQVL-NEXT: vpmullw %xmm2, %xmm1, %xmm1
3066 ; AVX512DQVL-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
3067 ; AVX512DQVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
3068 ; AVX512DQVL-NEXT: vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
3069 ; AVX512DQVL-NEXT: vpand %xmm1, %xmm0, %xmm2
3070 ; AVX512DQVL-NEXT: vpxor %xmm3, %xmm3, %xmm3
3071 ; AVX512DQVL-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
3072 ; AVX512DQVL-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
3073 ; AVX512DQVL-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
3074 ; AVX512DQVL-NEXT: vpmullw %xmm2, %xmm0, %xmm0
3075 ; AVX512DQVL-NEXT: vpand %xmm1, %xmm0, %xmm2
3076 ; AVX512DQVL-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
3077 ; AVX512DQVL-NEXT: vpsrld $16, %xmm2, %xmm2
3078 ; AVX512DQVL-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
3079 ; AVX512DQVL-NEXT: vpmullw %xmm2, %xmm0, %xmm0
3080 ; AVX512DQVL-NEXT: vpand %xmm1, %xmm0, %xmm1
3081 ; AVX512DQVL-NEXT: vpackuswb %xmm3, %xmm1, %xmm1
3082 ; AVX512DQVL-NEXT: vpsrlw $8, %xmm1, %xmm1
3083 ; AVX512DQVL-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
3084 ; AVX512DQVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
3085 ; AVX512DQVL-NEXT: vpextrb $0, %xmm0, %eax
3086 ; AVX512DQVL-NEXT: # kill: def $al killed $al killed $eax
3087 ; AVX512DQVL-NEXT: vzeroupper
3088 ; AVX512DQVL-NEXT: retq
3089 %1 = call i8 @llvm.experimental.vector.reduce.mul.v128i8(<128 x i8> %a0)
3090 ret i8 %1
3091 }
3093 declare i64 @llvm.experimental.vector.reduce.mul.v2i64(<2 x i64>)
3094 declare i64 @llvm.experimental.vector.reduce.mul.v4i64(<4 x i64>)
3095 declare i64 @llvm.experimental.vector.reduce.mul.v8i64(<8 x i64>)
3096 declare i64 @llvm.experimental.vector.reduce.mul.v16i64(<16 x i64>)
3098 declare i32 @llvm.experimental.vector.reduce.mul.v2i32(<2 x i32>)
3099 declare i32 @llvm.experimental.vector.reduce.mul.v4i32(<4 x i32>)
3100 declare i32 @llvm.experimental.vector.reduce.mul.v8i32(<8 x i32>)
3101 declare i32 @llvm.experimental.vector.reduce.mul.v16i32(<16 x i32>)
3102 declare i32 @llvm.experimental.vector.reduce.mul.v32i32(<32 x i32>)
3104 declare i16 @llvm.experimental.vector.reduce.mul.v2i16(<2 x i16>)
3105 declare i16 @llvm.experimental.vector.reduce.mul.v4i16(<4 x i16>)
3106 declare i16 @llvm.experimental.vector.reduce.mul.v8i16(<8 x i16>)
3107 declare i16 @llvm.experimental.vector.reduce.mul.v16i16(<16 x i16>)
3108 declare i16 @llvm.experimental.vector.reduce.mul.v32i16(<32 x i16>)
3109 declare i16 @llvm.experimental.vector.reduce.mul.v64i16(<64 x i16>)
3111 declare i8 @llvm.experimental.vector.reduce.mul.v2i8(<2 x i8>)
3112 declare i8 @llvm.experimental.vector.reduce.mul.v4i8(<4 x i8>)
3113 declare i8 @llvm.experimental.vector.reduce.mul.v8i8(<8 x i8>)
3114 declare i8 @llvm.experimental.vector.reduce.mul.v16i8(<16 x i8>)
3115 declare i8 @llvm.experimental.vector.reduce.mul.v32i8(<32 x i8>)
3116 declare i8 @llvm.experimental.vector.reduce.mul.v64i8(<64 x i8>)
3117 declare i8 @llvm.experimental.vector.reduce.mul.v128i8(<128 x i8>)