| blob | b8bbc5b2bd5670efc564aa0bfa325e3f8975bf1d |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
3 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
4 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
5 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
6 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
7 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL
8 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
9 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2
10 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW
11 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2
12 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefixes=XOP,XOPAVX1
13 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefixes=XOP,XOPAVX2
15 ; Just one 32-bit run to make sure we do reasonable things for i64 cases.
16 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=X32-SSE,X32-SSE2
18 declare <2 x i64> @llvm.fshl.v2i64(<2 x i64>, <2 x i64>, <2 x i64>)
19 declare <4 x i32> @llvm.fshl.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)
20 declare <8 x i16> @llvm.fshl.v8i16(<8 x i16>, <8 x i16>, <8 x i16>)
21 declare <16 x i8> @llvm.fshl.v16i8(<16 x i8>, <16 x i8>, <16 x i8>)
23 ;
24 ; Variable Shifts
25 ;
27 define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt) nounwind {
28 ; SSE2-LABEL: var_funnnel_v2i64:
29 ; SSE2: # %bb.0:
30 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm2
31 ; SSE2-NEXT: movdqa %xmm0, %xmm3
32 ; SSE2-NEXT: psllq %xmm2, %xmm3
33 ; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[2,3,0,1]
34 ; SSE2-NEXT: movdqa %xmm0, %xmm5
35 ; SSE2-NEXT: psllq %xmm4, %xmm5
36 ; SSE2-NEXT: movsd {{.*#+}} xmm5 = xmm3[0],xmm5[1]
37 ; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [64,64]
38 ; SSE2-NEXT: psubq %xmm2, %xmm3
39 ; SSE2-NEXT: movdqa %xmm1, %xmm4
40 ; SSE2-NEXT: psrlq %xmm3, %xmm4
41 ; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
42 ; SSE2-NEXT: psrlq %xmm3, %xmm1
43 ; SSE2-NEXT: movsd {{.*#+}} xmm1 = xmm4[0],xmm1[1]
44 ; SSE2-NEXT: orpd %xmm5, %xmm1
45 ; SSE2-NEXT: pxor %xmm3, %xmm3
46 ; SSE2-NEXT: pcmpeqd %xmm2, %xmm3
47 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,0,3,2]
48 ; SSE2-NEXT: pand %xmm3, %xmm2
49 ; SSE2-NEXT: pand %xmm2, %xmm0
50 ; SSE2-NEXT: pandn %xmm1, %xmm2
51 ; SSE2-NEXT: por %xmm2, %xmm0
52 ; SSE2-NEXT: retq
53 ;
54 ; SSE41-LABEL: var_funnnel_v2i64:
55 ; SSE41: # %bb.0:
56 ; SSE41-NEXT: movdqa %xmm0, %xmm3
57 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm2
58 ; SSE41-NEXT: psllq %xmm2, %xmm0
59 ; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm2[2,3,0,1]
60 ; SSE41-NEXT: movdqa %xmm3, %xmm4
61 ; SSE41-NEXT: psllq %xmm5, %xmm4
62 ; SSE41-NEXT: pblendw {{.*#+}} xmm4 = xmm0[0,1,2,3],xmm4[4,5,6,7]
63 ; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [64,64]
64 ; SSE41-NEXT: psubq %xmm2, %xmm0
65 ; SSE41-NEXT: movdqa %xmm1, %xmm5
66 ; SSE41-NEXT: psrlq %xmm0, %xmm5
67 ; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
68 ; SSE41-NEXT: psrlq %xmm0, %xmm1
69 ; SSE41-NEXT: pblendw {{.*#+}} xmm1 = xmm5[0,1,2,3],xmm1[4,5,6,7]
70 ; SSE41-NEXT: por %xmm1, %xmm4
71 ; SSE41-NEXT: pxor %xmm0, %xmm0
72 ; SSE41-NEXT: pcmpeqq %xmm2, %xmm0
73 ; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm4
74 ; SSE41-NEXT: movapd %xmm4, %xmm0
75 ; SSE41-NEXT: retq
76 ;
77 ; AVX1-LABEL: var_funnnel_v2i64:
78 ; AVX1: # %bb.0:
79 ; AVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
80 ; AVX1-NEXT: vpsllq %xmm2, %xmm0, %xmm3
81 ; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,0,1]
82 ; AVX1-NEXT: vpsllq %xmm4, %xmm0, %xmm4
83 ; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm4[4,5,6,7]
84 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
85 ; AVX1-NEXT: vpsubq %xmm2, %xmm4, %xmm4
86 ; AVX1-NEXT: vpsrlq %xmm4, %xmm1, %xmm5
87 ; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
88 ; AVX1-NEXT: vpsrlq %xmm4, %xmm1, %xmm1
89 ; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0,1,2,3],xmm1[4,5,6,7]
90 ; AVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
91 ; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
92 ; AVX1-NEXT: vpcmpeqq %xmm3, %xmm2, %xmm2
93 ; AVX1-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
94 ; AVX1-NEXT: retq
95 ;
96 ; AVX2-LABEL: var_funnnel_v2i64:
97 ; AVX2: # %bb.0:
98 ; AVX2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
99 ; AVX2-NEXT: vpsllvq %xmm2, %xmm0, %xmm3
100 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
101 ; AVX2-NEXT: vpsubq %xmm2, %xmm4, %xmm4
102 ; AVX2-NEXT: vpsrlvq %xmm4, %xmm1, %xmm1
103 ; AVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
104 ; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
105 ; AVX2-NEXT: vpcmpeqq %xmm3, %xmm2, %xmm2
106 ; AVX2-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
107 ; AVX2-NEXT: retq
108 ;
109 ; AVX512F-LABEL: var_funnnel_v2i64:
110 ; AVX512F: # %bb.0:
111 ; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
112 ; AVX512F-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
113 ; AVX512F-NEXT: vpsllvq %xmm2, %xmm0, %xmm3
114 ; AVX512F-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
115 ; AVX512F-NEXT: vpsubq %xmm2, %xmm4, %xmm4
116 ; AVX512F-NEXT: vpsrlvq %xmm4, %xmm1, %xmm1
117 ; AVX512F-NEXT: vpor %xmm1, %xmm3, %xmm1
118 ; AVX512F-NEXT: vptestnmq %zmm2, %zmm2, %k1
119 ; AVX512F-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
120 ; AVX512F-NEXT: vmovdqa %xmm1, %xmm0
121 ; AVX512F-NEXT: vzeroupper
122 ; AVX512F-NEXT: retq
123 ;
124 ; AVX512VL-LABEL: var_funnnel_v2i64:
125 ; AVX512VL: # %bb.0:
126 ; AVX512VL-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
127 ; AVX512VL-NEXT: vpsllvq %xmm2, %xmm0, %xmm3
128 ; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
129 ; AVX512VL-NEXT: vpsubq %xmm2, %xmm4, %xmm4
130 ; AVX512VL-NEXT: vpsrlvq %xmm4, %xmm1, %xmm1
131 ; AVX512VL-NEXT: vpor %xmm1, %xmm3, %xmm1
132 ; AVX512VL-NEXT: vptestnmq %xmm2, %xmm2, %k1
133 ; AVX512VL-NEXT: vmovdqa64 %xmm0, %xmm1 {%k1}
134 ; AVX512VL-NEXT: vmovdqa %xmm1, %xmm0
135 ; AVX512VL-NEXT: retq
136 ;
137 ; AVX512BW-LABEL: var_funnnel_v2i64:
138 ; AVX512BW: # %bb.0:
139 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
140 ; AVX512BW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
141 ; AVX512BW-NEXT: vpsllvq %xmm2, %xmm0, %xmm3
142 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
143 ; AVX512BW-NEXT: vpsubq %xmm2, %xmm4, %xmm4
144 ; AVX512BW-NEXT: vpsrlvq %xmm4, %xmm1, %xmm1
145 ; AVX512BW-NEXT: vpor %xmm1, %xmm3, %xmm1
146 ; AVX512BW-NEXT: vptestnmq %zmm2, %zmm2, %k1
147 ; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
148 ; AVX512BW-NEXT: vmovdqa %xmm1, %xmm0
149 ; AVX512BW-NEXT: vzeroupper
150 ; AVX512BW-NEXT: retq
151 ;
152 ; AVX512VBMI2-LABEL: var_funnnel_v2i64:
153 ; AVX512VBMI2: # %bb.0:
154 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
155 ; AVX512VBMI2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
156 ; AVX512VBMI2-NEXT: vpsllvq %xmm2, %xmm0, %xmm3
157 ; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
158 ; AVX512VBMI2-NEXT: vpsubq %xmm2, %xmm4, %xmm4
159 ; AVX512VBMI2-NEXT: vpsrlvq %xmm4, %xmm1, %xmm1
160 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm3, %xmm1
161 ; AVX512VBMI2-NEXT: vptestnmq %zmm2, %zmm2, %k1
162 ; AVX512VBMI2-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
163 ; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0
164 ; AVX512VBMI2-NEXT: vzeroupper
165 ; AVX512VBMI2-NEXT: retq
166 ;
167 ; AVX512VLBW-LABEL: var_funnnel_v2i64:
168 ; AVX512VLBW: # %bb.0:
169 ; AVX512VLBW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
170 ; AVX512VLBW-NEXT: vpsllvq %xmm2, %xmm0, %xmm3
171 ; AVX512VLBW-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
172 ; AVX512VLBW-NEXT: vpsubq %xmm2, %xmm4, %xmm4
173 ; AVX512VLBW-NEXT: vpsrlvq %xmm4, %xmm1, %xmm1
174 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm3, %xmm1
175 ; AVX512VLBW-NEXT: vptestnmq %xmm2, %xmm2, %k1
176 ; AVX512VLBW-NEXT: vmovdqa64 %xmm0, %xmm1 {%k1}
177 ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0
178 ; AVX512VLBW-NEXT: retq
179 ;
180 ; AVX512VLVBMI2-LABEL: var_funnnel_v2i64:
181 ; AVX512VLVBMI2: # %bb.0:
182 ; AVX512VLVBMI2-NEXT: vpshldvq %xmm2, %xmm1, %xmm0
183 ; AVX512VLVBMI2-NEXT: retq
184 ;
185 ; XOPAVX1-LABEL: var_funnnel_v2i64:
186 ; XOPAVX1: # %bb.0:
187 ; XOPAVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
188 ; XOPAVX1-NEXT: vpshlq %xmm2, %xmm0, %xmm3
189 ; XOPAVX1-NEXT: vpsubq {{.*}}(%rip), %xmm2, %xmm4
190 ; XOPAVX1-NEXT: vpshlq %xmm4, %xmm1, %xmm1
191 ; XOPAVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
192 ; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
193 ; XOPAVX1-NEXT: vpcomeqq %xmm3, %xmm2, %xmm2
194 ; XOPAVX1-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
195 ; XOPAVX1-NEXT: retq
196 ;
197 ; XOPAVX2-LABEL: var_funnnel_v2i64:
198 ; XOPAVX2: # %bb.0:
199 ; XOPAVX2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
200 ; XOPAVX2-NEXT: vpsllvq %xmm2, %xmm0, %xmm3
201 ; XOPAVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
202 ; XOPAVX2-NEXT: vpsubq %xmm2, %xmm4, %xmm4
203 ; XOPAVX2-NEXT: vpsrlvq %xmm4, %xmm1, %xmm1
204 ; XOPAVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
205 ; XOPAVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
206 ; XOPAVX2-NEXT: vpcomeqq %xmm3, %xmm2, %xmm2
207 ; XOPAVX2-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
208 ; XOPAVX2-NEXT: retq
209 ;
210 ; X32-SSE-LABEL: var_funnnel_v2i64:
211 ; X32-SSE: # %bb.0:
212 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm2
213 ; X32-SSE-NEXT: movdqa %xmm0, %xmm3
214 ; X32-SSE-NEXT: psllq %xmm2, %xmm3
215 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm2[2,3,0,1]
216 ; X32-SSE-NEXT: movdqa %xmm0, %xmm5
217 ; X32-SSE-NEXT: psllq %xmm4, %xmm5
218 ; X32-SSE-NEXT: movsd {{.*#+}} xmm5 = xmm3[0],xmm5[1]
219 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [64,0,64,0]
220 ; X32-SSE-NEXT: psubq %xmm2, %xmm3
221 ; X32-SSE-NEXT: movdqa %xmm1, %xmm4
222 ; X32-SSE-NEXT: psrlq %xmm3, %xmm4
223 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
224 ; X32-SSE-NEXT: psrlq %xmm3, %xmm1
225 ; X32-SSE-NEXT: movsd {{.*#+}} xmm1 = xmm4[0],xmm1[1]
226 ; X32-SSE-NEXT: orpd %xmm5, %xmm1
227 ; X32-SSE-NEXT: pxor %xmm3, %xmm3
228 ; X32-SSE-NEXT: pcmpeqd %xmm2, %xmm3
229 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,0,3,2]
230 ; X32-SSE-NEXT: pand %xmm3, %xmm2
231 ; X32-SSE-NEXT: pand %xmm2, %xmm0
232 ; X32-SSE-NEXT: pandn %xmm1, %xmm2
233 ; X32-SSE-NEXT: por %xmm2, %xmm0
234 ; X32-SSE-NEXT: retl
235 %res = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt)
236 ret <2 x i64> %res
237 }
239 define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %amt) nounwind {
240 ; SSE2-LABEL: var_funnnel_v4i32:
241 ; SSE2: # %bb.0:
242 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm2
243 ; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [32,32,32,32]
244 ; SSE2-NEXT: psubd %xmm2, %xmm4
245 ; SSE2-NEXT: pshuflw {{.*#+}} xmm3 = xmm4[2,3,3,3,4,5,6,7]
246 ; SSE2-NEXT: movdqa %xmm1, %xmm5
247 ; SSE2-NEXT: psrld %xmm3, %xmm5
248 ; SSE2-NEXT: pshuflw {{.*#+}} xmm6 = xmm4[0,1,1,1,4,5,6,7]
249 ; SSE2-NEXT: movdqa %xmm1, %xmm3
250 ; SSE2-NEXT: psrld %xmm6, %xmm3
251 ; SSE2-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
252 ; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
253 ; SSE2-NEXT: pshuflw {{.*#+}} xmm5 = xmm4[2,3,3,3,4,5,6,7]
254 ; SSE2-NEXT: movdqa %xmm1, %xmm6
255 ; SSE2-NEXT: psrld %xmm5, %xmm6
256 ; SSE2-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,1,4,5,6,7]
257 ; SSE2-NEXT: psrld %xmm4, %xmm1
258 ; SSE2-NEXT: punpckhqdq {{.*#+}} xmm1 = xmm1[1],xmm6[1]
259 ; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,3],xmm1[0,3]
260 ; SSE2-NEXT: pxor %xmm1, %xmm1
261 ; SSE2-NEXT: pcmpeqd %xmm2, %xmm1
262 ; SSE2-NEXT: pslld $23, %xmm2
263 ; SSE2-NEXT: paddd {{.*}}(%rip), %xmm2
264 ; SSE2-NEXT: cvttps2dq %xmm2, %xmm2
265 ; SSE2-NEXT: movdqa %xmm0, %xmm4
266 ; SSE2-NEXT: pmuludq %xmm2, %xmm4
267 ; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
268 ; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
269 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
270 ; SSE2-NEXT: pmuludq %xmm5, %xmm2
271 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
272 ; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
273 ; SSE2-NEXT: por %xmm3, %xmm4
274 ; SSE2-NEXT: pand %xmm1, %xmm0
275 ; SSE2-NEXT: pandn %xmm4, %xmm1
276 ; SSE2-NEXT: por %xmm1, %xmm0
277 ; SSE2-NEXT: retq
278 ;
279 ; SSE41-LABEL: var_funnnel_v4i32:
280 ; SSE41: # %bb.0:
281 ; SSE41-NEXT: movdqa %xmm0, %xmm3
282 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm2
283 ; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [32,32,32,32]
284 ; SSE41-NEXT: psubd %xmm2, %xmm0
285 ; SSE41-NEXT: pshuflw {{.*#+}} xmm4 = xmm0[2,3,3,3,4,5,6,7]
286 ; SSE41-NEXT: movdqa %xmm1, %xmm5
287 ; SSE41-NEXT: psrld %xmm4, %xmm5
288 ; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
289 ; SSE41-NEXT: pshuflw {{.*#+}} xmm6 = xmm4[2,3,3,3,4,5,6,7]
290 ; SSE41-NEXT: movdqa %xmm1, %xmm7
291 ; SSE41-NEXT: psrld %xmm6, %xmm7
292 ; SSE41-NEXT: pblendw {{.*#+}} xmm7 = xmm5[0,1,2,3],xmm7[4,5,6,7]
293 ; SSE41-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,1,4,5,6,7]
294 ; SSE41-NEXT: movdqa %xmm1, %xmm5
295 ; SSE41-NEXT: psrld %xmm0, %xmm5
296 ; SSE41-NEXT: pshuflw {{.*#+}} xmm0 = xmm4[0,1,1,1,4,5,6,7]
297 ; SSE41-NEXT: psrld %xmm0, %xmm1
298 ; SSE41-NEXT: pblendw {{.*#+}} xmm1 = xmm5[0,1,2,3],xmm1[4,5,6,7]
299 ; SSE41-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm7[2,3],xmm1[4,5],xmm7[6,7]
300 ; SSE41-NEXT: pxor %xmm0, %xmm0
301 ; SSE41-NEXT: pcmpeqd %xmm2, %xmm0
302 ; SSE41-NEXT: pslld $23, %xmm2
303 ; SSE41-NEXT: paddd {{.*}}(%rip), %xmm2
304 ; SSE41-NEXT: cvttps2dq %xmm2, %xmm2
305 ; SSE41-NEXT: pmulld %xmm3, %xmm2
306 ; SSE41-NEXT: por %xmm1, %xmm2
307 ; SSE41-NEXT: blendvps %xmm0, %xmm3, %xmm2
308 ; SSE41-NEXT: movaps %xmm2, %xmm0
309 ; SSE41-NEXT: retq
310 ;
311 ; AVX1-LABEL: var_funnnel_v4i32:
312 ; AVX1: # %bb.0:
313 ; AVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
314 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [32,32,32,32]
315 ; AVX1-NEXT: vpsubd %xmm2, %xmm3, %xmm3
316 ; AVX1-NEXT: vpsrldq {{.*#+}} xmm4 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
317 ; AVX1-NEXT: vpsrld %xmm4, %xmm1, %xmm4
318 ; AVX1-NEXT: vpsrlq $32, %xmm3, %xmm5
319 ; AVX1-NEXT: vpsrld %xmm5, %xmm1, %xmm5
320 ; AVX1-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
321 ; AVX1-NEXT: vpxor %xmm5, %xmm5, %xmm5
322 ; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
323 ; AVX1-NEXT: vpsrld %xmm6, %xmm1, %xmm6
324 ; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
325 ; AVX1-NEXT: vpsrld %xmm3, %xmm1, %xmm1
326 ; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm6[4,5,6,7]
327 ; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3],xmm1[4,5],xmm4[6,7]
328 ; AVX1-NEXT: vpslld $23, %xmm2, %xmm3
329 ; AVX1-NEXT: vpaddd {{.*}}(%rip), %xmm3, %xmm3
330 ; AVX1-NEXT: vcvttps2dq %xmm3, %xmm3
331 ; AVX1-NEXT: vpmulld %xmm3, %xmm0, %xmm3
332 ; AVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
333 ; AVX1-NEXT: vpcmpeqd %xmm5, %xmm2, %xmm2
334 ; AVX1-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
335 ; AVX1-NEXT: retq
336 ;
337 ; AVX2-LABEL: var_funnnel_v4i32:
338 ; AVX2: # %bb.0:
339 ; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
340 ; AVX2-NEXT: vpand %xmm3, %xmm2, %xmm2
341 ; AVX2-NEXT: vpsllvd %xmm2, %xmm0, %xmm3
342 ; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
343 ; AVX2-NEXT: vpsubd %xmm2, %xmm4, %xmm4
344 ; AVX2-NEXT: vpsrlvd %xmm4, %xmm1, %xmm1
345 ; AVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
346 ; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
347 ; AVX2-NEXT: vpcmpeqd %xmm3, %xmm2, %xmm2
348 ; AVX2-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
349 ; AVX2-NEXT: retq
350 ;
351 ; AVX512F-LABEL: var_funnnel_v4i32:
352 ; AVX512F: # %bb.0:
353 ; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
354 ; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
355 ; AVX512F-NEXT: vpand %xmm3, %xmm2, %xmm2
356 ; AVX512F-NEXT: vpsllvd %xmm2, %xmm0, %xmm3
357 ; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
358 ; AVX512F-NEXT: vpsubd %xmm2, %xmm4, %xmm4
359 ; AVX512F-NEXT: vpsrlvd %xmm4, %xmm1, %xmm1
360 ; AVX512F-NEXT: vpor %xmm1, %xmm3, %xmm1
361 ; AVX512F-NEXT: vptestnmd %zmm2, %zmm2, %k1
362 ; AVX512F-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1}
363 ; AVX512F-NEXT: vmovdqa %xmm1, %xmm0
364 ; AVX512F-NEXT: vzeroupper
365 ; AVX512F-NEXT: retq
366 ;
367 ; AVX512VL-LABEL: var_funnnel_v4i32:
368 ; AVX512VL: # %bb.0:
369 ; AVX512VL-NEXT: vpandd {{.*}}(%rip){1to4}, %xmm2, %xmm2
370 ; AVX512VL-NEXT: vpsllvd %xmm2, %xmm0, %xmm3
371 ; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
372 ; AVX512VL-NEXT: vpsubd %xmm2, %xmm4, %xmm4
373 ; AVX512VL-NEXT: vpsrlvd %xmm4, %xmm1, %xmm1
374 ; AVX512VL-NEXT: vpor %xmm1, %xmm3, %xmm1
375 ; AVX512VL-NEXT: vptestnmd %xmm2, %xmm2, %k1
376 ; AVX512VL-NEXT: vmovdqa32 %xmm0, %xmm1 {%k1}
377 ; AVX512VL-NEXT: vmovdqa %xmm1, %xmm0
378 ; AVX512VL-NEXT: retq
379 ;
380 ; AVX512BW-LABEL: var_funnnel_v4i32:
381 ; AVX512BW: # %bb.0:
382 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
383 ; AVX512BW-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
384 ; AVX512BW-NEXT: vpand %xmm3, %xmm2, %xmm2
385 ; AVX512BW-NEXT: vpsllvd %xmm2, %xmm0, %xmm3
386 ; AVX512BW-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
387 ; AVX512BW-NEXT: vpsubd %xmm2, %xmm4, %xmm4
388 ; AVX512BW-NEXT: vpsrlvd %xmm4, %xmm1, %xmm1
389 ; AVX512BW-NEXT: vpor %xmm1, %xmm3, %xmm1
390 ; AVX512BW-NEXT: vptestnmd %zmm2, %zmm2, %k1
391 ; AVX512BW-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1}
392 ; AVX512BW-NEXT: vmovdqa %xmm1, %xmm0
393 ; AVX512BW-NEXT: vzeroupper
394 ; AVX512BW-NEXT: retq
395 ;
396 ; AVX512VBMI2-LABEL: var_funnnel_v4i32:
397 ; AVX512VBMI2: # %bb.0:
398 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
399 ; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
400 ; AVX512VBMI2-NEXT: vpand %xmm3, %xmm2, %xmm2
401 ; AVX512VBMI2-NEXT: vpsllvd %xmm2, %xmm0, %xmm3
402 ; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
403 ; AVX512VBMI2-NEXT: vpsubd %xmm2, %xmm4, %xmm4
404 ; AVX512VBMI2-NEXT: vpsrlvd %xmm4, %xmm1, %xmm1
405 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm3, %xmm1
406 ; AVX512VBMI2-NEXT: vptestnmd %zmm2, %zmm2, %k1
407 ; AVX512VBMI2-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1}
408 ; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0
409 ; AVX512VBMI2-NEXT: vzeroupper
410 ; AVX512VBMI2-NEXT: retq
411 ;
412 ; AVX512VLBW-LABEL: var_funnnel_v4i32:
413 ; AVX512VLBW: # %bb.0:
414 ; AVX512VLBW-NEXT: vpandd {{.*}}(%rip){1to4}, %xmm2, %xmm2
415 ; AVX512VLBW-NEXT: vpsllvd %xmm2, %xmm0, %xmm3
416 ; AVX512VLBW-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
417 ; AVX512VLBW-NEXT: vpsubd %xmm2, %xmm4, %xmm4
418 ; AVX512VLBW-NEXT: vpsrlvd %xmm4, %xmm1, %xmm1
419 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm3, %xmm1
420 ; AVX512VLBW-NEXT: vptestnmd %xmm2, %xmm2, %k1
421 ; AVX512VLBW-NEXT: vmovdqa32 %xmm0, %xmm1 {%k1}
422 ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0
423 ; AVX512VLBW-NEXT: retq
424 ;
425 ; AVX512VLVBMI2-LABEL: var_funnnel_v4i32:
426 ; AVX512VLVBMI2: # %bb.0:
427 ; AVX512VLVBMI2-NEXT: vpshldvd %xmm2, %xmm1, %xmm0
428 ; AVX512VLVBMI2-NEXT: retq
429 ;
430 ; XOPAVX1-LABEL: var_funnnel_v4i32:
431 ; XOPAVX1: # %bb.0:
432 ; XOPAVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
433 ; XOPAVX1-NEXT: vpshld %xmm2, %xmm0, %xmm3
434 ; XOPAVX1-NEXT: vpsubd {{.*}}(%rip), %xmm2, %xmm4
435 ; XOPAVX1-NEXT: vpshld %xmm4, %xmm1, %xmm1
436 ; XOPAVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
437 ; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
438 ; XOPAVX1-NEXT: vpcomeqd %xmm3, %xmm2, %xmm2
439 ; XOPAVX1-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
440 ; XOPAVX1-NEXT: retq
441 ;
442 ; XOPAVX2-LABEL: var_funnnel_v4i32:
443 ; XOPAVX2: # %bb.0:
444 ; XOPAVX2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
445 ; XOPAVX2-NEXT: vpand %xmm3, %xmm2, %xmm2
446 ; XOPAVX2-NEXT: vpsllvd %xmm2, %xmm0, %xmm3
447 ; XOPAVX2-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
448 ; XOPAVX2-NEXT: vpsubd %xmm2, %xmm4, %xmm4
449 ; XOPAVX2-NEXT: vpsrlvd %xmm4, %xmm1, %xmm1
450 ; XOPAVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
451 ; XOPAVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
452 ; XOPAVX2-NEXT: vpcomeqd %xmm3, %xmm2, %xmm2
453 ; XOPAVX2-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
454 ; XOPAVX2-NEXT: retq
455 ;
456 ; X32-SSE-LABEL: var_funnnel_v4i32:
457 ; X32-SSE: # %bb.0:
458 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm2
459 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm4 = [32,32,32,32]
460 ; X32-SSE-NEXT: psubd %xmm2, %xmm4
461 ; X32-SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm4[2,3,3,3,4,5,6,7]
462 ; X32-SSE-NEXT: movdqa %xmm1, %xmm5
463 ; X32-SSE-NEXT: psrld %xmm3, %xmm5
464 ; X32-SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm4[0,1,1,1,4,5,6,7]
465 ; X32-SSE-NEXT: movdqa %xmm1, %xmm3
466 ; X32-SSE-NEXT: psrld %xmm6, %xmm3
467 ; X32-SSE-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
468 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
469 ; X32-SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm4[2,3,3,3,4,5,6,7]
470 ; X32-SSE-NEXT: movdqa %xmm1, %xmm6
471 ; X32-SSE-NEXT: psrld %xmm5, %xmm6
472 ; X32-SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,1,4,5,6,7]
473 ; X32-SSE-NEXT: psrld %xmm4, %xmm1
474 ; X32-SSE-NEXT: punpckhqdq {{.*#+}} xmm1 = xmm1[1],xmm6[1]
475 ; X32-SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,3],xmm1[0,3]
476 ; X32-SSE-NEXT: pxor %xmm1, %xmm1
477 ; X32-SSE-NEXT: pcmpeqd %xmm2, %xmm1
478 ; X32-SSE-NEXT: pslld $23, %xmm2
479 ; X32-SSE-NEXT: paddd {{\.LCPI.*}}, %xmm2
480 ; X32-SSE-NEXT: cvttps2dq %xmm2, %xmm2
481 ; X32-SSE-NEXT: movdqa %xmm0, %xmm4
482 ; X32-SSE-NEXT: pmuludq %xmm2, %xmm4
483 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
484 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
485 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
486 ; X32-SSE-NEXT: pmuludq %xmm5, %xmm2
487 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
488 ; X32-SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
489 ; X32-SSE-NEXT: por %xmm3, %xmm4
490 ; X32-SSE-NEXT: pand %xmm1, %xmm0
491 ; X32-SSE-NEXT: pandn %xmm4, %xmm1
492 ; X32-SSE-NEXT: por %xmm1, %xmm0
493 ; X32-SSE-NEXT: retl
494 %res = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %amt)
495 ret <4 x i32> %res
496 }
498 define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt) nounwind {
499 ; SSE2-LABEL: var_funnnel_v8i16:
500 ; SSE2: # %bb.0:
501 ; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
502 ; SSE2-NEXT: psubw %xmm2, %xmm3
503 ; SSE2-NEXT: psllw $12, %xmm3
504 ; SSE2-NEXT: movdqa %xmm3, %xmm4
505 ; SSE2-NEXT: psraw $15, %xmm4
506 ; SSE2-NEXT: movdqa %xmm4, %xmm5
507 ; SSE2-NEXT: pandn %xmm1, %xmm5
508 ; SSE2-NEXT: psrlw $8, %xmm1
509 ; SSE2-NEXT: pand %xmm4, %xmm1
510 ; SSE2-NEXT: por %xmm5, %xmm1
511 ; SSE2-NEXT: paddw %xmm3, %xmm3
512 ; SSE2-NEXT: movdqa %xmm3, %xmm4
513 ; SSE2-NEXT: psraw $15, %xmm4
514 ; SSE2-NEXT: movdqa %xmm4, %xmm5
515 ; SSE2-NEXT: pandn %xmm1, %xmm5
516 ; SSE2-NEXT: psrlw $4, %xmm1
517 ; SSE2-NEXT: pand %xmm4, %xmm1
518 ; SSE2-NEXT: por %xmm5, %xmm1
519 ; SSE2-NEXT: paddw %xmm3, %xmm3
520 ; SSE2-NEXT: movdqa %xmm3, %xmm4
521 ; SSE2-NEXT: psraw $15, %xmm4
522 ; SSE2-NEXT: movdqa %xmm4, %xmm5
523 ; SSE2-NEXT: pandn %xmm1, %xmm5
524 ; SSE2-NEXT: psrlw $2, %xmm1
525 ; SSE2-NEXT: pand %xmm4, %xmm1
526 ; SSE2-NEXT: por %xmm5, %xmm1
527 ; SSE2-NEXT: paddw %xmm3, %xmm3
528 ; SSE2-NEXT: psraw $15, %xmm3
529 ; SSE2-NEXT: movdqa %xmm3, %xmm4
530 ; SSE2-NEXT: pandn %xmm1, %xmm4
531 ; SSE2-NEXT: psrlw $1, %xmm1
532 ; SSE2-NEXT: pand %xmm3, %xmm1
533 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm2
534 ; SSE2-NEXT: movdqa %xmm2, %xmm3
535 ; SSE2-NEXT: pxor %xmm5, %xmm5
536 ; SSE2-NEXT: pcmpeqw %xmm2, %xmm5
537 ; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
538 ; SSE2-NEXT: pslld $23, %xmm2
539 ; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216]
540 ; SSE2-NEXT: paddd %xmm6, %xmm2
541 ; SSE2-NEXT: cvttps2dq %xmm2, %xmm2
542 ; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
543 ; SSE2-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
544 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
545 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
546 ; SSE2-NEXT: pslld $23, %xmm3
547 ; SSE2-NEXT: paddd %xmm6, %xmm3
548 ; SSE2-NEXT: cvttps2dq %xmm3, %xmm3
549 ; SSE2-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
550 ; SSE2-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
551 ; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
552 ; SSE2-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm2[0]
553 ; SSE2-NEXT: pmullw %xmm0, %xmm3
554 ; SSE2-NEXT: por %xmm4, %xmm3
555 ; SSE2-NEXT: por %xmm1, %xmm3
556 ; SSE2-NEXT: pand %xmm5, %xmm0
557 ; SSE2-NEXT: pandn %xmm3, %xmm5
558 ; SSE2-NEXT: por %xmm5, %xmm0
559 ; SSE2-NEXT: retq
560 ;
561 ; SSE41-LABEL: var_funnnel_v8i16:
562 ; SSE41: # %bb.0:
563 ; SSE41-NEXT: movdqa %xmm0, %xmm3
564 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm2
565 ; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [16,16,16,16,16,16,16,16]
566 ; SSE41-NEXT: psubw %xmm2, %xmm0
567 ; SSE41-NEXT: movdqa %xmm0, %xmm4
568 ; SSE41-NEXT: psllw $12, %xmm4
569 ; SSE41-NEXT: psllw $4, %xmm0
570 ; SSE41-NEXT: por %xmm4, %xmm0
571 ; SSE41-NEXT: movdqa %xmm0, %xmm4
572 ; SSE41-NEXT: paddw %xmm0, %xmm4
573 ; SSE41-NEXT: movdqa %xmm1, %xmm5
574 ; SSE41-NEXT: psrlw $8, %xmm5
575 ; SSE41-NEXT: pblendvb %xmm0, %xmm5, %xmm1
576 ; SSE41-NEXT: movdqa %xmm1, %xmm5
577 ; SSE41-NEXT: psrlw $4, %xmm5
578 ; SSE41-NEXT: movdqa %xmm4, %xmm0
579 ; SSE41-NEXT: pblendvb %xmm0, %xmm5, %xmm1
580 ; SSE41-NEXT: movdqa %xmm1, %xmm5
581 ; SSE41-NEXT: psrlw $2, %xmm5
582 ; SSE41-NEXT: paddw %xmm4, %xmm4
583 ; SSE41-NEXT: movdqa %xmm4, %xmm0
584 ; SSE41-NEXT: pblendvb %xmm0, %xmm5, %xmm1
585 ; SSE41-NEXT: movdqa %xmm1, %xmm5
586 ; SSE41-NEXT: psrlw $1, %xmm5
587 ; SSE41-NEXT: paddw %xmm4, %xmm4
588 ; SSE41-NEXT: movdqa %xmm4, %xmm0
589 ; SSE41-NEXT: pblendvb %xmm0, %xmm5, %xmm1
590 ; SSE41-NEXT: pmovzxwd {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
591 ; SSE41-NEXT: pxor %xmm0, %xmm0
592 ; SSE41-NEXT: pcmpeqw %xmm2, %xmm0
593 ; SSE41-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
594 ; SSE41-NEXT: pslld $23, %xmm2
595 ; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216]
596 ; SSE41-NEXT: paddd %xmm5, %xmm2
597 ; SSE41-NEXT: cvttps2dq %xmm2, %xmm6
598 ; SSE41-NEXT: pslld $23, %xmm4
599 ; SSE41-NEXT: paddd %xmm5, %xmm4
600 ; SSE41-NEXT: cvttps2dq %xmm4, %xmm2
601 ; SSE41-NEXT: packusdw %xmm6, %xmm2
602 ; SSE41-NEXT: pmullw %xmm3, %xmm2
603 ; SSE41-NEXT: por %xmm1, %xmm2
604 ; SSE41-NEXT: pblendvb %xmm0, %xmm3, %xmm2
605 ; SSE41-NEXT: movdqa %xmm2, %xmm0
606 ; SSE41-NEXT: retq
607 ;
608 ; AVX1-LABEL: var_funnnel_v8i16:
609 ; AVX1: # %bb.0:
610 ; AVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
611 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
612 ; AVX1-NEXT: vpsubw %xmm2, %xmm3, %xmm3
613 ; AVX1-NEXT: vpsllw $12, %xmm3, %xmm4
614 ; AVX1-NEXT: vpsllw $4, %xmm3, %xmm3
615 ; AVX1-NEXT: vpor %xmm4, %xmm3, %xmm3
616 ; AVX1-NEXT: vpaddw %xmm3, %xmm3, %xmm4
617 ; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm5
618 ; AVX1-NEXT: vpblendvb %xmm3, %xmm5, %xmm1, %xmm1
619 ; AVX1-NEXT: vpsrlw $4, %xmm1, %xmm3
620 ; AVX1-NEXT: vpblendvb %xmm4, %xmm3, %xmm1, %xmm1
621 ; AVX1-NEXT: vpsrlw $2, %xmm1, %xmm3
622 ; AVX1-NEXT: vpaddw %xmm4, %xmm4, %xmm4
623 ; AVX1-NEXT: vpblendvb %xmm4, %xmm3, %xmm1, %xmm1
624 ; AVX1-NEXT: vpsrlw $1, %xmm1, %xmm3
625 ; AVX1-NEXT: vpaddw %xmm4, %xmm4, %xmm4
626 ; AVX1-NEXT: vpblendvb %xmm4, %xmm3, %xmm1, %xmm1
627 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
628 ; AVX1-NEXT: vpslld $23, %xmm3, %xmm3
629 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
630 ; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
631 ; AVX1-NEXT: vcvttps2dq %xmm3, %xmm3
632 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm5 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
633 ; AVX1-NEXT: vpslld $23, %xmm5, %xmm5
634 ; AVX1-NEXT: vpaddd %xmm4, %xmm5, %xmm4
635 ; AVX1-NEXT: vcvttps2dq %xmm4, %xmm4
636 ; AVX1-NEXT: vpackusdw %xmm3, %xmm4, %xmm3
637 ; AVX1-NEXT: vpmullw %xmm3, %xmm0, %xmm3
638 ; AVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
639 ; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
640 ; AVX1-NEXT: vpcmpeqw %xmm3, %xmm2, %xmm2
641 ; AVX1-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
642 ; AVX1-NEXT: retq
643 ;
644 ; AVX2-LABEL: var_funnnel_v8i16:
645 ; AVX2: # %bb.0:
646 ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
647 ; AVX2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
648 ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
649 ; AVX2-NEXT: vpsllvd %ymm4, %ymm3, %ymm3
650 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
651 ; AVX2-NEXT: vpshufb %ymm4, %ymm3, %ymm3
652 ; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,2,3]
653 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm5 = [16,16,16,16,16,16,16,16]
654 ; AVX2-NEXT: vpsubw %xmm2, %xmm5, %xmm5
655 ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
656 ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
657 ; AVX2-NEXT: vpsrlvd %ymm5, %ymm1, %ymm1
658 ; AVX2-NEXT: vpshufb %ymm4, %ymm1, %ymm1
659 ; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
660 ; AVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
661 ; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
662 ; AVX2-NEXT: vpcmpeqw %xmm3, %xmm2, %xmm2
663 ; AVX2-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
664 ; AVX2-NEXT: vzeroupper
665 ; AVX2-NEXT: retq
666 ;
667 ; AVX512F-LABEL: var_funnnel_v8i16:
668 ; AVX512F: # %bb.0:
669 ; AVX512F-NEXT: vpmovzxwd {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
670 ; AVX512F-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
671 ; AVX512F-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
672 ; AVX512F-NEXT: vpsllvd %ymm4, %ymm3, %ymm3
673 ; AVX512F-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
674 ; AVX512F-NEXT: vpsubw %xmm2, %xmm4, %xmm4
675 ; AVX512F-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
676 ; AVX512F-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
677 ; AVX512F-NEXT: vpsrlvd %ymm4, %ymm1, %ymm1
678 ; AVX512F-NEXT: vpor %ymm1, %ymm3, %ymm1
679 ; AVX512F-NEXT: vpmovdw %zmm1, %ymm1
680 ; AVX512F-NEXT: vpxor %xmm3, %xmm3, %xmm3
681 ; AVX512F-NEXT: vpcmpeqw %xmm3, %xmm2, %xmm2
682 ; AVX512F-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
683 ; AVX512F-NEXT: vzeroupper
684 ; AVX512F-NEXT: retq
685 ;
686 ; AVX512VL-LABEL: var_funnnel_v8i16:
687 ; AVX512VL: # %bb.0:
688 ; AVX512VL-NEXT: vpmovzxwd {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
689 ; AVX512VL-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
690 ; AVX512VL-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
691 ; AVX512VL-NEXT: vpsllvd %ymm4, %ymm3, %ymm3
692 ; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
693 ; AVX512VL-NEXT: vpsubw %xmm2, %xmm4, %xmm4
694 ; AVX512VL-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
695 ; AVX512VL-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
696 ; AVX512VL-NEXT: vpsrlvd %ymm4, %ymm1, %ymm1
697 ; AVX512VL-NEXT: vpor %ymm1, %ymm3, %ymm1
698 ; AVX512VL-NEXT: vpmovdw %ymm1, %xmm1
699 ; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
700 ; AVX512VL-NEXT: vpcmpeqw %xmm3, %xmm2, %xmm2
701 ; AVX512VL-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
702 ; AVX512VL-NEXT: vzeroupper
703 ; AVX512VL-NEXT: retq
704 ;
705 ; AVX512BW-LABEL: var_funnnel_v8i16:
706 ; AVX512BW: # %bb.0:
707 ; AVX512BW-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
708 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
709 ; AVX512BW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
710 ; AVX512BW-NEXT: vpsllvw %zmm2, %zmm0, %zmm3
711 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
712 ; AVX512BW-NEXT: vpsubw %xmm2, %xmm4, %xmm4
713 ; AVX512BW-NEXT: vpsrlvw %zmm4, %zmm1, %zmm1
714 ; AVX512BW-NEXT: vpor %xmm1, %xmm3, %xmm1
715 ; AVX512BW-NEXT: vptestnmw %zmm2, %zmm2, %k1
716 ; AVX512BW-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1}
717 ; AVX512BW-NEXT: vmovdqa %xmm1, %xmm0
718 ; AVX512BW-NEXT: vzeroupper
719 ; AVX512BW-NEXT: retq
720 ;
721 ; AVX512VBMI2-LABEL: var_funnnel_v8i16:
722 ; AVX512VBMI2: # %bb.0:
723 ; AVX512VBMI2-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
724 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
725 ; AVX512VBMI2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
726 ; AVX512VBMI2-NEXT: vpsllvw %zmm2, %zmm0, %zmm3
727 ; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
728 ; AVX512VBMI2-NEXT: vpsubw %xmm2, %xmm4, %xmm4
729 ; AVX512VBMI2-NEXT: vpsrlvw %zmm4, %zmm1, %zmm1
730 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm3, %xmm1
731 ; AVX512VBMI2-NEXT: vptestnmw %zmm2, %zmm2, %k1
732 ; AVX512VBMI2-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1}
733 ; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0
734 ; AVX512VBMI2-NEXT: vzeroupper
735 ; AVX512VBMI2-NEXT: retq
736 ;
737 ; AVX512VLBW-LABEL: var_funnnel_v8i16:
738 ; AVX512VLBW: # %bb.0:
739 ; AVX512VLBW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
740 ; AVX512VLBW-NEXT: vpsllvw %xmm2, %xmm0, %xmm3
741 ; AVX512VLBW-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
742 ; AVX512VLBW-NEXT: vpsubw %xmm2, %xmm4, %xmm4
743 ; AVX512VLBW-NEXT: vpsrlvw %xmm4, %xmm1, %xmm1
744 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm3, %xmm1
745 ; AVX512VLBW-NEXT: vptestnmw %xmm2, %xmm2, %k1
746 ; AVX512VLBW-NEXT: vmovdqu16 %xmm0, %xmm1 {%k1}
747 ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0
748 ; AVX512VLBW-NEXT: retq
749 ;
750 ; AVX512VLVBMI2-LABEL: var_funnnel_v8i16:
751 ; AVX512VLVBMI2: # %bb.0:
752 ; AVX512VLVBMI2-NEXT: vpshldvw %xmm2, %xmm1, %xmm0
753 ; AVX512VLVBMI2-NEXT: retq
754 ;
755 ; XOP-LABEL: var_funnnel_v8i16:
756 ; XOP: # %bb.0:
757 ; XOP-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
758 ; XOP-NEXT: vpshlw %xmm2, %xmm0, %xmm3
759 ; XOP-NEXT: vpsubw {{.*}}(%rip), %xmm2, %xmm4
760 ; XOP-NEXT: vpshlw %xmm4, %xmm1, %xmm1
761 ; XOP-NEXT: vpor %xmm1, %xmm3, %xmm1
762 ; XOP-NEXT: vpxor %xmm3, %xmm3, %xmm3
763 ; XOP-NEXT: vpcomeqw %xmm3, %xmm2, %xmm2
764 ; XOP-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
765 ; XOP-NEXT: retq
766 ;
767 ; X32-SSE-LABEL: var_funnnel_v8i16:
768 ; X32-SSE: # %bb.0:
769 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
770 ; X32-SSE-NEXT: psubw %xmm2, %xmm3
771 ; X32-SSE-NEXT: psllw $12, %xmm3
772 ; X32-SSE-NEXT: movdqa %xmm3, %xmm4
773 ; X32-SSE-NEXT: psraw $15, %xmm4
774 ; X32-SSE-NEXT: movdqa %xmm4, %xmm5
775 ; X32-SSE-NEXT: pandn %xmm1, %xmm5
776 ; X32-SSE-NEXT: psrlw $8, %xmm1
777 ; X32-SSE-NEXT: pand %xmm4, %xmm1
778 ; X32-SSE-NEXT: por %xmm5, %xmm1
779 ; X32-SSE-NEXT: paddw %xmm3, %xmm3
780 ; X32-SSE-NEXT: movdqa %xmm3, %xmm4
781 ; X32-SSE-NEXT: psraw $15, %xmm4
782 ; X32-SSE-NEXT: movdqa %xmm4, %xmm5
783 ; X32-SSE-NEXT: pandn %xmm1, %xmm5
784 ; X32-SSE-NEXT: psrlw $4, %xmm1
785 ; X32-SSE-NEXT: pand %xmm4, %xmm1
786 ; X32-SSE-NEXT: por %xmm5, %xmm1
787 ; X32-SSE-NEXT: paddw %xmm3, %xmm3
788 ; X32-SSE-NEXT: movdqa %xmm3, %xmm4
789 ; X32-SSE-NEXT: psraw $15, %xmm4
790 ; X32-SSE-NEXT: movdqa %xmm4, %xmm5
791 ; X32-SSE-NEXT: pandn %xmm1, %xmm5
792 ; X32-SSE-NEXT: psrlw $2, %xmm1
793 ; X32-SSE-NEXT: pand %xmm4, %xmm1
794 ; X32-SSE-NEXT: por %xmm5, %xmm1
795 ; X32-SSE-NEXT: paddw %xmm3, %xmm3
796 ; X32-SSE-NEXT: psraw $15, %xmm3
797 ; X32-SSE-NEXT: movdqa %xmm3, %xmm4
798 ; X32-SSE-NEXT: pandn %xmm1, %xmm4
799 ; X32-SSE-NEXT: psrlw $1, %xmm1
800 ; X32-SSE-NEXT: pand %xmm3, %xmm1
801 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm2
802 ; X32-SSE-NEXT: movdqa %xmm2, %xmm3
803 ; X32-SSE-NEXT: pxor %xmm5, %xmm5
804 ; X32-SSE-NEXT: pcmpeqw %xmm2, %xmm5
805 ; X32-SSE-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
806 ; X32-SSE-NEXT: pslld $23, %xmm2
807 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216]
808 ; X32-SSE-NEXT: paddd %xmm6, %xmm2
809 ; X32-SSE-NEXT: cvttps2dq %xmm2, %xmm2
810 ; X32-SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
811 ; X32-SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
812 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
813 ; X32-SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
814 ; X32-SSE-NEXT: pslld $23, %xmm3
815 ; X32-SSE-NEXT: paddd %xmm6, %xmm3
816 ; X32-SSE-NEXT: cvttps2dq %xmm3, %xmm3
817 ; X32-SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
818 ; X32-SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
819 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
820 ; X32-SSE-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm2[0]
821 ; X32-SSE-NEXT: pmullw %xmm0, %xmm3
822 ; X32-SSE-NEXT: por %xmm4, %xmm3
823 ; X32-SSE-NEXT: por %xmm1, %xmm3
824 ; X32-SSE-NEXT: pand %xmm5, %xmm0
825 ; X32-SSE-NEXT: pandn %xmm3, %xmm5
826 ; X32-SSE-NEXT: por %xmm5, %xmm0
827 ; X32-SSE-NEXT: retl
828 %res = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
829 ret <8 x i16> %res
830 }
832 define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt) nounwind {
833 ; SSE2-LABEL: var_funnnel_v16i8:
834 ; SSE2: # %bb.0:
835 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm2
836 ; SSE2-NEXT: movdqa %xmm2, %xmm5
837 ; SSE2-NEXT: psllw $5, %xmm5
838 ; SSE2-NEXT: pxor %xmm4, %xmm4
839 ; SSE2-NEXT: pxor %xmm6, %xmm6
840 ; SSE2-NEXT: pcmpgtb %xmm5, %xmm6
841 ; SSE2-NEXT: movdqa %xmm0, %xmm3
842 ; SSE2-NEXT: psllw $4, %xmm3
843 ; SSE2-NEXT: pand %xmm6, %xmm3
844 ; SSE2-NEXT: pandn %xmm0, %xmm6
845 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm3
846 ; SSE2-NEXT: por %xmm6, %xmm3
847 ; SSE2-NEXT: paddb %xmm5, %xmm5
848 ; SSE2-NEXT: pxor %xmm6, %xmm6
849 ; SSE2-NEXT: pcmpgtb %xmm5, %xmm6
850 ; SSE2-NEXT: movdqa %xmm6, %xmm7
851 ; SSE2-NEXT: pandn %xmm3, %xmm7
852 ; SSE2-NEXT: psllw $2, %xmm3
853 ; SSE2-NEXT: pand %xmm6, %xmm3
854 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm3
855 ; SSE2-NEXT: por %xmm7, %xmm3
856 ; SSE2-NEXT: paddb %xmm5, %xmm5
857 ; SSE2-NEXT: pxor %xmm6, %xmm6
858 ; SSE2-NEXT: pcmpgtb %xmm5, %xmm6
859 ; SSE2-NEXT: movdqa %xmm6, %xmm8
860 ; SSE2-NEXT: pandn %xmm3, %xmm8
861 ; SSE2-NEXT: paddb %xmm3, %xmm3
862 ; SSE2-NEXT: pand %xmm6, %xmm3
863 ; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
864 ; SSE2-NEXT: psubb %xmm2, %xmm6
865 ; SSE2-NEXT: psllw $5, %xmm6
866 ; SSE2-NEXT: pxor %xmm7, %xmm7
867 ; SSE2-NEXT: pcmpgtb %xmm6, %xmm7
868 ; SSE2-NEXT: movdqa %xmm7, %xmm5
869 ; SSE2-NEXT: pandn %xmm1, %xmm5
870 ; SSE2-NEXT: psrlw $4, %xmm1
871 ; SSE2-NEXT: pand %xmm7, %xmm1
872 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm1
873 ; SSE2-NEXT: por %xmm5, %xmm1
874 ; SSE2-NEXT: paddb %xmm6, %xmm6
875 ; SSE2-NEXT: pxor %xmm5, %xmm5
876 ; SSE2-NEXT: pcmpgtb %xmm6, %xmm5
877 ; SSE2-NEXT: movdqa %xmm5, %xmm7
878 ; SSE2-NEXT: pandn %xmm1, %xmm7
879 ; SSE2-NEXT: psrlw $2, %xmm1
880 ; SSE2-NEXT: pand %xmm5, %xmm1
881 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm1
882 ; SSE2-NEXT: por %xmm7, %xmm1
883 ; SSE2-NEXT: paddb %xmm6, %xmm6
884 ; SSE2-NEXT: pcmpeqb %xmm4, %xmm2
885 ; SSE2-NEXT: pcmpgtb %xmm6, %xmm4
886 ; SSE2-NEXT: movdqa %xmm4, %xmm5
887 ; SSE2-NEXT: pandn %xmm1, %xmm5
888 ; SSE2-NEXT: psrlw $1, %xmm1
889 ; SSE2-NEXT: pand %xmm4, %xmm1
890 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm1
891 ; SSE2-NEXT: por %xmm5, %xmm1
892 ; SSE2-NEXT: por %xmm8, %xmm1
893 ; SSE2-NEXT: por %xmm3, %xmm1
894 ; SSE2-NEXT: pand %xmm2, %xmm0
895 ; SSE2-NEXT: pandn %xmm1, %xmm2
896 ; SSE2-NEXT: por %xmm2, %xmm0
897 ; SSE2-NEXT: retq
898 ;
899 ; SSE41-LABEL: var_funnnel_v16i8:
900 ; SSE41: # %bb.0:
901 ; SSE41-NEXT: movdqa %xmm0, %xmm3
902 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm2
903 ; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
904 ; SSE41-NEXT: psubb %xmm2, %xmm4
905 ; SSE41-NEXT: pxor %xmm5, %xmm5
906 ; SSE41-NEXT: pcmpeqb %xmm2, %xmm5
907 ; SSE41-NEXT: movdqa %xmm2, %xmm0
908 ; SSE41-NEXT: psllw $5, %xmm0
909 ; SSE41-NEXT: movdqa %xmm3, %xmm6
910 ; SSE41-NEXT: psllw $4, %xmm6
911 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm6
912 ; SSE41-NEXT: movdqa %xmm3, %xmm2
913 ; SSE41-NEXT: pblendvb %xmm0, %xmm6, %xmm2
914 ; SSE41-NEXT: movdqa %xmm2, %xmm6
915 ; SSE41-NEXT: psllw $2, %xmm6
916 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm6
917 ; SSE41-NEXT: paddb %xmm0, %xmm0
918 ; SSE41-NEXT: pblendvb %xmm0, %xmm6, %xmm2
919 ; SSE41-NEXT: movdqa %xmm2, %xmm6
920 ; SSE41-NEXT: paddb %xmm2, %xmm6
921 ; SSE41-NEXT: paddb %xmm0, %xmm0
922 ; SSE41-NEXT: pblendvb %xmm0, %xmm6, %xmm2
923 ; SSE41-NEXT: psllw $5, %xmm4
924 ; SSE41-NEXT: movdqa %xmm4, %xmm6
925 ; SSE41-NEXT: paddb %xmm4, %xmm6
926 ; SSE41-NEXT: movdqa %xmm1, %xmm7
927 ; SSE41-NEXT: psrlw $4, %xmm7
928 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm7
929 ; SSE41-NEXT: movdqa %xmm4, %xmm0
930 ; SSE41-NEXT: pblendvb %xmm0, %xmm7, %xmm1
931 ; SSE41-NEXT: movdqa %xmm1, %xmm4
932 ; SSE41-NEXT: psrlw $2, %xmm4
933 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm4
934 ; SSE41-NEXT: movdqa %xmm6, %xmm0
935 ; SSE41-NEXT: pblendvb %xmm0, %xmm4, %xmm1
936 ; SSE41-NEXT: movdqa %xmm1, %xmm4
937 ; SSE41-NEXT: psrlw $1, %xmm4
938 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm4
939 ; SSE41-NEXT: paddb %xmm6, %xmm6
940 ; SSE41-NEXT: movdqa %xmm6, %xmm0
941 ; SSE41-NEXT: pblendvb %xmm0, %xmm4, %xmm1
942 ; SSE41-NEXT: por %xmm1, %xmm2
943 ; SSE41-NEXT: movdqa %xmm5, %xmm0
944 ; SSE41-NEXT: pblendvb %xmm0, %xmm3, %xmm2
945 ; SSE41-NEXT: movdqa %xmm2, %xmm0
946 ; SSE41-NEXT: retq
947 ;
948 ; AVX-LABEL: var_funnnel_v16i8:
949 ; AVX: # %bb.0:
950 ; AVX-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
951 ; AVX-NEXT: vpsllw $5, %xmm2, %xmm3
952 ; AVX-NEXT: vpsllw $4, %xmm0, %xmm4
953 ; AVX-NEXT: vpand {{.*}}(%rip), %xmm4, %xmm4
954 ; AVX-NEXT: vpblendvb %xmm3, %xmm4, %xmm0, %xmm4
955 ; AVX-NEXT: vpsllw $2, %xmm4, %xmm5
956 ; AVX-NEXT: vpand {{.*}}(%rip), %xmm5, %xmm5
957 ; AVX-NEXT: vpaddb %xmm3, %xmm3, %xmm3
958 ; AVX-NEXT: vpblendvb %xmm3, %xmm5, %xmm4, %xmm4
959 ; AVX-NEXT: vpaddb %xmm4, %xmm4, %xmm5
960 ; AVX-NEXT: vpaddb %xmm3, %xmm3, %xmm3
961 ; AVX-NEXT: vpblendvb %xmm3, %xmm5, %xmm4, %xmm3
962 ; AVX-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
963 ; AVX-NEXT: vpsubb %xmm2, %xmm4, %xmm4
964 ; AVX-NEXT: vpsllw $5, %xmm4, %xmm4
965 ; AVX-NEXT: vpaddb %xmm4, %xmm4, %xmm5
966 ; AVX-NEXT: vpsrlw $4, %xmm1, %xmm6
967 ; AVX-NEXT: vpand {{.*}}(%rip), %xmm6, %xmm6
968 ; AVX-NEXT: vpblendvb %xmm4, %xmm6, %xmm1, %xmm1
969 ; AVX-NEXT: vpsrlw $2, %xmm1, %xmm4
970 ; AVX-NEXT: vpand {{.*}}(%rip), %xmm4, %xmm4
971 ; AVX-NEXT: vpblendvb %xmm5, %xmm4, %xmm1, %xmm1
972 ; AVX-NEXT: vpsrlw $1, %xmm1, %xmm4
973 ; AVX-NEXT: vpand {{.*}}(%rip), %xmm4, %xmm4
974 ; AVX-NEXT: vpaddb %xmm5, %xmm5, %xmm5
975 ; AVX-NEXT: vpblendvb %xmm5, %xmm4, %xmm1, %xmm1
976 ; AVX-NEXT: vpor %xmm1, %xmm3, %xmm1
977 ; AVX-NEXT: vpxor %xmm3, %xmm3, %xmm3
978 ; AVX-NEXT: vpcmpeqb %xmm3, %xmm2, %xmm2
979 ; AVX-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
980 ; AVX-NEXT: retq
981 ;
982 ; AVX512F-LABEL: var_funnnel_v16i8:
983 ; AVX512F: # %bb.0:
984 ; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
985 ; AVX512F-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
986 ; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
987 ; AVX512F-NEXT: vpsllvd %zmm4, %zmm3, %zmm3
988 ; AVX512F-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
989 ; AVX512F-NEXT: vpsubb %xmm2, %xmm4, %xmm4
990 ; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
991 ; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
992 ; AVX512F-NEXT: vpsrlvd %zmm4, %zmm1, %zmm1
993 ; AVX512F-NEXT: vpord %zmm1, %zmm3, %zmm1
994 ; AVX512F-NEXT: vpmovdb %zmm1, %xmm1
995 ; AVX512F-NEXT: vpxor %xmm3, %xmm3, %xmm3
996 ; AVX512F-NEXT: vpcmpeqb %xmm3, %xmm2, %xmm2
997 ; AVX512F-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
998 ; AVX512F-NEXT: vzeroupper
999 ; AVX512F-NEXT: retq
1000 ;
1001 ; AVX512VL-LABEL: var_funnnel_v16i8:
1002 ; AVX512VL: # %bb.0:
1003 ; AVX512VL-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
1004 ; AVX512VL-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1005 ; AVX512VL-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
1006 ; AVX512VL-NEXT: vpsllvd %zmm4, %zmm3, %zmm3
1007 ; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1008 ; AVX512VL-NEXT: vpsubb %xmm2, %xmm4, %xmm4
1009 ; AVX512VL-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
1010 ; AVX512VL-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
1011 ; AVX512VL-NEXT: vpsrlvd %zmm4, %zmm1, %zmm1
1012 ; AVX512VL-NEXT: vpord %zmm1, %zmm3, %zmm1
1013 ; AVX512VL-NEXT: vpmovdb %zmm1, %xmm1
1014 ; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
1015 ; AVX512VL-NEXT: vpcmpeqb %xmm3, %xmm2, %xmm2
1016 ; AVX512VL-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1017 ; AVX512VL-NEXT: vzeroupper
1018 ; AVX512VL-NEXT: retq
1019 ;
1020 ; AVX512BW-LABEL: var_funnnel_v16i8:
1021 ; AVX512BW: # %bb.0:
1022 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
1023 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1024 ; AVX512BW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1025 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
1026 ; AVX512BW-NEXT: vpsllvw %zmm4, %zmm3, %zmm3
1027 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1028 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1029 ; AVX512BW-NEXT: vpsubb %xmm2, %xmm4, %xmm4
1030 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
1031 ; AVX512BW-NEXT: vpsrlvw %zmm4, %zmm1, %zmm1
1032 ; AVX512BW-NEXT: vpor %ymm1, %ymm3, %ymm1
1033 ; AVX512BW-NEXT: vpmovwb %zmm1, %ymm1
1034 ; AVX512BW-NEXT: vptestnmb %zmm2, %zmm2, %k1
1035 ; AVX512BW-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1}
1036 ; AVX512BW-NEXT: vmovdqa %xmm1, %xmm0
1037 ; AVX512BW-NEXT: vzeroupper
1038 ; AVX512BW-NEXT: retq
1039 ;
1040 ; AVX512VBMI2-LABEL: var_funnnel_v16i8:
1041 ; AVX512VBMI2: # %bb.0:
1042 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
1043 ; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1044 ; AVX512VBMI2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1045 ; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
1046 ; AVX512VBMI2-NEXT: vpsllvw %zmm4, %zmm3, %zmm3
1047 ; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1048 ; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1049 ; AVX512VBMI2-NEXT: vpsubb %xmm2, %xmm4, %xmm4
1050 ; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
1051 ; AVX512VBMI2-NEXT: vpsrlvw %zmm4, %zmm1, %zmm1
1052 ; AVX512VBMI2-NEXT: vpor %ymm1, %ymm3, %ymm1
1053 ; AVX512VBMI2-NEXT: vpmovwb %zmm1, %ymm1
1054 ; AVX512VBMI2-NEXT: vptestnmb %zmm2, %zmm2, %k1
1055 ; AVX512VBMI2-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1}
1056 ; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0
1057 ; AVX512VBMI2-NEXT: vzeroupper
1058 ; AVX512VBMI2-NEXT: retq
1059 ;
1060 ; AVX512VLBW-LABEL: var_funnnel_v16i8:
1061 ; AVX512VLBW: # %bb.0:
1062 ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1063 ; AVX512VLBW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1064 ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
1065 ; AVX512VLBW-NEXT: vpsllvw %ymm4, %ymm3, %ymm3
1066 ; AVX512VLBW-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1067 ; AVX512VLBW-NEXT: vpsubb %xmm2, %xmm4, %xmm4
1068 ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
1069 ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1070 ; AVX512VLBW-NEXT: vpsrlvw %ymm4, %ymm1, %ymm1
1071 ; AVX512VLBW-NEXT: vpor %ymm1, %ymm3, %ymm1
1072 ; AVX512VLBW-NEXT: vpmovwb %ymm1, %xmm1
1073 ; AVX512VLBW-NEXT: vptestnmb %xmm2, %xmm2, %k1
1074 ; AVX512VLBW-NEXT: vmovdqu8 %xmm0, %xmm1 {%k1}
1075 ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0
1076 ; AVX512VLBW-NEXT: vzeroupper
1077 ; AVX512VLBW-NEXT: retq
1078 ;
1079 ; AVX512VLVBMI2-LABEL: var_funnnel_v16i8:
1080 ; AVX512VLVBMI2: # %bb.0:
1081 ; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1082 ; AVX512VLVBMI2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1083 ; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
1084 ; AVX512VLVBMI2-NEXT: vpsllvw %ymm4, %ymm3, %ymm3
1085 ; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1086 ; AVX512VLVBMI2-NEXT: vpsubb %xmm2, %xmm4, %xmm4
1087 ; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
1088 ; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1089 ; AVX512VLVBMI2-NEXT: vpsrlvw %ymm4, %ymm1, %ymm1
1090 ; AVX512VLVBMI2-NEXT: vpor %ymm1, %ymm3, %ymm1
1091 ; AVX512VLVBMI2-NEXT: vpmovwb %ymm1, %xmm1
1092 ; AVX512VLVBMI2-NEXT: vptestnmb %xmm2, %xmm2, %k1
1093 ; AVX512VLVBMI2-NEXT: vmovdqu8 %xmm0, %xmm1 {%k1}
1094 ; AVX512VLVBMI2-NEXT: vmovdqa %xmm1, %xmm0
1095 ; AVX512VLVBMI2-NEXT: vzeroupper
1096 ; AVX512VLVBMI2-NEXT: retq
1097 ;
1098 ; XOP-LABEL: var_funnnel_v16i8:
1099 ; XOP: # %bb.0:
1100 ; XOP-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1101 ; XOP-NEXT: vpshlb %xmm2, %xmm0, %xmm3
1102 ; XOP-NEXT: vpsubb {{.*}}(%rip), %xmm2, %xmm4
1103 ; XOP-NEXT: vpshlb %xmm4, %xmm1, %xmm1
1104 ; XOP-NEXT: vpor %xmm1, %xmm3, %xmm1
1105 ; XOP-NEXT: vpxor %xmm3, %xmm3, %xmm3
1106 ; XOP-NEXT: vpcomeqb %xmm3, %xmm2, %xmm2
1107 ; XOP-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1108 ; XOP-NEXT: retq
1109 ;
1110 ; X32-SSE-LABEL: var_funnnel_v16i8:
1111 ; X32-SSE: # %bb.0:
1112 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm2
1113 ; X32-SSE-NEXT: movdqa %xmm2, %xmm4
1114 ; X32-SSE-NEXT: psllw $5, %xmm4
1115 ; X32-SSE-NEXT: pxor %xmm5, %xmm5
1116 ; X32-SSE-NEXT: pcmpgtb %xmm4, %xmm5
1117 ; X32-SSE-NEXT: movdqa %xmm0, %xmm3
1118 ; X32-SSE-NEXT: psllw $4, %xmm3
1119 ; X32-SSE-NEXT: pand %xmm5, %xmm3
1120 ; X32-SSE-NEXT: pandn %xmm0, %xmm5
1121 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm3
1122 ; X32-SSE-NEXT: por %xmm5, %xmm3
1123 ; X32-SSE-NEXT: paddb %xmm4, %xmm4
1124 ; X32-SSE-NEXT: pxor %xmm5, %xmm5
1125 ; X32-SSE-NEXT: pcmpgtb %xmm4, %xmm5
1126 ; X32-SSE-NEXT: movdqa %xmm5, %xmm6
1127 ; X32-SSE-NEXT: pandn %xmm3, %xmm6
1128 ; X32-SSE-NEXT: psllw $2, %xmm3
1129 ; X32-SSE-NEXT: pand %xmm5, %xmm3
1130 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm3
1131 ; X32-SSE-NEXT: por %xmm6, %xmm3
1132 ; X32-SSE-NEXT: paddb %xmm4, %xmm4
1133 ; X32-SSE-NEXT: pxor %xmm5, %xmm5
1134 ; X32-SSE-NEXT: pcmpgtb %xmm4, %xmm5
1135 ; X32-SSE-NEXT: movdqa %xmm5, %xmm4
1136 ; X32-SSE-NEXT: pandn %xmm3, %xmm4
1137 ; X32-SSE-NEXT: paddb %xmm3, %xmm3
1138 ; X32-SSE-NEXT: pand %xmm5, %xmm3
1139 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm5 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1140 ; X32-SSE-NEXT: psubb %xmm2, %xmm5
1141 ; X32-SSE-NEXT: psllw $5, %xmm5
1142 ; X32-SSE-NEXT: pxor %xmm6, %xmm6
1143 ; X32-SSE-NEXT: pcmpgtb %xmm5, %xmm6
1144 ; X32-SSE-NEXT: movdqa %xmm6, %xmm7
1145 ; X32-SSE-NEXT: pandn %xmm1, %xmm7
1146 ; X32-SSE-NEXT: psrlw $4, %xmm1
1147 ; X32-SSE-NEXT: pand %xmm6, %xmm1
1148 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm1
1149 ; X32-SSE-NEXT: por %xmm7, %xmm1
1150 ; X32-SSE-NEXT: paddb %xmm5, %xmm5
1151 ; X32-SSE-NEXT: pxor %xmm6, %xmm6
1152 ; X32-SSE-NEXT: pcmpgtb %xmm5, %xmm6
1153 ; X32-SSE-NEXT: movdqa %xmm6, %xmm7
1154 ; X32-SSE-NEXT: pandn %xmm1, %xmm7
1155 ; X32-SSE-NEXT: psrlw $2, %xmm1
1156 ; X32-SSE-NEXT: pand %xmm6, %xmm1
1157 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm1
1158 ; X32-SSE-NEXT: por %xmm7, %xmm1
1159 ; X32-SSE-NEXT: pxor %xmm6, %xmm6
1160 ; X32-SSE-NEXT: paddb %xmm5, %xmm5
1161 ; X32-SSE-NEXT: pcmpeqb %xmm6, %xmm2
1162 ; X32-SSE-NEXT: pcmpgtb %xmm5, %xmm6
1163 ; X32-SSE-NEXT: movdqa %xmm6, %xmm5
1164 ; X32-SSE-NEXT: pandn %xmm1, %xmm5
1165 ; X32-SSE-NEXT: psrlw $1, %xmm1
1166 ; X32-SSE-NEXT: pand %xmm6, %xmm1
1167 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm1
1168 ; X32-SSE-NEXT: por %xmm5, %xmm1
1169 ; X32-SSE-NEXT: por %xmm4, %xmm1
1170 ; X32-SSE-NEXT: por %xmm3, %xmm1
1171 ; X32-SSE-NEXT: pand %xmm2, %xmm0
1172 ; X32-SSE-NEXT: pandn %xmm1, %xmm2
1173 ; X32-SSE-NEXT: por %xmm2, %xmm0
1174 ; X32-SSE-NEXT: retl
1175 %res = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt)
1176 ret <16 x i8> %res
1177 }
1179 ;
1180 ; Uniform Variable Shifts
1181 ;
1183 define <2 x i64> @splatvar_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt) nounwind {
1184 ; SSE2-LABEL: splatvar_funnnel_v2i64:
1185 ; SSE2: # %bb.0:
1186 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
1187 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm2
1188 ; SSE2-NEXT: movdqa %xmm0, %xmm3
1189 ; SSE2-NEXT: psllq %xmm2, %xmm3
1190 ; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [64,64]
1191 ; SSE2-NEXT: psubq %xmm2, %xmm4
1192 ; SSE2-NEXT: psrlq %xmm4, %xmm1
1193 ; SSE2-NEXT: por %xmm3, %xmm1
1194 ; SSE2-NEXT: pxor %xmm3, %xmm3
1195 ; SSE2-NEXT: pcmpeqd %xmm2, %xmm3
1196 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,0,3,2]
1197 ; SSE2-NEXT: pand %xmm3, %xmm2
1198 ; SSE2-NEXT: pand %xmm2, %xmm0
1199 ; SSE2-NEXT: pandn %xmm1, %xmm2
1200 ; SSE2-NEXT: por %xmm2, %xmm0
1201 ; SSE2-NEXT: retq
1202 ;
1203 ; SSE41-LABEL: splatvar_funnnel_v2i64:
1204 ; SSE41: # %bb.0:
1205 ; SSE41-NEXT: movdqa %xmm0, %xmm3
1206 ; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm2[0,1,0,1]
1207 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm4
1208 ; SSE41-NEXT: movdqa %xmm0, %xmm2
1209 ; SSE41-NEXT: psllq %xmm4, %xmm2
1210 ; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [64,64]
1211 ; SSE41-NEXT: psubq %xmm4, %xmm0
1212 ; SSE41-NEXT: psrlq %xmm0, %xmm1
1213 ; SSE41-NEXT: por %xmm1, %xmm2
1214 ; SSE41-NEXT: pxor %xmm0, %xmm0
1215 ; SSE41-NEXT: pcmpeqq %xmm4, %xmm0
1216 ; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm2
1217 ; SSE41-NEXT: movapd %xmm2, %xmm0
1218 ; SSE41-NEXT: retq
1219 ;
1220 ; AVX1-LABEL: splatvar_funnnel_v2i64:
1221 ; AVX1: # %bb.0:
1222 ; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
1223 ; AVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1224 ; AVX1-NEXT: vpsllq %xmm2, %xmm0, %xmm3
1225 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
1226 ; AVX1-NEXT: vpsubq %xmm2, %xmm4, %xmm4
1227 ; AVX1-NEXT: vpsrlq %xmm4, %xmm1, %xmm1
1228 ; AVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
1229 ; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
1230 ; AVX1-NEXT: vpcmpeqq %xmm3, %xmm2, %xmm2
1231 ; AVX1-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
1232 ; AVX1-NEXT: retq
1233 ;
1234 ; AVX2-LABEL: splatvar_funnnel_v2i64:
1235 ; AVX2: # %bb.0:
1236 ; AVX2-NEXT: vpbroadcastq %xmm2, %xmm2
1237 ; AVX2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1238 ; AVX2-NEXT: vpsllq %xmm2, %xmm0, %xmm3
1239 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
1240 ; AVX2-NEXT: vpsubq %xmm2, %xmm4, %xmm4
1241 ; AVX2-NEXT: vpsrlq %xmm4, %xmm1, %xmm1
1242 ; AVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
1243 ; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
1244 ; AVX2-NEXT: vpcmpeqq %xmm3, %xmm2, %xmm2
1245 ; AVX2-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
1246 ; AVX2-NEXT: retq
1247 ;
1248 ; AVX512F-LABEL: splatvar_funnnel_v2i64:
1249 ; AVX512F: # %bb.0:
1250 ; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
1251 ; AVX512F-NEXT: vpbroadcastq %xmm2, %xmm2
1252 ; AVX512F-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1253 ; AVX512F-NEXT: vpsllq %xmm2, %xmm0, %xmm3
1254 ; AVX512F-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
1255 ; AVX512F-NEXT: vpsubq %xmm2, %xmm4, %xmm4
1256 ; AVX512F-NEXT: vpsrlq %xmm4, %xmm1, %xmm1
1257 ; AVX512F-NEXT: vpor %xmm1, %xmm3, %xmm1
1258 ; AVX512F-NEXT: vptestnmq %zmm2, %zmm2, %k1
1259 ; AVX512F-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
1260 ; AVX512F-NEXT: vmovdqa %xmm1, %xmm0
1261 ; AVX512F-NEXT: vzeroupper
1262 ; AVX512F-NEXT: retq
1263 ;
1264 ; AVX512VL-LABEL: splatvar_funnnel_v2i64:
1265 ; AVX512VL: # %bb.0:
1266 ; AVX512VL-NEXT: vpbroadcastq %xmm2, %xmm2
1267 ; AVX512VL-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1268 ; AVX512VL-NEXT: vpsllq %xmm2, %xmm0, %xmm3
1269 ; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
1270 ; AVX512VL-NEXT: vpsubq %xmm2, %xmm4, %xmm4
1271 ; AVX512VL-NEXT: vpsrlq %xmm4, %xmm1, %xmm1
1272 ; AVX512VL-NEXT: vpor %xmm1, %xmm3, %xmm1
1273 ; AVX512VL-NEXT: vptestnmq %xmm2, %xmm2, %k1
1274 ; AVX512VL-NEXT: vmovdqa64 %xmm0, %xmm1 {%k1}
1275 ; AVX512VL-NEXT: vmovdqa %xmm1, %xmm0
1276 ; AVX512VL-NEXT: retq
1277 ;
1278 ; AVX512BW-LABEL: splatvar_funnnel_v2i64:
1279 ; AVX512BW: # %bb.0:
1280 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
1281 ; AVX512BW-NEXT: vpbroadcastq %xmm2, %xmm2
1282 ; AVX512BW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1283 ; AVX512BW-NEXT: vpsllq %xmm2, %xmm0, %xmm3
1284 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
1285 ; AVX512BW-NEXT: vpsubq %xmm2, %xmm4, %xmm4
1286 ; AVX512BW-NEXT: vpsrlq %xmm4, %xmm1, %xmm1
1287 ; AVX512BW-NEXT: vpor %xmm1, %xmm3, %xmm1
1288 ; AVX512BW-NEXT: vptestnmq %zmm2, %zmm2, %k1
1289 ; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
1290 ; AVX512BW-NEXT: vmovdqa %xmm1, %xmm0
1291 ; AVX512BW-NEXT: vzeroupper
1292 ; AVX512BW-NEXT: retq
1293 ;
1294 ; AVX512VBMI2-LABEL: splatvar_funnnel_v2i64:
1295 ; AVX512VBMI2: # %bb.0:
1296 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
1297 ; AVX512VBMI2-NEXT: vpbroadcastq %xmm2, %xmm2
1298 ; AVX512VBMI2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1299 ; AVX512VBMI2-NEXT: vpsllq %xmm2, %xmm0, %xmm3
1300 ; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
1301 ; AVX512VBMI2-NEXT: vpsubq %xmm2, %xmm4, %xmm4
1302 ; AVX512VBMI2-NEXT: vpsrlq %xmm4, %xmm1, %xmm1
1303 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm3, %xmm1
1304 ; AVX512VBMI2-NEXT: vptestnmq %zmm2, %zmm2, %k1
1305 ; AVX512VBMI2-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
1306 ; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0
1307 ; AVX512VBMI2-NEXT: vzeroupper
1308 ; AVX512VBMI2-NEXT: retq
1309 ;
1310 ; AVX512VLBW-LABEL: splatvar_funnnel_v2i64:
1311 ; AVX512VLBW: # %bb.0:
1312 ; AVX512VLBW-NEXT: vpbroadcastq %xmm2, %xmm2
1313 ; AVX512VLBW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1314 ; AVX512VLBW-NEXT: vpsllq %xmm2, %xmm0, %xmm3
1315 ; AVX512VLBW-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
1316 ; AVX512VLBW-NEXT: vpsubq %xmm2, %xmm4, %xmm4
1317 ; AVX512VLBW-NEXT: vpsrlq %xmm4, %xmm1, %xmm1
1318 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm3, %xmm1
1319 ; AVX512VLBW-NEXT: vptestnmq %xmm2, %xmm2, %k1
1320 ; AVX512VLBW-NEXT: vmovdqa64 %xmm0, %xmm1 {%k1}
1321 ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0
1322 ; AVX512VLBW-NEXT: retq
1323 ;
1324 ; AVX512VLVBMI2-LABEL: splatvar_funnnel_v2i64:
1325 ; AVX512VLVBMI2: # %bb.0:
1326 ; AVX512VLVBMI2-NEXT: vpbroadcastq %xmm2, %xmm2
1327 ; AVX512VLVBMI2-NEXT: vpshldvq %xmm2, %xmm1, %xmm0
1328 ; AVX512VLVBMI2-NEXT: retq
1329 ;
1330 ; XOPAVX1-LABEL: splatvar_funnnel_v2i64:
1331 ; XOPAVX1: # %bb.0:
1332 ; XOPAVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
1333 ; XOPAVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1334 ; XOPAVX1-NEXT: vpsllq %xmm2, %xmm0, %xmm3
1335 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
1336 ; XOPAVX1-NEXT: vpsubq %xmm2, %xmm4, %xmm4
1337 ; XOPAVX1-NEXT: vpsrlq %xmm4, %xmm1, %xmm1
1338 ; XOPAVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
1339 ; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
1340 ; XOPAVX1-NEXT: vpcomeqq %xmm3, %xmm2, %xmm2
1341 ; XOPAVX1-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
1342 ; XOPAVX1-NEXT: retq
1343 ;
1344 ; XOPAVX2-LABEL: splatvar_funnnel_v2i64:
1345 ; XOPAVX2: # %bb.0:
1346 ; XOPAVX2-NEXT: vpbroadcastq %xmm2, %xmm2
1347 ; XOPAVX2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1348 ; XOPAVX2-NEXT: vpsllq %xmm2, %xmm0, %xmm3
1349 ; XOPAVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
1350 ; XOPAVX2-NEXT: vpsubq %xmm2, %xmm4, %xmm4
1351 ; XOPAVX2-NEXT: vpsrlq %xmm4, %xmm1, %xmm1
1352 ; XOPAVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
1353 ; XOPAVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
1354 ; XOPAVX2-NEXT: vpcomeqq %xmm3, %xmm2, %xmm2
1355 ; XOPAVX2-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
1356 ; XOPAVX2-NEXT: retq
1357 ;
1358 ; X32-SSE-LABEL: splatvar_funnnel_v2i64:
1359 ; X32-SSE: # %bb.0:
1360 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
1361 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm2
1362 ; X32-SSE-NEXT: movdqa %xmm0, %xmm3
1363 ; X32-SSE-NEXT: psllq %xmm2, %xmm3
1364 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm2[2,3,0,1]
1365 ; X32-SSE-NEXT: movdqa %xmm0, %xmm5
1366 ; X32-SSE-NEXT: psllq %xmm4, %xmm5
1367 ; X32-SSE-NEXT: movsd {{.*#+}} xmm5 = xmm3[0],xmm5[1]
1368 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [64,0,64,0]
1369 ; X32-SSE-NEXT: psubq %xmm2, %xmm3
1370 ; X32-SSE-NEXT: movdqa %xmm1, %xmm4
1371 ; X32-SSE-NEXT: psrlq %xmm3, %xmm4
1372 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
1373 ; X32-SSE-NEXT: psrlq %xmm3, %xmm1
1374 ; X32-SSE-NEXT: movsd {{.*#+}} xmm1 = xmm4[0],xmm1[1]
1375 ; X32-SSE-NEXT: orpd %xmm5, %xmm1
1376 ; X32-SSE-NEXT: pxor %xmm3, %xmm3
1377 ; X32-SSE-NEXT: pcmpeqd %xmm2, %xmm3
1378 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,0,3,2]
1379 ; X32-SSE-NEXT: pand %xmm3, %xmm2
1380 ; X32-SSE-NEXT: pand %xmm2, %xmm0
1381 ; X32-SSE-NEXT: pandn %xmm1, %xmm2
1382 ; X32-SSE-NEXT: por %xmm2, %xmm0
1383 ; X32-SSE-NEXT: retl
1384 %splat = shufflevector <2 x i64> %amt, <2 x i64> undef, <2 x i32> zeroinitializer
1385 %res = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %splat)
1386 ret <2 x i64> %res
1387 }
1389 define <4 x i32> @splatvar_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %amt) nounwind {
1390 ; SSE2-LABEL: splatvar_funnnel_v4i32:
1391 ; SSE2: # %bb.0:
1392 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
1393 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm2
1394 ; SSE2-NEXT: pxor %xmm3, %xmm3
1395 ; SSE2-NEXT: xorps %xmm4, %xmm4
1396 ; SSE2-NEXT: movss {{.*#+}} xmm4 = xmm2[0],xmm4[1,2,3]
1397 ; SSE2-NEXT: movdqa %xmm0, %xmm5
1398 ; SSE2-NEXT: pslld %xmm4, %xmm5
1399 ; SSE2-NEXT: movd %xmm2, %eax
1400 ; SSE2-NEXT: movl $32, %ecx
1401 ; SSE2-NEXT: subl %eax, %ecx
1402 ; SSE2-NEXT: movd %ecx, %xmm4
1403 ; SSE2-NEXT: psrld %xmm4, %xmm1
1404 ; SSE2-NEXT: por %xmm5, %xmm1
1405 ; SSE2-NEXT: pcmpeqd %xmm3, %xmm2
1406 ; SSE2-NEXT: pand %xmm2, %xmm0
1407 ; SSE2-NEXT: pandn %xmm1, %xmm2
1408 ; SSE2-NEXT: por %xmm2, %xmm0
1409 ; SSE2-NEXT: retq
1410 ;
1411 ; SSE41-LABEL: splatvar_funnnel_v4i32:
1412 ; SSE41: # %bb.0:
1413 ; SSE41-NEXT: movdqa %xmm0, %xmm3
1414 ; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm2[0,0,0,0]
1415 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm4
1416 ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm4[0],zero,xmm4[1],zero
1417 ; SSE41-NEXT: movdqa %xmm3, %xmm2
1418 ; SSE41-NEXT: pslld %xmm0, %xmm2
1419 ; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [32,32,32,32]
1420 ; SSE41-NEXT: psubd %xmm4, %xmm0
1421 ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
1422 ; SSE41-NEXT: psrld %xmm0, %xmm1
1423 ; SSE41-NEXT: por %xmm1, %xmm2
1424 ; SSE41-NEXT: pxor %xmm0, %xmm0
1425 ; SSE41-NEXT: pcmpeqd %xmm4, %xmm0
1426 ; SSE41-NEXT: blendvps %xmm0, %xmm3, %xmm2
1427 ; SSE41-NEXT: movaps %xmm2, %xmm0
1428 ; SSE41-NEXT: retq
1429 ;
1430 ; AVX1-LABEL: splatvar_funnnel_v4i32:
1431 ; AVX1: # %bb.0:
1432 ; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
1433 ; AVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1434 ; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
1435 ; AVX1-NEXT: vpslld %xmm3, %xmm0, %xmm3
1436 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [32,32,32,32]
1437 ; AVX1-NEXT: vpsubd %xmm2, %xmm4, %xmm4
1438 ; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
1439 ; AVX1-NEXT: vpsrld %xmm4, %xmm1, %xmm1
1440 ; AVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
1441 ; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
1442 ; AVX1-NEXT: vpcmpeqd %xmm3, %xmm2, %xmm2
1443 ; AVX1-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
1444 ; AVX1-NEXT: retq
1445 ;
1446 ; AVX2-LABEL: splatvar_funnnel_v4i32:
1447 ; AVX2: # %bb.0:
1448 ; AVX2-NEXT: vpbroadcastd %xmm2, %xmm2
1449 ; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
1450 ; AVX2-NEXT: vpand %xmm3, %xmm2, %xmm2
1451 ; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
1452 ; AVX2-NEXT: vpslld %xmm3, %xmm0, %xmm3
1453 ; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
1454 ; AVX2-NEXT: vpsubd %xmm2, %xmm4, %xmm4
1455 ; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
1456 ; AVX2-NEXT: vpsrld %xmm4, %xmm1, %xmm1
1457 ; AVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
1458 ; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
1459 ; AVX2-NEXT: vpcmpeqd %xmm3, %xmm2, %xmm2
1460 ; AVX2-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
1461 ; AVX2-NEXT: retq
1462 ;
1463 ; AVX512F-LABEL: splatvar_funnnel_v4i32:
1464 ; AVX512F: # %bb.0:
1465 ; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
1466 ; AVX512F-NEXT: vpbroadcastd %xmm2, %xmm2
1467 ; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
1468 ; AVX512F-NEXT: vpand %xmm3, %xmm2, %xmm2
1469 ; AVX512F-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
1470 ; AVX512F-NEXT: vpslld %xmm3, %xmm0, %xmm3
1471 ; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
1472 ; AVX512F-NEXT: vpsubd %xmm2, %xmm4, %xmm4
1473 ; AVX512F-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
1474 ; AVX512F-NEXT: vpsrld %xmm4, %xmm1, %xmm1
1475 ; AVX512F-NEXT: vpor %xmm1, %xmm3, %xmm1
1476 ; AVX512F-NEXT: vptestnmd %zmm2, %zmm2, %k1
1477 ; AVX512F-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1}
1478 ; AVX512F-NEXT: vmovdqa %xmm1, %xmm0
1479 ; AVX512F-NEXT: vzeroupper
1480 ; AVX512F-NEXT: retq
1481 ;
1482 ; AVX512VL-LABEL: splatvar_funnnel_v4i32:
1483 ; AVX512VL: # %bb.0:
1484 ; AVX512VL-NEXT: vpbroadcastd %xmm2, %xmm2
1485 ; AVX512VL-NEXT: vpandd {{.*}}(%rip){1to4}, %xmm2, %xmm2
1486 ; AVX512VL-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
1487 ; AVX512VL-NEXT: vpslld %xmm3, %xmm0, %xmm3
1488 ; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
1489 ; AVX512VL-NEXT: vpsubd %xmm2, %xmm4, %xmm4
1490 ; AVX512VL-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
1491 ; AVX512VL-NEXT: vpsrld %xmm4, %xmm1, %xmm1
1492 ; AVX512VL-NEXT: vpor %xmm1, %xmm3, %xmm1
1493 ; AVX512VL-NEXT: vptestnmd %xmm2, %xmm2, %k1
1494 ; AVX512VL-NEXT: vmovdqa32 %xmm0, %xmm1 {%k1}
1495 ; AVX512VL-NEXT: vmovdqa %xmm1, %xmm0
1496 ; AVX512VL-NEXT: retq
1497 ;
1498 ; AVX512BW-LABEL: splatvar_funnnel_v4i32:
1499 ; AVX512BW: # %bb.0:
1500 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
1501 ; AVX512BW-NEXT: vpbroadcastd %xmm2, %xmm2
1502 ; AVX512BW-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
1503 ; AVX512BW-NEXT: vpand %xmm3, %xmm2, %xmm2
1504 ; AVX512BW-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
1505 ; AVX512BW-NEXT: vpslld %xmm3, %xmm0, %xmm3
1506 ; AVX512BW-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
1507 ; AVX512BW-NEXT: vpsubd %xmm2, %xmm4, %xmm4
1508 ; AVX512BW-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
1509 ; AVX512BW-NEXT: vpsrld %xmm4, %xmm1, %xmm1
1510 ; AVX512BW-NEXT: vpor %xmm1, %xmm3, %xmm1
1511 ; AVX512BW-NEXT: vptestnmd %zmm2, %zmm2, %k1
1512 ; AVX512BW-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1}
1513 ; AVX512BW-NEXT: vmovdqa %xmm1, %xmm0
1514 ; AVX512BW-NEXT: vzeroupper
1515 ; AVX512BW-NEXT: retq
1516 ;
1517 ; AVX512VBMI2-LABEL: splatvar_funnnel_v4i32:
1518 ; AVX512VBMI2: # %bb.0:
1519 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
1520 ; AVX512VBMI2-NEXT: vpbroadcastd %xmm2, %xmm2
1521 ; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
1522 ; AVX512VBMI2-NEXT: vpand %xmm3, %xmm2, %xmm2
1523 ; AVX512VBMI2-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
1524 ; AVX512VBMI2-NEXT: vpslld %xmm3, %xmm0, %xmm3
1525 ; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
1526 ; AVX512VBMI2-NEXT: vpsubd %xmm2, %xmm4, %xmm4
1527 ; AVX512VBMI2-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
1528 ; AVX512VBMI2-NEXT: vpsrld %xmm4, %xmm1, %xmm1
1529 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm3, %xmm1
1530 ; AVX512VBMI2-NEXT: vptestnmd %zmm2, %zmm2, %k1
1531 ; AVX512VBMI2-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1}
1532 ; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0
1533 ; AVX512VBMI2-NEXT: vzeroupper
1534 ; AVX512VBMI2-NEXT: retq
1535 ;
1536 ; AVX512VLBW-LABEL: splatvar_funnnel_v4i32:
1537 ; AVX512VLBW: # %bb.0:
1538 ; AVX512VLBW-NEXT: vpbroadcastd %xmm2, %xmm2
1539 ; AVX512VLBW-NEXT: vpandd {{.*}}(%rip){1to4}, %xmm2, %xmm2
1540 ; AVX512VLBW-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
1541 ; AVX512VLBW-NEXT: vpslld %xmm3, %xmm0, %xmm3
1542 ; AVX512VLBW-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
1543 ; AVX512VLBW-NEXT: vpsubd %xmm2, %xmm4, %xmm4
1544 ; AVX512VLBW-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
1545 ; AVX512VLBW-NEXT: vpsrld %xmm4, %xmm1, %xmm1
1546 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm3, %xmm1
1547 ; AVX512VLBW-NEXT: vptestnmd %xmm2, %xmm2, %k1
1548 ; AVX512VLBW-NEXT: vmovdqa32 %xmm0, %xmm1 {%k1}
1549 ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0
1550 ; AVX512VLBW-NEXT: retq
1551 ;
1552 ; AVX512VLVBMI2-LABEL: splatvar_funnnel_v4i32:
1553 ; AVX512VLVBMI2: # %bb.0:
1554 ; AVX512VLVBMI2-NEXT: vpbroadcastd %xmm2, %xmm2
1555 ; AVX512VLVBMI2-NEXT: vpshldvd %xmm2, %xmm1, %xmm0
1556 ; AVX512VLVBMI2-NEXT: retq
1557 ;
1558 ; XOPAVX1-LABEL: splatvar_funnnel_v4i32:
1559 ; XOPAVX1: # %bb.0:
1560 ; XOPAVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
1561 ; XOPAVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1562 ; XOPAVX1-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
1563 ; XOPAVX1-NEXT: vpslld %xmm3, %xmm0, %xmm3
1564 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [32,32,32,32]
1565 ; XOPAVX1-NEXT: vpsubd %xmm2, %xmm4, %xmm4
1566 ; XOPAVX1-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
1567 ; XOPAVX1-NEXT: vpsrld %xmm4, %xmm1, %xmm1
1568 ; XOPAVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
1569 ; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
1570 ; XOPAVX1-NEXT: vpcomeqd %xmm3, %xmm2, %xmm2
1571 ; XOPAVX1-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
1572 ; XOPAVX1-NEXT: retq
1573 ;
1574 ; XOPAVX2-LABEL: splatvar_funnnel_v4i32:
1575 ; XOPAVX2: # %bb.0:
1576 ; XOPAVX2-NEXT: vpbroadcastd %xmm2, %xmm2
1577 ; XOPAVX2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
1578 ; XOPAVX2-NEXT: vpand %xmm3, %xmm2, %xmm2
1579 ; XOPAVX2-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
1580 ; XOPAVX2-NEXT: vpslld %xmm3, %xmm0, %xmm3
1581 ; XOPAVX2-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
1582 ; XOPAVX2-NEXT: vpsubd %xmm2, %xmm4, %xmm4
1583 ; XOPAVX2-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
1584 ; XOPAVX2-NEXT: vpsrld %xmm4, %xmm1, %xmm1
1585 ; XOPAVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
1586 ; XOPAVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
1587 ; XOPAVX2-NEXT: vpcomeqd %xmm3, %xmm2, %xmm2
1588 ; XOPAVX2-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
1589 ; XOPAVX2-NEXT: retq
1590 ;
1591 ; X32-SSE-LABEL: splatvar_funnnel_v4i32:
1592 ; X32-SSE: # %bb.0:
1593 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
1594 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm2
1595 ; X32-SSE-NEXT: pxor %xmm3, %xmm3
1596 ; X32-SSE-NEXT: xorps %xmm4, %xmm4
1597 ; X32-SSE-NEXT: movss {{.*#+}} xmm4 = xmm2[0],xmm4[1,2,3]
1598 ; X32-SSE-NEXT: movdqa %xmm0, %xmm5
1599 ; X32-SSE-NEXT: pslld %xmm4, %xmm5
1600 ; X32-SSE-NEXT: movd %xmm2, %eax
1601 ; X32-SSE-NEXT: movl $32, %ecx
1602 ; X32-SSE-NEXT: subl %eax, %ecx
1603 ; X32-SSE-NEXT: movd %ecx, %xmm4
1604 ; X32-SSE-NEXT: psrld %xmm4, %xmm1
1605 ; X32-SSE-NEXT: por %xmm5, %xmm1
1606 ; X32-SSE-NEXT: pcmpeqd %xmm3, %xmm2
1607 ; X32-SSE-NEXT: pand %xmm2, %xmm0
1608 ; X32-SSE-NEXT: pandn %xmm1, %xmm2
1609 ; X32-SSE-NEXT: por %xmm2, %xmm0
1610 ; X32-SSE-NEXT: retl
1611 %splat = shufflevector <4 x i32> %amt, <4 x i32> undef, <4 x i32> zeroinitializer
1612 %res = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %splat)
1613 ret <4 x i32> %res
1614 }
1616 define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt) nounwind {
1617 ; SSE2-LABEL: splatvar_funnnel_v8i16:
1618 ; SSE2: # %bb.0:
1619 ; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
1620 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
1621 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm2
1622 ; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
1623 ; SSE2-NEXT: psubw %xmm2, %xmm3
1624 ; SSE2-NEXT: pxor %xmm4, %xmm4
1625 ; SSE2-NEXT: pcmpeqw %xmm2, %xmm4
1626 ; SSE2-NEXT: pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1]
1627 ; SSE2-NEXT: psrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1628 ; SSE2-NEXT: movdqa %xmm0, %xmm5
1629 ; SSE2-NEXT: psllw %xmm2, %xmm5
1630 ; SSE2-NEXT: pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1]
1631 ; SSE2-NEXT: psrldq {{.*#+}} xmm3 = xmm3[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1632 ; SSE2-NEXT: psrlw %xmm3, %xmm1
1633 ; SSE2-NEXT: por %xmm5, %xmm1
1634 ; SSE2-NEXT: pand %xmm4, %xmm0
1635 ; SSE2-NEXT: pandn %xmm1, %xmm4
1636 ; SSE2-NEXT: por %xmm4, %xmm0
1637 ; SSE2-NEXT: retq
1638 ;
1639 ; SSE41-LABEL: splatvar_funnnel_v8i16:
1640 ; SSE41: # %bb.0:
1641 ; SSE41-NEXT: movdqa %xmm0, %xmm3
1642 ; SSE41-NEXT: pshuflw {{.*#+}} xmm0 = xmm2[0,0,2,3,4,5,6,7]
1643 ; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,0,0,0]
1644 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm4
1645 ; SSE41-NEXT: pmovzxwq {{.*#+}} xmm0 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
1646 ; SSE41-NEXT: movdqa %xmm3, %xmm2
1647 ; SSE41-NEXT: psllw %xmm0, %xmm2
1648 ; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [16,16,16,16,16,16,16,16]
1649 ; SSE41-NEXT: psubw %xmm4, %xmm0
1650 ; SSE41-NEXT: pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
1651 ; SSE41-NEXT: psrlw %xmm0, %xmm1
1652 ; SSE41-NEXT: por %xmm1, %xmm2
1653 ; SSE41-NEXT: pxor %xmm0, %xmm0
1654 ; SSE41-NEXT: pcmpeqw %xmm4, %xmm0
1655 ; SSE41-NEXT: pblendvb %xmm0, %xmm3, %xmm2
1656 ; SSE41-NEXT: movdqa %xmm2, %xmm0
1657 ; SSE41-NEXT: retq
1658 ;
1659 ; AVX1-LABEL: splatvar_funnnel_v8i16:
1660 ; AVX1: # %bb.0:
1661 ; AVX1-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
1662 ; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
1663 ; AVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1664 ; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
1665 ; AVX1-NEXT: vpsllw %xmm3, %xmm0, %xmm3
1666 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
1667 ; AVX1-NEXT: vpsubw %xmm2, %xmm4, %xmm4
1668 ; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
1669 ; AVX1-NEXT: vpsrlw %xmm4, %xmm1, %xmm1
1670 ; AVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
1671 ; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
1672 ; AVX1-NEXT: vpcmpeqw %xmm3, %xmm2, %xmm2
1673 ; AVX1-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1674 ; AVX1-NEXT: retq
1675 ;
1676 ; AVX2-LABEL: splatvar_funnnel_v8i16:
1677 ; AVX2: # %bb.0:
1678 ; AVX2-NEXT: vpbroadcastw %xmm2, %xmm2
1679 ; AVX2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1680 ; AVX2-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
1681 ; AVX2-NEXT: vpsllw %xmm3, %xmm0, %xmm3
1682 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
1683 ; AVX2-NEXT: vpsubw %xmm2, %xmm4, %xmm4
1684 ; AVX2-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
1685 ; AVX2-NEXT: vpsrlw %xmm4, %xmm1, %xmm1
1686 ; AVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
1687 ; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
1688 ; AVX2-NEXT: vpcmpeqw %xmm3, %xmm2, %xmm2
1689 ; AVX2-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1690 ; AVX2-NEXT: retq
1691 ;
1692 ; AVX512F-LABEL: splatvar_funnnel_v8i16:
1693 ; AVX512F: # %bb.0:
1694 ; AVX512F-NEXT: vpbroadcastw %xmm2, %xmm2
1695 ; AVX512F-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1696 ; AVX512F-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
1697 ; AVX512F-NEXT: vpsllw %xmm3, %xmm0, %xmm3
1698 ; AVX512F-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
1699 ; AVX512F-NEXT: vpsubw %xmm2, %xmm4, %xmm4
1700 ; AVX512F-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
1701 ; AVX512F-NEXT: vpsrlw %xmm4, %xmm1, %xmm1
1702 ; AVX512F-NEXT: vpor %xmm1, %xmm3, %xmm1
1703 ; AVX512F-NEXT: vpxor %xmm3, %xmm3, %xmm3
1704 ; AVX512F-NEXT: vpcmpeqw %xmm3, %xmm2, %xmm2
1705 ; AVX512F-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1706 ; AVX512F-NEXT: retq
1707 ;
1708 ; AVX512VL-LABEL: splatvar_funnnel_v8i16:
1709 ; AVX512VL: # %bb.0:
1710 ; AVX512VL-NEXT: vpbroadcastw %xmm2, %xmm2
1711 ; AVX512VL-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1712 ; AVX512VL-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
1713 ; AVX512VL-NEXT: vpsllw %xmm3, %xmm0, %xmm3
1714 ; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
1715 ; AVX512VL-NEXT: vpsubw %xmm2, %xmm4, %xmm4
1716 ; AVX512VL-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
1717 ; AVX512VL-NEXT: vpsrlw %xmm4, %xmm1, %xmm1
1718 ; AVX512VL-NEXT: vpor %xmm1, %xmm3, %xmm1
1719 ; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
1720 ; AVX512VL-NEXT: vpcmpeqw %xmm3, %xmm2, %xmm2
1721 ; AVX512VL-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1722 ; AVX512VL-NEXT: retq
1723 ;
1724 ; AVX512BW-LABEL: splatvar_funnnel_v8i16:
1725 ; AVX512BW: # %bb.0:
1726 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
1727 ; AVX512BW-NEXT: vpbroadcastw %xmm2, %xmm2
1728 ; AVX512BW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1729 ; AVX512BW-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
1730 ; AVX512BW-NEXT: vpsllw %xmm3, %xmm0, %xmm3
1731 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
1732 ; AVX512BW-NEXT: vpsubw %xmm2, %xmm4, %xmm4
1733 ; AVX512BW-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
1734 ; AVX512BW-NEXT: vpsrlw %xmm4, %xmm1, %xmm1
1735 ; AVX512BW-NEXT: vpor %xmm1, %xmm3, %xmm1
1736 ; AVX512BW-NEXT: vptestnmw %zmm2, %zmm2, %k1
1737 ; AVX512BW-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1}
1738 ; AVX512BW-NEXT: vmovdqa %xmm1, %xmm0
1739 ; AVX512BW-NEXT: vzeroupper
1740 ; AVX512BW-NEXT: retq
1741 ;
1742 ; AVX512VBMI2-LABEL: splatvar_funnnel_v8i16:
1743 ; AVX512VBMI2: # %bb.0:
1744 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
1745 ; AVX512VBMI2-NEXT: vpbroadcastw %xmm2, %xmm2
1746 ; AVX512VBMI2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1747 ; AVX512VBMI2-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
1748 ; AVX512VBMI2-NEXT: vpsllw %xmm3, %xmm0, %xmm3
1749 ; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
1750 ; AVX512VBMI2-NEXT: vpsubw %xmm2, %xmm4, %xmm4
1751 ; AVX512VBMI2-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
1752 ; AVX512VBMI2-NEXT: vpsrlw %xmm4, %xmm1, %xmm1
1753 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm3, %xmm1
1754 ; AVX512VBMI2-NEXT: vptestnmw %zmm2, %zmm2, %k1
1755 ; AVX512VBMI2-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1}
1756 ; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0
1757 ; AVX512VBMI2-NEXT: vzeroupper
1758 ; AVX512VBMI2-NEXT: retq
1759 ;
1760 ; AVX512VLBW-LABEL: splatvar_funnnel_v8i16:
1761 ; AVX512VLBW: # %bb.0:
1762 ; AVX512VLBW-NEXT: vpbroadcastw %xmm2, %xmm2
1763 ; AVX512VLBW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1764 ; AVX512VLBW-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
1765 ; AVX512VLBW-NEXT: vpsllw %xmm3, %xmm0, %xmm3
1766 ; AVX512VLBW-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
1767 ; AVX512VLBW-NEXT: vpsubw %xmm2, %xmm4, %xmm4
1768 ; AVX512VLBW-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
1769 ; AVX512VLBW-NEXT: vpsrlw %xmm4, %xmm1, %xmm1
1770 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm3, %xmm1
1771 ; AVX512VLBW-NEXT: vptestnmw %xmm2, %xmm2, %k1
1772 ; AVX512VLBW-NEXT: vmovdqu16 %xmm0, %xmm1 {%k1}
1773 ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0
1774 ; AVX512VLBW-NEXT: retq
1775 ;
1776 ; AVX512VLVBMI2-LABEL: splatvar_funnnel_v8i16:
1777 ; AVX512VLVBMI2: # %bb.0:
1778 ; AVX512VLVBMI2-NEXT: vpbroadcastw %xmm2, %xmm2
1779 ; AVX512VLVBMI2-NEXT: vpshldvw %xmm2, %xmm1, %xmm0
1780 ; AVX512VLVBMI2-NEXT: retq
1781 ;
1782 ; XOPAVX1-LABEL: splatvar_funnnel_v8i16:
1783 ; XOPAVX1: # %bb.0:
1784 ; XOPAVX1-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
1785 ; XOPAVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
1786 ; XOPAVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1787 ; XOPAVX1-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
1788 ; XOPAVX1-NEXT: vpsllw %xmm3, %xmm0, %xmm3
1789 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
1790 ; XOPAVX1-NEXT: vpsubw %xmm2, %xmm4, %xmm4
1791 ; XOPAVX1-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
1792 ; XOPAVX1-NEXT: vpsrlw %xmm4, %xmm1, %xmm1
1793 ; XOPAVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
1794 ; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
1795 ; XOPAVX1-NEXT: vpcomeqw %xmm3, %xmm2, %xmm2
1796 ; XOPAVX1-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1797 ; XOPAVX1-NEXT: retq
1798 ;
1799 ; XOPAVX2-LABEL: splatvar_funnnel_v8i16:
1800 ; XOPAVX2: # %bb.0:
1801 ; XOPAVX2-NEXT: vpbroadcastw %xmm2, %xmm2
1802 ; XOPAVX2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1803 ; XOPAVX2-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
1804 ; XOPAVX2-NEXT: vpsllw %xmm3, %xmm0, %xmm3
1805 ; XOPAVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
1806 ; XOPAVX2-NEXT: vpsubw %xmm2, %xmm4, %xmm4
1807 ; XOPAVX2-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
1808 ; XOPAVX2-NEXT: vpsrlw %xmm4, %xmm1, %xmm1
1809 ; XOPAVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
1810 ; XOPAVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
1811 ; XOPAVX2-NEXT: vpcomeqw %xmm3, %xmm2, %xmm2
1812 ; XOPAVX2-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1813 ; XOPAVX2-NEXT: retq
1814 ;
1815 ; X32-SSE-LABEL: splatvar_funnnel_v8i16:
1816 ; X32-SSE: # %bb.0:
1817 ; X32-SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
1818 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
1819 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm2
1820 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
1821 ; X32-SSE-NEXT: psubw %xmm2, %xmm3
1822 ; X32-SSE-NEXT: pxor %xmm4, %xmm4
1823 ; X32-SSE-NEXT: pcmpeqw %xmm2, %xmm4
1824 ; X32-SSE-NEXT: pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1]
1825 ; X32-SSE-NEXT: psrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1826 ; X32-SSE-NEXT: movdqa %xmm0, %xmm5
1827 ; X32-SSE-NEXT: psllw %xmm2, %xmm5
1828 ; X32-SSE-NEXT: pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1]
1829 ; X32-SSE-NEXT: psrldq {{.*#+}} xmm3 = xmm3[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1830 ; X32-SSE-NEXT: psrlw %xmm3, %xmm1
1831 ; X32-SSE-NEXT: por %xmm5, %xmm1
1832 ; X32-SSE-NEXT: pand %xmm4, %xmm0
1833 ; X32-SSE-NEXT: pandn %xmm1, %xmm4
1834 ; X32-SSE-NEXT: por %xmm4, %xmm0
1835 ; X32-SSE-NEXT: retl
1836 %splat = shufflevector <8 x i16> %amt, <8 x i16> undef, <8 x i32> zeroinitializer
1837 %res = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %splat)
1838 ret <8 x i16> %res
1839 }
1841 define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt) nounwind {
1842 ; SSE2-LABEL: splatvar_funnnel_v16i8:
1843 ; SSE2: # %bb.0:
1844 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1845 ; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
1846 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
1847 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm2
1848 ; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1849 ; SSE2-NEXT: psubb %xmm2, %xmm3
1850 ; SSE2-NEXT: pxor %xmm4, %xmm4
1851 ; SSE2-NEXT: pcmpeqb %xmm2, %xmm4
1852 ; SSE2-NEXT: pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0]
1853 ; SSE2-NEXT: psrldq {{.*#+}} xmm2 = xmm2[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1854 ; SSE2-NEXT: movdqa %xmm0, %xmm5
1855 ; SSE2-NEXT: psllw %xmm2, %xmm5
1856 ; SSE2-NEXT: pcmpeqd %xmm6, %xmm6
1857 ; SSE2-NEXT: psllw %xmm2, %xmm6
1858 ; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
1859 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1860 ; SSE2-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,0,2,3,4,5,6,7]
1861 ; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,0,0,0]
1862 ; SSE2-NEXT: pand %xmm5, %xmm6
1863 ; SSE2-NEXT: pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0]
1864 ; SSE2-NEXT: psrldq {{.*#+}} xmm3 = xmm3[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1865 ; SSE2-NEXT: psrlw %xmm3, %xmm1
1866 ; SSE2-NEXT: psrlw %xmm3, %xmm2
1867 ; SSE2-NEXT: psrlw $8, %xmm2
1868 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1869 ; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
1870 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
1871 ; SSE2-NEXT: pand %xmm1, %xmm2
1872 ; SSE2-NEXT: por %xmm6, %xmm2
1873 ; SSE2-NEXT: pand %xmm4, %xmm0
1874 ; SSE2-NEXT: pandn %xmm2, %xmm4
1875 ; SSE2-NEXT: por %xmm4, %xmm0
1876 ; SSE2-NEXT: retq
1877 ;
1878 ; SSE41-LABEL: splatvar_funnnel_v16i8:
1879 ; SSE41: # %bb.0:
1880 ; SSE41-NEXT: movdqa %xmm0, %xmm3
1881 ; SSE41-NEXT: pxor %xmm0, %xmm0
1882 ; SSE41-NEXT: pshufb %xmm0, %xmm2
1883 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm2
1884 ; SSE41-NEXT: pmovzxbq {{.*#+}} xmm5 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
1885 ; SSE41-NEXT: movdqa %xmm3, %xmm4
1886 ; SSE41-NEXT: psllw %xmm5, %xmm4
1887 ; SSE41-NEXT: pcmpeqd %xmm6, %xmm6
1888 ; SSE41-NEXT: pcmpeqd %xmm7, %xmm7
1889 ; SSE41-NEXT: psllw %xmm5, %xmm7
1890 ; SSE41-NEXT: pshufb %xmm0, %xmm7
1891 ; SSE41-NEXT: pand %xmm7, %xmm4
1892 ; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1893 ; SSE41-NEXT: psubb %xmm2, %xmm5
1894 ; SSE41-NEXT: pmovzxbq {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,zero,zero,zero,zero,xmm5[1],zero,zero,zero,zero,zero,zero,zero
1895 ; SSE41-NEXT: psrlw %xmm5, %xmm1
1896 ; SSE41-NEXT: psrlw %xmm5, %xmm6
1897 ; SSE41-NEXT: pshufb {{.*#+}} xmm6 = xmm6[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
1898 ; SSE41-NEXT: pand %xmm1, %xmm6
1899 ; SSE41-NEXT: por %xmm6, %xmm4
1900 ; SSE41-NEXT: pcmpeqb %xmm2, %xmm0
1901 ; SSE41-NEXT: pblendvb %xmm0, %xmm3, %xmm4
1902 ; SSE41-NEXT: movdqa %xmm4, %xmm0
1903 ; SSE41-NEXT: retq
1904 ;
1905 ; AVX1-LABEL: splatvar_funnnel_v16i8:
1906 ; AVX1: # %bb.0:
1907 ; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
1908 ; AVX1-NEXT: vpshufb %xmm3, %xmm2, %xmm2
1909 ; AVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1910 ; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
1911 ; AVX1-NEXT: vpsllw %xmm4, %xmm0, %xmm5
1912 ; AVX1-NEXT: vpcmpeqd %xmm6, %xmm6, %xmm6
1913 ; AVX1-NEXT: vpsllw %xmm4, %xmm6, %xmm4
1914 ; AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm4
1915 ; AVX1-NEXT: vpand %xmm4, %xmm5, %xmm4
1916 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm5 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1917 ; AVX1-NEXT: vpsubb %xmm2, %xmm5, %xmm5
1918 ; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,zero,zero,zero,zero,xmm5[1],zero,zero,zero,zero,zero,zero,zero
1919 ; AVX1-NEXT: vpsrlw %xmm5, %xmm1, %xmm1
1920 ; AVX1-NEXT: vpsrlw %xmm5, %xmm6, %xmm5
1921 ; AVX1-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
1922 ; AVX1-NEXT: vpand %xmm5, %xmm1, %xmm1
1923 ; AVX1-NEXT: vpor %xmm1, %xmm4, %xmm1
1924 ; AVX1-NEXT: vpcmpeqb %xmm3, %xmm2, %xmm2
1925 ; AVX1-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1926 ; AVX1-NEXT: retq
1927 ;
1928 ; AVX2-LABEL: splatvar_funnnel_v16i8:
1929 ; AVX2: # %bb.0:
1930 ; AVX2-NEXT: vpbroadcastb %xmm2, %xmm2
1931 ; AVX2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1932 ; AVX2-NEXT: vpmovzxbq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
1933 ; AVX2-NEXT: vpsllw %xmm3, %xmm0, %xmm4
1934 ; AVX2-NEXT: vpcmpeqd %xmm5, %xmm5, %xmm5
1935 ; AVX2-NEXT: vpsllw %xmm3, %xmm5, %xmm3
1936 ; AVX2-NEXT: vpbroadcastb %xmm3, %xmm3
1937 ; AVX2-NEXT: vpand %xmm3, %xmm4, %xmm3
1938 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1939 ; AVX2-NEXT: vpsubb %xmm2, %xmm4, %xmm4
1940 ; AVX2-NEXT: vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
1941 ; AVX2-NEXT: vpsrlw %xmm4, %xmm1, %xmm1
1942 ; AVX2-NEXT: vpsrlw %xmm4, %xmm5, %xmm4
1943 ; AVX2-NEXT: vpsrlw $8, %xmm4, %xmm4
1944 ; AVX2-NEXT: vpbroadcastb %xmm4, %xmm4
1945 ; AVX2-NEXT: vpand %xmm4, %xmm1, %xmm1
1946 ; AVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
1947 ; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
1948 ; AVX2-NEXT: vpcmpeqb %xmm3, %xmm2, %xmm2
1949 ; AVX2-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1950 ; AVX2-NEXT: retq
1951 ;
1952 ; AVX512F-LABEL: splatvar_funnnel_v16i8:
1953 ; AVX512F: # %bb.0:
1954 ; AVX512F-NEXT: vpbroadcastb %xmm2, %xmm2
1955 ; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
1956 ; AVX512F-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1957 ; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
1958 ; AVX512F-NEXT: vpsllvd %zmm4, %zmm3, %zmm3
1959 ; AVX512F-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1960 ; AVX512F-NEXT: vpsubb %xmm2, %xmm4, %xmm4
1961 ; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
1962 ; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
1963 ; AVX512F-NEXT: vpsrlvd %zmm4, %zmm1, %zmm1
1964 ; AVX512F-NEXT: vpord %zmm1, %zmm3, %zmm1
1965 ; AVX512F-NEXT: vpmovdb %zmm1, %xmm1
1966 ; AVX512F-NEXT: vpxor %xmm3, %xmm3, %xmm3
1967 ; AVX512F-NEXT: vpcmpeqb %xmm3, %xmm2, %xmm2
1968 ; AVX512F-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1969 ; AVX512F-NEXT: vzeroupper
1970 ; AVX512F-NEXT: retq
1971 ;
1972 ; AVX512VL-LABEL: splatvar_funnnel_v16i8:
1973 ; AVX512VL: # %bb.0:
1974 ; AVX512VL-NEXT: vpbroadcastb %xmm2, %xmm2
1975 ; AVX512VL-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
1976 ; AVX512VL-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1977 ; AVX512VL-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
1978 ; AVX512VL-NEXT: vpsllvd %zmm4, %zmm3, %zmm3
1979 ; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1980 ; AVX512VL-NEXT: vpsubb %xmm2, %xmm4, %xmm4
1981 ; AVX512VL-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
1982 ; AVX512VL-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
1983 ; AVX512VL-NEXT: vpsrlvd %zmm4, %zmm1, %zmm1
1984 ; AVX512VL-NEXT: vpord %zmm1, %zmm3, %zmm1
1985 ; AVX512VL-NEXT: vpmovdb %zmm1, %xmm1
1986 ; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
1987 ; AVX512VL-NEXT: vpcmpeqb %xmm3, %xmm2, %xmm2
1988 ; AVX512VL-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1989 ; AVX512VL-NEXT: vzeroupper
1990 ; AVX512VL-NEXT: retq
1991 ;
1992 ; AVX512BW-LABEL: splatvar_funnnel_v16i8:
1993 ; AVX512BW: # %bb.0:
1994 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
1995 ; AVX512BW-NEXT: vpbroadcastb %xmm2, %xmm2
1996 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1997 ; AVX512BW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1998 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
1999 ; AVX512BW-NEXT: vpsllvw %zmm4, %zmm3, %zmm3
2000 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
2001 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
2002 ; AVX512BW-NEXT: vpsubb %xmm2, %xmm4, %xmm4
2003 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
2004 ; AVX512BW-NEXT: vpsrlvw %zmm4, %zmm1, %zmm1
2005 ; AVX512BW-NEXT: vpor %ymm1, %ymm3, %ymm1
2006 ; AVX512BW-NEXT: vpmovwb %zmm1, %ymm1
2007 ; AVX512BW-NEXT: vptestnmb %zmm2, %zmm2, %k1
2008 ; AVX512BW-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1}
2009 ; AVX512BW-NEXT: vmovdqa %xmm1, %xmm0
2010 ; AVX512BW-NEXT: vzeroupper
2011 ; AVX512BW-NEXT: retq
2012 ;
2013 ; AVX512VBMI2-LABEL: splatvar_funnnel_v16i8:
2014 ; AVX512VBMI2: # %bb.0:
2015 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
2016 ; AVX512VBMI2-NEXT: vpbroadcastb %xmm2, %xmm2
2017 ; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
2018 ; AVX512VBMI2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
2019 ; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
2020 ; AVX512VBMI2-NEXT: vpsllvw %zmm4, %zmm3, %zmm3
2021 ; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
2022 ; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
2023 ; AVX512VBMI2-NEXT: vpsubb %xmm2, %xmm4, %xmm4
2024 ; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
2025 ; AVX512VBMI2-NEXT: vpsrlvw %zmm4, %zmm1, %zmm1
2026 ; AVX512VBMI2-NEXT: vpor %ymm1, %ymm3, %ymm1
2027 ; AVX512VBMI2-NEXT: vpmovwb %zmm1, %ymm1
2028 ; AVX512VBMI2-NEXT: vptestnmb %zmm2, %zmm2, %k1
2029 ; AVX512VBMI2-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1}
2030 ; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0
2031 ; AVX512VBMI2-NEXT: vzeroupper
2032 ; AVX512VBMI2-NEXT: retq
2033 ;
2034 ; AVX512VLBW-LABEL: splatvar_funnnel_v16i8:
2035 ; AVX512VLBW: # %bb.0:
2036 ; AVX512VLBW-NEXT: vpbroadcastb %xmm2, %xmm2
2037 ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
2038 ; AVX512VLBW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
2039 ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
2040 ; AVX512VLBW-NEXT: vpsllvw %ymm4, %ymm3, %ymm3
2041 ; AVX512VLBW-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
2042 ; AVX512VLBW-NEXT: vpsubb %xmm2, %xmm4, %xmm4
2043 ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
2044 ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
2045 ; AVX512VLBW-NEXT: vpsrlvw %ymm4, %ymm1, %ymm1
2046 ; AVX512VLBW-NEXT: vpor %ymm1, %ymm3, %ymm1
2047 ; AVX512VLBW-NEXT: vpmovwb %ymm1, %xmm1
2048 ; AVX512VLBW-NEXT: vptestnmb %xmm2, %xmm2, %k1
2049 ; AVX512VLBW-NEXT: vmovdqu8 %xmm0, %xmm1 {%k1}
2050 ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0
2051 ; AVX512VLBW-NEXT: vzeroupper
2052 ; AVX512VLBW-NEXT: retq
2053 ;
2054 ; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i8:
2055 ; AVX512VLVBMI2: # %bb.0:
2056 ; AVX512VLVBMI2-NEXT: vpbroadcastb %xmm2, %xmm2
2057 ; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
2058 ; AVX512VLVBMI2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
2059 ; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
2060 ; AVX512VLVBMI2-NEXT: vpsllvw %ymm4, %ymm3, %ymm3
2061 ; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
2062 ; AVX512VLVBMI2-NEXT: vpsubb %xmm2, %xmm4, %xmm4
2063 ; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
2064 ; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
2065 ; AVX512VLVBMI2-NEXT: vpsrlvw %ymm4, %ymm1, %ymm1
2066 ; AVX512VLVBMI2-NEXT: vpor %ymm1, %ymm3, %ymm1
2067 ; AVX512VLVBMI2-NEXT: vpmovwb %ymm1, %xmm1
2068 ; AVX512VLVBMI2-NEXT: vptestnmb %xmm2, %xmm2, %k1
2069 ; AVX512VLVBMI2-NEXT: vmovdqu8 %xmm0, %xmm1 {%k1}
2070 ; AVX512VLVBMI2-NEXT: vmovdqa %xmm1, %xmm0
2071 ; AVX512VLVBMI2-NEXT: vzeroupper
2072 ; AVX512VLVBMI2-NEXT: retq
2073 ;
2074 ; XOPAVX1-LABEL: splatvar_funnnel_v16i8:
2075 ; XOPAVX1: # %bb.0:
2076 ; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
2077 ; XOPAVX1-NEXT: vpshufb %xmm3, %xmm2, %xmm2
2078 ; XOPAVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
2079 ; XOPAVX1-NEXT: vpshlb %xmm2, %xmm0, %xmm4
2080 ; XOPAVX1-NEXT: vpsubb {{.*}}(%rip), %xmm2, %xmm5
2081 ; XOPAVX1-NEXT: vpshlb %xmm5, %xmm1, %xmm1
2082 ; XOPAVX1-NEXT: vpor %xmm1, %xmm4, %xmm1
2083 ; XOPAVX1-NEXT: vpcomeqb %xmm3, %xmm2, %xmm2
2084 ; XOPAVX1-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
2085 ; XOPAVX1-NEXT: retq
2086 ;
2087 ; XOPAVX2-LABEL: splatvar_funnnel_v16i8:
2088 ; XOPAVX2: # %bb.0:
2089 ; XOPAVX2-NEXT: vpbroadcastb %xmm2, %xmm2
2090 ; XOPAVX2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
2091 ; XOPAVX2-NEXT: vpshlb %xmm2, %xmm0, %xmm3
2092 ; XOPAVX2-NEXT: vpsubb {{.*}}(%rip), %xmm2, %xmm4
2093 ; XOPAVX2-NEXT: vpshlb %xmm4, %xmm1, %xmm1
2094 ; XOPAVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
2095 ; XOPAVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
2096 ; XOPAVX2-NEXT: vpcomeqb %xmm3, %xmm2, %xmm2
2097 ; XOPAVX2-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
2098 ; XOPAVX2-NEXT: retq
2099 ;
2100 ; X32-SSE-LABEL: splatvar_funnnel_v16i8:
2101 ; X32-SSE: # %bb.0:
2102 ; X32-SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2103 ; X32-SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
2104 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
2105 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm2
2106 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
2107 ; X32-SSE-NEXT: psubb %xmm2, %xmm3
2108 ; X32-SSE-NEXT: pxor %xmm4, %xmm4
2109 ; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm4
2110 ; X32-SSE-NEXT: pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0]
2111 ; X32-SSE-NEXT: psrldq {{.*#+}} xmm2 = xmm2[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
2112 ; X32-SSE-NEXT: movdqa %xmm0, %xmm5
2113 ; X32-SSE-NEXT: psllw %xmm2, %xmm5
2114 ; X32-SSE-NEXT: pcmpeqd %xmm6, %xmm6
2115 ; X32-SSE-NEXT: psllw %xmm2, %xmm6
2116 ; X32-SSE-NEXT: pcmpeqd %xmm2, %xmm2
2117 ; X32-SSE-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2118 ; X32-SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,0,2,3,4,5,6,7]
2119 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,0,0,0]
2120 ; X32-SSE-NEXT: pand %xmm5, %xmm6
2121 ; X32-SSE-NEXT: pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0]
2122 ; X32-SSE-NEXT: psrldq {{.*#+}} xmm3 = xmm3[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
2123 ; X32-SSE-NEXT: psrlw %xmm3, %xmm1
2124 ; X32-SSE-NEXT: psrlw %xmm3, %xmm2
2125 ; X32-SSE-NEXT: psrlw $8, %xmm2
2126 ; X32-SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2127 ; X32-SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
2128 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
2129 ; X32-SSE-NEXT: pand %xmm1, %xmm2
2130 ; X32-SSE-NEXT: por %xmm6, %xmm2
2131 ; X32-SSE-NEXT: pand %xmm4, %xmm0
2132 ; X32-SSE-NEXT: pandn %xmm2, %xmm4
2133 ; X32-SSE-NEXT: por %xmm4, %xmm0
2134 ; X32-SSE-NEXT: retl
2135 %splat = shufflevector <16 x i8> %amt, <16 x i8> undef, <16 x i32> zeroinitializer
2136 %res = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %splat)
2137 ret <16 x i8> %res
2138 }
2140 ;
2141 ; Constant Shifts
2142 ;
2144 define <2 x i64> @constant_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
2145 ; SSE2-LABEL: constant_funnnel_v2i64:
2146 ; SSE2: # %bb.0:
2147 ; SSE2-NEXT: movdqa %xmm1, %xmm2
2148 ; SSE2-NEXT: psrlq $60, %xmm2
2149 ; SSE2-NEXT: psrlq $50, %xmm1
2150 ; SSE2-NEXT: movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
2151 ; SSE2-NEXT: movdqa %xmm0, %xmm2
2152 ; SSE2-NEXT: psllq $4, %xmm2
2153 ; SSE2-NEXT: psllq $14, %xmm0
2154 ; SSE2-NEXT: movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
2155 ; SSE2-NEXT: orpd %xmm1, %xmm0
2156 ; SSE2-NEXT: retq
2157 ;
2158 ; SSE41-LABEL: constant_funnnel_v2i64:
2159 ; SSE41: # %bb.0:
2160 ; SSE41-NEXT: movdqa %xmm1, %xmm2
2161 ; SSE41-NEXT: psrlq $50, %xmm2
2162 ; SSE41-NEXT: psrlq $60, %xmm1
2163 ; SSE41-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
2164 ; SSE41-NEXT: movdqa %xmm0, %xmm2
2165 ; SSE41-NEXT: psllq $14, %xmm2
2166 ; SSE41-NEXT: psllq $4, %xmm0
2167 ; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
2168 ; SSE41-NEXT: por %xmm1, %xmm0
2169 ; SSE41-NEXT: retq
2170 ;
2171 ; AVX1-LABEL: constant_funnnel_v2i64:
2172 ; AVX1: # %bb.0:
2173 ; AVX1-NEXT: vpsrlq $50, %xmm1, %xmm2
2174 ; AVX1-NEXT: vpsrlq $60, %xmm1, %xmm1
2175 ; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
2176 ; AVX1-NEXT: vpsllq $14, %xmm0, %xmm2
2177 ; AVX1-NEXT: vpsllq $4, %xmm0, %xmm0
2178 ; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
2179 ; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0
2180 ; AVX1-NEXT: retq
2181 ;
2182 ; AVX2-LABEL: constant_funnnel_v2i64:
2183 ; AVX2: # %bb.0:
2184 ; AVX2-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
2185 ; AVX2-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0
2186 ; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0
2187 ; AVX2-NEXT: retq
2188 ;
2189 ; AVX512F-LABEL: constant_funnnel_v2i64:
2190 ; AVX512F: # %bb.0:
2191 ; AVX512F-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
2192 ; AVX512F-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0
2193 ; AVX512F-NEXT: vpor %xmm1, %xmm0, %xmm0
2194 ; AVX512F-NEXT: retq
2195 ;
2196 ; AVX512VL-LABEL: constant_funnnel_v2i64:
2197 ; AVX512VL: # %bb.0:
2198 ; AVX512VL-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
2199 ; AVX512VL-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0
2200 ; AVX512VL-NEXT: vpor %xmm1, %xmm0, %xmm0
2201 ; AVX512VL-NEXT: retq
2202 ;
2203 ; AVX512BW-LABEL: constant_funnnel_v2i64:
2204 ; AVX512BW: # %bb.0:
2205 ; AVX512BW-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
2206 ; AVX512BW-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0
2207 ; AVX512BW-NEXT: vpor %xmm1, %xmm0, %xmm0
2208 ; AVX512BW-NEXT: retq
2209 ;
2210 ; AVX512VBMI2-LABEL: constant_funnnel_v2i64:
2211 ; AVX512VBMI2: # %bb.0:
2212 ; AVX512VBMI2-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
2213 ; AVX512VBMI2-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0
2214 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm0, %xmm0
2215 ; AVX512VBMI2-NEXT: retq
2216 ;
2217 ; AVX512VLBW-LABEL: constant_funnnel_v2i64:
2218 ; AVX512VLBW: # %bb.0:
2219 ; AVX512VLBW-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
2220 ; AVX512VLBW-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0
2221 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm0, %xmm0
2222 ; AVX512VLBW-NEXT: retq
2223 ;
2224 ; AVX512VLVBMI2-LABEL: constant_funnnel_v2i64:
2225 ; AVX512VLVBMI2: # %bb.0:
2226 ; AVX512VLVBMI2-NEXT: vpshldvq {{.*}}(%rip), %xmm1, %xmm0
2227 ; AVX512VLVBMI2-NEXT: retq
2228 ;
2229 ; XOPAVX1-LABEL: constant_funnnel_v2i64:
2230 ; XOPAVX1: # %bb.0:
2231 ; XOPAVX1-NEXT: vpshlq {{.*}}(%rip), %xmm1, %xmm1
2232 ; XOPAVX1-NEXT: vpshlq {{.*}}(%rip), %xmm0, %xmm0
2233 ; XOPAVX1-NEXT: vpor %xmm1, %xmm0, %xmm0
2234 ; XOPAVX1-NEXT: retq
2235 ;
2236 ; XOPAVX2-LABEL: constant_funnnel_v2i64:
2237 ; XOPAVX2: # %bb.0:
2238 ; XOPAVX2-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
2239 ; XOPAVX2-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0
2240 ; XOPAVX2-NEXT: vpor %xmm1, %xmm0, %xmm0
2241 ; XOPAVX2-NEXT: retq
2242 ;
2243 ; X32-SSE-LABEL: constant_funnnel_v2i64:
2244 ; X32-SSE: # %bb.0:
2245 ; X32-SSE-NEXT: movdqa %xmm1, %xmm2
2246 ; X32-SSE-NEXT: psrlq $60, %xmm2
2247 ; X32-SSE-NEXT: psrlq $50, %xmm1
2248 ; X32-SSE-NEXT: movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
2249 ; X32-SSE-NEXT: movdqa %xmm0, %xmm2
2250 ; X32-SSE-NEXT: psllq $4, %xmm2
2251 ; X32-SSE-NEXT: psllq $14, %xmm0
2252 ; X32-SSE-NEXT: movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
2253 ; X32-SSE-NEXT: orpd %xmm1, %xmm0
2254 ; X32-SSE-NEXT: retl
2255 %res = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> <i64 4, i64 14>)
2256 ret <2 x i64> %res
2257 }
2259 define <4 x i32> @constant_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
2260 ; SSE2-LABEL: constant_funnnel_v4i32:
2261 ; SSE2: # %bb.0:
2262 ; SSE2-NEXT: movdqa %xmm1, %xmm2
2263 ; SSE2-NEXT: psrld $25, %xmm2
2264 ; SSE2-NEXT: movdqa %xmm1, %xmm3
2265 ; SSE2-NEXT: psrld $26, %xmm3
2266 ; SSE2-NEXT: punpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm2[1]
2267 ; SSE2-NEXT: movdqa %xmm1, %xmm2
2268 ; SSE2-NEXT: psrld $27, %xmm2
2269 ; SSE2-NEXT: psrld $28, %xmm1
2270 ; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
2271 ; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,3],xmm3[0,3]
2272 ; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [16,32,64,128]
2273 ; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
2274 ; SSE2-NEXT: pmuludq %xmm2, %xmm0
2275 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2276 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
2277 ; SSE2-NEXT: pmuludq %xmm3, %xmm2
2278 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
2279 ; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
2280 ; SSE2-NEXT: por %xmm1, %xmm0
2281 ; SSE2-NEXT: retq
2282 ;
2283 ; SSE41-LABEL: constant_funnnel_v4i32:
2284 ; SSE41: # %bb.0:
2285 ; SSE41-NEXT: movdqa %xmm1, %xmm2
2286 ; SSE41-NEXT: psrld $25, %xmm2
2287 ; SSE41-NEXT: movdqa %xmm1, %xmm3
2288 ; SSE41-NEXT: psrld $27, %xmm3
2289 ; SSE41-NEXT: pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm2[4,5,6,7]
2290 ; SSE41-NEXT: movdqa %xmm1, %xmm2
2291 ; SSE41-NEXT: psrld $26, %xmm2
2292 ; SSE41-NEXT: psrld $28, %xmm1
2293 ; SSE41-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
2294 ; SSE41-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
2295 ; SSE41-NEXT: pmulld {{.*}}(%rip), %xmm0
2296 ; SSE41-NEXT: por %xmm1, %xmm0
2297 ; SSE41-NEXT: retq
2298 ;
2299 ; AVX1-LABEL: constant_funnnel_v4i32:
2300 ; AVX1: # %bb.0:
2301 ; AVX1-NEXT: vpsrld $25, %xmm1, %xmm2
2302 ; AVX1-NEXT: vpsrld $27, %xmm1, %xmm3
2303 ; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
2304 ; AVX1-NEXT: vpsrld $26, %xmm1, %xmm3
2305 ; AVX1-NEXT: vpsrld $28, %xmm1, %xmm1
2306 ; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5,6,7]
2307 ; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
2308 ; AVX1-NEXT: vpmulld {{.*}}(%rip), %xmm0, %xmm0
2309 ; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0
2310 ; AVX1-NEXT: retq
2311 ;
2312 ; AVX2-LABEL: constant_funnnel_v4i32:
2313 ; AVX2: # %bb.0:
2314 ; AVX2-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
2315 ; AVX2-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0
2316 ; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0
2317 ; AVX2-NEXT: retq
2318 ;
2319 ; AVX512F-LABEL: constant_funnnel_v4i32:
2320 ; AVX512F: # %bb.0:
2321 ; AVX512F-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
2322 ; AVX512F-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0
2323 ; AVX512F-NEXT: vpor %xmm1, %xmm0, %xmm0
2324 ; AVX512F-NEXT: retq
2325 ;
2326 ; AVX512VL-LABEL: constant_funnnel_v4i32:
2327 ; AVX512VL: # %bb.0:
2328 ; AVX512VL-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
2329 ; AVX512VL-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0
2330 ; AVX512VL-NEXT: vpor %xmm1, %xmm0, %xmm0
2331 ; AVX512VL-NEXT: retq
2332 ;
2333 ; AVX512BW-LABEL: constant_funnnel_v4i32:
2334 ; AVX512BW: # %bb.0:
2335 ; AVX512BW-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
2336 ; AVX512BW-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0
2337 ; AVX512BW-NEXT: vpor %xmm1, %xmm0, %xmm0
2338 ; AVX512BW-NEXT: retq
2339 ;
2340 ; AVX512VBMI2-LABEL: constant_funnnel_v4i32:
2341 ; AVX512VBMI2: # %bb.0:
2342 ; AVX512VBMI2-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
2343 ; AVX512VBMI2-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0
2344 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm0, %xmm0
2345 ; AVX512VBMI2-NEXT: retq
2346 ;
2347 ; AVX512VLBW-LABEL: constant_funnnel_v4i32:
2348 ; AVX512VLBW: # %bb.0:
2349 ; AVX512VLBW-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
2350 ; AVX512VLBW-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0
2351 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm0, %xmm0
2352 ; AVX512VLBW-NEXT: retq
2353 ;
2354 ; AVX512VLVBMI2-LABEL: constant_funnnel_v4i32:
2355 ; AVX512VLVBMI2: # %bb.0:
2356 ; AVX512VLVBMI2-NEXT: vpshldvd {{.*}}(%rip), %xmm1, %xmm0
2357 ; AVX512VLVBMI2-NEXT: retq
2358 ;
2359 ; XOPAVX1-LABEL: constant_funnnel_v4i32:
2360 ; XOPAVX1: # %bb.0:
2361 ; XOPAVX1-NEXT: vpshld {{.*}}(%rip), %xmm1, %xmm1
2362 ; XOPAVX1-NEXT: vpshld {{.*}}(%rip), %xmm0, %xmm0
2363 ; XOPAVX1-NEXT: vpor %xmm1, %xmm0, %xmm0
2364 ; XOPAVX1-NEXT: retq
2365 ;
2366 ; XOPAVX2-LABEL: constant_funnnel_v4i32:
2367 ; XOPAVX2: # %bb.0:
2368 ; XOPAVX2-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
2369 ; XOPAVX2-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0
2370 ; XOPAVX2-NEXT: vpor %xmm1, %xmm0, %xmm0
2371 ; XOPAVX2-NEXT: retq
2372 ;
2373 ; X32-SSE-LABEL: constant_funnnel_v4i32:
2374 ; X32-SSE: # %bb.0:
2375 ; X32-SSE-NEXT: movdqa %xmm1, %xmm2
2376 ; X32-SSE-NEXT: psrld $25, %xmm2
2377 ; X32-SSE-NEXT: movdqa %xmm1, %xmm3
2378 ; X32-SSE-NEXT: psrld $26, %xmm3
2379 ; X32-SSE-NEXT: punpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm2[1]
2380 ; X32-SSE-NEXT: movdqa %xmm1, %xmm2
2381 ; X32-SSE-NEXT: psrld $27, %xmm2
2382 ; X32-SSE-NEXT: psrld $28, %xmm1
2383 ; X32-SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
2384 ; X32-SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,3],xmm3[0,3]
2385 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm2 = [16,32,64,128]
2386 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
2387 ; X32-SSE-NEXT: pmuludq %xmm2, %xmm0
2388 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2389 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
2390 ; X32-SSE-NEXT: pmuludq %xmm3, %xmm2
2391 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
2392 ; X32-SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
2393 ; X32-SSE-NEXT: por %xmm1, %xmm0
2394 ; X32-SSE-NEXT: retl
2395 %res = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 4, i32 5, i32 6, i32 7>)
2396 ret <4 x i32> %res
2397 }
2399 define <8 x i16> @constant_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
2400 ; SSE2-LABEL: constant_funnnel_v8i16:
2401 ; SSE2: # %bb.0:
2402 ; SSE2-NEXT: movdqa {{.*#+}} xmm2 = <u,2,4,8,16,32,64,128>
2403 ; SSE2-NEXT: pmulhuw %xmm2, %xmm1
2404 ; SSE2-NEXT: pmullw %xmm0, %xmm2
2405 ; SSE2-NEXT: por %xmm1, %xmm2
2406 ; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [0,65535,65535,65535,65535,65535,65535,65535]
2407 ; SSE2-NEXT: pand %xmm1, %xmm2
2408 ; SSE2-NEXT: pandn %xmm0, %xmm1
2409 ; SSE2-NEXT: por %xmm2, %xmm1
2410 ; SSE2-NEXT: movdqa %xmm1, %xmm0
2411 ; SSE2-NEXT: retq
2412 ;
2413 ; SSE41-LABEL: constant_funnnel_v8i16:
2414 ; SSE41: # %bb.0:
2415 ; SSE41-NEXT: movdqa {{.*#+}} xmm2 = <u,2,4,8,16,32,64,128>
2416 ; SSE41-NEXT: pmulhuw %xmm2, %xmm1
2417 ; SSE41-NEXT: pmullw %xmm0, %xmm2
2418 ; SSE41-NEXT: por %xmm1, %xmm2
2419 ; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7]
2420 ; SSE41-NEXT: retq
2421 ;
2422 ; AVX-LABEL: constant_funnnel_v8i16:
2423 ; AVX: # %bb.0:
2424 ; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = <u,2,4,8,16,32,64,128>
2425 ; AVX-NEXT: vpmulhuw %xmm2, %xmm1, %xmm1
2426 ; AVX-NEXT: vpmullw %xmm2, %xmm0, %xmm2
2427 ; AVX-NEXT: vpor %xmm1, %xmm2, %xmm1
2428 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
2429 ; AVX-NEXT: retq
2430 ;
2431 ; AVX512F-LABEL: constant_funnnel_v8i16:
2432 ; AVX512F: # %bb.0:
2433 ; AVX512F-NEXT: vmovdqa {{.*#+}} xmm2 = <u,2,4,8,16,32,64,128>
2434 ; AVX512F-NEXT: vpmulhuw %xmm2, %xmm1, %xmm1
2435 ; AVX512F-NEXT: vpmullw %xmm2, %xmm0, %xmm2
2436 ; AVX512F-NEXT: vpor %xmm1, %xmm2, %xmm1
2437 ; AVX512F-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
2438 ; AVX512F-NEXT: retq
2439 ;
2440 ; AVX512VL-LABEL: constant_funnnel_v8i16:
2441 ; AVX512VL: # %bb.0:
2442 ; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm2 = <u,2,4,8,16,32,64,128>
2443 ; AVX512VL-NEXT: vpmulhuw %xmm2, %xmm1, %xmm1
2444 ; AVX512VL-NEXT: vpmullw %xmm2, %xmm0, %xmm2
2445 ; AVX512VL-NEXT: vpor %xmm1, %xmm2, %xmm1
2446 ; AVX512VL-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
2447 ; AVX512VL-NEXT: retq
2448 ;
2449 ; AVX512BW-LABEL: constant_funnnel_v8i16:
2450 ; AVX512BW: # %bb.0:
2451 ; AVX512BW-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
2452 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
2453 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm2 = <u,15,14,13,12,11,10,9>
2454 ; AVX512BW-NEXT: vpsrlvw %zmm2, %zmm1, %zmm1
2455 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm2 = <u,1,2,3,4,5,6,7>
2456 ; AVX512BW-NEXT: vpsllvw %zmm2, %zmm0, %zmm2
2457 ; AVX512BW-NEXT: vpor %xmm1, %xmm2, %xmm1
2458 ; AVX512BW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
2459 ; AVX512BW-NEXT: vzeroupper
2460 ; AVX512BW-NEXT: retq
2461 ;
2462 ; AVX512VBMI2-LABEL: constant_funnnel_v8i16:
2463 ; AVX512VBMI2: # %bb.0:
2464 ; AVX512VBMI2-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
2465 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
2466 ; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm2 = <u,15,14,13,12,11,10,9>
2467 ; AVX512VBMI2-NEXT: vpsrlvw %zmm2, %zmm1, %zmm1
2468 ; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm2 = <u,1,2,3,4,5,6,7>
2469 ; AVX512VBMI2-NEXT: vpsllvw %zmm2, %zmm0, %zmm2
2470 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm2, %xmm1
2471 ; AVX512VBMI2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
2472 ; AVX512VBMI2-NEXT: vzeroupper
2473 ; AVX512VBMI2-NEXT: retq
2474 ;
2475 ; AVX512VLBW-LABEL: constant_funnnel_v8i16:
2476 ; AVX512VLBW: # %bb.0:
2477 ; AVX512VLBW-NEXT: vpsrlvw {{.*}}(%rip), %xmm1, %xmm1
2478 ; AVX512VLBW-NEXT: vpsllvw {{.*}}(%rip), %xmm0, %xmm2
2479 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm2, %xmm1
2480 ; AVX512VLBW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
2481 ; AVX512VLBW-NEXT: retq
2482 ;
2483 ; AVX512VLVBMI2-LABEL: constant_funnnel_v8i16:
2484 ; AVX512VLVBMI2: # %bb.0:
2485 ; AVX512VLVBMI2-NEXT: vpshldvw {{.*}}(%rip), %xmm1, %xmm0
2486 ; AVX512VLVBMI2-NEXT: retq
2487 ;
2488 ; XOP-LABEL: constant_funnnel_v8i16:
2489 ; XOP: # %bb.0:
2490 ; XOP-NEXT: vpshlw {{.*}}(%rip), %xmm1, %xmm1
2491 ; XOP-NEXT: vpshlw {{.*}}(%rip), %xmm0, %xmm2
2492 ; XOP-NEXT: vpor %xmm1, %xmm2, %xmm1
2493 ; XOP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
2494 ; XOP-NEXT: retq
2495 ;
2496 ; X32-SSE-LABEL: constant_funnnel_v8i16:
2497 ; X32-SSE: # %bb.0:
2498 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm2 = <u,2,4,8,16,32,64,128>
2499 ; X32-SSE-NEXT: pmulhuw %xmm2, %xmm1
2500 ; X32-SSE-NEXT: pmullw %xmm0, %xmm2
2501 ; X32-SSE-NEXT: por %xmm1, %xmm2
2502 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm1 = [0,65535,65535,65535,65535,65535,65535,65535]
2503 ; X32-SSE-NEXT: pand %xmm1, %xmm2
2504 ; X32-SSE-NEXT: pandn %xmm0, %xmm1
2505 ; X32-SSE-NEXT: por %xmm2, %xmm1
2506 ; X32-SSE-NEXT: movdqa %xmm1, %xmm0
2507 ; X32-SSE-NEXT: retl
2508 %res = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
2509 ret <8 x i16> %res
2510 }
2512 define <16 x i8> @constant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
2513 ; SSE2-LABEL: constant_funnnel_v16i8:
2514 ; SSE2: # %bb.0:
2515 ; SSE2-NEXT: pxor %xmm2, %xmm2
2516 ; SSE2-NEXT: movdqa %xmm1, %xmm3
2517 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
2518 ; SSE2-NEXT: movdqa {{.*#+}} xmm4 = <u,128,64,32,16,8,4,2>
2519 ; SSE2-NEXT: pmullw %xmm4, %xmm3
2520 ; SSE2-NEXT: psrlw $8, %xmm3
2521 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
2522 ; SSE2-NEXT: movdqa {{.*#+}} xmm5 = <u,2,4,8,16,32,64,128>
2523 ; SSE2-NEXT: pmullw %xmm5, %xmm1
2524 ; SSE2-NEXT: psrlw $8, %xmm1
2525 ; SSE2-NEXT: packuswb %xmm3, %xmm1
2526 ; SSE2-NEXT: movdqa %xmm0, %xmm3
2527 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
2528 ; SSE2-NEXT: pmullw %xmm4, %xmm3
2529 ; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
2530 ; SSE2-NEXT: pand %xmm4, %xmm3
2531 ; SSE2-NEXT: movdqa %xmm0, %xmm2
2532 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
2533 ; SSE2-NEXT: pmullw %xmm5, %xmm2
2534 ; SSE2-NEXT: pand %xmm4, %xmm2
2535 ; SSE2-NEXT: packuswb %xmm3, %xmm2
2536 ; SSE2-NEXT: por %xmm1, %xmm2
2537 ; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
2538 ; SSE2-NEXT: pand %xmm1, %xmm2
2539 ; SSE2-NEXT: pandn %xmm0, %xmm1
2540 ; SSE2-NEXT: por %xmm1, %xmm2
2541 ; SSE2-NEXT: movdqa %xmm2, %xmm0
2542 ; SSE2-NEXT: retq
2543 ;
2544 ; SSE41-LABEL: constant_funnnel_v16i8:
2545 ; SSE41: # %bb.0:
2546 ; SSE41-NEXT: movdqa %xmm0, %xmm2
2547 ; SSE41-NEXT: pxor %xmm0, %xmm0
2548 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2549 ; SSE41-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2550 ; SSE41-NEXT: movdqa {{.*#+}} xmm0 = <u,128,64,32,16,8,4,2>
2551 ; SSE41-NEXT: pmullw %xmm0, %xmm1
2552 ; SSE41-NEXT: psrlw $8, %xmm1
2553 ; SSE41-NEXT: movdqa {{.*#+}} xmm4 = <u,2,4,8,16,32,64,128>
2554 ; SSE41-NEXT: pmullw %xmm4, %xmm3
2555 ; SSE41-NEXT: psrlw $8, %xmm3
2556 ; SSE41-NEXT: packuswb %xmm1, %xmm3
2557 ; SSE41-NEXT: movdqa %xmm2, %xmm1
2558 ; SSE41-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2559 ; SSE41-NEXT: pmullw %xmm0, %xmm1
2560 ; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
2561 ; SSE41-NEXT: pand %xmm0, %xmm1
2562 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm5 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2563 ; SSE41-NEXT: pmullw %xmm4, %xmm5
2564 ; SSE41-NEXT: pand %xmm0, %xmm5
2565 ; SSE41-NEXT: packuswb %xmm1, %xmm5
2566 ; SSE41-NEXT: por %xmm3, %xmm5
2567 ; SSE41-NEXT: movaps {{.*#+}} xmm0 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
2568 ; SSE41-NEXT: pblendvb %xmm0, %xmm5, %xmm2
2569 ; SSE41-NEXT: movdqa %xmm2, %xmm0
2570 ; SSE41-NEXT: retq
2571 ;
2572 ; AVX1-LABEL: constant_funnnel_v16i8:
2573 ; AVX1: # %bb.0:
2574 ; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
2575 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
2576 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = <u,128,64,32,16,8,4,2>
2577 ; AVX1-NEXT: vpmullw %xmm3, %xmm2, %xmm2
2578 ; AVX1-NEXT: vpsrlw $8, %xmm2, %xmm2
2579 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2580 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = <u,2,4,8,16,32,64,128>
2581 ; AVX1-NEXT: vpmullw %xmm4, %xmm1, %xmm1
2582 ; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm1
2583 ; AVX1-NEXT: vpackuswb %xmm2, %xmm1, %xmm1
2584 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2585 ; AVX1-NEXT: vpmullw %xmm3, %xmm2, %xmm2
2586 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
2587 ; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2
2588 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2589 ; AVX1-NEXT: vpmullw %xmm4, %xmm5, %xmm4
2590 ; AVX1-NEXT: vpand %xmm3, %xmm4, %xmm3
2591 ; AVX1-NEXT: vpackuswb %xmm2, %xmm3, %xmm2
2592 ; AVX1-NEXT: vpor %xmm1, %xmm2, %xmm1
2593 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
2594 ; AVX1-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
2595 ; AVX1-NEXT: retq
2596 ;
2597 ; AVX2-LABEL: constant_funnnel_v16i8:
2598 ; AVX2: # %bb.0:
2599 ; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
2600 ; AVX2-NEXT: vpmullw {{.*}}(%rip), %ymm1, %ymm1
2601 ; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
2602 ; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
2603 ; AVX2-NEXT: vpackuswb %xmm2, %xmm1, %xmm1
2604 ; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
2605 ; AVX2-NEXT: vpmullw {{.*}}(%rip), %ymm2, %ymm2
2606 ; AVX2-NEXT: vpand {{.*}}(%rip), %ymm2, %ymm2
2607 ; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3
2608 ; AVX2-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
2609 ; AVX2-NEXT: vpor %xmm1, %xmm2, %xmm1
2610 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
2611 ; AVX2-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
2612 ; AVX2-NEXT: vzeroupper
2613 ; AVX2-NEXT: retq
2614 ;
2615 ; AVX512F-LABEL: constant_funnnel_v16i8:
2616 ; AVX512F: # %bb.0:
2617 ; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
2618 ; AVX512F-NEXT: vpsrlvd {{.*}}(%rip), %zmm1, %zmm1
2619 ; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
2620 ; AVX512F-NEXT: vpsllvd {{.*}}(%rip), %zmm2, %zmm2
2621 ; AVX512F-NEXT: vpord %zmm1, %zmm2, %zmm1
2622 ; AVX512F-NEXT: vpmovdb %zmm1, %xmm1
2623 ; AVX512F-NEXT: vmovdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
2624 ; AVX512F-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
2625 ; AVX512F-NEXT: vzeroupper
2626 ; AVX512F-NEXT: retq
2627 ;
2628 ; AVX512VL-LABEL: constant_funnnel_v16i8:
2629 ; AVX512VL: # %bb.0:
2630 ; AVX512VL-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
2631 ; AVX512VL-NEXT: vpsrlvd {{.*}}(%rip), %zmm1, %zmm1
2632 ; AVX512VL-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
2633 ; AVX512VL-NEXT: vpsllvd {{.*}}(%rip), %zmm2, %zmm2
2634 ; AVX512VL-NEXT: vpord %zmm1, %zmm2, %zmm1
2635 ; AVX512VL-NEXT: vpmovdb %zmm1, %xmm1
2636 ; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
2637 ; AVX512VL-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
2638 ; AVX512VL-NEXT: vzeroupper
2639 ; AVX512VL-NEXT: retq
2640 ;
2641 ; AVX512BW-LABEL: constant_funnnel_v16i8:
2642 ; AVX512BW: # %bb.0:
2643 ; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm2 = <u,7,6,5,4,3,2,1,u,1,2,3,4,5,6,7>
2644 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
2645 ; AVX512BW-NEXT: vpsrlvw %zmm2, %zmm1, %zmm1
2646 ; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm2 = <u,1,2,3,4,5,6,7,u,7,6,5,4,3,2,1>
2647 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
2648 ; AVX512BW-NEXT: vpsllvw %zmm2, %zmm3, %zmm2
2649 ; AVX512BW-NEXT: vpor %ymm1, %ymm2, %ymm1
2650 ; AVX512BW-NEXT: vpmovwb %zmm1, %ymm1
2651 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
2652 ; AVX512BW-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
2653 ; AVX512BW-NEXT: vzeroupper
2654 ; AVX512BW-NEXT: retq
2655 ;
2656 ; AVX512VBMI2-LABEL: constant_funnnel_v16i8:
2657 ; AVX512VBMI2: # %bb.0:
2658 ; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm2 = <u,7,6,5,4,3,2,1,u,1,2,3,4,5,6,7>
2659 ; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
2660 ; AVX512VBMI2-NEXT: vpsrlvw %zmm2, %zmm1, %zmm1
2661 ; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm2 = <u,1,2,3,4,5,6,7,u,7,6,5,4,3,2,1>
2662 ; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
2663 ; AVX512VBMI2-NEXT: vpsllvw %zmm2, %zmm3, %zmm2
2664 ; AVX512VBMI2-NEXT: vpor %ymm1, %ymm2, %ymm1
2665 ; AVX512VBMI2-NEXT: vpmovwb %zmm1, %ymm1
2666 ; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
2667 ; AVX512VBMI2-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
2668 ; AVX512VBMI2-NEXT: vzeroupper
2669 ; AVX512VBMI2-NEXT: retq
2670 ;
2671 ; AVX512VLBW-LABEL: constant_funnnel_v16i8:
2672 ; AVX512VLBW: # %bb.0:
2673 ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
2674 ; AVX512VLBW-NEXT: vpsrlvw {{.*}}(%rip), %ymm1, %ymm1
2675 ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
2676 ; AVX512VLBW-NEXT: vpsllvw {{.*}}(%rip), %ymm2, %ymm2
2677 ; AVX512VLBW-NEXT: vpor %ymm1, %ymm2, %ymm1
2678 ; AVX512VLBW-NEXT: vpmovwb %ymm1, %xmm1
2679 ; AVX512VLBW-NEXT: movw $257, %ax # imm = 0x101
2680 ; AVX512VLBW-NEXT: kmovd %eax, %k1
2681 ; AVX512VLBW-NEXT: vmovdqu8 %xmm0, %xmm1 {%k1}
2682 ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0
2683 ; AVX512VLBW-NEXT: vzeroupper
2684 ; AVX512VLBW-NEXT: retq
2685 ;
2686 ; AVX512VLVBMI2-LABEL: constant_funnnel_v16i8:
2687 ; AVX512VLVBMI2: # %bb.0:
2688 ; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
2689 ; AVX512VLVBMI2-NEXT: vpsrlvw {{.*}}(%rip), %ymm1, %ymm1
2690 ; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
2691 ; AVX512VLVBMI2-NEXT: vpsllvw {{.*}}(%rip), %ymm2, %ymm2
2692 ; AVX512VLVBMI2-NEXT: vpor %ymm1, %ymm2, %ymm1
2693 ; AVX512VLVBMI2-NEXT: vpmovwb %ymm1, %xmm1
2694 ; AVX512VLVBMI2-NEXT: movw $257, %ax # imm = 0x101
2695 ; AVX512VLVBMI2-NEXT: kmovd %eax, %k1
2696 ; AVX512VLVBMI2-NEXT: vmovdqu8 %xmm0, %xmm1 {%k1}
2697 ; AVX512VLVBMI2-NEXT: vmovdqa %xmm1, %xmm0
2698 ; AVX512VLVBMI2-NEXT: vzeroupper
2699 ; AVX512VLVBMI2-NEXT: retq
2700 ;
2701 ; XOP-LABEL: constant_funnnel_v16i8:
2702 ; XOP: # %bb.0:
2703 ; XOP-NEXT: vpshlb {{.*}}(%rip), %xmm1, %xmm1
2704 ; XOP-NEXT: vpshlb {{.*}}(%rip), %xmm0, %xmm2
2705 ; XOP-NEXT: vpor %xmm1, %xmm2, %xmm1
2706 ; XOP-NEXT: vmovdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
2707 ; XOP-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
2708 ; XOP-NEXT: retq
2709 ;
2710 ; X32-SSE-LABEL: constant_funnnel_v16i8:
2711 ; X32-SSE: # %bb.0:
2712 ; X32-SSE-NEXT: pxor %xmm2, %xmm2
2713 ; X32-SSE-NEXT: movdqa %xmm1, %xmm3
2714 ; X32-SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
2715 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm4 = <u,128,64,32,16,8,4,2>
2716 ; X32-SSE-NEXT: pmullw %xmm4, %xmm3
2717 ; X32-SSE-NEXT: psrlw $8, %xmm3
2718 ; X32-SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
2719 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm5 = <u,2,4,8,16,32,64,128>
2720 ; X32-SSE-NEXT: pmullw %xmm5, %xmm1
2721 ; X32-SSE-NEXT: psrlw $8, %xmm1
2722 ; X32-SSE-NEXT: packuswb %xmm3, %xmm1
2723 ; X32-SSE-NEXT: movdqa %xmm0, %xmm3
2724 ; X32-SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
2725 ; X32-SSE-NEXT: pmullw %xmm4, %xmm3
2726 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
2727 ; X32-SSE-NEXT: pand %xmm4, %xmm3
2728 ; X32-SSE-NEXT: movdqa %xmm0, %xmm2
2729 ; X32-SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
2730 ; X32-SSE-NEXT: pmullw %xmm5, %xmm2
2731 ; X32-SSE-NEXT: pand %xmm4, %xmm2
2732 ; X32-SSE-NEXT: packuswb %xmm3, %xmm2
2733 ; X32-SSE-NEXT: por %xmm1, %xmm2
2734 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm1 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
2735 ; X32-SSE-NEXT: pand %xmm1, %xmm2
2736 ; X32-SSE-NEXT: pandn %xmm0, %xmm1
2737 ; X32-SSE-NEXT: por %xmm1, %xmm2
2738 ; X32-SSE-NEXT: movdqa %xmm2, %xmm0
2739 ; X32-SSE-NEXT: retl
2740 %res = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1>)
2741 ret <16 x i8> %res
2742 }
2744 ;
2745 ; Uniform Constant Shifts
2746 ;
2748 define <2 x i64> @splatconstant_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
2749 ; SSE-LABEL: splatconstant_funnnel_v2i64:
2750 ; SSE: # %bb.0:
2751 ; SSE-NEXT: psrlq $50, %xmm1
2752 ; SSE-NEXT: psllq $14, %xmm0
2753 ; SSE-NEXT: por %xmm1, %xmm0
2754 ; SSE-NEXT: retq
2755 ;
2756 ; AVX-LABEL: splatconstant_funnnel_v2i64:
2757 ; AVX: # %bb.0:
2758 ; AVX-NEXT: vpsrlq $50, %xmm1, %xmm1
2759 ; AVX-NEXT: vpsllq $14, %xmm0, %xmm0
2760 ; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0
2761 ; AVX-NEXT: retq
2762 ;
2763 ; AVX512F-LABEL: splatconstant_funnnel_v2i64:
2764 ; AVX512F: # %bb.0:
2765 ; AVX512F-NEXT: vpsrlq $50, %xmm1, %xmm1
2766 ; AVX512F-NEXT: vpsllq $14, %xmm0, %xmm0
2767 ; AVX512F-NEXT: vpor %xmm1, %xmm0, %xmm0
2768 ; AVX512F-NEXT: retq
2769 ;
2770 ; AVX512VL-LABEL: splatconstant_funnnel_v2i64:
2771 ; AVX512VL: # %bb.0:
2772 ; AVX512VL-NEXT: vpsrlq $50, %xmm1, %xmm1
2773 ; AVX512VL-NEXT: vpsllq $14, %xmm0, %xmm0
2774 ; AVX512VL-NEXT: vpor %xmm1, %xmm0, %xmm0
2775 ; AVX512VL-NEXT: retq
2776 ;
2777 ; AVX512BW-LABEL: splatconstant_funnnel_v2i64:
2778 ; AVX512BW: # %bb.0:
2779 ; AVX512BW-NEXT: vpsrlq $50, %xmm1, %xmm1
2780 ; AVX512BW-NEXT: vpsllq $14, %xmm0, %xmm0
2781 ; AVX512BW-NEXT: vpor %xmm1, %xmm0, %xmm0
2782 ; AVX512BW-NEXT: retq
2783 ;
2784 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v2i64:
2785 ; AVX512VBMI2: # %bb.0:
2786 ; AVX512VBMI2-NEXT: vpsrlq $50, %xmm1, %xmm1
2787 ; AVX512VBMI2-NEXT: vpsllq $14, %xmm0, %xmm0
2788 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm0, %xmm0
2789 ; AVX512VBMI2-NEXT: retq
2790 ;
2791 ; AVX512VLBW-LABEL: splatconstant_funnnel_v2i64:
2792 ; AVX512VLBW: # %bb.0:
2793 ; AVX512VLBW-NEXT: vpsrlq $50, %xmm1, %xmm1
2794 ; AVX512VLBW-NEXT: vpsllq $14, %xmm0, %xmm0
2795 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm0, %xmm0
2796 ; AVX512VLBW-NEXT: retq
2797 ;
2798 ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v2i64:
2799 ; AVX512VLVBMI2: # %bb.0:
2800 ; AVX512VLVBMI2-NEXT: vpshldq $14, %xmm1, %xmm0, %xmm0
2801 ; AVX512VLVBMI2-NEXT: retq
2802 ;
2803 ; XOP-LABEL: splatconstant_funnnel_v2i64:
2804 ; XOP: # %bb.0:
2805 ; XOP-NEXT: vpsrlq $50, %xmm1, %xmm1
2806 ; XOP-NEXT: vpsllq $14, %xmm0, %xmm0
2807 ; XOP-NEXT: vpor %xmm1, %xmm0, %xmm0
2808 ; XOP-NEXT: retq
2809 ;
2810 ; X32-SSE-LABEL: splatconstant_funnnel_v2i64:
2811 ; X32-SSE: # %bb.0:
2812 ; X32-SSE-NEXT: psrlq $50, %xmm1
2813 ; X32-SSE-NEXT: psllq $14, %xmm0
2814 ; X32-SSE-NEXT: por %xmm1, %xmm0
2815 ; X32-SSE-NEXT: retl
2816 %res = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> <i64 14, i64 14>)
2817 ret <2 x i64> %res
2818 }
2820 define <4 x i32> @splatconstant_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
2821 ; SSE-LABEL: splatconstant_funnnel_v4i32:
2822 ; SSE: # %bb.0:
2823 ; SSE-NEXT: psrld $28, %xmm1
2824 ; SSE-NEXT: pslld $4, %xmm0
2825 ; SSE-NEXT: por %xmm1, %xmm0
2826 ; SSE-NEXT: retq
2827 ;
2828 ; AVX-LABEL: splatconstant_funnnel_v4i32:
2829 ; AVX: # %bb.0:
2830 ; AVX-NEXT: vpsrld $28, %xmm1, %xmm1
2831 ; AVX-NEXT: vpslld $4, %xmm0, %xmm0
2832 ; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0
2833 ; AVX-NEXT: retq
2834 ;
2835 ; AVX512F-LABEL: splatconstant_funnnel_v4i32:
2836 ; AVX512F: # %bb.0:
2837 ; AVX512F-NEXT: vpsrld $28, %xmm1, %xmm1
2838 ; AVX512F-NEXT: vpslld $4, %xmm0, %xmm0
2839 ; AVX512F-NEXT: vpor %xmm1, %xmm0, %xmm0
2840 ; AVX512F-NEXT: retq
2841 ;
2842 ; AVX512VL-LABEL: splatconstant_funnnel_v4i32:
2843 ; AVX512VL: # %bb.0:
2844 ; AVX512VL-NEXT: vpsrld $28, %xmm1, %xmm1
2845 ; AVX512VL-NEXT: vpslld $4, %xmm0, %xmm0
2846 ; AVX512VL-NEXT: vpor %xmm1, %xmm0, %xmm0
2847 ; AVX512VL-NEXT: retq
2848 ;
2849 ; AVX512BW-LABEL: splatconstant_funnnel_v4i32:
2850 ; AVX512BW: # %bb.0:
2851 ; AVX512BW-NEXT: vpsrld $28, %xmm1, %xmm1
2852 ; AVX512BW-NEXT: vpslld $4, %xmm0, %xmm0
2853 ; AVX512BW-NEXT: vpor %xmm1, %xmm0, %xmm0
2854 ; AVX512BW-NEXT: retq
2855 ;
2856 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v4i32:
2857 ; AVX512VBMI2: # %bb.0:
2858 ; AVX512VBMI2-NEXT: vpsrld $28, %xmm1, %xmm1
2859 ; AVX512VBMI2-NEXT: vpslld $4, %xmm0, %xmm0
2860 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm0, %xmm0
2861 ; AVX512VBMI2-NEXT: retq
2862 ;
2863 ; AVX512VLBW-LABEL: splatconstant_funnnel_v4i32:
2864 ; AVX512VLBW: # %bb.0:
2865 ; AVX512VLBW-NEXT: vpsrld $28, %xmm1, %xmm1
2866 ; AVX512VLBW-NEXT: vpslld $4, %xmm0, %xmm0
2867 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm0, %xmm0
2868 ; AVX512VLBW-NEXT: retq
2869 ;
2870 ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v4i32:
2871 ; AVX512VLVBMI2: # %bb.0:
2872 ; AVX512VLVBMI2-NEXT: vpshldd $4, %xmm1, %xmm0, %xmm0
2873 ; AVX512VLVBMI2-NEXT: retq
2874 ;
2875 ; XOP-LABEL: splatconstant_funnnel_v4i32:
2876 ; XOP: # %bb.0:
2877 ; XOP-NEXT: vpsrld $28, %xmm1, %xmm1
2878 ; XOP-NEXT: vpslld $4, %xmm0, %xmm0
2879 ; XOP-NEXT: vpor %xmm1, %xmm0, %xmm0
2880 ; XOP-NEXT: retq
2881 ;
2882 ; X32-SSE-LABEL: splatconstant_funnnel_v4i32:
2883 ; X32-SSE: # %bb.0:
2884 ; X32-SSE-NEXT: psrld $28, %xmm1
2885 ; X32-SSE-NEXT: pslld $4, %xmm0
2886 ; X32-SSE-NEXT: por %xmm1, %xmm0
2887 ; X32-SSE-NEXT: retl
2888 %res = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 4, i32 4, i32 4, i32 4>)
2889 ret <4 x i32> %res
2890 }
2892 define <8 x i16> @splatconstant_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
2893 ; SSE-LABEL: splatconstant_funnnel_v8i16:
2894 ; SSE: # %bb.0:
2895 ; SSE-NEXT: psrlw $9, %xmm1
2896 ; SSE-NEXT: psllw $7, %xmm0
2897 ; SSE-NEXT: por %xmm1, %xmm0
2898 ; SSE-NEXT: retq
2899 ;
2900 ; AVX-LABEL: splatconstant_funnnel_v8i16:
2901 ; AVX: # %bb.0:
2902 ; AVX-NEXT: vpsrlw $9, %xmm1, %xmm1
2903 ; AVX-NEXT: vpsllw $7, %xmm0, %xmm0
2904 ; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0
2905 ; AVX-NEXT: retq
2906 ;
2907 ; AVX512F-LABEL: splatconstant_funnnel_v8i16:
2908 ; AVX512F: # %bb.0:
2909 ; AVX512F-NEXT: vpsrlw $9, %xmm1, %xmm1
2910 ; AVX512F-NEXT: vpsllw $7, %xmm0, %xmm0
2911 ; AVX512F-NEXT: vpor %xmm1, %xmm0, %xmm0
2912 ; AVX512F-NEXT: retq
2913 ;
2914 ; AVX512VL-LABEL: splatconstant_funnnel_v8i16:
2915 ; AVX512VL: # %bb.0:
2916 ; AVX512VL-NEXT: vpsrlw $9, %xmm1, %xmm1
2917 ; AVX512VL-NEXT: vpsllw $7, %xmm0, %xmm0
2918 ; AVX512VL-NEXT: vpor %xmm1, %xmm0, %xmm0
2919 ; AVX512VL-NEXT: retq
2920 ;
2921 ; AVX512BW-LABEL: splatconstant_funnnel_v8i16:
2922 ; AVX512BW: # %bb.0:
2923 ; AVX512BW-NEXT: vpsrlw $9, %xmm1, %xmm1
2924 ; AVX512BW-NEXT: vpsllw $7, %xmm0, %xmm0
2925 ; AVX512BW-NEXT: vpor %xmm1, %xmm0, %xmm0
2926 ; AVX512BW-NEXT: retq
2927 ;
2928 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i16:
2929 ; AVX512VBMI2: # %bb.0:
2930 ; AVX512VBMI2-NEXT: vpsrlw $9, %xmm1, %xmm1
2931 ; AVX512VBMI2-NEXT: vpsllw $7, %xmm0, %xmm0
2932 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm0, %xmm0
2933 ; AVX512VBMI2-NEXT: retq
2934 ;
2935 ; AVX512VLBW-LABEL: splatconstant_funnnel_v8i16:
2936 ; AVX512VLBW: # %bb.0:
2937 ; AVX512VLBW-NEXT: vpsrlw $9, %xmm1, %xmm1
2938 ; AVX512VLBW-NEXT: vpsllw $7, %xmm0, %xmm0
2939 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm0, %xmm0
2940 ; AVX512VLBW-NEXT: retq
2941 ;
2942 ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v8i16:
2943 ; AVX512VLVBMI2: # %bb.0:
2944 ; AVX512VLVBMI2-NEXT: vpshldw $7, %xmm1, %xmm0, %xmm0
2945 ; AVX512VLVBMI2-NEXT: retq
2946 ;
2947 ; XOP-LABEL: splatconstant_funnnel_v8i16:
2948 ; XOP: # %bb.0:
2949 ; XOP-NEXT: vpsrlw $9, %xmm1, %xmm1
2950 ; XOP-NEXT: vpsllw $7, %xmm0, %xmm0
2951 ; XOP-NEXT: vpor %xmm1, %xmm0, %xmm0
2952 ; XOP-NEXT: retq
2953 ;
2954 ; X32-SSE-LABEL: splatconstant_funnnel_v8i16:
2955 ; X32-SSE: # %bb.0:
2956 ; X32-SSE-NEXT: psrlw $9, %xmm1
2957 ; X32-SSE-NEXT: psllw $7, %xmm0
2958 ; X32-SSE-NEXT: por %xmm1, %xmm0
2959 ; X32-SSE-NEXT: retl
2960 %res = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>)
2961 ret <8 x i16> %res
2962 }
2964 define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
2965 ; SSE-LABEL: splatconstant_funnnel_v16i8:
2966 ; SSE: # %bb.0:
2967 ; SSE-NEXT: psrlw $4, %xmm1
2968 ; SSE-NEXT: pand {{.*}}(%rip), %xmm1
2969 ; SSE-NEXT: psllw $4, %xmm0
2970 ; SSE-NEXT: pand {{.*}}(%rip), %xmm0
2971 ; SSE-NEXT: por %xmm1, %xmm0
2972 ; SSE-NEXT: retq
2973 ;
2974 ; AVX-LABEL: splatconstant_funnnel_v16i8:
2975 ; AVX: # %bb.0:
2976 ; AVX-NEXT: vpsrlw $4, %xmm1, %xmm1
2977 ; AVX-NEXT: vpand {{.*}}(%rip), %xmm1, %xmm1
2978 ; AVX-NEXT: vpsllw $4, %xmm0, %xmm0
2979 ; AVX-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0
2980 ; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0
2981 ; AVX-NEXT: retq
2982 ;
2983 ; AVX512-LABEL: splatconstant_funnnel_v16i8:
2984 ; AVX512: # %bb.0:
2985 ; AVX512-NEXT: vpsrlw $4, %xmm1, %xmm1
2986 ; AVX512-NEXT: vpand {{.*}}(%rip), %xmm1, %xmm1
2987 ; AVX512-NEXT: vpsllw $4, %xmm0, %xmm0
2988 ; AVX512-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0
2989 ; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0
2990 ; AVX512-NEXT: retq
2991 ;
2992 ; XOP-LABEL: splatconstant_funnnel_v16i8:
2993 ; XOP: # %bb.0:
2994 ; XOP-NEXT: vpshlb {{.*}}(%rip), %xmm1, %xmm1
2995 ; XOP-NEXT: vpshlb {{.*}}(%rip), %xmm0, %xmm0
2996 ; XOP-NEXT: vpor %xmm1, %xmm0, %xmm0
2997 ; XOP-NEXT: retq
2998 ;
2999 ; X32-SSE-LABEL: splatconstant_funnnel_v16i8:
3000 ; X32-SSE: # %bb.0:
3001 ; X32-SSE-NEXT: psrlw $4, %xmm1
3002 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm1
3003 ; X32-SSE-NEXT: psllw $4, %xmm0
3004 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm0
3005 ; X32-SSE-NEXT: por %xmm1, %xmm0
3006 ; X32-SSE-NEXT: retl
3007 %res = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
3008 ret <16 x i8> %res
3009 }