| blob | b40c88169172724efcb2225bc55e3484b874045a |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
3 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
4 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
5 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
6 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
7 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL
8 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
9 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2
10 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW
11 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2
12 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefixes=XOP,XOPAVX1
13 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefixes=XOP,XOPAVX2
15 ; Just one 32-bit run to make sure we do reasonable things for i64 cases.
16 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=X32-SSE,X32-SSE2
18 declare <2 x i64> @llvm.fshl.v2i64(<2 x i64>, <2 x i64>, <2 x i64>)
19 declare <4 x i32> @llvm.fshl.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)
20 declare <8 x i16> @llvm.fshl.v8i16(<8 x i16>, <8 x i16>, <8 x i16>)
21 declare <16 x i8> @llvm.fshl.v16i8(<16 x i8>, <16 x i8>, <16 x i8>)
23 ;
24 ; Variable Shifts
25 ;
27 define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt) nounwind {
28 ; SSE2-LABEL: var_funnnel_v2i64:
29 ; SSE2: # %bb.0:
30 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm2
31 ; SSE2-NEXT: movdqa %xmm0, %xmm3
32 ; SSE2-NEXT: psllq %xmm2, %xmm3
33 ; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[2,3,0,1]
34 ; SSE2-NEXT: movdqa %xmm0, %xmm5
35 ; SSE2-NEXT: psllq %xmm4, %xmm5
36 ; SSE2-NEXT: movsd {{.*#+}} xmm5 = xmm3[0],xmm5[1]
37 ; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [64,64]
38 ; SSE2-NEXT: psubq %xmm2, %xmm3
39 ; SSE2-NEXT: movdqa %xmm1, %xmm4
40 ; SSE2-NEXT: psrlq %xmm3, %xmm4
41 ; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
42 ; SSE2-NEXT: psrlq %xmm3, %xmm1
43 ; SSE2-NEXT: movsd {{.*#+}} xmm1 = xmm4[0],xmm1[1]
44 ; SSE2-NEXT: orpd %xmm5, %xmm1
45 ; SSE2-NEXT: pxor %xmm3, %xmm3
46 ; SSE2-NEXT: pcmpeqd %xmm2, %xmm3
47 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,0,3,2]
48 ; SSE2-NEXT: pand %xmm3, %xmm2
49 ; SSE2-NEXT: pand %xmm2, %xmm0
50 ; SSE2-NEXT: pandn %xmm1, %xmm2
51 ; SSE2-NEXT: por %xmm2, %xmm0
52 ; SSE2-NEXT: retq
53 ;
54 ; SSE41-LABEL: var_funnnel_v2i64:
55 ; SSE41: # %bb.0:
56 ; SSE41-NEXT: movdqa %xmm0, %xmm3
57 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm2
58 ; SSE41-NEXT: psllq %xmm2, %xmm0
59 ; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm2[2,3,0,1]
60 ; SSE41-NEXT: movdqa %xmm3, %xmm4
61 ; SSE41-NEXT: psllq %xmm5, %xmm4
62 ; SSE41-NEXT: pblendw {{.*#+}} xmm4 = xmm0[0,1,2,3],xmm4[4,5,6,7]
63 ; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [64,64]
64 ; SSE41-NEXT: psubq %xmm2, %xmm0
65 ; SSE41-NEXT: movdqa %xmm1, %xmm5
66 ; SSE41-NEXT: psrlq %xmm0, %xmm5
67 ; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
68 ; SSE41-NEXT: psrlq %xmm0, %xmm1
69 ; SSE41-NEXT: pblendw {{.*#+}} xmm1 = xmm5[0,1,2,3],xmm1[4,5,6,7]
70 ; SSE41-NEXT: por %xmm1, %xmm4
71 ; SSE41-NEXT: pxor %xmm0, %xmm0
72 ; SSE41-NEXT: pcmpeqq %xmm2, %xmm0
73 ; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm4
74 ; SSE41-NEXT: movapd %xmm4, %xmm0
75 ; SSE41-NEXT: retq
76 ;
77 ; AVX1-LABEL: var_funnnel_v2i64:
78 ; AVX1: # %bb.0:
79 ; AVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
80 ; AVX1-NEXT: vpsllq %xmm2, %xmm0, %xmm3
81 ; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,0,1]
82 ; AVX1-NEXT: vpsllq %xmm4, %xmm0, %xmm4
83 ; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm4[4,5,6,7]
84 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
85 ; AVX1-NEXT: vpsubq %xmm2, %xmm4, %xmm4
86 ; AVX1-NEXT: vpsrlq %xmm4, %xmm1, %xmm5
87 ; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
88 ; AVX1-NEXT: vpsrlq %xmm4, %xmm1, %xmm1
89 ; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0,1,2,3],xmm1[4,5,6,7]
90 ; AVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
91 ; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
92 ; AVX1-NEXT: vpcmpeqq %xmm3, %xmm2, %xmm2
93 ; AVX1-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
94 ; AVX1-NEXT: retq
95 ;
96 ; AVX2-LABEL: var_funnnel_v2i64:
97 ; AVX2: # %bb.0:
98 ; AVX2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
99 ; AVX2-NEXT: vpsllvq %xmm2, %xmm0, %xmm3
100 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
101 ; AVX2-NEXT: vpsubq %xmm2, %xmm4, %xmm4
102 ; AVX2-NEXT: vpsrlvq %xmm4, %xmm1, %xmm1
103 ; AVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
104 ; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
105 ; AVX2-NEXT: vpcmpeqq %xmm3, %xmm2, %xmm2
106 ; AVX2-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
107 ; AVX2-NEXT: retq
108 ;
109 ; AVX512F-LABEL: var_funnnel_v2i64:
110 ; AVX512F: # %bb.0:
111 ; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
112 ; AVX512F-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
113 ; AVX512F-NEXT: vpsllvq %xmm2, %xmm0, %xmm3
114 ; AVX512F-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
115 ; AVX512F-NEXT: vpsubq %xmm2, %xmm4, %xmm4
116 ; AVX512F-NEXT: vpsrlvq %xmm4, %xmm1, %xmm1
117 ; AVX512F-NEXT: vpor %xmm1, %xmm3, %xmm1
118 ; AVX512F-NEXT: vptestnmq %zmm2, %zmm2, %k1
119 ; AVX512F-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
120 ; AVX512F-NEXT: vmovdqa %xmm1, %xmm0
121 ; AVX512F-NEXT: vzeroupper
122 ; AVX512F-NEXT: retq
123 ;
124 ; AVX512VL-LABEL: var_funnnel_v2i64:
125 ; AVX512VL: # %bb.0:
126 ; AVX512VL-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
127 ; AVX512VL-NEXT: vpsllvq %xmm2, %xmm0, %xmm3
128 ; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
129 ; AVX512VL-NEXT: vpsubq %xmm2, %xmm4, %xmm4
130 ; AVX512VL-NEXT: vpsrlvq %xmm4, %xmm1, %xmm1
131 ; AVX512VL-NEXT: vpor %xmm1, %xmm3, %xmm1
132 ; AVX512VL-NEXT: vptestnmq %xmm2, %xmm2, %k1
133 ; AVX512VL-NEXT: vmovdqa64 %xmm0, %xmm1 {%k1}
134 ; AVX512VL-NEXT: vmovdqa %xmm1, %xmm0
135 ; AVX512VL-NEXT: retq
136 ;
137 ; AVX512BW-LABEL: var_funnnel_v2i64:
138 ; AVX512BW: # %bb.0:
139 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
140 ; AVX512BW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
141 ; AVX512BW-NEXT: vpsllvq %xmm2, %xmm0, %xmm3
142 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
143 ; AVX512BW-NEXT: vpsubq %xmm2, %xmm4, %xmm4
144 ; AVX512BW-NEXT: vpsrlvq %xmm4, %xmm1, %xmm1
145 ; AVX512BW-NEXT: vpor %xmm1, %xmm3, %xmm1
146 ; AVX512BW-NEXT: vptestnmq %zmm2, %zmm2, %k1
147 ; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
148 ; AVX512BW-NEXT: vmovdqa %xmm1, %xmm0
149 ; AVX512BW-NEXT: vzeroupper
150 ; AVX512BW-NEXT: retq
151 ;
152 ; AVX512VBMI2-LABEL: var_funnnel_v2i64:
153 ; AVX512VBMI2: # %bb.0:
154 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
155 ; AVX512VBMI2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
156 ; AVX512VBMI2-NEXT: vpsllvq %xmm2, %xmm0, %xmm3
157 ; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
158 ; AVX512VBMI2-NEXT: vpsubq %xmm2, %xmm4, %xmm4
159 ; AVX512VBMI2-NEXT: vpsrlvq %xmm4, %xmm1, %xmm1
160 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm3, %xmm1
161 ; AVX512VBMI2-NEXT: vptestnmq %zmm2, %zmm2, %k1
162 ; AVX512VBMI2-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
163 ; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0
164 ; AVX512VBMI2-NEXT: vzeroupper
165 ; AVX512VBMI2-NEXT: retq
166 ;
167 ; AVX512VLBW-LABEL: var_funnnel_v2i64:
168 ; AVX512VLBW: # %bb.0:
169 ; AVX512VLBW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
170 ; AVX512VLBW-NEXT: vpsllvq %xmm2, %xmm0, %xmm3
171 ; AVX512VLBW-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
172 ; AVX512VLBW-NEXT: vpsubq %xmm2, %xmm4, %xmm4
173 ; AVX512VLBW-NEXT: vpsrlvq %xmm4, %xmm1, %xmm1
174 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm3, %xmm1
175 ; AVX512VLBW-NEXT: vptestnmq %xmm2, %xmm2, %k1
176 ; AVX512VLBW-NEXT: vmovdqa64 %xmm0, %xmm1 {%k1}
177 ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0
178 ; AVX512VLBW-NEXT: retq
179 ;
180 ; AVX512VLVBMI2-LABEL: var_funnnel_v2i64:
181 ; AVX512VLVBMI2: # %bb.0:
182 ; AVX512VLVBMI2-NEXT: vpshldvq %xmm2, %xmm1, %xmm0
183 ; AVX512VLVBMI2-NEXT: retq
184 ;
185 ; XOPAVX1-LABEL: var_funnnel_v2i64:
186 ; XOPAVX1: # %bb.0:
187 ; XOPAVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
188 ; XOPAVX1-NEXT: vpshlq %xmm2, %xmm0, %xmm3
189 ; XOPAVX1-NEXT: vpsubq {{.*}}(%rip), %xmm2, %xmm4
190 ; XOPAVX1-NEXT: vpshlq %xmm4, %xmm1, %xmm1
191 ; XOPAVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
192 ; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
193 ; XOPAVX1-NEXT: vpcomeqq %xmm3, %xmm2, %xmm2
194 ; XOPAVX1-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
195 ; XOPAVX1-NEXT: retq
196 ;
197 ; XOPAVX2-LABEL: var_funnnel_v2i64:
198 ; XOPAVX2: # %bb.0:
199 ; XOPAVX2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
200 ; XOPAVX2-NEXT: vpsllvq %xmm2, %xmm0, %xmm3
201 ; XOPAVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
202 ; XOPAVX2-NEXT: vpsubq %xmm2, %xmm4, %xmm4
203 ; XOPAVX2-NEXT: vpsrlvq %xmm4, %xmm1, %xmm1
204 ; XOPAVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
205 ; XOPAVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
206 ; XOPAVX2-NEXT: vpcomeqq %xmm3, %xmm2, %xmm2
207 ; XOPAVX2-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
208 ; XOPAVX2-NEXT: retq
209 ;
210 ; X32-SSE-LABEL: var_funnnel_v2i64:
211 ; X32-SSE: # %bb.0:
212 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm2
213 ; X32-SSE-NEXT: movdqa %xmm0, %xmm3
214 ; X32-SSE-NEXT: psllq %xmm2, %xmm3
215 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm2[2,3,0,1]
216 ; X32-SSE-NEXT: movdqa %xmm0, %xmm5
217 ; X32-SSE-NEXT: psllq %xmm4, %xmm5
218 ; X32-SSE-NEXT: movsd {{.*#+}} xmm5 = xmm3[0],xmm5[1]
219 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [64,0,64,0]
220 ; X32-SSE-NEXT: psubq %xmm2, %xmm3
221 ; X32-SSE-NEXT: movdqa %xmm1, %xmm4
222 ; X32-SSE-NEXT: psrlq %xmm3, %xmm4
223 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
224 ; X32-SSE-NEXT: psrlq %xmm3, %xmm1
225 ; X32-SSE-NEXT: movsd {{.*#+}} xmm1 = xmm4[0],xmm1[1]
226 ; X32-SSE-NEXT: orpd %xmm5, %xmm1
227 ; X32-SSE-NEXT: pxor %xmm3, %xmm3
228 ; X32-SSE-NEXT: pcmpeqd %xmm2, %xmm3
229 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,0,3,2]
230 ; X32-SSE-NEXT: pand %xmm3, %xmm2
231 ; X32-SSE-NEXT: pand %xmm2, %xmm0
232 ; X32-SSE-NEXT: pandn %xmm1, %xmm2
233 ; X32-SSE-NEXT: por %xmm2, %xmm0
234 ; X32-SSE-NEXT: retl
235 %res = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt)
236 ret <2 x i64> %res
237 }
239 define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %amt) nounwind {
240 ; SSE2-LABEL: var_funnnel_v4i32:
241 ; SSE2: # %bb.0:
242 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm2
243 ; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [32,32,32,32]
244 ; SSE2-NEXT: psubd %xmm2, %xmm4
245 ; SSE2-NEXT: pshuflw {{.*#+}} xmm3 = xmm4[2,3,3,3,4,5,6,7]
246 ; SSE2-NEXT: movdqa %xmm1, %xmm5
247 ; SSE2-NEXT: psrld %xmm3, %xmm5
248 ; SSE2-NEXT: pshuflw {{.*#+}} xmm6 = xmm4[0,1,1,1,4,5,6,7]
249 ; SSE2-NEXT: movdqa %xmm1, %xmm3
250 ; SSE2-NEXT: psrld %xmm6, %xmm3
251 ; SSE2-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
252 ; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
253 ; SSE2-NEXT: pshuflw {{.*#+}} xmm5 = xmm4[2,3,3,3,4,5,6,7]
254 ; SSE2-NEXT: movdqa %xmm1, %xmm6
255 ; SSE2-NEXT: psrld %xmm5, %xmm6
256 ; SSE2-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,1,4,5,6,7]
257 ; SSE2-NEXT: psrld %xmm4, %xmm1
258 ; SSE2-NEXT: punpckhqdq {{.*#+}} xmm1 = xmm1[1],xmm6[1]
259 ; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,3],xmm1[0,3]
260 ; SSE2-NEXT: pxor %xmm1, %xmm1
261 ; SSE2-NEXT: pcmpeqd %xmm2, %xmm1
262 ; SSE2-NEXT: pslld $23, %xmm2
263 ; SSE2-NEXT: paddd {{.*}}(%rip), %xmm2
264 ; SSE2-NEXT: cvttps2dq %xmm2, %xmm2
265 ; SSE2-NEXT: movdqa %xmm0, %xmm4
266 ; SSE2-NEXT: pmuludq %xmm2, %xmm4
267 ; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
268 ; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
269 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
270 ; SSE2-NEXT: pmuludq %xmm5, %xmm2
271 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
272 ; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
273 ; SSE2-NEXT: por %xmm3, %xmm4
274 ; SSE2-NEXT: pand %xmm1, %xmm0
275 ; SSE2-NEXT: pandn %xmm4, %xmm1
276 ; SSE2-NEXT: por %xmm1, %xmm0
277 ; SSE2-NEXT: retq
278 ;
279 ; SSE41-LABEL: var_funnnel_v4i32:
280 ; SSE41: # %bb.0:
281 ; SSE41-NEXT: movdqa %xmm0, %xmm3
282 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm2
283 ; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [32,32,32,32]
284 ; SSE41-NEXT: psubd %xmm2, %xmm0
285 ; SSE41-NEXT: pshuflw {{.*#+}} xmm4 = xmm0[2,3,3,3,4,5,6,7]
286 ; SSE41-NEXT: movdqa %xmm1, %xmm5
287 ; SSE41-NEXT: psrld %xmm4, %xmm5
288 ; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
289 ; SSE41-NEXT: pshuflw {{.*#+}} xmm6 = xmm4[2,3,3,3,4,5,6,7]
290 ; SSE41-NEXT: movdqa %xmm1, %xmm7
291 ; SSE41-NEXT: psrld %xmm6, %xmm7
292 ; SSE41-NEXT: pblendw {{.*#+}} xmm7 = xmm5[0,1,2,3],xmm7[4,5,6,7]
293 ; SSE41-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,1,4,5,6,7]
294 ; SSE41-NEXT: movdqa %xmm1, %xmm5
295 ; SSE41-NEXT: psrld %xmm0, %xmm5
296 ; SSE41-NEXT: pshuflw {{.*#+}} xmm0 = xmm4[0,1,1,1,4,5,6,7]
297 ; SSE41-NEXT: psrld %xmm0, %xmm1
298 ; SSE41-NEXT: pblendw {{.*#+}} xmm1 = xmm5[0,1,2,3],xmm1[4,5,6,7]
299 ; SSE41-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm7[2,3],xmm1[4,5],xmm7[6,7]
300 ; SSE41-NEXT: pxor %xmm0, %xmm0
301 ; SSE41-NEXT: pcmpeqd %xmm2, %xmm0
302 ; SSE41-NEXT: pslld $23, %xmm2
303 ; SSE41-NEXT: paddd {{.*}}(%rip), %xmm2
304 ; SSE41-NEXT: cvttps2dq %xmm2, %xmm2
305 ; SSE41-NEXT: pmulld %xmm3, %xmm2
306 ; SSE41-NEXT: por %xmm1, %xmm2
307 ; SSE41-NEXT: blendvps %xmm0, %xmm3, %xmm2
308 ; SSE41-NEXT: movaps %xmm2, %xmm0
309 ; SSE41-NEXT: retq
310 ;
311 ; AVX1-LABEL: var_funnnel_v4i32:
312 ; AVX1: # %bb.0:
313 ; AVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
314 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [32,32,32,32]
315 ; AVX1-NEXT: vpsubd %xmm2, %xmm3, %xmm3
316 ; AVX1-NEXT: vpsrldq {{.*#+}} xmm4 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
317 ; AVX1-NEXT: vpsrld %xmm4, %xmm1, %xmm4
318 ; AVX1-NEXT: vpsrlq $32, %xmm3, %xmm5
319 ; AVX1-NEXT: vpsrld %xmm5, %xmm1, %xmm5
320 ; AVX1-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
321 ; AVX1-NEXT: vpxor %xmm5, %xmm5, %xmm5
322 ; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
323 ; AVX1-NEXT: vpsrld %xmm6, %xmm1, %xmm6
324 ; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
325 ; AVX1-NEXT: vpsrld %xmm3, %xmm1, %xmm1
326 ; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm6[4,5,6,7]
327 ; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3],xmm1[4,5],xmm4[6,7]
328 ; AVX1-NEXT: vpslld $23, %xmm2, %xmm3
329 ; AVX1-NEXT: vpaddd {{.*}}(%rip), %xmm3, %xmm3
330 ; AVX1-NEXT: vcvttps2dq %xmm3, %xmm3
331 ; AVX1-NEXT: vpmulld %xmm3, %xmm0, %xmm3
332 ; AVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
333 ; AVX1-NEXT: vpcmpeqd %xmm5, %xmm2, %xmm2
334 ; AVX1-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
335 ; AVX1-NEXT: retq
336 ;
337 ; AVX2-LABEL: var_funnnel_v4i32:
338 ; AVX2: # %bb.0:
339 ; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
340 ; AVX2-NEXT: vpand %xmm3, %xmm2, %xmm2
341 ; AVX2-NEXT: vpsllvd %xmm2, %xmm0, %xmm3
342 ; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
343 ; AVX2-NEXT: vpsubd %xmm2, %xmm4, %xmm4
344 ; AVX2-NEXT: vpsrlvd %xmm4, %xmm1, %xmm1
345 ; AVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
346 ; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
347 ; AVX2-NEXT: vpcmpeqd %xmm3, %xmm2, %xmm2
348 ; AVX2-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
349 ; AVX2-NEXT: retq
350 ;
351 ; AVX512F-LABEL: var_funnnel_v4i32:
352 ; AVX512F: # %bb.0:
353 ; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
354 ; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
355 ; AVX512F-NEXT: vpand %xmm3, %xmm2, %xmm2
356 ; AVX512F-NEXT: vpsllvd %xmm2, %xmm0, %xmm3
357 ; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
358 ; AVX512F-NEXT: vpsubd %xmm2, %xmm4, %xmm4
359 ; AVX512F-NEXT: vpsrlvd %xmm4, %xmm1, %xmm1
360 ; AVX512F-NEXT: vpor %xmm1, %xmm3, %xmm1
361 ; AVX512F-NEXT: vptestnmd %zmm2, %zmm2, %k1
362 ; AVX512F-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1}
363 ; AVX512F-NEXT: vmovdqa %xmm1, %xmm0
364 ; AVX512F-NEXT: vzeroupper
365 ; AVX512F-NEXT: retq
366 ;
367 ; AVX512VL-LABEL: var_funnnel_v4i32:
368 ; AVX512VL: # %bb.0:
369 ; AVX512VL-NEXT: vpandd {{.*}}(%rip){1to4}, %xmm2, %xmm2
370 ; AVX512VL-NEXT: vpsllvd %xmm2, %xmm0, %xmm3
371 ; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
372 ; AVX512VL-NEXT: vpsubd %xmm2, %xmm4, %xmm4
373 ; AVX512VL-NEXT: vpsrlvd %xmm4, %xmm1, %xmm1
374 ; AVX512VL-NEXT: vpor %xmm1, %xmm3, %xmm1
375 ; AVX512VL-NEXT: vptestnmd %xmm2, %xmm2, %k1
376 ; AVX512VL-NEXT: vmovdqa32 %xmm0, %xmm1 {%k1}
377 ; AVX512VL-NEXT: vmovdqa %xmm1, %xmm0
378 ; AVX512VL-NEXT: retq
379 ;
380 ; AVX512BW-LABEL: var_funnnel_v4i32:
381 ; AVX512BW: # %bb.0:
382 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
383 ; AVX512BW-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
384 ; AVX512BW-NEXT: vpand %xmm3, %xmm2, %xmm2
385 ; AVX512BW-NEXT: vpsllvd %xmm2, %xmm0, %xmm3
386 ; AVX512BW-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
387 ; AVX512BW-NEXT: vpsubd %xmm2, %xmm4, %xmm4
388 ; AVX512BW-NEXT: vpsrlvd %xmm4, %xmm1, %xmm1
389 ; AVX512BW-NEXT: vpor %xmm1, %xmm3, %xmm1
390 ; AVX512BW-NEXT: vptestnmd %zmm2, %zmm2, %k1
391 ; AVX512BW-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1}
392 ; AVX512BW-NEXT: vmovdqa %xmm1, %xmm0
393 ; AVX512BW-NEXT: vzeroupper
394 ; AVX512BW-NEXT: retq
395 ;
396 ; AVX512VBMI2-LABEL: var_funnnel_v4i32:
397 ; AVX512VBMI2: # %bb.0:
398 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
399 ; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
400 ; AVX512VBMI2-NEXT: vpand %xmm3, %xmm2, %xmm2
401 ; AVX512VBMI2-NEXT: vpsllvd %xmm2, %xmm0, %xmm3
402 ; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
403 ; AVX512VBMI2-NEXT: vpsubd %xmm2, %xmm4, %xmm4
404 ; AVX512VBMI2-NEXT: vpsrlvd %xmm4, %xmm1, %xmm1
405 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm3, %xmm1
406 ; AVX512VBMI2-NEXT: vptestnmd %zmm2, %zmm2, %k1
407 ; AVX512VBMI2-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1}
408 ; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0
409 ; AVX512VBMI2-NEXT: vzeroupper
410 ; AVX512VBMI2-NEXT: retq
411 ;
412 ; AVX512VLBW-LABEL: var_funnnel_v4i32:
413 ; AVX512VLBW: # %bb.0:
414 ; AVX512VLBW-NEXT: vpandd {{.*}}(%rip){1to4}, %xmm2, %xmm2
415 ; AVX512VLBW-NEXT: vpsllvd %xmm2, %xmm0, %xmm3
416 ; AVX512VLBW-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
417 ; AVX512VLBW-NEXT: vpsubd %xmm2, %xmm4, %xmm4
418 ; AVX512VLBW-NEXT: vpsrlvd %xmm4, %xmm1, %xmm1
419 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm3, %xmm1
420 ; AVX512VLBW-NEXT: vptestnmd %xmm2, %xmm2, %k1
421 ; AVX512VLBW-NEXT: vmovdqa32 %xmm0, %xmm1 {%k1}
422 ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0
423 ; AVX512VLBW-NEXT: retq
424 ;
425 ; AVX512VLVBMI2-LABEL: var_funnnel_v4i32:
426 ; AVX512VLVBMI2: # %bb.0:
427 ; AVX512VLVBMI2-NEXT: vpshldvd %xmm2, %xmm1, %xmm0
428 ; AVX512VLVBMI2-NEXT: retq
429 ;
430 ; XOPAVX1-LABEL: var_funnnel_v4i32:
431 ; XOPAVX1: # %bb.0:
432 ; XOPAVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
433 ; XOPAVX1-NEXT: vpshld %xmm2, %xmm0, %xmm3
434 ; XOPAVX1-NEXT: vpsubd {{.*}}(%rip), %xmm2, %xmm4
435 ; XOPAVX1-NEXT: vpshld %xmm4, %xmm1, %xmm1
436 ; XOPAVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
437 ; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
438 ; XOPAVX1-NEXT: vpcomeqd %xmm3, %xmm2, %xmm2
439 ; XOPAVX1-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
440 ; XOPAVX1-NEXT: retq
441 ;
442 ; XOPAVX2-LABEL: var_funnnel_v4i32:
443 ; XOPAVX2: # %bb.0:
444 ; XOPAVX2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
445 ; XOPAVX2-NEXT: vpand %xmm3, %xmm2, %xmm2
446 ; XOPAVX2-NEXT: vpsllvd %xmm2, %xmm0, %xmm3
447 ; XOPAVX2-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
448 ; XOPAVX2-NEXT: vpsubd %xmm2, %xmm4, %xmm4
449 ; XOPAVX2-NEXT: vpsrlvd %xmm4, %xmm1, %xmm1
450 ; XOPAVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
451 ; XOPAVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
452 ; XOPAVX2-NEXT: vpcomeqd %xmm3, %xmm2, %xmm2
453 ; XOPAVX2-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
454 ; XOPAVX2-NEXT: retq
455 ;
456 ; X32-SSE-LABEL: var_funnnel_v4i32:
457 ; X32-SSE: # %bb.0:
458 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm2
459 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm4 = [32,32,32,32]
460 ; X32-SSE-NEXT: psubd %xmm2, %xmm4
461 ; X32-SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm4[2,3,3,3,4,5,6,7]
462 ; X32-SSE-NEXT: movdqa %xmm1, %xmm5
463 ; X32-SSE-NEXT: psrld %xmm3, %xmm5
464 ; X32-SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm4[0,1,1,1,4,5,6,7]
465 ; X32-SSE-NEXT: movdqa %xmm1, %xmm3
466 ; X32-SSE-NEXT: psrld %xmm6, %xmm3
467 ; X32-SSE-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
468 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
469 ; X32-SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm4[2,3,3,3,4,5,6,7]
470 ; X32-SSE-NEXT: movdqa %xmm1, %xmm6
471 ; X32-SSE-NEXT: psrld %xmm5, %xmm6
472 ; X32-SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,1,4,5,6,7]
473 ; X32-SSE-NEXT: psrld %xmm4, %xmm1
474 ; X32-SSE-NEXT: punpckhqdq {{.*#+}} xmm1 = xmm1[1],xmm6[1]
475 ; X32-SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,3],xmm1[0,3]
476 ; X32-SSE-NEXT: pxor %xmm1, %xmm1
477 ; X32-SSE-NEXT: pcmpeqd %xmm2, %xmm1
478 ; X32-SSE-NEXT: pslld $23, %xmm2
479 ; X32-SSE-NEXT: paddd {{\.LCPI.*}}, %xmm2
480 ; X32-SSE-NEXT: cvttps2dq %xmm2, %xmm2
481 ; X32-SSE-NEXT: movdqa %xmm0, %xmm4
482 ; X32-SSE-NEXT: pmuludq %xmm2, %xmm4
483 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
484 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
485 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
486 ; X32-SSE-NEXT: pmuludq %xmm5, %xmm2
487 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
488 ; X32-SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
489 ; X32-SSE-NEXT: por %xmm3, %xmm4
490 ; X32-SSE-NEXT: pand %xmm1, %xmm0
491 ; X32-SSE-NEXT: pandn %xmm4, %xmm1
492 ; X32-SSE-NEXT: por %xmm1, %xmm0
493 ; X32-SSE-NEXT: retl
494 %res = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %amt)
495 ret <4 x i32> %res
496 }
498 define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt) nounwind {
499 ; SSE2-LABEL: var_funnnel_v8i16:
500 ; SSE2: # %bb.0:
501 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm2
502 ; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
503 ; SSE2-NEXT: psubw %xmm2, %xmm3
504 ; SSE2-NEXT: psllw $12, %xmm3
505 ; SSE2-NEXT: movdqa %xmm3, %xmm4
506 ; SSE2-NEXT: psraw $15, %xmm4
507 ; SSE2-NEXT: movdqa %xmm4, %xmm5
508 ; SSE2-NEXT: pandn %xmm1, %xmm5
509 ; SSE2-NEXT: psrlw $8, %xmm1
510 ; SSE2-NEXT: pand %xmm4, %xmm1
511 ; SSE2-NEXT: por %xmm5, %xmm1
512 ; SSE2-NEXT: paddw %xmm3, %xmm3
513 ; SSE2-NEXT: movdqa %xmm3, %xmm4
514 ; SSE2-NEXT: psraw $15, %xmm4
515 ; SSE2-NEXT: movdqa %xmm4, %xmm5
516 ; SSE2-NEXT: pandn %xmm1, %xmm5
517 ; SSE2-NEXT: psrlw $4, %xmm1
518 ; SSE2-NEXT: pand %xmm4, %xmm1
519 ; SSE2-NEXT: por %xmm5, %xmm1
520 ; SSE2-NEXT: paddw %xmm3, %xmm3
521 ; SSE2-NEXT: movdqa %xmm3, %xmm4
522 ; SSE2-NEXT: psraw $15, %xmm4
523 ; SSE2-NEXT: movdqa %xmm4, %xmm5
524 ; SSE2-NEXT: pandn %xmm1, %xmm5
525 ; SSE2-NEXT: psrlw $2, %xmm1
526 ; SSE2-NEXT: pand %xmm4, %xmm1
527 ; SSE2-NEXT: por %xmm5, %xmm1
528 ; SSE2-NEXT: paddw %xmm3, %xmm3
529 ; SSE2-NEXT: psraw $15, %xmm3
530 ; SSE2-NEXT: movdqa %xmm3, %xmm4
531 ; SSE2-NEXT: pandn %xmm1, %xmm4
532 ; SSE2-NEXT: psrlw $1, %xmm1
533 ; SSE2-NEXT: pand %xmm3, %xmm1
534 ; SSE2-NEXT: pxor %xmm3, %xmm3
535 ; SSE2-NEXT: movdqa %xmm2, %xmm5
536 ; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
537 ; SSE2-NEXT: pslld $23, %xmm5
538 ; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216]
539 ; SSE2-NEXT: paddd %xmm6, %xmm5
540 ; SSE2-NEXT: cvttps2dq %xmm5, %xmm5
541 ; SSE2-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[0,2,2,3,4,5,6,7]
542 ; SSE2-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,6,6,7]
543 ; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3]
544 ; SSE2-NEXT: movdqa %xmm2, %xmm7
545 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3]
546 ; SSE2-NEXT: pslld $23, %xmm7
547 ; SSE2-NEXT: paddd %xmm6, %xmm7
548 ; SSE2-NEXT: cvttps2dq %xmm7, %xmm6
549 ; SSE2-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,2,2,3,4,5,6,7]
550 ; SSE2-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,6,6,7]
551 ; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,2,2,3]
552 ; SSE2-NEXT: punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm5[0]
553 ; SSE2-NEXT: pmullw %xmm0, %xmm6
554 ; SSE2-NEXT: por %xmm4, %xmm6
555 ; SSE2-NEXT: por %xmm1, %xmm6
556 ; SSE2-NEXT: pcmpeqw %xmm3, %xmm2
557 ; SSE2-NEXT: pand %xmm2, %xmm0
558 ; SSE2-NEXT: pandn %xmm6, %xmm2
559 ; SSE2-NEXT: por %xmm2, %xmm0
560 ; SSE2-NEXT: retq
561 ;
562 ; SSE41-LABEL: var_funnnel_v8i16:
563 ; SSE41: # %bb.0:
564 ; SSE41-NEXT: movdqa %xmm0, %xmm3
565 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm2
566 ; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [16,16,16,16,16,16,16,16]
567 ; SSE41-NEXT: psubw %xmm2, %xmm0
568 ; SSE41-NEXT: movdqa %xmm0, %xmm4
569 ; SSE41-NEXT: psllw $12, %xmm4
570 ; SSE41-NEXT: psllw $4, %xmm0
571 ; SSE41-NEXT: por %xmm4, %xmm0
572 ; SSE41-NEXT: movdqa %xmm0, %xmm4
573 ; SSE41-NEXT: paddw %xmm0, %xmm4
574 ; SSE41-NEXT: movdqa %xmm1, %xmm5
575 ; SSE41-NEXT: psrlw $8, %xmm5
576 ; SSE41-NEXT: pblendvb %xmm0, %xmm5, %xmm1
577 ; SSE41-NEXT: movdqa %xmm1, %xmm5
578 ; SSE41-NEXT: psrlw $4, %xmm5
579 ; SSE41-NEXT: movdqa %xmm4, %xmm0
580 ; SSE41-NEXT: pblendvb %xmm0, %xmm5, %xmm1
581 ; SSE41-NEXT: movdqa %xmm1, %xmm5
582 ; SSE41-NEXT: psrlw $2, %xmm5
583 ; SSE41-NEXT: paddw %xmm4, %xmm4
584 ; SSE41-NEXT: movdqa %xmm4, %xmm0
585 ; SSE41-NEXT: pblendvb %xmm0, %xmm5, %xmm1
586 ; SSE41-NEXT: movdqa %xmm1, %xmm5
587 ; SSE41-NEXT: psrlw $1, %xmm5
588 ; SSE41-NEXT: paddw %xmm4, %xmm4
589 ; SSE41-NEXT: movdqa %xmm4, %xmm0
590 ; SSE41-NEXT: pblendvb %xmm0, %xmm5, %xmm1
591 ; SSE41-NEXT: pxor %xmm0, %xmm0
592 ; SSE41-NEXT: movdqa %xmm2, %xmm4
593 ; SSE41-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
594 ; SSE41-NEXT: pslld $23, %xmm4
595 ; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216]
596 ; SSE41-NEXT: paddd %xmm5, %xmm4
597 ; SSE41-NEXT: cvttps2dq %xmm4, %xmm6
598 ; SSE41-NEXT: pmovzxwd {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
599 ; SSE41-NEXT: pslld $23, %xmm4
600 ; SSE41-NEXT: paddd %xmm5, %xmm4
601 ; SSE41-NEXT: cvttps2dq %xmm4, %xmm4
602 ; SSE41-NEXT: packusdw %xmm6, %xmm4
603 ; SSE41-NEXT: pmullw %xmm3, %xmm4
604 ; SSE41-NEXT: por %xmm1, %xmm4
605 ; SSE41-NEXT: pcmpeqw %xmm2, %xmm0
606 ; SSE41-NEXT: pblendvb %xmm0, %xmm3, %xmm4
607 ; SSE41-NEXT: movdqa %xmm4, %xmm0
608 ; SSE41-NEXT: retq
609 ;
610 ; AVX1-LABEL: var_funnnel_v8i16:
611 ; AVX1: # %bb.0:
612 ; AVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
613 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
614 ; AVX1-NEXT: vpsubw %xmm2, %xmm3, %xmm3
615 ; AVX1-NEXT: vpsllw $12, %xmm3, %xmm4
616 ; AVX1-NEXT: vpsllw $4, %xmm3, %xmm3
617 ; AVX1-NEXT: vpor %xmm4, %xmm3, %xmm3
618 ; AVX1-NEXT: vpaddw %xmm3, %xmm3, %xmm4
619 ; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm5
620 ; AVX1-NEXT: vpblendvb %xmm3, %xmm5, %xmm1, %xmm1
621 ; AVX1-NEXT: vpsrlw $4, %xmm1, %xmm3
622 ; AVX1-NEXT: vpblendvb %xmm4, %xmm3, %xmm1, %xmm1
623 ; AVX1-NEXT: vpsrlw $2, %xmm1, %xmm3
624 ; AVX1-NEXT: vpaddw %xmm4, %xmm4, %xmm4
625 ; AVX1-NEXT: vpblendvb %xmm4, %xmm3, %xmm1, %xmm1
626 ; AVX1-NEXT: vpsrlw $1, %xmm1, %xmm3
627 ; AVX1-NEXT: vpaddw %xmm4, %xmm4, %xmm4
628 ; AVX1-NEXT: vpblendvb %xmm4, %xmm3, %xmm1, %xmm1
629 ; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
630 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
631 ; AVX1-NEXT: vpslld $23, %xmm4, %xmm4
632 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216]
633 ; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4
634 ; AVX1-NEXT: vcvttps2dq %xmm4, %xmm4
635 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm6 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
636 ; AVX1-NEXT: vpslld $23, %xmm6, %xmm6
637 ; AVX1-NEXT: vpaddd %xmm5, %xmm6, %xmm5
638 ; AVX1-NEXT: vcvttps2dq %xmm5, %xmm5
639 ; AVX1-NEXT: vpackusdw %xmm4, %xmm5, %xmm4
640 ; AVX1-NEXT: vpmullw %xmm4, %xmm0, %xmm4
641 ; AVX1-NEXT: vpor %xmm1, %xmm4, %xmm1
642 ; AVX1-NEXT: vpcmpeqw %xmm3, %xmm2, %xmm2
643 ; AVX1-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
644 ; AVX1-NEXT: retq
645 ;
646 ; AVX2-LABEL: var_funnnel_v8i16:
647 ; AVX2: # %bb.0:
648 ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
649 ; AVX2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
650 ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
651 ; AVX2-NEXT: vpsllvd %ymm4, %ymm3, %ymm3
652 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
653 ; AVX2-NEXT: vpshufb %ymm4, %ymm3, %ymm3
654 ; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,2,3]
655 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm5 = [16,16,16,16,16,16,16,16]
656 ; AVX2-NEXT: vpsubw %xmm2, %xmm5, %xmm5
657 ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
658 ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
659 ; AVX2-NEXT: vpsrlvd %ymm5, %ymm1, %ymm1
660 ; AVX2-NEXT: vpshufb %ymm4, %ymm1, %ymm1
661 ; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
662 ; AVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
663 ; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
664 ; AVX2-NEXT: vpcmpeqw %xmm3, %xmm2, %xmm2
665 ; AVX2-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
666 ; AVX2-NEXT: vzeroupper
667 ; AVX2-NEXT: retq
668 ;
669 ; AVX512F-LABEL: var_funnnel_v8i16:
670 ; AVX512F: # %bb.0:
671 ; AVX512F-NEXT: vpmovzxwd {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
672 ; AVX512F-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
673 ; AVX512F-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
674 ; AVX512F-NEXT: vpsllvd %ymm4, %ymm3, %ymm3
675 ; AVX512F-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
676 ; AVX512F-NEXT: vpsubw %xmm2, %xmm4, %xmm4
677 ; AVX512F-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
678 ; AVX512F-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
679 ; AVX512F-NEXT: vpsrlvd %ymm4, %ymm1, %ymm1
680 ; AVX512F-NEXT: vpor %ymm1, %ymm3, %ymm1
681 ; AVX512F-NEXT: vpmovdw %zmm1, %ymm1
682 ; AVX512F-NEXT: vpxor %xmm3, %xmm3, %xmm3
683 ; AVX512F-NEXT: vpcmpeqw %xmm3, %xmm2, %xmm2
684 ; AVX512F-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
685 ; AVX512F-NEXT: vzeroupper
686 ; AVX512F-NEXT: retq
687 ;
688 ; AVX512VL-LABEL: var_funnnel_v8i16:
689 ; AVX512VL: # %bb.0:
690 ; AVX512VL-NEXT: vpmovzxwd {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
691 ; AVX512VL-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
692 ; AVX512VL-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
693 ; AVX512VL-NEXT: vpsllvd %ymm4, %ymm3, %ymm3
694 ; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
695 ; AVX512VL-NEXT: vpsubw %xmm2, %xmm4, %xmm4
696 ; AVX512VL-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
697 ; AVX512VL-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
698 ; AVX512VL-NEXT: vpsrlvd %ymm4, %ymm1, %ymm1
699 ; AVX512VL-NEXT: vpor %ymm1, %ymm3, %ymm1
700 ; AVX512VL-NEXT: vpmovdw %ymm1, %xmm1
701 ; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
702 ; AVX512VL-NEXT: vpcmpeqw %xmm3, %xmm2, %xmm2
703 ; AVX512VL-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
704 ; AVX512VL-NEXT: vzeroupper
705 ; AVX512VL-NEXT: retq
706 ;
707 ; AVX512BW-LABEL: var_funnnel_v8i16:
708 ; AVX512BW: # %bb.0:
709 ; AVX512BW-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
710 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
711 ; AVX512BW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
712 ; AVX512BW-NEXT: vpsllvw %zmm2, %zmm0, %zmm3
713 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
714 ; AVX512BW-NEXT: vpsubw %xmm2, %xmm4, %xmm4
715 ; AVX512BW-NEXT: vpsrlvw %zmm4, %zmm1, %zmm1
716 ; AVX512BW-NEXT: vpor %xmm1, %xmm3, %xmm1
717 ; AVX512BW-NEXT: vptestnmw %zmm2, %zmm2, %k1
718 ; AVX512BW-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1}
719 ; AVX512BW-NEXT: vmovdqa %xmm1, %xmm0
720 ; AVX512BW-NEXT: vzeroupper
721 ; AVX512BW-NEXT: retq
722 ;
723 ; AVX512VBMI2-LABEL: var_funnnel_v8i16:
724 ; AVX512VBMI2: # %bb.0:
725 ; AVX512VBMI2-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
726 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
727 ; AVX512VBMI2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
728 ; AVX512VBMI2-NEXT: vpsllvw %zmm2, %zmm0, %zmm3
729 ; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
730 ; AVX512VBMI2-NEXT: vpsubw %xmm2, %xmm4, %xmm4
731 ; AVX512VBMI2-NEXT: vpsrlvw %zmm4, %zmm1, %zmm1
732 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm3, %xmm1
733 ; AVX512VBMI2-NEXT: vptestnmw %zmm2, %zmm2, %k1
734 ; AVX512VBMI2-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1}
735 ; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0
736 ; AVX512VBMI2-NEXT: vzeroupper
737 ; AVX512VBMI2-NEXT: retq
738 ;
739 ; AVX512VLBW-LABEL: var_funnnel_v8i16:
740 ; AVX512VLBW: # %bb.0:
741 ; AVX512VLBW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
742 ; AVX512VLBW-NEXT: vpsllvw %xmm2, %xmm0, %xmm3
743 ; AVX512VLBW-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
744 ; AVX512VLBW-NEXT: vpsubw %xmm2, %xmm4, %xmm4
745 ; AVX512VLBW-NEXT: vpsrlvw %xmm4, %xmm1, %xmm1
746 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm3, %xmm1
747 ; AVX512VLBW-NEXT: vptestnmw %xmm2, %xmm2, %k1
748 ; AVX512VLBW-NEXT: vmovdqu16 %xmm0, %xmm1 {%k1}
749 ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0
750 ; AVX512VLBW-NEXT: retq
751 ;
752 ; AVX512VLVBMI2-LABEL: var_funnnel_v8i16:
753 ; AVX512VLVBMI2: # %bb.0:
754 ; AVX512VLVBMI2-NEXT: vpshldvw %xmm2, %xmm1, %xmm0
755 ; AVX512VLVBMI2-NEXT: retq
756 ;
757 ; XOP-LABEL: var_funnnel_v8i16:
758 ; XOP: # %bb.0:
759 ; XOP-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
760 ; XOP-NEXT: vpshlw %xmm2, %xmm0, %xmm3
761 ; XOP-NEXT: vpsubw {{.*}}(%rip), %xmm2, %xmm4
762 ; XOP-NEXT: vpshlw %xmm4, %xmm1, %xmm1
763 ; XOP-NEXT: vpor %xmm1, %xmm3, %xmm1
764 ; XOP-NEXT: vpxor %xmm3, %xmm3, %xmm3
765 ; XOP-NEXT: vpcomeqw %xmm3, %xmm2, %xmm2
766 ; XOP-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
767 ; XOP-NEXT: retq
768 ;
769 ; X32-SSE-LABEL: var_funnnel_v8i16:
770 ; X32-SSE: # %bb.0:
771 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm2
772 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
773 ; X32-SSE-NEXT: psubw %xmm2, %xmm3
774 ; X32-SSE-NEXT: psllw $12, %xmm3
775 ; X32-SSE-NEXT: movdqa %xmm3, %xmm4
776 ; X32-SSE-NEXT: psraw $15, %xmm4
777 ; X32-SSE-NEXT: movdqa %xmm4, %xmm5
778 ; X32-SSE-NEXT: pandn %xmm1, %xmm5
779 ; X32-SSE-NEXT: psrlw $8, %xmm1
780 ; X32-SSE-NEXT: pand %xmm4, %xmm1
781 ; X32-SSE-NEXT: por %xmm5, %xmm1
782 ; X32-SSE-NEXT: paddw %xmm3, %xmm3
783 ; X32-SSE-NEXT: movdqa %xmm3, %xmm4
784 ; X32-SSE-NEXT: psraw $15, %xmm4
785 ; X32-SSE-NEXT: movdqa %xmm4, %xmm5
786 ; X32-SSE-NEXT: pandn %xmm1, %xmm5
787 ; X32-SSE-NEXT: psrlw $4, %xmm1
788 ; X32-SSE-NEXT: pand %xmm4, %xmm1
789 ; X32-SSE-NEXT: por %xmm5, %xmm1
790 ; X32-SSE-NEXT: paddw %xmm3, %xmm3
791 ; X32-SSE-NEXT: movdqa %xmm3, %xmm4
792 ; X32-SSE-NEXT: psraw $15, %xmm4
793 ; X32-SSE-NEXT: movdqa %xmm4, %xmm5
794 ; X32-SSE-NEXT: pandn %xmm1, %xmm5
795 ; X32-SSE-NEXT: psrlw $2, %xmm1
796 ; X32-SSE-NEXT: pand %xmm4, %xmm1
797 ; X32-SSE-NEXT: por %xmm5, %xmm1
798 ; X32-SSE-NEXT: paddw %xmm3, %xmm3
799 ; X32-SSE-NEXT: psraw $15, %xmm3
800 ; X32-SSE-NEXT: movdqa %xmm3, %xmm4
801 ; X32-SSE-NEXT: pandn %xmm1, %xmm4
802 ; X32-SSE-NEXT: psrlw $1, %xmm1
803 ; X32-SSE-NEXT: pand %xmm3, %xmm1
804 ; X32-SSE-NEXT: pxor %xmm3, %xmm3
805 ; X32-SSE-NEXT: movdqa %xmm2, %xmm5
806 ; X32-SSE-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
807 ; X32-SSE-NEXT: pslld $23, %xmm5
808 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216]
809 ; X32-SSE-NEXT: paddd %xmm6, %xmm5
810 ; X32-SSE-NEXT: cvttps2dq %xmm5, %xmm5
811 ; X32-SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[0,2,2,3,4,5,6,7]
812 ; X32-SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,6,6,7]
813 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3]
814 ; X32-SSE-NEXT: movdqa %xmm2, %xmm7
815 ; X32-SSE-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3]
816 ; X32-SSE-NEXT: pslld $23, %xmm7
817 ; X32-SSE-NEXT: paddd %xmm6, %xmm7
818 ; X32-SSE-NEXT: cvttps2dq %xmm7, %xmm6
819 ; X32-SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,2,2,3,4,5,6,7]
820 ; X32-SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,6,6,7]
821 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,2,2,3]
822 ; X32-SSE-NEXT: punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm5[0]
823 ; X32-SSE-NEXT: pmullw %xmm0, %xmm6
824 ; X32-SSE-NEXT: por %xmm4, %xmm6
825 ; X32-SSE-NEXT: por %xmm1, %xmm6
826 ; X32-SSE-NEXT: pcmpeqw %xmm3, %xmm2
827 ; X32-SSE-NEXT: pand %xmm2, %xmm0
828 ; X32-SSE-NEXT: pandn %xmm6, %xmm2
829 ; X32-SSE-NEXT: por %xmm2, %xmm0
830 ; X32-SSE-NEXT: retl
831 %res = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
832 ret <8 x i16> %res
833 }
835 define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt) nounwind {
836 ; SSE2-LABEL: var_funnnel_v16i8:
837 ; SSE2: # %bb.0:
838 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm2
839 ; SSE2-NEXT: movdqa %xmm2, %xmm5
840 ; SSE2-NEXT: psllw $5, %xmm5
841 ; SSE2-NEXT: pxor %xmm4, %xmm4
842 ; SSE2-NEXT: pxor %xmm6, %xmm6
843 ; SSE2-NEXT: pcmpgtb %xmm5, %xmm6
844 ; SSE2-NEXT: movdqa %xmm0, %xmm3
845 ; SSE2-NEXT: psllw $4, %xmm3
846 ; SSE2-NEXT: pand %xmm6, %xmm3
847 ; SSE2-NEXT: pandn %xmm0, %xmm6
848 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm3
849 ; SSE2-NEXT: por %xmm6, %xmm3
850 ; SSE2-NEXT: paddb %xmm5, %xmm5
851 ; SSE2-NEXT: pxor %xmm6, %xmm6
852 ; SSE2-NEXT: pcmpgtb %xmm5, %xmm6
853 ; SSE2-NEXT: movdqa %xmm6, %xmm7
854 ; SSE2-NEXT: pandn %xmm3, %xmm7
855 ; SSE2-NEXT: psllw $2, %xmm3
856 ; SSE2-NEXT: pand %xmm6, %xmm3
857 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm3
858 ; SSE2-NEXT: por %xmm7, %xmm3
859 ; SSE2-NEXT: paddb %xmm5, %xmm5
860 ; SSE2-NEXT: pxor %xmm6, %xmm6
861 ; SSE2-NEXT: pcmpgtb %xmm5, %xmm6
862 ; SSE2-NEXT: movdqa %xmm6, %xmm8
863 ; SSE2-NEXT: pandn %xmm3, %xmm8
864 ; SSE2-NEXT: paddb %xmm3, %xmm3
865 ; SSE2-NEXT: pand %xmm6, %xmm3
866 ; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
867 ; SSE2-NEXT: psubb %xmm2, %xmm6
868 ; SSE2-NEXT: psllw $5, %xmm6
869 ; SSE2-NEXT: pxor %xmm7, %xmm7
870 ; SSE2-NEXT: pcmpgtb %xmm6, %xmm7
871 ; SSE2-NEXT: movdqa %xmm7, %xmm5
872 ; SSE2-NEXT: pandn %xmm1, %xmm5
873 ; SSE2-NEXT: psrlw $4, %xmm1
874 ; SSE2-NEXT: pand %xmm7, %xmm1
875 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm1
876 ; SSE2-NEXT: por %xmm5, %xmm1
877 ; SSE2-NEXT: paddb %xmm6, %xmm6
878 ; SSE2-NEXT: pxor %xmm5, %xmm5
879 ; SSE2-NEXT: pcmpgtb %xmm6, %xmm5
880 ; SSE2-NEXT: movdqa %xmm5, %xmm7
881 ; SSE2-NEXT: pandn %xmm1, %xmm7
882 ; SSE2-NEXT: psrlw $2, %xmm1
883 ; SSE2-NEXT: pand %xmm5, %xmm1
884 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm1
885 ; SSE2-NEXT: por %xmm7, %xmm1
886 ; SSE2-NEXT: paddb %xmm6, %xmm6
887 ; SSE2-NEXT: pcmpeqb %xmm4, %xmm2
888 ; SSE2-NEXT: pcmpgtb %xmm6, %xmm4
889 ; SSE2-NEXT: movdqa %xmm4, %xmm5
890 ; SSE2-NEXT: pandn %xmm1, %xmm5
891 ; SSE2-NEXT: psrlw $1, %xmm1
892 ; SSE2-NEXT: pand %xmm4, %xmm1
893 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm1
894 ; SSE2-NEXT: por %xmm5, %xmm1
895 ; SSE2-NEXT: por %xmm8, %xmm1
896 ; SSE2-NEXT: por %xmm3, %xmm1
897 ; SSE2-NEXT: pand %xmm2, %xmm0
898 ; SSE2-NEXT: pandn %xmm1, %xmm2
899 ; SSE2-NEXT: por %xmm2, %xmm0
900 ; SSE2-NEXT: retq
901 ;
902 ; SSE41-LABEL: var_funnnel_v16i8:
903 ; SSE41: # %bb.0:
904 ; SSE41-NEXT: movdqa %xmm0, %xmm3
905 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm2
906 ; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
907 ; SSE41-NEXT: psubb %xmm2, %xmm4
908 ; SSE41-NEXT: pxor %xmm5, %xmm5
909 ; SSE41-NEXT: pcmpeqb %xmm2, %xmm5
910 ; SSE41-NEXT: movdqa %xmm2, %xmm0
911 ; SSE41-NEXT: psllw $5, %xmm0
912 ; SSE41-NEXT: movdqa %xmm3, %xmm6
913 ; SSE41-NEXT: psllw $4, %xmm6
914 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm6
915 ; SSE41-NEXT: movdqa %xmm3, %xmm2
916 ; SSE41-NEXT: pblendvb %xmm0, %xmm6, %xmm2
917 ; SSE41-NEXT: movdqa %xmm2, %xmm6
918 ; SSE41-NEXT: psllw $2, %xmm6
919 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm6
920 ; SSE41-NEXT: paddb %xmm0, %xmm0
921 ; SSE41-NEXT: pblendvb %xmm0, %xmm6, %xmm2
922 ; SSE41-NEXT: movdqa %xmm2, %xmm6
923 ; SSE41-NEXT: paddb %xmm2, %xmm6
924 ; SSE41-NEXT: paddb %xmm0, %xmm0
925 ; SSE41-NEXT: pblendvb %xmm0, %xmm6, %xmm2
926 ; SSE41-NEXT: psllw $5, %xmm4
927 ; SSE41-NEXT: movdqa %xmm4, %xmm6
928 ; SSE41-NEXT: paddb %xmm4, %xmm6
929 ; SSE41-NEXT: movdqa %xmm1, %xmm7
930 ; SSE41-NEXT: psrlw $4, %xmm7
931 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm7
932 ; SSE41-NEXT: movdqa %xmm4, %xmm0
933 ; SSE41-NEXT: pblendvb %xmm0, %xmm7, %xmm1
934 ; SSE41-NEXT: movdqa %xmm1, %xmm4
935 ; SSE41-NEXT: psrlw $2, %xmm4
936 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm4
937 ; SSE41-NEXT: movdqa %xmm6, %xmm0
938 ; SSE41-NEXT: pblendvb %xmm0, %xmm4, %xmm1
939 ; SSE41-NEXT: movdqa %xmm1, %xmm4
940 ; SSE41-NEXT: psrlw $1, %xmm4
941 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm4
942 ; SSE41-NEXT: paddb %xmm6, %xmm6
943 ; SSE41-NEXT: movdqa %xmm6, %xmm0
944 ; SSE41-NEXT: pblendvb %xmm0, %xmm4, %xmm1
945 ; SSE41-NEXT: por %xmm1, %xmm2
946 ; SSE41-NEXT: movdqa %xmm5, %xmm0
947 ; SSE41-NEXT: pblendvb %xmm0, %xmm3, %xmm2
948 ; SSE41-NEXT: movdqa %xmm2, %xmm0
949 ; SSE41-NEXT: retq
950 ;
951 ; AVX-LABEL: var_funnnel_v16i8:
952 ; AVX: # %bb.0:
953 ; AVX-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
954 ; AVX-NEXT: vpsllw $5, %xmm2, %xmm3
955 ; AVX-NEXT: vpsllw $4, %xmm0, %xmm4
956 ; AVX-NEXT: vpand {{.*}}(%rip), %xmm4, %xmm4
957 ; AVX-NEXT: vpblendvb %xmm3, %xmm4, %xmm0, %xmm4
958 ; AVX-NEXT: vpsllw $2, %xmm4, %xmm5
959 ; AVX-NEXT: vpand {{.*}}(%rip), %xmm5, %xmm5
960 ; AVX-NEXT: vpaddb %xmm3, %xmm3, %xmm3
961 ; AVX-NEXT: vpblendvb %xmm3, %xmm5, %xmm4, %xmm4
962 ; AVX-NEXT: vpaddb %xmm4, %xmm4, %xmm5
963 ; AVX-NEXT: vpaddb %xmm3, %xmm3, %xmm3
964 ; AVX-NEXT: vpblendvb %xmm3, %xmm5, %xmm4, %xmm3
965 ; AVX-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
966 ; AVX-NEXT: vpsubb %xmm2, %xmm4, %xmm4
967 ; AVX-NEXT: vpsllw $5, %xmm4, %xmm4
968 ; AVX-NEXT: vpaddb %xmm4, %xmm4, %xmm5
969 ; AVX-NEXT: vpsrlw $4, %xmm1, %xmm6
970 ; AVX-NEXT: vpand {{.*}}(%rip), %xmm6, %xmm6
971 ; AVX-NEXT: vpblendvb %xmm4, %xmm6, %xmm1, %xmm1
972 ; AVX-NEXT: vpsrlw $2, %xmm1, %xmm4
973 ; AVX-NEXT: vpand {{.*}}(%rip), %xmm4, %xmm4
974 ; AVX-NEXT: vpblendvb %xmm5, %xmm4, %xmm1, %xmm1
975 ; AVX-NEXT: vpsrlw $1, %xmm1, %xmm4
976 ; AVX-NEXT: vpand {{.*}}(%rip), %xmm4, %xmm4
977 ; AVX-NEXT: vpaddb %xmm5, %xmm5, %xmm5
978 ; AVX-NEXT: vpblendvb %xmm5, %xmm4, %xmm1, %xmm1
979 ; AVX-NEXT: vpor %xmm1, %xmm3, %xmm1
980 ; AVX-NEXT: vpxor %xmm3, %xmm3, %xmm3
981 ; AVX-NEXT: vpcmpeqb %xmm3, %xmm2, %xmm2
982 ; AVX-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
983 ; AVX-NEXT: retq
984 ;
985 ; AVX512F-LABEL: var_funnnel_v16i8:
986 ; AVX512F: # %bb.0:
987 ; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
988 ; AVX512F-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
989 ; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
990 ; AVX512F-NEXT: vpsllvd %zmm4, %zmm3, %zmm3
991 ; AVX512F-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
992 ; AVX512F-NEXT: vpsubb %xmm2, %xmm4, %xmm4
993 ; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
994 ; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
995 ; AVX512F-NEXT: vpsrlvd %zmm4, %zmm1, %zmm1
996 ; AVX512F-NEXT: vpord %zmm1, %zmm3, %zmm1
997 ; AVX512F-NEXT: vpmovdb %zmm1, %xmm1
998 ; AVX512F-NEXT: vpxor %xmm3, %xmm3, %xmm3
999 ; AVX512F-NEXT: vpcmpeqb %xmm3, %xmm2, %xmm2
1000 ; AVX512F-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1001 ; AVX512F-NEXT: vzeroupper
1002 ; AVX512F-NEXT: retq
1003 ;
1004 ; AVX512VL-LABEL: var_funnnel_v16i8:
1005 ; AVX512VL: # %bb.0:
1006 ; AVX512VL-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
1007 ; AVX512VL-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1008 ; AVX512VL-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
1009 ; AVX512VL-NEXT: vpsllvd %zmm4, %zmm3, %zmm3
1010 ; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1011 ; AVX512VL-NEXT: vpsubb %xmm2, %xmm4, %xmm4
1012 ; AVX512VL-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
1013 ; AVX512VL-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
1014 ; AVX512VL-NEXT: vpsrlvd %zmm4, %zmm1, %zmm1
1015 ; AVX512VL-NEXT: vpord %zmm1, %zmm3, %zmm1
1016 ; AVX512VL-NEXT: vpmovdb %zmm1, %xmm1
1017 ; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
1018 ; AVX512VL-NEXT: vpcmpeqb %xmm3, %xmm2, %xmm2
1019 ; AVX512VL-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1020 ; AVX512VL-NEXT: vzeroupper
1021 ; AVX512VL-NEXT: retq
1022 ;
1023 ; AVX512BW-LABEL: var_funnnel_v16i8:
1024 ; AVX512BW: # %bb.0:
1025 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
1026 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1027 ; AVX512BW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1028 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
1029 ; AVX512BW-NEXT: vpsllvw %zmm4, %zmm3, %zmm3
1030 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1031 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1032 ; AVX512BW-NEXT: vpsubb %xmm2, %xmm4, %xmm4
1033 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
1034 ; AVX512BW-NEXT: vpsrlvw %zmm4, %zmm1, %zmm1
1035 ; AVX512BW-NEXT: vpor %ymm1, %ymm3, %ymm1
1036 ; AVX512BW-NEXT: vpmovwb %zmm1, %ymm1
1037 ; AVX512BW-NEXT: vptestnmb %zmm2, %zmm2, %k1
1038 ; AVX512BW-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1}
1039 ; AVX512BW-NEXT: vmovdqa %xmm1, %xmm0
1040 ; AVX512BW-NEXT: vzeroupper
1041 ; AVX512BW-NEXT: retq
1042 ;
1043 ; AVX512VBMI2-LABEL: var_funnnel_v16i8:
1044 ; AVX512VBMI2: # %bb.0:
1045 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
1046 ; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1047 ; AVX512VBMI2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1048 ; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
1049 ; AVX512VBMI2-NEXT: vpsllvw %zmm4, %zmm3, %zmm3
1050 ; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1051 ; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1052 ; AVX512VBMI2-NEXT: vpsubb %xmm2, %xmm4, %xmm4
1053 ; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
1054 ; AVX512VBMI2-NEXT: vpsrlvw %zmm4, %zmm1, %zmm1
1055 ; AVX512VBMI2-NEXT: vpor %ymm1, %ymm3, %ymm1
1056 ; AVX512VBMI2-NEXT: vpmovwb %zmm1, %ymm1
1057 ; AVX512VBMI2-NEXT: vptestnmb %zmm2, %zmm2, %k1
1058 ; AVX512VBMI2-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1}
1059 ; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0
1060 ; AVX512VBMI2-NEXT: vzeroupper
1061 ; AVX512VBMI2-NEXT: retq
1062 ;
1063 ; AVX512VLBW-LABEL: var_funnnel_v16i8:
1064 ; AVX512VLBW: # %bb.0:
1065 ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1066 ; AVX512VLBW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1067 ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
1068 ; AVX512VLBW-NEXT: vpsllvw %ymm4, %ymm3, %ymm3
1069 ; AVX512VLBW-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1070 ; AVX512VLBW-NEXT: vpsubb %xmm2, %xmm4, %xmm4
1071 ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
1072 ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1073 ; AVX512VLBW-NEXT: vpsrlvw %ymm4, %ymm1, %ymm1
1074 ; AVX512VLBW-NEXT: vpor %ymm1, %ymm3, %ymm1
1075 ; AVX512VLBW-NEXT: vpmovwb %ymm1, %xmm1
1076 ; AVX512VLBW-NEXT: vptestnmb %xmm2, %xmm2, %k1
1077 ; AVX512VLBW-NEXT: vmovdqu8 %xmm0, %xmm1 {%k1}
1078 ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0
1079 ; AVX512VLBW-NEXT: vzeroupper
1080 ; AVX512VLBW-NEXT: retq
1081 ;
1082 ; AVX512VLVBMI2-LABEL: var_funnnel_v16i8:
1083 ; AVX512VLVBMI2: # %bb.0:
1084 ; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
1085 ; AVX512VLVBMI2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1086 ; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
1087 ; AVX512VLVBMI2-NEXT: vpsllvw %ymm4, %ymm3, %ymm3
1088 ; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1089 ; AVX512VLVBMI2-NEXT: vpsubb %xmm2, %xmm4, %xmm4
1090 ; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
1091 ; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
1092 ; AVX512VLVBMI2-NEXT: vpsrlvw %ymm4, %ymm1, %ymm1
1093 ; AVX512VLVBMI2-NEXT: vpor %ymm1, %ymm3, %ymm1
1094 ; AVX512VLVBMI2-NEXT: vpmovwb %ymm1, %xmm1
1095 ; AVX512VLVBMI2-NEXT: vptestnmb %xmm2, %xmm2, %k1
1096 ; AVX512VLVBMI2-NEXT: vmovdqu8 %xmm0, %xmm1 {%k1}
1097 ; AVX512VLVBMI2-NEXT: vmovdqa %xmm1, %xmm0
1098 ; AVX512VLVBMI2-NEXT: vzeroupper
1099 ; AVX512VLVBMI2-NEXT: retq
1100 ;
1101 ; XOP-LABEL: var_funnnel_v16i8:
1102 ; XOP: # %bb.0:
1103 ; XOP-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1104 ; XOP-NEXT: vpshlb %xmm2, %xmm0, %xmm3
1105 ; XOP-NEXT: vpsubb {{.*}}(%rip), %xmm2, %xmm4
1106 ; XOP-NEXT: vpshlb %xmm4, %xmm1, %xmm1
1107 ; XOP-NEXT: vpor %xmm1, %xmm3, %xmm1
1108 ; XOP-NEXT: vpxor %xmm3, %xmm3, %xmm3
1109 ; XOP-NEXT: vpcomeqb %xmm3, %xmm2, %xmm2
1110 ; XOP-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1111 ; XOP-NEXT: retq
1112 ;
1113 ; X32-SSE-LABEL: var_funnnel_v16i8:
1114 ; X32-SSE: # %bb.0:
1115 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm2
1116 ; X32-SSE-NEXT: movdqa %xmm2, %xmm4
1117 ; X32-SSE-NEXT: psllw $5, %xmm4
1118 ; X32-SSE-NEXT: pxor %xmm5, %xmm5
1119 ; X32-SSE-NEXT: pcmpgtb %xmm4, %xmm5
1120 ; X32-SSE-NEXT: movdqa %xmm0, %xmm3
1121 ; X32-SSE-NEXT: psllw $4, %xmm3
1122 ; X32-SSE-NEXT: pand %xmm5, %xmm3
1123 ; X32-SSE-NEXT: pandn %xmm0, %xmm5
1124 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm3
1125 ; X32-SSE-NEXT: por %xmm5, %xmm3
1126 ; X32-SSE-NEXT: paddb %xmm4, %xmm4
1127 ; X32-SSE-NEXT: pxor %xmm5, %xmm5
1128 ; X32-SSE-NEXT: pcmpgtb %xmm4, %xmm5
1129 ; X32-SSE-NEXT: movdqa %xmm5, %xmm6
1130 ; X32-SSE-NEXT: pandn %xmm3, %xmm6
1131 ; X32-SSE-NEXT: psllw $2, %xmm3
1132 ; X32-SSE-NEXT: pand %xmm5, %xmm3
1133 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm3
1134 ; X32-SSE-NEXT: por %xmm6, %xmm3
1135 ; X32-SSE-NEXT: paddb %xmm4, %xmm4
1136 ; X32-SSE-NEXT: pxor %xmm5, %xmm5
1137 ; X32-SSE-NEXT: pcmpgtb %xmm4, %xmm5
1138 ; X32-SSE-NEXT: movdqa %xmm5, %xmm4
1139 ; X32-SSE-NEXT: pandn %xmm3, %xmm4
1140 ; X32-SSE-NEXT: paddb %xmm3, %xmm3
1141 ; X32-SSE-NEXT: pand %xmm5, %xmm3
1142 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm5 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1143 ; X32-SSE-NEXT: psubb %xmm2, %xmm5
1144 ; X32-SSE-NEXT: psllw $5, %xmm5
1145 ; X32-SSE-NEXT: pxor %xmm6, %xmm6
1146 ; X32-SSE-NEXT: pcmpgtb %xmm5, %xmm6
1147 ; X32-SSE-NEXT: movdqa %xmm6, %xmm7
1148 ; X32-SSE-NEXT: pandn %xmm1, %xmm7
1149 ; X32-SSE-NEXT: psrlw $4, %xmm1
1150 ; X32-SSE-NEXT: pand %xmm6, %xmm1
1151 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm1
1152 ; X32-SSE-NEXT: por %xmm7, %xmm1
1153 ; X32-SSE-NEXT: paddb %xmm5, %xmm5
1154 ; X32-SSE-NEXT: pxor %xmm6, %xmm6
1155 ; X32-SSE-NEXT: pcmpgtb %xmm5, %xmm6
1156 ; X32-SSE-NEXT: movdqa %xmm6, %xmm7
1157 ; X32-SSE-NEXT: pandn %xmm1, %xmm7
1158 ; X32-SSE-NEXT: psrlw $2, %xmm1
1159 ; X32-SSE-NEXT: pand %xmm6, %xmm1
1160 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm1
1161 ; X32-SSE-NEXT: por %xmm7, %xmm1
1162 ; X32-SSE-NEXT: pxor %xmm6, %xmm6
1163 ; X32-SSE-NEXT: paddb %xmm5, %xmm5
1164 ; X32-SSE-NEXT: pcmpeqb %xmm6, %xmm2
1165 ; X32-SSE-NEXT: pcmpgtb %xmm5, %xmm6
1166 ; X32-SSE-NEXT: movdqa %xmm6, %xmm5
1167 ; X32-SSE-NEXT: pandn %xmm1, %xmm5
1168 ; X32-SSE-NEXT: psrlw $1, %xmm1
1169 ; X32-SSE-NEXT: pand %xmm6, %xmm1
1170 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm1
1171 ; X32-SSE-NEXT: por %xmm5, %xmm1
1172 ; X32-SSE-NEXT: por %xmm4, %xmm1
1173 ; X32-SSE-NEXT: por %xmm3, %xmm1
1174 ; X32-SSE-NEXT: pand %xmm2, %xmm0
1175 ; X32-SSE-NEXT: pandn %xmm1, %xmm2
1176 ; X32-SSE-NEXT: por %xmm2, %xmm0
1177 ; X32-SSE-NEXT: retl
1178 %res = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt)
1179 ret <16 x i8> %res
1180 }
1182 ;
1183 ; Uniform Variable Shifts
1184 ;
1186 define <2 x i64> @splatvar_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt) nounwind {
1187 ; SSE2-LABEL: splatvar_funnnel_v2i64:
1188 ; SSE2: # %bb.0:
1189 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
1190 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm2
1191 ; SSE2-NEXT: movdqa %xmm0, %xmm3
1192 ; SSE2-NEXT: psllq %xmm2, %xmm3
1193 ; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [64,64]
1194 ; SSE2-NEXT: psubq %xmm2, %xmm4
1195 ; SSE2-NEXT: psrlq %xmm4, %xmm1
1196 ; SSE2-NEXT: por %xmm3, %xmm1
1197 ; SSE2-NEXT: pxor %xmm3, %xmm3
1198 ; SSE2-NEXT: pcmpeqd %xmm2, %xmm3
1199 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,0,3,2]
1200 ; SSE2-NEXT: pand %xmm3, %xmm2
1201 ; SSE2-NEXT: pand %xmm2, %xmm0
1202 ; SSE2-NEXT: pandn %xmm1, %xmm2
1203 ; SSE2-NEXT: por %xmm2, %xmm0
1204 ; SSE2-NEXT: retq
1205 ;
1206 ; SSE41-LABEL: splatvar_funnnel_v2i64:
1207 ; SSE41: # %bb.0:
1208 ; SSE41-NEXT: movdqa %xmm0, %xmm3
1209 ; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm2[0,1,0,1]
1210 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm4
1211 ; SSE41-NEXT: movdqa %xmm0, %xmm2
1212 ; SSE41-NEXT: psllq %xmm4, %xmm2
1213 ; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [64,64]
1214 ; SSE41-NEXT: psubq %xmm4, %xmm0
1215 ; SSE41-NEXT: psrlq %xmm0, %xmm1
1216 ; SSE41-NEXT: por %xmm1, %xmm2
1217 ; SSE41-NEXT: pxor %xmm0, %xmm0
1218 ; SSE41-NEXT: pcmpeqq %xmm4, %xmm0
1219 ; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm2
1220 ; SSE41-NEXT: movapd %xmm2, %xmm0
1221 ; SSE41-NEXT: retq
1222 ;
1223 ; AVX1-LABEL: splatvar_funnnel_v2i64:
1224 ; AVX1: # %bb.0:
1225 ; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
1226 ; AVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1227 ; AVX1-NEXT: vpsllq %xmm2, %xmm0, %xmm3
1228 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
1229 ; AVX1-NEXT: vpsubq %xmm2, %xmm4, %xmm4
1230 ; AVX1-NEXT: vpsrlq %xmm4, %xmm1, %xmm1
1231 ; AVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
1232 ; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
1233 ; AVX1-NEXT: vpcmpeqq %xmm3, %xmm2, %xmm2
1234 ; AVX1-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
1235 ; AVX1-NEXT: retq
1236 ;
1237 ; AVX2-LABEL: splatvar_funnnel_v2i64:
1238 ; AVX2: # %bb.0:
1239 ; AVX2-NEXT: vpbroadcastq %xmm2, %xmm2
1240 ; AVX2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1241 ; AVX2-NEXT: vpsllq %xmm2, %xmm0, %xmm3
1242 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
1243 ; AVX2-NEXT: vpsubq %xmm2, %xmm4, %xmm4
1244 ; AVX2-NEXT: vpsrlq %xmm4, %xmm1, %xmm1
1245 ; AVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
1246 ; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
1247 ; AVX2-NEXT: vpcmpeqq %xmm3, %xmm2, %xmm2
1248 ; AVX2-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
1249 ; AVX2-NEXT: retq
1250 ;
1251 ; AVX512F-LABEL: splatvar_funnnel_v2i64:
1252 ; AVX512F: # %bb.0:
1253 ; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
1254 ; AVX512F-NEXT: vpbroadcastq %xmm2, %xmm2
1255 ; AVX512F-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1256 ; AVX512F-NEXT: vpsllq %xmm2, %xmm0, %xmm3
1257 ; AVX512F-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
1258 ; AVX512F-NEXT: vpsubq %xmm2, %xmm4, %xmm4
1259 ; AVX512F-NEXT: vpsrlq %xmm4, %xmm1, %xmm1
1260 ; AVX512F-NEXT: vpor %xmm1, %xmm3, %xmm1
1261 ; AVX512F-NEXT: vptestnmq %zmm2, %zmm2, %k1
1262 ; AVX512F-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
1263 ; AVX512F-NEXT: vmovdqa %xmm1, %xmm0
1264 ; AVX512F-NEXT: vzeroupper
1265 ; AVX512F-NEXT: retq
1266 ;
1267 ; AVX512VL-LABEL: splatvar_funnnel_v2i64:
1268 ; AVX512VL: # %bb.0:
1269 ; AVX512VL-NEXT: vpbroadcastq %xmm2, %xmm2
1270 ; AVX512VL-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1271 ; AVX512VL-NEXT: vpsllq %xmm2, %xmm0, %xmm3
1272 ; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
1273 ; AVX512VL-NEXT: vpsubq %xmm2, %xmm4, %xmm4
1274 ; AVX512VL-NEXT: vpsrlq %xmm4, %xmm1, %xmm1
1275 ; AVX512VL-NEXT: vpor %xmm1, %xmm3, %xmm1
1276 ; AVX512VL-NEXT: vptestnmq %xmm2, %xmm2, %k1
1277 ; AVX512VL-NEXT: vmovdqa64 %xmm0, %xmm1 {%k1}
1278 ; AVX512VL-NEXT: vmovdqa %xmm1, %xmm0
1279 ; AVX512VL-NEXT: retq
1280 ;
1281 ; AVX512BW-LABEL: splatvar_funnnel_v2i64:
1282 ; AVX512BW: # %bb.0:
1283 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
1284 ; AVX512BW-NEXT: vpbroadcastq %xmm2, %xmm2
1285 ; AVX512BW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1286 ; AVX512BW-NEXT: vpsllq %xmm2, %xmm0, %xmm3
1287 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
1288 ; AVX512BW-NEXT: vpsubq %xmm2, %xmm4, %xmm4
1289 ; AVX512BW-NEXT: vpsrlq %xmm4, %xmm1, %xmm1
1290 ; AVX512BW-NEXT: vpor %xmm1, %xmm3, %xmm1
1291 ; AVX512BW-NEXT: vptestnmq %zmm2, %zmm2, %k1
1292 ; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
1293 ; AVX512BW-NEXT: vmovdqa %xmm1, %xmm0
1294 ; AVX512BW-NEXT: vzeroupper
1295 ; AVX512BW-NEXT: retq
1296 ;
1297 ; AVX512VBMI2-LABEL: splatvar_funnnel_v2i64:
1298 ; AVX512VBMI2: # %bb.0:
1299 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
1300 ; AVX512VBMI2-NEXT: vpbroadcastq %xmm2, %xmm2
1301 ; AVX512VBMI2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1302 ; AVX512VBMI2-NEXT: vpsllq %xmm2, %xmm0, %xmm3
1303 ; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
1304 ; AVX512VBMI2-NEXT: vpsubq %xmm2, %xmm4, %xmm4
1305 ; AVX512VBMI2-NEXT: vpsrlq %xmm4, %xmm1, %xmm1
1306 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm3, %xmm1
1307 ; AVX512VBMI2-NEXT: vptestnmq %zmm2, %zmm2, %k1
1308 ; AVX512VBMI2-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
1309 ; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0
1310 ; AVX512VBMI2-NEXT: vzeroupper
1311 ; AVX512VBMI2-NEXT: retq
1312 ;
1313 ; AVX512VLBW-LABEL: splatvar_funnnel_v2i64:
1314 ; AVX512VLBW: # %bb.0:
1315 ; AVX512VLBW-NEXT: vpbroadcastq %xmm2, %xmm2
1316 ; AVX512VLBW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1317 ; AVX512VLBW-NEXT: vpsllq %xmm2, %xmm0, %xmm3
1318 ; AVX512VLBW-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
1319 ; AVX512VLBW-NEXT: vpsubq %xmm2, %xmm4, %xmm4
1320 ; AVX512VLBW-NEXT: vpsrlq %xmm4, %xmm1, %xmm1
1321 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm3, %xmm1
1322 ; AVX512VLBW-NEXT: vptestnmq %xmm2, %xmm2, %k1
1323 ; AVX512VLBW-NEXT: vmovdqa64 %xmm0, %xmm1 {%k1}
1324 ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0
1325 ; AVX512VLBW-NEXT: retq
1326 ;
1327 ; AVX512VLVBMI2-LABEL: splatvar_funnnel_v2i64:
1328 ; AVX512VLVBMI2: # %bb.0:
1329 ; AVX512VLVBMI2-NEXT: vpbroadcastq %xmm2, %xmm2
1330 ; AVX512VLVBMI2-NEXT: vpshldvq %xmm2, %xmm1, %xmm0
1331 ; AVX512VLVBMI2-NEXT: retq
1332 ;
1333 ; XOPAVX1-LABEL: splatvar_funnnel_v2i64:
1334 ; XOPAVX1: # %bb.0:
1335 ; XOPAVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
1336 ; XOPAVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1337 ; XOPAVX1-NEXT: vpsllq %xmm2, %xmm0, %xmm3
1338 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
1339 ; XOPAVX1-NEXT: vpsubq %xmm2, %xmm4, %xmm4
1340 ; XOPAVX1-NEXT: vpsrlq %xmm4, %xmm1, %xmm1
1341 ; XOPAVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
1342 ; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
1343 ; XOPAVX1-NEXT: vpcomeqq %xmm3, %xmm2, %xmm2
1344 ; XOPAVX1-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
1345 ; XOPAVX1-NEXT: retq
1346 ;
1347 ; XOPAVX2-LABEL: splatvar_funnnel_v2i64:
1348 ; XOPAVX2: # %bb.0:
1349 ; XOPAVX2-NEXT: vpbroadcastq %xmm2, %xmm2
1350 ; XOPAVX2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1351 ; XOPAVX2-NEXT: vpsllq %xmm2, %xmm0, %xmm3
1352 ; XOPAVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [64,64]
1353 ; XOPAVX2-NEXT: vpsubq %xmm2, %xmm4, %xmm4
1354 ; XOPAVX2-NEXT: vpsrlq %xmm4, %xmm1, %xmm1
1355 ; XOPAVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
1356 ; XOPAVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
1357 ; XOPAVX2-NEXT: vpcomeqq %xmm3, %xmm2, %xmm2
1358 ; XOPAVX2-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
1359 ; XOPAVX2-NEXT: retq
1360 ;
1361 ; X32-SSE-LABEL: splatvar_funnnel_v2i64:
1362 ; X32-SSE: # %bb.0:
1363 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
1364 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm2
1365 ; X32-SSE-NEXT: movdqa %xmm0, %xmm3
1366 ; X32-SSE-NEXT: psllq %xmm2, %xmm3
1367 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm2[2,3,0,1]
1368 ; X32-SSE-NEXT: movdqa %xmm0, %xmm5
1369 ; X32-SSE-NEXT: psllq %xmm4, %xmm5
1370 ; X32-SSE-NEXT: movsd {{.*#+}} xmm5 = xmm3[0],xmm5[1]
1371 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [64,0,64,0]
1372 ; X32-SSE-NEXT: psubq %xmm2, %xmm3
1373 ; X32-SSE-NEXT: movdqa %xmm1, %xmm4
1374 ; X32-SSE-NEXT: psrlq %xmm3, %xmm4
1375 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
1376 ; X32-SSE-NEXT: psrlq %xmm3, %xmm1
1377 ; X32-SSE-NEXT: movsd {{.*#+}} xmm1 = xmm4[0],xmm1[1]
1378 ; X32-SSE-NEXT: orpd %xmm5, %xmm1
1379 ; X32-SSE-NEXT: pxor %xmm3, %xmm3
1380 ; X32-SSE-NEXT: pcmpeqd %xmm2, %xmm3
1381 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,0,3,2]
1382 ; X32-SSE-NEXT: pand %xmm3, %xmm2
1383 ; X32-SSE-NEXT: pand %xmm2, %xmm0
1384 ; X32-SSE-NEXT: pandn %xmm1, %xmm2
1385 ; X32-SSE-NEXT: por %xmm2, %xmm0
1386 ; X32-SSE-NEXT: retl
1387 %splat = shufflevector <2 x i64> %amt, <2 x i64> undef, <2 x i32> zeroinitializer
1388 %res = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %splat)
1389 ret <2 x i64> %res
1390 }
1392 define <4 x i32> @splatvar_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %amt) nounwind {
1393 ; SSE2-LABEL: splatvar_funnnel_v4i32:
1394 ; SSE2: # %bb.0:
1395 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
1396 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm2
1397 ; SSE2-NEXT: pxor %xmm3, %xmm3
1398 ; SSE2-NEXT: xorps %xmm4, %xmm4
1399 ; SSE2-NEXT: movss {{.*#+}} xmm4 = xmm2[0],xmm4[1,2,3]
1400 ; SSE2-NEXT: movdqa %xmm0, %xmm5
1401 ; SSE2-NEXT: pslld %xmm4, %xmm5
1402 ; SSE2-NEXT: movd %xmm2, %eax
1403 ; SSE2-NEXT: movl $32, %ecx
1404 ; SSE2-NEXT: subl %eax, %ecx
1405 ; SSE2-NEXT: movd %ecx, %xmm4
1406 ; SSE2-NEXT: psrld %xmm4, %xmm1
1407 ; SSE2-NEXT: por %xmm5, %xmm1
1408 ; SSE2-NEXT: pcmpeqd %xmm3, %xmm2
1409 ; SSE2-NEXT: pand %xmm2, %xmm0
1410 ; SSE2-NEXT: pandn %xmm1, %xmm2
1411 ; SSE2-NEXT: por %xmm2, %xmm0
1412 ; SSE2-NEXT: retq
1413 ;
1414 ; SSE41-LABEL: splatvar_funnnel_v4i32:
1415 ; SSE41: # %bb.0:
1416 ; SSE41-NEXT: movdqa %xmm0, %xmm3
1417 ; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm2[0,0,0,0]
1418 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm4
1419 ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm4[0],zero,xmm4[1],zero
1420 ; SSE41-NEXT: movdqa %xmm3, %xmm2
1421 ; SSE41-NEXT: pslld %xmm0, %xmm2
1422 ; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [32,32,32,32]
1423 ; SSE41-NEXT: psubd %xmm4, %xmm0
1424 ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
1425 ; SSE41-NEXT: psrld %xmm0, %xmm1
1426 ; SSE41-NEXT: por %xmm1, %xmm2
1427 ; SSE41-NEXT: pxor %xmm0, %xmm0
1428 ; SSE41-NEXT: pcmpeqd %xmm4, %xmm0
1429 ; SSE41-NEXT: blendvps %xmm0, %xmm3, %xmm2
1430 ; SSE41-NEXT: movaps %xmm2, %xmm0
1431 ; SSE41-NEXT: retq
1432 ;
1433 ; AVX1-LABEL: splatvar_funnnel_v4i32:
1434 ; AVX1: # %bb.0:
1435 ; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
1436 ; AVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1437 ; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
1438 ; AVX1-NEXT: vpslld %xmm3, %xmm0, %xmm3
1439 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [32,32,32,32]
1440 ; AVX1-NEXT: vpsubd %xmm2, %xmm4, %xmm4
1441 ; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
1442 ; AVX1-NEXT: vpsrld %xmm4, %xmm1, %xmm1
1443 ; AVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
1444 ; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
1445 ; AVX1-NEXT: vpcmpeqd %xmm3, %xmm2, %xmm2
1446 ; AVX1-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
1447 ; AVX1-NEXT: retq
1448 ;
1449 ; AVX2-LABEL: splatvar_funnnel_v4i32:
1450 ; AVX2: # %bb.0:
1451 ; AVX2-NEXT: vpbroadcastd %xmm2, %xmm2
1452 ; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
1453 ; AVX2-NEXT: vpand %xmm3, %xmm2, %xmm2
1454 ; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
1455 ; AVX2-NEXT: vpslld %xmm3, %xmm0, %xmm3
1456 ; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
1457 ; AVX2-NEXT: vpsubd %xmm2, %xmm4, %xmm4
1458 ; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
1459 ; AVX2-NEXT: vpsrld %xmm4, %xmm1, %xmm1
1460 ; AVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
1461 ; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
1462 ; AVX2-NEXT: vpcmpeqd %xmm3, %xmm2, %xmm2
1463 ; AVX2-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
1464 ; AVX2-NEXT: retq
1465 ;
1466 ; AVX512F-LABEL: splatvar_funnnel_v4i32:
1467 ; AVX512F: # %bb.0:
1468 ; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
1469 ; AVX512F-NEXT: vpbroadcastd %xmm2, %xmm2
1470 ; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
1471 ; AVX512F-NEXT: vpand %xmm3, %xmm2, %xmm2
1472 ; AVX512F-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
1473 ; AVX512F-NEXT: vpslld %xmm3, %xmm0, %xmm3
1474 ; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
1475 ; AVX512F-NEXT: vpsubd %xmm2, %xmm4, %xmm4
1476 ; AVX512F-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
1477 ; AVX512F-NEXT: vpsrld %xmm4, %xmm1, %xmm1
1478 ; AVX512F-NEXT: vpor %xmm1, %xmm3, %xmm1
1479 ; AVX512F-NEXT: vptestnmd %zmm2, %zmm2, %k1
1480 ; AVX512F-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1}
1481 ; AVX512F-NEXT: vmovdqa %xmm1, %xmm0
1482 ; AVX512F-NEXT: vzeroupper
1483 ; AVX512F-NEXT: retq
1484 ;
1485 ; AVX512VL-LABEL: splatvar_funnnel_v4i32:
1486 ; AVX512VL: # %bb.0:
1487 ; AVX512VL-NEXT: vpbroadcastd %xmm2, %xmm2
1488 ; AVX512VL-NEXT: vpandd {{.*}}(%rip){1to4}, %xmm2, %xmm2
1489 ; AVX512VL-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
1490 ; AVX512VL-NEXT: vpslld %xmm3, %xmm0, %xmm3
1491 ; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
1492 ; AVX512VL-NEXT: vpsubd %xmm2, %xmm4, %xmm4
1493 ; AVX512VL-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
1494 ; AVX512VL-NEXT: vpsrld %xmm4, %xmm1, %xmm1
1495 ; AVX512VL-NEXT: vpor %xmm1, %xmm3, %xmm1
1496 ; AVX512VL-NEXT: vptestnmd %xmm2, %xmm2, %k1
1497 ; AVX512VL-NEXT: vmovdqa32 %xmm0, %xmm1 {%k1}
1498 ; AVX512VL-NEXT: vmovdqa %xmm1, %xmm0
1499 ; AVX512VL-NEXT: retq
1500 ;
1501 ; AVX512BW-LABEL: splatvar_funnnel_v4i32:
1502 ; AVX512BW: # %bb.0:
1503 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
1504 ; AVX512BW-NEXT: vpbroadcastd %xmm2, %xmm2
1505 ; AVX512BW-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
1506 ; AVX512BW-NEXT: vpand %xmm3, %xmm2, %xmm2
1507 ; AVX512BW-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
1508 ; AVX512BW-NEXT: vpslld %xmm3, %xmm0, %xmm3
1509 ; AVX512BW-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
1510 ; AVX512BW-NEXT: vpsubd %xmm2, %xmm4, %xmm4
1511 ; AVX512BW-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
1512 ; AVX512BW-NEXT: vpsrld %xmm4, %xmm1, %xmm1
1513 ; AVX512BW-NEXT: vpor %xmm1, %xmm3, %xmm1
1514 ; AVX512BW-NEXT: vptestnmd %zmm2, %zmm2, %k1
1515 ; AVX512BW-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1}
1516 ; AVX512BW-NEXT: vmovdqa %xmm1, %xmm0
1517 ; AVX512BW-NEXT: vzeroupper
1518 ; AVX512BW-NEXT: retq
1519 ;
1520 ; AVX512VBMI2-LABEL: splatvar_funnnel_v4i32:
1521 ; AVX512VBMI2: # %bb.0:
1522 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
1523 ; AVX512VBMI2-NEXT: vpbroadcastd %xmm2, %xmm2
1524 ; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
1525 ; AVX512VBMI2-NEXT: vpand %xmm3, %xmm2, %xmm2
1526 ; AVX512VBMI2-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
1527 ; AVX512VBMI2-NEXT: vpslld %xmm3, %xmm0, %xmm3
1528 ; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
1529 ; AVX512VBMI2-NEXT: vpsubd %xmm2, %xmm4, %xmm4
1530 ; AVX512VBMI2-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
1531 ; AVX512VBMI2-NEXT: vpsrld %xmm4, %xmm1, %xmm1
1532 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm3, %xmm1
1533 ; AVX512VBMI2-NEXT: vptestnmd %zmm2, %zmm2, %k1
1534 ; AVX512VBMI2-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1}
1535 ; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0
1536 ; AVX512VBMI2-NEXT: vzeroupper
1537 ; AVX512VBMI2-NEXT: retq
1538 ;
1539 ; AVX512VLBW-LABEL: splatvar_funnnel_v4i32:
1540 ; AVX512VLBW: # %bb.0:
1541 ; AVX512VLBW-NEXT: vpbroadcastd %xmm2, %xmm2
1542 ; AVX512VLBW-NEXT: vpandd {{.*}}(%rip){1to4}, %xmm2, %xmm2
1543 ; AVX512VLBW-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
1544 ; AVX512VLBW-NEXT: vpslld %xmm3, %xmm0, %xmm3
1545 ; AVX512VLBW-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
1546 ; AVX512VLBW-NEXT: vpsubd %xmm2, %xmm4, %xmm4
1547 ; AVX512VLBW-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
1548 ; AVX512VLBW-NEXT: vpsrld %xmm4, %xmm1, %xmm1
1549 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm3, %xmm1
1550 ; AVX512VLBW-NEXT: vptestnmd %xmm2, %xmm2, %k1
1551 ; AVX512VLBW-NEXT: vmovdqa32 %xmm0, %xmm1 {%k1}
1552 ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0
1553 ; AVX512VLBW-NEXT: retq
1554 ;
1555 ; AVX512VLVBMI2-LABEL: splatvar_funnnel_v4i32:
1556 ; AVX512VLVBMI2: # %bb.0:
1557 ; AVX512VLVBMI2-NEXT: vpbroadcastd %xmm2, %xmm2
1558 ; AVX512VLVBMI2-NEXT: vpshldvd %xmm2, %xmm1, %xmm0
1559 ; AVX512VLVBMI2-NEXT: retq
1560 ;
1561 ; XOPAVX1-LABEL: splatvar_funnnel_v4i32:
1562 ; XOPAVX1: # %bb.0:
1563 ; XOPAVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
1564 ; XOPAVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1565 ; XOPAVX1-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
1566 ; XOPAVX1-NEXT: vpslld %xmm3, %xmm0, %xmm3
1567 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [32,32,32,32]
1568 ; XOPAVX1-NEXT: vpsubd %xmm2, %xmm4, %xmm4
1569 ; XOPAVX1-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
1570 ; XOPAVX1-NEXT: vpsrld %xmm4, %xmm1, %xmm1
1571 ; XOPAVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
1572 ; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
1573 ; XOPAVX1-NEXT: vpcomeqd %xmm3, %xmm2, %xmm2
1574 ; XOPAVX1-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
1575 ; XOPAVX1-NEXT: retq
1576 ;
1577 ; XOPAVX2-LABEL: splatvar_funnnel_v4i32:
1578 ; XOPAVX2: # %bb.0:
1579 ; XOPAVX2-NEXT: vpbroadcastd %xmm2, %xmm2
1580 ; XOPAVX2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
1581 ; XOPAVX2-NEXT: vpand %xmm3, %xmm2, %xmm2
1582 ; XOPAVX2-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
1583 ; XOPAVX2-NEXT: vpslld %xmm3, %xmm0, %xmm3
1584 ; XOPAVX2-NEXT: vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
1585 ; XOPAVX2-NEXT: vpsubd %xmm2, %xmm4, %xmm4
1586 ; XOPAVX2-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
1587 ; XOPAVX2-NEXT: vpsrld %xmm4, %xmm1, %xmm1
1588 ; XOPAVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
1589 ; XOPAVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
1590 ; XOPAVX2-NEXT: vpcomeqd %xmm3, %xmm2, %xmm2
1591 ; XOPAVX2-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
1592 ; XOPAVX2-NEXT: retq
1593 ;
1594 ; X32-SSE-LABEL: splatvar_funnnel_v4i32:
1595 ; X32-SSE: # %bb.0:
1596 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
1597 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm2
1598 ; X32-SSE-NEXT: pxor %xmm3, %xmm3
1599 ; X32-SSE-NEXT: xorps %xmm4, %xmm4
1600 ; X32-SSE-NEXT: movss {{.*#+}} xmm4 = xmm2[0],xmm4[1,2,3]
1601 ; X32-SSE-NEXT: movdqa %xmm0, %xmm5
1602 ; X32-SSE-NEXT: pslld %xmm4, %xmm5
1603 ; X32-SSE-NEXT: movd %xmm2, %eax
1604 ; X32-SSE-NEXT: movl $32, %ecx
1605 ; X32-SSE-NEXT: subl %eax, %ecx
1606 ; X32-SSE-NEXT: movd %ecx, %xmm4
1607 ; X32-SSE-NEXT: psrld %xmm4, %xmm1
1608 ; X32-SSE-NEXT: por %xmm5, %xmm1
1609 ; X32-SSE-NEXT: pcmpeqd %xmm3, %xmm2
1610 ; X32-SSE-NEXT: pand %xmm2, %xmm0
1611 ; X32-SSE-NEXT: pandn %xmm1, %xmm2
1612 ; X32-SSE-NEXT: por %xmm2, %xmm0
1613 ; X32-SSE-NEXT: retl
1614 %splat = shufflevector <4 x i32> %amt, <4 x i32> undef, <4 x i32> zeroinitializer
1615 %res = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %splat)
1616 ret <4 x i32> %res
1617 }
1619 define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt) nounwind {
1620 ; SSE2-LABEL: splatvar_funnnel_v8i16:
1621 ; SSE2: # %bb.0:
1622 ; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
1623 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
1624 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm2
1625 ; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
1626 ; SSE2-NEXT: psubw %xmm2, %xmm3
1627 ; SSE2-NEXT: pxor %xmm4, %xmm4
1628 ; SSE2-NEXT: pcmpeqw %xmm2, %xmm4
1629 ; SSE2-NEXT: pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1]
1630 ; SSE2-NEXT: psrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1631 ; SSE2-NEXT: movdqa %xmm0, %xmm5
1632 ; SSE2-NEXT: psllw %xmm2, %xmm5
1633 ; SSE2-NEXT: pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1]
1634 ; SSE2-NEXT: psrldq {{.*#+}} xmm3 = xmm3[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1635 ; SSE2-NEXT: psrlw %xmm3, %xmm1
1636 ; SSE2-NEXT: por %xmm5, %xmm1
1637 ; SSE2-NEXT: pand %xmm4, %xmm0
1638 ; SSE2-NEXT: pandn %xmm1, %xmm4
1639 ; SSE2-NEXT: por %xmm4, %xmm0
1640 ; SSE2-NEXT: retq
1641 ;
1642 ; SSE41-LABEL: splatvar_funnnel_v8i16:
1643 ; SSE41: # %bb.0:
1644 ; SSE41-NEXT: movdqa %xmm0, %xmm3
1645 ; SSE41-NEXT: pshuflw {{.*#+}} xmm0 = xmm2[0,0,2,3,4,5,6,7]
1646 ; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,0,0,0]
1647 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm4
1648 ; SSE41-NEXT: pmovzxwq {{.*#+}} xmm0 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
1649 ; SSE41-NEXT: movdqa %xmm3, %xmm2
1650 ; SSE41-NEXT: psllw %xmm0, %xmm2
1651 ; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [16,16,16,16,16,16,16,16]
1652 ; SSE41-NEXT: psubw %xmm4, %xmm0
1653 ; SSE41-NEXT: pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
1654 ; SSE41-NEXT: psrlw %xmm0, %xmm1
1655 ; SSE41-NEXT: por %xmm1, %xmm2
1656 ; SSE41-NEXT: pxor %xmm0, %xmm0
1657 ; SSE41-NEXT: pcmpeqw %xmm4, %xmm0
1658 ; SSE41-NEXT: pblendvb %xmm0, %xmm3, %xmm2
1659 ; SSE41-NEXT: movdqa %xmm2, %xmm0
1660 ; SSE41-NEXT: retq
1661 ;
1662 ; AVX1-LABEL: splatvar_funnnel_v8i16:
1663 ; AVX1: # %bb.0:
1664 ; AVX1-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
1665 ; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
1666 ; AVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1667 ; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
1668 ; AVX1-NEXT: vpsllw %xmm3, %xmm0, %xmm3
1669 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
1670 ; AVX1-NEXT: vpsubw %xmm2, %xmm4, %xmm4
1671 ; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
1672 ; AVX1-NEXT: vpsrlw %xmm4, %xmm1, %xmm1
1673 ; AVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
1674 ; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
1675 ; AVX1-NEXT: vpcmpeqw %xmm3, %xmm2, %xmm2
1676 ; AVX1-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1677 ; AVX1-NEXT: retq
1678 ;
1679 ; AVX2-LABEL: splatvar_funnnel_v8i16:
1680 ; AVX2: # %bb.0:
1681 ; AVX2-NEXT: vpbroadcastw %xmm2, %xmm2
1682 ; AVX2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1683 ; AVX2-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
1684 ; AVX2-NEXT: vpsllw %xmm3, %xmm0, %xmm3
1685 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
1686 ; AVX2-NEXT: vpsubw %xmm2, %xmm4, %xmm4
1687 ; AVX2-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
1688 ; AVX2-NEXT: vpsrlw %xmm4, %xmm1, %xmm1
1689 ; AVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
1690 ; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
1691 ; AVX2-NEXT: vpcmpeqw %xmm3, %xmm2, %xmm2
1692 ; AVX2-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1693 ; AVX2-NEXT: retq
1694 ;
1695 ; AVX512F-LABEL: splatvar_funnnel_v8i16:
1696 ; AVX512F: # %bb.0:
1697 ; AVX512F-NEXT: vpbroadcastw %xmm2, %xmm2
1698 ; AVX512F-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1699 ; AVX512F-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
1700 ; AVX512F-NEXT: vpsllw %xmm3, %xmm0, %xmm3
1701 ; AVX512F-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
1702 ; AVX512F-NEXT: vpsubw %xmm2, %xmm4, %xmm4
1703 ; AVX512F-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
1704 ; AVX512F-NEXT: vpsrlw %xmm4, %xmm1, %xmm1
1705 ; AVX512F-NEXT: vpor %xmm1, %xmm3, %xmm1
1706 ; AVX512F-NEXT: vpxor %xmm3, %xmm3, %xmm3
1707 ; AVX512F-NEXT: vpcmpeqw %xmm3, %xmm2, %xmm2
1708 ; AVX512F-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1709 ; AVX512F-NEXT: retq
1710 ;
1711 ; AVX512VL-LABEL: splatvar_funnnel_v8i16:
1712 ; AVX512VL: # %bb.0:
1713 ; AVX512VL-NEXT: vpbroadcastw %xmm2, %xmm2
1714 ; AVX512VL-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1715 ; AVX512VL-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
1716 ; AVX512VL-NEXT: vpsllw %xmm3, %xmm0, %xmm3
1717 ; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
1718 ; AVX512VL-NEXT: vpsubw %xmm2, %xmm4, %xmm4
1719 ; AVX512VL-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
1720 ; AVX512VL-NEXT: vpsrlw %xmm4, %xmm1, %xmm1
1721 ; AVX512VL-NEXT: vpor %xmm1, %xmm3, %xmm1
1722 ; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
1723 ; AVX512VL-NEXT: vpcmpeqw %xmm3, %xmm2, %xmm2
1724 ; AVX512VL-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1725 ; AVX512VL-NEXT: retq
1726 ;
1727 ; AVX512BW-LABEL: splatvar_funnnel_v8i16:
1728 ; AVX512BW: # %bb.0:
1729 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
1730 ; AVX512BW-NEXT: vpbroadcastw %xmm2, %xmm2
1731 ; AVX512BW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1732 ; AVX512BW-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
1733 ; AVX512BW-NEXT: vpsllw %xmm3, %xmm0, %xmm3
1734 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
1735 ; AVX512BW-NEXT: vpsubw %xmm2, %xmm4, %xmm4
1736 ; AVX512BW-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
1737 ; AVX512BW-NEXT: vpsrlw %xmm4, %xmm1, %xmm1
1738 ; AVX512BW-NEXT: vpor %xmm1, %xmm3, %xmm1
1739 ; AVX512BW-NEXT: vptestnmw %zmm2, %zmm2, %k1
1740 ; AVX512BW-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1}
1741 ; AVX512BW-NEXT: vmovdqa %xmm1, %xmm0
1742 ; AVX512BW-NEXT: vzeroupper
1743 ; AVX512BW-NEXT: retq
1744 ;
1745 ; AVX512VBMI2-LABEL: splatvar_funnnel_v8i16:
1746 ; AVX512VBMI2: # %bb.0:
1747 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
1748 ; AVX512VBMI2-NEXT: vpbroadcastw %xmm2, %xmm2
1749 ; AVX512VBMI2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1750 ; AVX512VBMI2-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
1751 ; AVX512VBMI2-NEXT: vpsllw %xmm3, %xmm0, %xmm3
1752 ; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
1753 ; AVX512VBMI2-NEXT: vpsubw %xmm2, %xmm4, %xmm4
1754 ; AVX512VBMI2-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
1755 ; AVX512VBMI2-NEXT: vpsrlw %xmm4, %xmm1, %xmm1
1756 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm3, %xmm1
1757 ; AVX512VBMI2-NEXT: vptestnmw %zmm2, %zmm2, %k1
1758 ; AVX512VBMI2-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1}
1759 ; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0
1760 ; AVX512VBMI2-NEXT: vzeroupper
1761 ; AVX512VBMI2-NEXT: retq
1762 ;
1763 ; AVX512VLBW-LABEL: splatvar_funnnel_v8i16:
1764 ; AVX512VLBW: # %bb.0:
1765 ; AVX512VLBW-NEXT: vpbroadcastw %xmm2, %xmm2
1766 ; AVX512VLBW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1767 ; AVX512VLBW-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
1768 ; AVX512VLBW-NEXT: vpsllw %xmm3, %xmm0, %xmm3
1769 ; AVX512VLBW-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
1770 ; AVX512VLBW-NEXT: vpsubw %xmm2, %xmm4, %xmm4
1771 ; AVX512VLBW-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
1772 ; AVX512VLBW-NEXT: vpsrlw %xmm4, %xmm1, %xmm1
1773 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm3, %xmm1
1774 ; AVX512VLBW-NEXT: vptestnmw %xmm2, %xmm2, %k1
1775 ; AVX512VLBW-NEXT: vmovdqu16 %xmm0, %xmm1 {%k1}
1776 ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0
1777 ; AVX512VLBW-NEXT: retq
1778 ;
1779 ; AVX512VLVBMI2-LABEL: splatvar_funnnel_v8i16:
1780 ; AVX512VLVBMI2: # %bb.0:
1781 ; AVX512VLVBMI2-NEXT: vpbroadcastw %xmm2, %xmm2
1782 ; AVX512VLVBMI2-NEXT: vpshldvw %xmm2, %xmm1, %xmm0
1783 ; AVX512VLVBMI2-NEXT: retq
1784 ;
1785 ; XOPAVX1-LABEL: splatvar_funnnel_v8i16:
1786 ; XOPAVX1: # %bb.0:
1787 ; XOPAVX1-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
1788 ; XOPAVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
1789 ; XOPAVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1790 ; XOPAVX1-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
1791 ; XOPAVX1-NEXT: vpsllw %xmm3, %xmm0, %xmm3
1792 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
1793 ; XOPAVX1-NEXT: vpsubw %xmm2, %xmm4, %xmm4
1794 ; XOPAVX1-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
1795 ; XOPAVX1-NEXT: vpsrlw %xmm4, %xmm1, %xmm1
1796 ; XOPAVX1-NEXT: vpor %xmm1, %xmm3, %xmm1
1797 ; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
1798 ; XOPAVX1-NEXT: vpcomeqw %xmm3, %xmm2, %xmm2
1799 ; XOPAVX1-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1800 ; XOPAVX1-NEXT: retq
1801 ;
1802 ; XOPAVX2-LABEL: splatvar_funnnel_v8i16:
1803 ; XOPAVX2: # %bb.0:
1804 ; XOPAVX2-NEXT: vpbroadcastw %xmm2, %xmm2
1805 ; XOPAVX2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1806 ; XOPAVX2-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
1807 ; XOPAVX2-NEXT: vpsllw %xmm3, %xmm0, %xmm3
1808 ; XOPAVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
1809 ; XOPAVX2-NEXT: vpsubw %xmm2, %xmm4, %xmm4
1810 ; XOPAVX2-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
1811 ; XOPAVX2-NEXT: vpsrlw %xmm4, %xmm1, %xmm1
1812 ; XOPAVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
1813 ; XOPAVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
1814 ; XOPAVX2-NEXT: vpcomeqw %xmm3, %xmm2, %xmm2
1815 ; XOPAVX2-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1816 ; XOPAVX2-NEXT: retq
1817 ;
1818 ; X32-SSE-LABEL: splatvar_funnnel_v8i16:
1819 ; X32-SSE: # %bb.0:
1820 ; X32-SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
1821 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
1822 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm2
1823 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
1824 ; X32-SSE-NEXT: psubw %xmm2, %xmm3
1825 ; X32-SSE-NEXT: pxor %xmm4, %xmm4
1826 ; X32-SSE-NEXT: pcmpeqw %xmm2, %xmm4
1827 ; X32-SSE-NEXT: pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1]
1828 ; X32-SSE-NEXT: psrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1829 ; X32-SSE-NEXT: movdqa %xmm0, %xmm5
1830 ; X32-SSE-NEXT: psllw %xmm2, %xmm5
1831 ; X32-SSE-NEXT: pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1]
1832 ; X32-SSE-NEXT: psrldq {{.*#+}} xmm3 = xmm3[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1833 ; X32-SSE-NEXT: psrlw %xmm3, %xmm1
1834 ; X32-SSE-NEXT: por %xmm5, %xmm1
1835 ; X32-SSE-NEXT: pand %xmm4, %xmm0
1836 ; X32-SSE-NEXT: pandn %xmm1, %xmm4
1837 ; X32-SSE-NEXT: por %xmm4, %xmm0
1838 ; X32-SSE-NEXT: retl
1839 %splat = shufflevector <8 x i16> %amt, <8 x i16> undef, <8 x i32> zeroinitializer
1840 %res = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %splat)
1841 ret <8 x i16> %res
1842 }
1844 define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt) nounwind {
1845 ; SSE2-LABEL: splatvar_funnnel_v16i8:
1846 ; SSE2: # %bb.0:
1847 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1848 ; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
1849 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
1850 ; SSE2-NEXT: pand {{.*}}(%rip), %xmm2
1851 ; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1852 ; SSE2-NEXT: psubb %xmm2, %xmm3
1853 ; SSE2-NEXT: pxor %xmm4, %xmm4
1854 ; SSE2-NEXT: pcmpeqb %xmm2, %xmm4
1855 ; SSE2-NEXT: pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0]
1856 ; SSE2-NEXT: psrldq {{.*#+}} xmm2 = xmm2[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1857 ; SSE2-NEXT: movdqa %xmm0, %xmm5
1858 ; SSE2-NEXT: psllw %xmm2, %xmm5
1859 ; SSE2-NEXT: pcmpeqd %xmm6, %xmm6
1860 ; SSE2-NEXT: psllw %xmm2, %xmm6
1861 ; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
1862 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1863 ; SSE2-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,0,2,3,4,5,6,7]
1864 ; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,0,0,0]
1865 ; SSE2-NEXT: pand %xmm5, %xmm6
1866 ; SSE2-NEXT: pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0]
1867 ; SSE2-NEXT: psrldq {{.*#+}} xmm3 = xmm3[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
1868 ; SSE2-NEXT: psrlw %xmm3, %xmm1
1869 ; SSE2-NEXT: psrlw %xmm3, %xmm2
1870 ; SSE2-NEXT: psrlw $8, %xmm2
1871 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
1872 ; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
1873 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
1874 ; SSE2-NEXT: pand %xmm1, %xmm2
1875 ; SSE2-NEXT: por %xmm6, %xmm2
1876 ; SSE2-NEXT: pand %xmm4, %xmm0
1877 ; SSE2-NEXT: pandn %xmm2, %xmm4
1878 ; SSE2-NEXT: por %xmm4, %xmm0
1879 ; SSE2-NEXT: retq
1880 ;
1881 ; SSE41-LABEL: splatvar_funnnel_v16i8:
1882 ; SSE41: # %bb.0:
1883 ; SSE41-NEXT: movdqa %xmm0, %xmm3
1884 ; SSE41-NEXT: pxor %xmm0, %xmm0
1885 ; SSE41-NEXT: pshufb %xmm0, %xmm2
1886 ; SSE41-NEXT: pand {{.*}}(%rip), %xmm2
1887 ; SSE41-NEXT: pmovzxbq {{.*#+}} xmm5 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
1888 ; SSE41-NEXT: movdqa %xmm3, %xmm4
1889 ; SSE41-NEXT: psllw %xmm5, %xmm4
1890 ; SSE41-NEXT: pcmpeqd %xmm6, %xmm6
1891 ; SSE41-NEXT: pcmpeqd %xmm7, %xmm7
1892 ; SSE41-NEXT: psllw %xmm5, %xmm7
1893 ; SSE41-NEXT: pshufb %xmm0, %xmm7
1894 ; SSE41-NEXT: pand %xmm7, %xmm4
1895 ; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1896 ; SSE41-NEXT: psubb %xmm2, %xmm5
1897 ; SSE41-NEXT: pmovzxbq {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,zero,zero,zero,zero,xmm5[1],zero,zero,zero,zero,zero,zero,zero
1898 ; SSE41-NEXT: psrlw %xmm5, %xmm1
1899 ; SSE41-NEXT: psrlw %xmm5, %xmm6
1900 ; SSE41-NEXT: pshufb {{.*#+}} xmm6 = xmm6[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
1901 ; SSE41-NEXT: pand %xmm1, %xmm6
1902 ; SSE41-NEXT: por %xmm6, %xmm4
1903 ; SSE41-NEXT: pcmpeqb %xmm2, %xmm0
1904 ; SSE41-NEXT: pblendvb %xmm0, %xmm3, %xmm4
1905 ; SSE41-NEXT: movdqa %xmm4, %xmm0
1906 ; SSE41-NEXT: retq
1907 ;
1908 ; AVX1-LABEL: splatvar_funnnel_v16i8:
1909 ; AVX1: # %bb.0:
1910 ; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
1911 ; AVX1-NEXT: vpshufb %xmm3, %xmm2, %xmm2
1912 ; AVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1913 ; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
1914 ; AVX1-NEXT: vpsllw %xmm4, %xmm0, %xmm5
1915 ; AVX1-NEXT: vpcmpeqd %xmm6, %xmm6, %xmm6
1916 ; AVX1-NEXT: vpsllw %xmm4, %xmm6, %xmm4
1917 ; AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm4
1918 ; AVX1-NEXT: vpand %xmm4, %xmm5, %xmm4
1919 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm5 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1920 ; AVX1-NEXT: vpsubb %xmm2, %xmm5, %xmm5
1921 ; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,zero,zero,zero,zero,xmm5[1],zero,zero,zero,zero,zero,zero,zero
1922 ; AVX1-NEXT: vpsrlw %xmm5, %xmm1, %xmm1
1923 ; AVX1-NEXT: vpsrlw %xmm5, %xmm6, %xmm5
1924 ; AVX1-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
1925 ; AVX1-NEXT: vpand %xmm5, %xmm1, %xmm1
1926 ; AVX1-NEXT: vpor %xmm1, %xmm4, %xmm1
1927 ; AVX1-NEXT: vpcmpeqb %xmm3, %xmm2, %xmm2
1928 ; AVX1-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1929 ; AVX1-NEXT: retq
1930 ;
1931 ; AVX2-LABEL: splatvar_funnnel_v16i8:
1932 ; AVX2: # %bb.0:
1933 ; AVX2-NEXT: vpbroadcastb %xmm2, %xmm2
1934 ; AVX2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1935 ; AVX2-NEXT: vpmovzxbq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
1936 ; AVX2-NEXT: vpsllw %xmm3, %xmm0, %xmm4
1937 ; AVX2-NEXT: vpcmpeqd %xmm5, %xmm5, %xmm5
1938 ; AVX2-NEXT: vpsllw %xmm3, %xmm5, %xmm3
1939 ; AVX2-NEXT: vpbroadcastb %xmm3, %xmm3
1940 ; AVX2-NEXT: vpand %xmm3, %xmm4, %xmm3
1941 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1942 ; AVX2-NEXT: vpsubb %xmm2, %xmm4, %xmm4
1943 ; AVX2-NEXT: vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
1944 ; AVX2-NEXT: vpsrlw %xmm4, %xmm1, %xmm1
1945 ; AVX2-NEXT: vpsrlw %xmm4, %xmm5, %xmm4
1946 ; AVX2-NEXT: vpsrlw $8, %xmm4, %xmm4
1947 ; AVX2-NEXT: vpbroadcastb %xmm4, %xmm4
1948 ; AVX2-NEXT: vpand %xmm4, %xmm1, %xmm1
1949 ; AVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
1950 ; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
1951 ; AVX2-NEXT: vpcmpeqb %xmm3, %xmm2, %xmm2
1952 ; AVX2-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1953 ; AVX2-NEXT: retq
1954 ;
1955 ; AVX512F-LABEL: splatvar_funnnel_v16i8:
1956 ; AVX512F: # %bb.0:
1957 ; AVX512F-NEXT: vpbroadcastb %xmm2, %xmm2
1958 ; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
1959 ; AVX512F-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1960 ; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
1961 ; AVX512F-NEXT: vpsllvd %zmm4, %zmm3, %zmm3
1962 ; AVX512F-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1963 ; AVX512F-NEXT: vpsubb %xmm2, %xmm4, %xmm4
1964 ; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
1965 ; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
1966 ; AVX512F-NEXT: vpsrlvd %zmm4, %zmm1, %zmm1
1967 ; AVX512F-NEXT: vpord %zmm1, %zmm3, %zmm1
1968 ; AVX512F-NEXT: vpmovdb %zmm1, %xmm1
1969 ; AVX512F-NEXT: vpxor %xmm3, %xmm3, %xmm3
1970 ; AVX512F-NEXT: vpcmpeqb %xmm3, %xmm2, %xmm2
1971 ; AVX512F-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1972 ; AVX512F-NEXT: vzeroupper
1973 ; AVX512F-NEXT: retq
1974 ;
1975 ; AVX512VL-LABEL: splatvar_funnnel_v16i8:
1976 ; AVX512VL: # %bb.0:
1977 ; AVX512VL-NEXT: vpbroadcastb %xmm2, %xmm2
1978 ; AVX512VL-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
1979 ; AVX512VL-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
1980 ; AVX512VL-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
1981 ; AVX512VL-NEXT: vpsllvd %zmm4, %zmm3, %zmm3
1982 ; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
1983 ; AVX512VL-NEXT: vpsubb %xmm2, %xmm4, %xmm4
1984 ; AVX512VL-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
1985 ; AVX512VL-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
1986 ; AVX512VL-NEXT: vpsrlvd %zmm4, %zmm1, %zmm1
1987 ; AVX512VL-NEXT: vpord %zmm1, %zmm3, %zmm1
1988 ; AVX512VL-NEXT: vpmovdb %zmm1, %xmm1
1989 ; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
1990 ; AVX512VL-NEXT: vpcmpeqb %xmm3, %xmm2, %xmm2
1991 ; AVX512VL-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
1992 ; AVX512VL-NEXT: vzeroupper
1993 ; AVX512VL-NEXT: retq
1994 ;
1995 ; AVX512BW-LABEL: splatvar_funnnel_v16i8:
1996 ; AVX512BW: # %bb.0:
1997 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
1998 ; AVX512BW-NEXT: vpbroadcastb %xmm2, %xmm2
1999 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
2000 ; AVX512BW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
2001 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
2002 ; AVX512BW-NEXT: vpsllvw %zmm4, %zmm3, %zmm3
2003 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
2004 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
2005 ; AVX512BW-NEXT: vpsubb %xmm2, %xmm4, %xmm4
2006 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
2007 ; AVX512BW-NEXT: vpsrlvw %zmm4, %zmm1, %zmm1
2008 ; AVX512BW-NEXT: vpor %ymm1, %ymm3, %ymm1
2009 ; AVX512BW-NEXT: vpmovwb %zmm1, %ymm1
2010 ; AVX512BW-NEXT: vptestnmb %zmm2, %zmm2, %k1
2011 ; AVX512BW-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1}
2012 ; AVX512BW-NEXT: vmovdqa %xmm1, %xmm0
2013 ; AVX512BW-NEXT: vzeroupper
2014 ; AVX512BW-NEXT: retq
2015 ;
2016 ; AVX512VBMI2-LABEL: splatvar_funnnel_v16i8:
2017 ; AVX512VBMI2: # %bb.0:
2018 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
2019 ; AVX512VBMI2-NEXT: vpbroadcastb %xmm2, %xmm2
2020 ; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
2021 ; AVX512VBMI2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
2022 ; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
2023 ; AVX512VBMI2-NEXT: vpsllvw %zmm4, %zmm3, %zmm3
2024 ; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
2025 ; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
2026 ; AVX512VBMI2-NEXT: vpsubb %xmm2, %xmm4, %xmm4
2027 ; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
2028 ; AVX512VBMI2-NEXT: vpsrlvw %zmm4, %zmm1, %zmm1
2029 ; AVX512VBMI2-NEXT: vpor %ymm1, %ymm3, %ymm1
2030 ; AVX512VBMI2-NEXT: vpmovwb %zmm1, %ymm1
2031 ; AVX512VBMI2-NEXT: vptestnmb %zmm2, %zmm2, %k1
2032 ; AVX512VBMI2-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1}
2033 ; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0
2034 ; AVX512VBMI2-NEXT: vzeroupper
2035 ; AVX512VBMI2-NEXT: retq
2036 ;
2037 ; AVX512VLBW-LABEL: splatvar_funnnel_v16i8:
2038 ; AVX512VLBW: # %bb.0:
2039 ; AVX512VLBW-NEXT: vpbroadcastb %xmm2, %xmm2
2040 ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
2041 ; AVX512VLBW-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
2042 ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
2043 ; AVX512VLBW-NEXT: vpsllvw %ymm4, %ymm3, %ymm3
2044 ; AVX512VLBW-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
2045 ; AVX512VLBW-NEXT: vpsubb %xmm2, %xmm4, %xmm4
2046 ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
2047 ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
2048 ; AVX512VLBW-NEXT: vpsrlvw %ymm4, %ymm1, %ymm1
2049 ; AVX512VLBW-NEXT: vpor %ymm1, %ymm3, %ymm1
2050 ; AVX512VLBW-NEXT: vpmovwb %ymm1, %xmm1
2051 ; AVX512VLBW-NEXT: vptestnmb %xmm2, %xmm2, %k1
2052 ; AVX512VLBW-NEXT: vmovdqu8 %xmm0, %xmm1 {%k1}
2053 ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0
2054 ; AVX512VLBW-NEXT: vzeroupper
2055 ; AVX512VLBW-NEXT: retq
2056 ;
2057 ; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i8:
2058 ; AVX512VLVBMI2: # %bb.0:
2059 ; AVX512VLVBMI2-NEXT: vpbroadcastb %xmm2, %xmm2
2060 ; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
2061 ; AVX512VLVBMI2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
2062 ; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
2063 ; AVX512VLVBMI2-NEXT: vpsllvw %ymm4, %ymm3, %ymm3
2064 ; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
2065 ; AVX512VLVBMI2-NEXT: vpsubb %xmm2, %xmm4, %xmm4
2066 ; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
2067 ; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
2068 ; AVX512VLVBMI2-NEXT: vpsrlvw %ymm4, %ymm1, %ymm1
2069 ; AVX512VLVBMI2-NEXT: vpor %ymm1, %ymm3, %ymm1
2070 ; AVX512VLVBMI2-NEXT: vpmovwb %ymm1, %xmm1
2071 ; AVX512VLVBMI2-NEXT: vptestnmb %xmm2, %xmm2, %k1
2072 ; AVX512VLVBMI2-NEXT: vmovdqu8 %xmm0, %xmm1 {%k1}
2073 ; AVX512VLVBMI2-NEXT: vmovdqa %xmm1, %xmm0
2074 ; AVX512VLVBMI2-NEXT: vzeroupper
2075 ; AVX512VLVBMI2-NEXT: retq
2076 ;
2077 ; XOPAVX1-LABEL: splatvar_funnnel_v16i8:
2078 ; XOPAVX1: # %bb.0:
2079 ; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
2080 ; XOPAVX1-NEXT: vpshufb %xmm3, %xmm2, %xmm2
2081 ; XOPAVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
2082 ; XOPAVX1-NEXT: vpshlb %xmm2, %xmm0, %xmm4
2083 ; XOPAVX1-NEXT: vpsubb {{.*}}(%rip), %xmm2, %xmm5
2084 ; XOPAVX1-NEXT: vpshlb %xmm5, %xmm1, %xmm1
2085 ; XOPAVX1-NEXT: vpor %xmm1, %xmm4, %xmm1
2086 ; XOPAVX1-NEXT: vpcomeqb %xmm3, %xmm2, %xmm2
2087 ; XOPAVX1-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
2088 ; XOPAVX1-NEXT: retq
2089 ;
2090 ; XOPAVX2-LABEL: splatvar_funnnel_v16i8:
2091 ; XOPAVX2: # %bb.0:
2092 ; XOPAVX2-NEXT: vpbroadcastb %xmm2, %xmm2
2093 ; XOPAVX2-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2
2094 ; XOPAVX2-NEXT: vpshlb %xmm2, %xmm0, %xmm3
2095 ; XOPAVX2-NEXT: vpsubb {{.*}}(%rip), %xmm2, %xmm4
2096 ; XOPAVX2-NEXT: vpshlb %xmm4, %xmm1, %xmm1
2097 ; XOPAVX2-NEXT: vpor %xmm1, %xmm3, %xmm1
2098 ; XOPAVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
2099 ; XOPAVX2-NEXT: vpcomeqb %xmm3, %xmm2, %xmm2
2100 ; XOPAVX2-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
2101 ; XOPAVX2-NEXT: retq
2102 ;
2103 ; X32-SSE-LABEL: splatvar_funnnel_v16i8:
2104 ; X32-SSE: # %bb.0:
2105 ; X32-SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2106 ; X32-SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
2107 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
2108 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm2
2109 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
2110 ; X32-SSE-NEXT: psubb %xmm2, %xmm3
2111 ; X32-SSE-NEXT: pxor %xmm4, %xmm4
2112 ; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm4
2113 ; X32-SSE-NEXT: pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0]
2114 ; X32-SSE-NEXT: psrldq {{.*#+}} xmm2 = xmm2[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
2115 ; X32-SSE-NEXT: movdqa %xmm0, %xmm5
2116 ; X32-SSE-NEXT: psllw %xmm2, %xmm5
2117 ; X32-SSE-NEXT: pcmpeqd %xmm6, %xmm6
2118 ; X32-SSE-NEXT: psllw %xmm2, %xmm6
2119 ; X32-SSE-NEXT: pcmpeqd %xmm2, %xmm2
2120 ; X32-SSE-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2121 ; X32-SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,0,2,3,4,5,6,7]
2122 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,0,0,0]
2123 ; X32-SSE-NEXT: pand %xmm5, %xmm6
2124 ; X32-SSE-NEXT: pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0]
2125 ; X32-SSE-NEXT: psrldq {{.*#+}} xmm3 = xmm3[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
2126 ; X32-SSE-NEXT: psrlw %xmm3, %xmm1
2127 ; X32-SSE-NEXT: psrlw %xmm3, %xmm2
2128 ; X32-SSE-NEXT: psrlw $8, %xmm2
2129 ; X32-SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2130 ; X32-SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
2131 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
2132 ; X32-SSE-NEXT: pand %xmm1, %xmm2
2133 ; X32-SSE-NEXT: por %xmm6, %xmm2
2134 ; X32-SSE-NEXT: pand %xmm4, %xmm0
2135 ; X32-SSE-NEXT: pandn %xmm2, %xmm4
2136 ; X32-SSE-NEXT: por %xmm4, %xmm0
2137 ; X32-SSE-NEXT: retl
2138 %splat = shufflevector <16 x i8> %amt, <16 x i8> undef, <16 x i32> zeroinitializer
2139 %res = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %splat)
2140 ret <16 x i8> %res
2141 }
2143 ;
2144 ; Constant Shifts
2145 ;
2147 define <2 x i64> @constant_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
2148 ; SSE2-LABEL: constant_funnnel_v2i64:
2149 ; SSE2: # %bb.0:
2150 ; SSE2-NEXT: movdqa %xmm1, %xmm2
2151 ; SSE2-NEXT: psrlq $60, %xmm2
2152 ; SSE2-NEXT: psrlq $50, %xmm1
2153 ; SSE2-NEXT: movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
2154 ; SSE2-NEXT: movdqa %xmm0, %xmm2
2155 ; SSE2-NEXT: psllq $4, %xmm2
2156 ; SSE2-NEXT: psllq $14, %xmm0
2157 ; SSE2-NEXT: movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
2158 ; SSE2-NEXT: orpd %xmm1, %xmm0
2159 ; SSE2-NEXT: retq
2160 ;
2161 ; SSE41-LABEL: constant_funnnel_v2i64:
2162 ; SSE41: # %bb.0:
2163 ; SSE41-NEXT: movdqa %xmm1, %xmm2
2164 ; SSE41-NEXT: psrlq $50, %xmm2
2165 ; SSE41-NEXT: psrlq $60, %xmm1
2166 ; SSE41-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
2167 ; SSE41-NEXT: movdqa %xmm0, %xmm2
2168 ; SSE41-NEXT: psllq $14, %xmm2
2169 ; SSE41-NEXT: psllq $4, %xmm0
2170 ; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
2171 ; SSE41-NEXT: por %xmm1, %xmm0
2172 ; SSE41-NEXT: retq
2173 ;
2174 ; AVX1-LABEL: constant_funnnel_v2i64:
2175 ; AVX1: # %bb.0:
2176 ; AVX1-NEXT: vpsrlq $50, %xmm1, %xmm2
2177 ; AVX1-NEXT: vpsrlq $60, %xmm1, %xmm1
2178 ; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
2179 ; AVX1-NEXT: vpsllq $14, %xmm0, %xmm2
2180 ; AVX1-NEXT: vpsllq $4, %xmm0, %xmm0
2181 ; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
2182 ; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0
2183 ; AVX1-NEXT: retq
2184 ;
2185 ; AVX2-LABEL: constant_funnnel_v2i64:
2186 ; AVX2: # %bb.0:
2187 ; AVX2-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
2188 ; AVX2-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0
2189 ; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0
2190 ; AVX2-NEXT: retq
2191 ;
2192 ; AVX512F-LABEL: constant_funnnel_v2i64:
2193 ; AVX512F: # %bb.0:
2194 ; AVX512F-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
2195 ; AVX512F-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0
2196 ; AVX512F-NEXT: vpor %xmm1, %xmm0, %xmm0
2197 ; AVX512F-NEXT: retq
2198 ;
2199 ; AVX512VL-LABEL: constant_funnnel_v2i64:
2200 ; AVX512VL: # %bb.0:
2201 ; AVX512VL-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
2202 ; AVX512VL-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0
2203 ; AVX512VL-NEXT: vpor %xmm1, %xmm0, %xmm0
2204 ; AVX512VL-NEXT: retq
2205 ;
2206 ; AVX512BW-LABEL: constant_funnnel_v2i64:
2207 ; AVX512BW: # %bb.0:
2208 ; AVX512BW-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
2209 ; AVX512BW-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0
2210 ; AVX512BW-NEXT: vpor %xmm1, %xmm0, %xmm0
2211 ; AVX512BW-NEXT: retq
2212 ;
2213 ; AVX512VBMI2-LABEL: constant_funnnel_v2i64:
2214 ; AVX512VBMI2: # %bb.0:
2215 ; AVX512VBMI2-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
2216 ; AVX512VBMI2-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0
2217 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm0, %xmm0
2218 ; AVX512VBMI2-NEXT: retq
2219 ;
2220 ; AVX512VLBW-LABEL: constant_funnnel_v2i64:
2221 ; AVX512VLBW: # %bb.0:
2222 ; AVX512VLBW-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
2223 ; AVX512VLBW-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0
2224 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm0, %xmm0
2225 ; AVX512VLBW-NEXT: retq
2226 ;
2227 ; AVX512VLVBMI2-LABEL: constant_funnnel_v2i64:
2228 ; AVX512VLVBMI2: # %bb.0:
2229 ; AVX512VLVBMI2-NEXT: vpshldvq {{.*}}(%rip), %xmm1, %xmm0
2230 ; AVX512VLVBMI2-NEXT: retq
2231 ;
2232 ; XOPAVX1-LABEL: constant_funnnel_v2i64:
2233 ; XOPAVX1: # %bb.0:
2234 ; XOPAVX1-NEXT: vpshlq {{.*}}(%rip), %xmm1, %xmm1
2235 ; XOPAVX1-NEXT: vpshlq {{.*}}(%rip), %xmm0, %xmm0
2236 ; XOPAVX1-NEXT: vpor %xmm1, %xmm0, %xmm0
2237 ; XOPAVX1-NEXT: retq
2238 ;
2239 ; XOPAVX2-LABEL: constant_funnnel_v2i64:
2240 ; XOPAVX2: # %bb.0:
2241 ; XOPAVX2-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
2242 ; XOPAVX2-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0
2243 ; XOPAVX2-NEXT: vpor %xmm1, %xmm0, %xmm0
2244 ; XOPAVX2-NEXT: retq
2245 ;
2246 ; X32-SSE-LABEL: constant_funnnel_v2i64:
2247 ; X32-SSE: # %bb.0:
2248 ; X32-SSE-NEXT: movdqa %xmm1, %xmm2
2249 ; X32-SSE-NEXT: psrlq $60, %xmm2
2250 ; X32-SSE-NEXT: psrlq $50, %xmm1
2251 ; X32-SSE-NEXT: movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
2252 ; X32-SSE-NEXT: movdqa %xmm0, %xmm2
2253 ; X32-SSE-NEXT: psllq $4, %xmm2
2254 ; X32-SSE-NEXT: psllq $14, %xmm0
2255 ; X32-SSE-NEXT: movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
2256 ; X32-SSE-NEXT: orpd %xmm1, %xmm0
2257 ; X32-SSE-NEXT: retl
2258 %res = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> <i64 4, i64 14>)
2259 ret <2 x i64> %res
2260 }
2262 define <4 x i32> @constant_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
2263 ; SSE2-LABEL: constant_funnnel_v4i32:
2264 ; SSE2: # %bb.0:
2265 ; SSE2-NEXT: movdqa %xmm1, %xmm2
2266 ; SSE2-NEXT: psrld $25, %xmm2
2267 ; SSE2-NEXT: movdqa %xmm1, %xmm3
2268 ; SSE2-NEXT: psrld $26, %xmm3
2269 ; SSE2-NEXT: punpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm2[1]
2270 ; SSE2-NEXT: movdqa %xmm1, %xmm2
2271 ; SSE2-NEXT: psrld $27, %xmm2
2272 ; SSE2-NEXT: psrld $28, %xmm1
2273 ; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
2274 ; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,3],xmm3[0,3]
2275 ; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [16,32,64,128]
2276 ; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
2277 ; SSE2-NEXT: pmuludq %xmm2, %xmm0
2278 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2279 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
2280 ; SSE2-NEXT: pmuludq %xmm3, %xmm2
2281 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
2282 ; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
2283 ; SSE2-NEXT: por %xmm1, %xmm0
2284 ; SSE2-NEXT: retq
2285 ;
2286 ; SSE41-LABEL: constant_funnnel_v4i32:
2287 ; SSE41: # %bb.0:
2288 ; SSE41-NEXT: movdqa %xmm1, %xmm2
2289 ; SSE41-NEXT: psrld $25, %xmm2
2290 ; SSE41-NEXT: movdqa %xmm1, %xmm3
2291 ; SSE41-NEXT: psrld $27, %xmm3
2292 ; SSE41-NEXT: pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm2[4,5,6,7]
2293 ; SSE41-NEXT: movdqa %xmm1, %xmm2
2294 ; SSE41-NEXT: psrld $26, %xmm2
2295 ; SSE41-NEXT: psrld $28, %xmm1
2296 ; SSE41-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
2297 ; SSE41-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
2298 ; SSE41-NEXT: pmulld {{.*}}(%rip), %xmm0
2299 ; SSE41-NEXT: por %xmm1, %xmm0
2300 ; SSE41-NEXT: retq
2301 ;
2302 ; AVX1-LABEL: constant_funnnel_v4i32:
2303 ; AVX1: # %bb.0:
2304 ; AVX1-NEXT: vpsrld $25, %xmm1, %xmm2
2305 ; AVX1-NEXT: vpsrld $27, %xmm1, %xmm3
2306 ; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
2307 ; AVX1-NEXT: vpsrld $26, %xmm1, %xmm3
2308 ; AVX1-NEXT: vpsrld $28, %xmm1, %xmm1
2309 ; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5,6,7]
2310 ; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
2311 ; AVX1-NEXT: vpmulld {{.*}}(%rip), %xmm0, %xmm0
2312 ; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0
2313 ; AVX1-NEXT: retq
2314 ;
2315 ; AVX2-LABEL: constant_funnnel_v4i32:
2316 ; AVX2: # %bb.0:
2317 ; AVX2-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
2318 ; AVX2-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0
2319 ; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0
2320 ; AVX2-NEXT: retq
2321 ;
2322 ; AVX512F-LABEL: constant_funnnel_v4i32:
2323 ; AVX512F: # %bb.0:
2324 ; AVX512F-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
2325 ; AVX512F-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0
2326 ; AVX512F-NEXT: vpor %xmm1, %xmm0, %xmm0
2327 ; AVX512F-NEXT: retq
2328 ;
2329 ; AVX512VL-LABEL: constant_funnnel_v4i32:
2330 ; AVX512VL: # %bb.0:
2331 ; AVX512VL-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
2332 ; AVX512VL-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0
2333 ; AVX512VL-NEXT: vpor %xmm1, %xmm0, %xmm0
2334 ; AVX512VL-NEXT: retq
2335 ;
2336 ; AVX512BW-LABEL: constant_funnnel_v4i32:
2337 ; AVX512BW: # %bb.0:
2338 ; AVX512BW-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
2339 ; AVX512BW-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0
2340 ; AVX512BW-NEXT: vpor %xmm1, %xmm0, %xmm0
2341 ; AVX512BW-NEXT: retq
2342 ;
2343 ; AVX512VBMI2-LABEL: constant_funnnel_v4i32:
2344 ; AVX512VBMI2: # %bb.0:
2345 ; AVX512VBMI2-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
2346 ; AVX512VBMI2-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0
2347 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm0, %xmm0
2348 ; AVX512VBMI2-NEXT: retq
2349 ;
2350 ; AVX512VLBW-LABEL: constant_funnnel_v4i32:
2351 ; AVX512VLBW: # %bb.0:
2352 ; AVX512VLBW-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
2353 ; AVX512VLBW-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0
2354 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm0, %xmm0
2355 ; AVX512VLBW-NEXT: retq
2356 ;
2357 ; AVX512VLVBMI2-LABEL: constant_funnnel_v4i32:
2358 ; AVX512VLVBMI2: # %bb.0:
2359 ; AVX512VLVBMI2-NEXT: vpshldvd {{.*}}(%rip), %xmm1, %xmm0
2360 ; AVX512VLVBMI2-NEXT: retq
2361 ;
2362 ; XOPAVX1-LABEL: constant_funnnel_v4i32:
2363 ; XOPAVX1: # %bb.0:
2364 ; XOPAVX1-NEXT: vpshld {{.*}}(%rip), %xmm1, %xmm1
2365 ; XOPAVX1-NEXT: vpshld {{.*}}(%rip), %xmm0, %xmm0
2366 ; XOPAVX1-NEXT: vpor %xmm1, %xmm0, %xmm0
2367 ; XOPAVX1-NEXT: retq
2368 ;
2369 ; XOPAVX2-LABEL: constant_funnnel_v4i32:
2370 ; XOPAVX2: # %bb.0:
2371 ; XOPAVX2-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
2372 ; XOPAVX2-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0
2373 ; XOPAVX2-NEXT: vpor %xmm1, %xmm0, %xmm0
2374 ; XOPAVX2-NEXT: retq
2375 ;
2376 ; X32-SSE-LABEL: constant_funnnel_v4i32:
2377 ; X32-SSE: # %bb.0:
2378 ; X32-SSE-NEXT: movdqa %xmm1, %xmm2
2379 ; X32-SSE-NEXT: psrld $25, %xmm2
2380 ; X32-SSE-NEXT: movdqa %xmm1, %xmm3
2381 ; X32-SSE-NEXT: psrld $26, %xmm3
2382 ; X32-SSE-NEXT: punpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm2[1]
2383 ; X32-SSE-NEXT: movdqa %xmm1, %xmm2
2384 ; X32-SSE-NEXT: psrld $27, %xmm2
2385 ; X32-SSE-NEXT: psrld $28, %xmm1
2386 ; X32-SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
2387 ; X32-SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,3],xmm3[0,3]
2388 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm2 = [16,32,64,128]
2389 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
2390 ; X32-SSE-NEXT: pmuludq %xmm2, %xmm0
2391 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2392 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
2393 ; X32-SSE-NEXT: pmuludq %xmm3, %xmm2
2394 ; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
2395 ; X32-SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
2396 ; X32-SSE-NEXT: por %xmm1, %xmm0
2397 ; X32-SSE-NEXT: retl
2398 %res = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 4, i32 5, i32 6, i32 7>)
2399 ret <4 x i32> %res
2400 }
2402 define <8 x i16> @constant_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
2403 ; SSE2-LABEL: constant_funnnel_v8i16:
2404 ; SSE2: # %bb.0:
2405 ; SSE2-NEXT: movdqa {{.*#+}} xmm2 = <u,2,4,8,16,32,64,128>
2406 ; SSE2-NEXT: pmulhuw %xmm2, %xmm1
2407 ; SSE2-NEXT: pmullw %xmm0, %xmm2
2408 ; SSE2-NEXT: por %xmm1, %xmm2
2409 ; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [0,65535,65535,65535,65535,65535,65535,65535]
2410 ; SSE2-NEXT: pand %xmm1, %xmm2
2411 ; SSE2-NEXT: pandn %xmm0, %xmm1
2412 ; SSE2-NEXT: por %xmm2, %xmm1
2413 ; SSE2-NEXT: movdqa %xmm1, %xmm0
2414 ; SSE2-NEXT: retq
2415 ;
2416 ; SSE41-LABEL: constant_funnnel_v8i16:
2417 ; SSE41: # %bb.0:
2418 ; SSE41-NEXT: movdqa {{.*#+}} xmm2 = <u,2,4,8,16,32,64,128>
2419 ; SSE41-NEXT: pmulhuw %xmm2, %xmm1
2420 ; SSE41-NEXT: pmullw %xmm0, %xmm2
2421 ; SSE41-NEXT: por %xmm1, %xmm2
2422 ; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7]
2423 ; SSE41-NEXT: retq
2424 ;
2425 ; AVX-LABEL: constant_funnnel_v8i16:
2426 ; AVX: # %bb.0:
2427 ; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = <u,2,4,8,16,32,64,128>
2428 ; AVX-NEXT: vpmulhuw %xmm2, %xmm1, %xmm1
2429 ; AVX-NEXT: vpmullw %xmm2, %xmm0, %xmm2
2430 ; AVX-NEXT: vpor %xmm1, %xmm2, %xmm1
2431 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
2432 ; AVX-NEXT: retq
2433 ;
2434 ; AVX512F-LABEL: constant_funnnel_v8i16:
2435 ; AVX512F: # %bb.0:
2436 ; AVX512F-NEXT: vmovdqa {{.*#+}} xmm2 = <u,2,4,8,16,32,64,128>
2437 ; AVX512F-NEXT: vpmulhuw %xmm2, %xmm1, %xmm1
2438 ; AVX512F-NEXT: vpmullw %xmm2, %xmm0, %xmm2
2439 ; AVX512F-NEXT: vpor %xmm1, %xmm2, %xmm1
2440 ; AVX512F-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
2441 ; AVX512F-NEXT: retq
2442 ;
2443 ; AVX512VL-LABEL: constant_funnnel_v8i16:
2444 ; AVX512VL: # %bb.0:
2445 ; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm2 = <u,2,4,8,16,32,64,128>
2446 ; AVX512VL-NEXT: vpmulhuw %xmm2, %xmm1, %xmm1
2447 ; AVX512VL-NEXT: vpmullw %xmm2, %xmm0, %xmm2
2448 ; AVX512VL-NEXT: vpor %xmm1, %xmm2, %xmm1
2449 ; AVX512VL-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
2450 ; AVX512VL-NEXT: retq
2451 ;
2452 ; AVX512BW-LABEL: constant_funnnel_v8i16:
2453 ; AVX512BW: # %bb.0:
2454 ; AVX512BW-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
2455 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
2456 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm2 = <u,15,14,13,12,11,10,9>
2457 ; AVX512BW-NEXT: vpsrlvw %zmm2, %zmm1, %zmm1
2458 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm2 = <u,1,2,3,4,5,6,7>
2459 ; AVX512BW-NEXT: vpsllvw %zmm2, %zmm0, %zmm2
2460 ; AVX512BW-NEXT: vpor %xmm1, %xmm2, %xmm1
2461 ; AVX512BW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
2462 ; AVX512BW-NEXT: vzeroupper
2463 ; AVX512BW-NEXT: retq
2464 ;
2465 ; AVX512VBMI2-LABEL: constant_funnnel_v8i16:
2466 ; AVX512VBMI2: # %bb.0:
2467 ; AVX512VBMI2-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
2468 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
2469 ; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm2 = <u,15,14,13,12,11,10,9>
2470 ; AVX512VBMI2-NEXT: vpsrlvw %zmm2, %zmm1, %zmm1
2471 ; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm2 = <u,1,2,3,4,5,6,7>
2472 ; AVX512VBMI2-NEXT: vpsllvw %zmm2, %zmm0, %zmm2
2473 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm2, %xmm1
2474 ; AVX512VBMI2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
2475 ; AVX512VBMI2-NEXT: vzeroupper
2476 ; AVX512VBMI2-NEXT: retq
2477 ;
2478 ; AVX512VLBW-LABEL: constant_funnnel_v8i16:
2479 ; AVX512VLBW: # %bb.0:
2480 ; AVX512VLBW-NEXT: vpsrlvw {{.*}}(%rip), %xmm1, %xmm1
2481 ; AVX512VLBW-NEXT: vpsllvw {{.*}}(%rip), %xmm0, %xmm2
2482 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm2, %xmm1
2483 ; AVX512VLBW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
2484 ; AVX512VLBW-NEXT: retq
2485 ;
2486 ; AVX512VLVBMI2-LABEL: constant_funnnel_v8i16:
2487 ; AVX512VLVBMI2: # %bb.0:
2488 ; AVX512VLVBMI2-NEXT: vpshldvw {{.*}}(%rip), %xmm1, %xmm0
2489 ; AVX512VLVBMI2-NEXT: retq
2490 ;
2491 ; XOP-LABEL: constant_funnnel_v8i16:
2492 ; XOP: # %bb.0:
2493 ; XOP-NEXT: vpshlw {{.*}}(%rip), %xmm1, %xmm1
2494 ; XOP-NEXT: vpshlw {{.*}}(%rip), %xmm0, %xmm2
2495 ; XOP-NEXT: vpor %xmm1, %xmm2, %xmm1
2496 ; XOP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
2497 ; XOP-NEXT: retq
2498 ;
2499 ; X32-SSE-LABEL: constant_funnnel_v8i16:
2500 ; X32-SSE: # %bb.0:
2501 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm2 = <u,2,4,8,16,32,64,128>
2502 ; X32-SSE-NEXT: pmulhuw %xmm2, %xmm1
2503 ; X32-SSE-NEXT: pmullw %xmm0, %xmm2
2504 ; X32-SSE-NEXT: por %xmm1, %xmm2
2505 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm1 = [0,65535,65535,65535,65535,65535,65535,65535]
2506 ; X32-SSE-NEXT: pand %xmm1, %xmm2
2507 ; X32-SSE-NEXT: pandn %xmm0, %xmm1
2508 ; X32-SSE-NEXT: por %xmm2, %xmm1
2509 ; X32-SSE-NEXT: movdqa %xmm1, %xmm0
2510 ; X32-SSE-NEXT: retl
2511 %res = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
2512 ret <8 x i16> %res
2513 }
2515 define <16 x i8> @constant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
2516 ; SSE2-LABEL: constant_funnnel_v16i8:
2517 ; SSE2: # %bb.0:
2518 ; SSE2-NEXT: pxor %xmm2, %xmm2
2519 ; SSE2-NEXT: movdqa %xmm1, %xmm3
2520 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
2521 ; SSE2-NEXT: movdqa {{.*#+}} xmm4 = <u,128,64,32,16,8,4,2>
2522 ; SSE2-NEXT: pmullw %xmm4, %xmm3
2523 ; SSE2-NEXT: psrlw $8, %xmm3
2524 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
2525 ; SSE2-NEXT: movdqa {{.*#+}} xmm5 = <u,2,4,8,16,32,64,128>
2526 ; SSE2-NEXT: pmullw %xmm5, %xmm1
2527 ; SSE2-NEXT: psrlw $8, %xmm1
2528 ; SSE2-NEXT: packuswb %xmm3, %xmm1
2529 ; SSE2-NEXT: movdqa %xmm0, %xmm3
2530 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
2531 ; SSE2-NEXT: pmullw %xmm4, %xmm3
2532 ; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
2533 ; SSE2-NEXT: pand %xmm4, %xmm3
2534 ; SSE2-NEXT: movdqa %xmm0, %xmm2
2535 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
2536 ; SSE2-NEXT: pmullw %xmm5, %xmm2
2537 ; SSE2-NEXT: pand %xmm4, %xmm2
2538 ; SSE2-NEXT: packuswb %xmm3, %xmm2
2539 ; SSE2-NEXT: por %xmm1, %xmm2
2540 ; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
2541 ; SSE2-NEXT: pand %xmm1, %xmm2
2542 ; SSE2-NEXT: pandn %xmm0, %xmm1
2543 ; SSE2-NEXT: por %xmm1, %xmm2
2544 ; SSE2-NEXT: movdqa %xmm2, %xmm0
2545 ; SSE2-NEXT: retq
2546 ;
2547 ; SSE41-LABEL: constant_funnnel_v16i8:
2548 ; SSE41: # %bb.0:
2549 ; SSE41-NEXT: movdqa %xmm0, %xmm2
2550 ; SSE41-NEXT: pxor %xmm0, %xmm0
2551 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2552 ; SSE41-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2553 ; SSE41-NEXT: movdqa {{.*#+}} xmm0 = <u,128,64,32,16,8,4,2>
2554 ; SSE41-NEXT: pmullw %xmm0, %xmm1
2555 ; SSE41-NEXT: psrlw $8, %xmm1
2556 ; SSE41-NEXT: movdqa {{.*#+}} xmm4 = <u,2,4,8,16,32,64,128>
2557 ; SSE41-NEXT: pmullw %xmm4, %xmm3
2558 ; SSE41-NEXT: psrlw $8, %xmm3
2559 ; SSE41-NEXT: packuswb %xmm1, %xmm3
2560 ; SSE41-NEXT: movdqa %xmm2, %xmm1
2561 ; SSE41-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
2562 ; SSE41-NEXT: pmullw %xmm0, %xmm1
2563 ; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
2564 ; SSE41-NEXT: pand %xmm0, %xmm1
2565 ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm5 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2566 ; SSE41-NEXT: pmullw %xmm4, %xmm5
2567 ; SSE41-NEXT: pand %xmm0, %xmm5
2568 ; SSE41-NEXT: packuswb %xmm1, %xmm5
2569 ; SSE41-NEXT: por %xmm3, %xmm5
2570 ; SSE41-NEXT: movaps {{.*#+}} xmm0 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
2571 ; SSE41-NEXT: pblendvb %xmm0, %xmm5, %xmm2
2572 ; SSE41-NEXT: movdqa %xmm2, %xmm0
2573 ; SSE41-NEXT: retq
2574 ;
2575 ; AVX1-LABEL: constant_funnnel_v16i8:
2576 ; AVX1: # %bb.0:
2577 ; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
2578 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
2579 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = <u,128,64,32,16,8,4,2>
2580 ; AVX1-NEXT: vpmullw %xmm3, %xmm2, %xmm2
2581 ; AVX1-NEXT: vpsrlw $8, %xmm2, %xmm2
2582 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2583 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = <u,2,4,8,16,32,64,128>
2584 ; AVX1-NEXT: vpmullw %xmm4, %xmm1, %xmm1
2585 ; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm1
2586 ; AVX1-NEXT: vpackuswb %xmm2, %xmm1, %xmm1
2587 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2588 ; AVX1-NEXT: vpmullw %xmm3, %xmm2, %xmm2
2589 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
2590 ; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2
2591 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2592 ; AVX1-NEXT: vpmullw %xmm4, %xmm5, %xmm4
2593 ; AVX1-NEXT: vpand %xmm3, %xmm4, %xmm3
2594 ; AVX1-NEXT: vpackuswb %xmm2, %xmm3, %xmm2
2595 ; AVX1-NEXT: vpor %xmm1, %xmm2, %xmm1
2596 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
2597 ; AVX1-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
2598 ; AVX1-NEXT: retq
2599 ;
2600 ; AVX2-LABEL: constant_funnnel_v16i8:
2601 ; AVX2: # %bb.0:
2602 ; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
2603 ; AVX2-NEXT: vpmullw {{.*}}(%rip), %ymm1, %ymm1
2604 ; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
2605 ; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
2606 ; AVX2-NEXT: vpackuswb %xmm2, %xmm1, %xmm1
2607 ; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
2608 ; AVX2-NEXT: vpmullw {{.*}}(%rip), %ymm2, %ymm2
2609 ; AVX2-NEXT: vpand {{.*}}(%rip), %ymm2, %ymm2
2610 ; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3
2611 ; AVX2-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
2612 ; AVX2-NEXT: vpor %xmm1, %xmm2, %xmm1
2613 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
2614 ; AVX2-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
2615 ; AVX2-NEXT: vzeroupper
2616 ; AVX2-NEXT: retq
2617 ;
2618 ; AVX512F-LABEL: constant_funnnel_v16i8:
2619 ; AVX512F: # %bb.0:
2620 ; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
2621 ; AVX512F-NEXT: vpsrlvd {{.*}}(%rip), %zmm1, %zmm1
2622 ; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
2623 ; AVX512F-NEXT: vpsllvd {{.*}}(%rip), %zmm2, %zmm2
2624 ; AVX512F-NEXT: vpord %zmm1, %zmm2, %zmm1
2625 ; AVX512F-NEXT: vpmovdb %zmm1, %xmm1
2626 ; AVX512F-NEXT: vmovdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
2627 ; AVX512F-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
2628 ; AVX512F-NEXT: vzeroupper
2629 ; AVX512F-NEXT: retq
2630 ;
2631 ; AVX512VL-LABEL: constant_funnnel_v16i8:
2632 ; AVX512VL: # %bb.0:
2633 ; AVX512VL-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
2634 ; AVX512VL-NEXT: vpsrlvd {{.*}}(%rip), %zmm1, %zmm1
2635 ; AVX512VL-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
2636 ; AVX512VL-NEXT: vpsllvd {{.*}}(%rip), %zmm2, %zmm2
2637 ; AVX512VL-NEXT: vpord %zmm1, %zmm2, %zmm1
2638 ; AVX512VL-NEXT: vpmovdb %zmm1, %xmm1
2639 ; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
2640 ; AVX512VL-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
2641 ; AVX512VL-NEXT: vzeroupper
2642 ; AVX512VL-NEXT: retq
2643 ;
2644 ; AVX512BW-LABEL: constant_funnnel_v16i8:
2645 ; AVX512BW: # %bb.0:
2646 ; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm2 = <u,7,6,5,4,3,2,1,u,1,2,3,4,5,6,7>
2647 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
2648 ; AVX512BW-NEXT: vpsrlvw %zmm2, %zmm1, %zmm1
2649 ; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm2 = <u,1,2,3,4,5,6,7,u,7,6,5,4,3,2,1>
2650 ; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
2651 ; AVX512BW-NEXT: vpsllvw %zmm2, %zmm3, %zmm2
2652 ; AVX512BW-NEXT: vpor %ymm1, %ymm2, %ymm1
2653 ; AVX512BW-NEXT: vpmovwb %zmm1, %ymm1
2654 ; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
2655 ; AVX512BW-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
2656 ; AVX512BW-NEXT: vzeroupper
2657 ; AVX512BW-NEXT: retq
2658 ;
2659 ; AVX512VBMI2-LABEL: constant_funnnel_v16i8:
2660 ; AVX512VBMI2: # %bb.0:
2661 ; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm2 = <u,7,6,5,4,3,2,1,u,1,2,3,4,5,6,7>
2662 ; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
2663 ; AVX512VBMI2-NEXT: vpsrlvw %zmm2, %zmm1, %zmm1
2664 ; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm2 = <u,1,2,3,4,5,6,7,u,7,6,5,4,3,2,1>
2665 ; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
2666 ; AVX512VBMI2-NEXT: vpsllvw %zmm2, %zmm3, %zmm2
2667 ; AVX512VBMI2-NEXT: vpor %ymm1, %ymm2, %ymm1
2668 ; AVX512VBMI2-NEXT: vpmovwb %zmm1, %ymm1
2669 ; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
2670 ; AVX512VBMI2-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
2671 ; AVX512VBMI2-NEXT: vzeroupper
2672 ; AVX512VBMI2-NEXT: retq
2673 ;
2674 ; AVX512VLBW-LABEL: constant_funnnel_v16i8:
2675 ; AVX512VLBW: # %bb.0:
2676 ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
2677 ; AVX512VLBW-NEXT: vpsrlvw {{.*}}(%rip), %ymm1, %ymm1
2678 ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
2679 ; AVX512VLBW-NEXT: vpsllvw {{.*}}(%rip), %ymm2, %ymm2
2680 ; AVX512VLBW-NEXT: vpor %ymm1, %ymm2, %ymm1
2681 ; AVX512VLBW-NEXT: vpmovwb %ymm1, %xmm1
2682 ; AVX512VLBW-NEXT: movw $257, %ax # imm = 0x101
2683 ; AVX512VLBW-NEXT: kmovd %eax, %k1
2684 ; AVX512VLBW-NEXT: vmovdqu8 %xmm0, %xmm1 {%k1}
2685 ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0
2686 ; AVX512VLBW-NEXT: vzeroupper
2687 ; AVX512VLBW-NEXT: retq
2688 ;
2689 ; AVX512VLVBMI2-LABEL: constant_funnnel_v16i8:
2690 ; AVX512VLVBMI2: # %bb.0:
2691 ; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
2692 ; AVX512VLVBMI2-NEXT: vpsrlvw {{.*}}(%rip), %ymm1, %ymm1
2693 ; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
2694 ; AVX512VLVBMI2-NEXT: vpsllvw {{.*}}(%rip), %ymm2, %ymm2
2695 ; AVX512VLVBMI2-NEXT: vpor %ymm1, %ymm2, %ymm1
2696 ; AVX512VLVBMI2-NEXT: vpmovwb %ymm1, %xmm1
2697 ; AVX512VLVBMI2-NEXT: movw $257, %ax # imm = 0x101
2698 ; AVX512VLVBMI2-NEXT: kmovd %eax, %k1
2699 ; AVX512VLVBMI2-NEXT: vmovdqu8 %xmm0, %xmm1 {%k1}
2700 ; AVX512VLVBMI2-NEXT: vmovdqa %xmm1, %xmm0
2701 ; AVX512VLVBMI2-NEXT: vzeroupper
2702 ; AVX512VLVBMI2-NEXT: retq
2703 ;
2704 ; XOP-LABEL: constant_funnnel_v16i8:
2705 ; XOP: # %bb.0:
2706 ; XOP-NEXT: vpshlb {{.*}}(%rip), %xmm1, %xmm1
2707 ; XOP-NEXT: vpshlb {{.*}}(%rip), %xmm0, %xmm2
2708 ; XOP-NEXT: vpor %xmm1, %xmm2, %xmm1
2709 ; XOP-NEXT: vmovdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
2710 ; XOP-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
2711 ; XOP-NEXT: retq
2712 ;
2713 ; X32-SSE-LABEL: constant_funnnel_v16i8:
2714 ; X32-SSE: # %bb.0:
2715 ; X32-SSE-NEXT: pxor %xmm2, %xmm2
2716 ; X32-SSE-NEXT: movdqa %xmm1, %xmm3
2717 ; X32-SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
2718 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm4 = <u,128,64,32,16,8,4,2>
2719 ; X32-SSE-NEXT: pmullw %xmm4, %xmm3
2720 ; X32-SSE-NEXT: psrlw $8, %xmm3
2721 ; X32-SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
2722 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm5 = <u,2,4,8,16,32,64,128>
2723 ; X32-SSE-NEXT: pmullw %xmm5, %xmm1
2724 ; X32-SSE-NEXT: psrlw $8, %xmm1
2725 ; X32-SSE-NEXT: packuswb %xmm3, %xmm1
2726 ; X32-SSE-NEXT: movdqa %xmm0, %xmm3
2727 ; X32-SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
2728 ; X32-SSE-NEXT: pmullw %xmm4, %xmm3
2729 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
2730 ; X32-SSE-NEXT: pand %xmm4, %xmm3
2731 ; X32-SSE-NEXT: movdqa %xmm0, %xmm2
2732 ; X32-SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
2733 ; X32-SSE-NEXT: pmullw %xmm5, %xmm2
2734 ; X32-SSE-NEXT: pand %xmm4, %xmm2
2735 ; X32-SSE-NEXT: packuswb %xmm3, %xmm2
2736 ; X32-SSE-NEXT: por %xmm1, %xmm2
2737 ; X32-SSE-NEXT: movdqa {{.*#+}} xmm1 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
2738 ; X32-SSE-NEXT: pand %xmm1, %xmm2
2739 ; X32-SSE-NEXT: pandn %xmm0, %xmm1
2740 ; X32-SSE-NEXT: por %xmm1, %xmm2
2741 ; X32-SSE-NEXT: movdqa %xmm2, %xmm0
2742 ; X32-SSE-NEXT: retl
2743 %res = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1>)
2744 ret <16 x i8> %res
2745 }
2747 ;
2748 ; Uniform Constant Shifts
2749 ;
2751 define <2 x i64> @splatconstant_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
2752 ; SSE-LABEL: splatconstant_funnnel_v2i64:
2753 ; SSE: # %bb.0:
2754 ; SSE-NEXT: psrlq $50, %xmm1
2755 ; SSE-NEXT: psllq $14, %xmm0
2756 ; SSE-NEXT: por %xmm1, %xmm0
2757 ; SSE-NEXT: retq
2758 ;
2759 ; AVX-LABEL: splatconstant_funnnel_v2i64:
2760 ; AVX: # %bb.0:
2761 ; AVX-NEXT: vpsrlq $50, %xmm1, %xmm1
2762 ; AVX-NEXT: vpsllq $14, %xmm0, %xmm0
2763 ; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0
2764 ; AVX-NEXT: retq
2765 ;
2766 ; AVX512F-LABEL: splatconstant_funnnel_v2i64:
2767 ; AVX512F: # %bb.0:
2768 ; AVX512F-NEXT: vpsrlq $50, %xmm1, %xmm1
2769 ; AVX512F-NEXT: vpsllq $14, %xmm0, %xmm0
2770 ; AVX512F-NEXT: vpor %xmm1, %xmm0, %xmm0
2771 ; AVX512F-NEXT: retq
2772 ;
2773 ; AVX512VL-LABEL: splatconstant_funnnel_v2i64:
2774 ; AVX512VL: # %bb.0:
2775 ; AVX512VL-NEXT: vpsrlq $50, %xmm1, %xmm1
2776 ; AVX512VL-NEXT: vpsllq $14, %xmm0, %xmm0
2777 ; AVX512VL-NEXT: vpor %xmm1, %xmm0, %xmm0
2778 ; AVX512VL-NEXT: retq
2779 ;
2780 ; AVX512BW-LABEL: splatconstant_funnnel_v2i64:
2781 ; AVX512BW: # %bb.0:
2782 ; AVX512BW-NEXT: vpsrlq $50, %xmm1, %xmm1
2783 ; AVX512BW-NEXT: vpsllq $14, %xmm0, %xmm0
2784 ; AVX512BW-NEXT: vpor %xmm1, %xmm0, %xmm0
2785 ; AVX512BW-NEXT: retq
2786 ;
2787 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v2i64:
2788 ; AVX512VBMI2: # %bb.0:
2789 ; AVX512VBMI2-NEXT: vpsrlq $50, %xmm1, %xmm1
2790 ; AVX512VBMI2-NEXT: vpsllq $14, %xmm0, %xmm0
2791 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm0, %xmm0
2792 ; AVX512VBMI2-NEXT: retq
2793 ;
2794 ; AVX512VLBW-LABEL: splatconstant_funnnel_v2i64:
2795 ; AVX512VLBW: # %bb.0:
2796 ; AVX512VLBW-NEXT: vpsrlq $50, %xmm1, %xmm1
2797 ; AVX512VLBW-NEXT: vpsllq $14, %xmm0, %xmm0
2798 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm0, %xmm0
2799 ; AVX512VLBW-NEXT: retq
2800 ;
2801 ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v2i64:
2802 ; AVX512VLVBMI2: # %bb.0:
2803 ; AVX512VLVBMI2-NEXT: vpshldq $14, %xmm1, %xmm0, %xmm0
2804 ; AVX512VLVBMI2-NEXT: retq
2805 ;
2806 ; XOP-LABEL: splatconstant_funnnel_v2i64:
2807 ; XOP: # %bb.0:
2808 ; XOP-NEXT: vpsrlq $50, %xmm1, %xmm1
2809 ; XOP-NEXT: vpsllq $14, %xmm0, %xmm0
2810 ; XOP-NEXT: vpor %xmm1, %xmm0, %xmm0
2811 ; XOP-NEXT: retq
2812 ;
2813 ; X32-SSE-LABEL: splatconstant_funnnel_v2i64:
2814 ; X32-SSE: # %bb.0:
2815 ; X32-SSE-NEXT: psrlq $50, %xmm1
2816 ; X32-SSE-NEXT: psllq $14, %xmm0
2817 ; X32-SSE-NEXT: por %xmm1, %xmm0
2818 ; X32-SSE-NEXT: retl
2819 %res = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> <i64 14, i64 14>)
2820 ret <2 x i64> %res
2821 }
2823 define <4 x i32> @splatconstant_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
2824 ; SSE-LABEL: splatconstant_funnnel_v4i32:
2825 ; SSE: # %bb.0:
2826 ; SSE-NEXT: psrld $28, %xmm1
2827 ; SSE-NEXT: pslld $4, %xmm0
2828 ; SSE-NEXT: por %xmm1, %xmm0
2829 ; SSE-NEXT: retq
2830 ;
2831 ; AVX-LABEL: splatconstant_funnnel_v4i32:
2832 ; AVX: # %bb.0:
2833 ; AVX-NEXT: vpsrld $28, %xmm1, %xmm1
2834 ; AVX-NEXT: vpslld $4, %xmm0, %xmm0
2835 ; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0
2836 ; AVX-NEXT: retq
2837 ;
2838 ; AVX512F-LABEL: splatconstant_funnnel_v4i32:
2839 ; AVX512F: # %bb.0:
2840 ; AVX512F-NEXT: vpsrld $28, %xmm1, %xmm1
2841 ; AVX512F-NEXT: vpslld $4, %xmm0, %xmm0
2842 ; AVX512F-NEXT: vpor %xmm1, %xmm0, %xmm0
2843 ; AVX512F-NEXT: retq
2844 ;
2845 ; AVX512VL-LABEL: splatconstant_funnnel_v4i32:
2846 ; AVX512VL: # %bb.0:
2847 ; AVX512VL-NEXT: vpsrld $28, %xmm1, %xmm1
2848 ; AVX512VL-NEXT: vpslld $4, %xmm0, %xmm0
2849 ; AVX512VL-NEXT: vpor %xmm1, %xmm0, %xmm0
2850 ; AVX512VL-NEXT: retq
2851 ;
2852 ; AVX512BW-LABEL: splatconstant_funnnel_v4i32:
2853 ; AVX512BW: # %bb.0:
2854 ; AVX512BW-NEXT: vpsrld $28, %xmm1, %xmm1
2855 ; AVX512BW-NEXT: vpslld $4, %xmm0, %xmm0
2856 ; AVX512BW-NEXT: vpor %xmm1, %xmm0, %xmm0
2857 ; AVX512BW-NEXT: retq
2858 ;
2859 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v4i32:
2860 ; AVX512VBMI2: # %bb.0:
2861 ; AVX512VBMI2-NEXT: vpsrld $28, %xmm1, %xmm1
2862 ; AVX512VBMI2-NEXT: vpslld $4, %xmm0, %xmm0
2863 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm0, %xmm0
2864 ; AVX512VBMI2-NEXT: retq
2865 ;
2866 ; AVX512VLBW-LABEL: splatconstant_funnnel_v4i32:
2867 ; AVX512VLBW: # %bb.0:
2868 ; AVX512VLBW-NEXT: vpsrld $28, %xmm1, %xmm1
2869 ; AVX512VLBW-NEXT: vpslld $4, %xmm0, %xmm0
2870 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm0, %xmm0
2871 ; AVX512VLBW-NEXT: retq
2872 ;
2873 ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v4i32:
2874 ; AVX512VLVBMI2: # %bb.0:
2875 ; AVX512VLVBMI2-NEXT: vpshldd $4, %xmm1, %xmm0, %xmm0
2876 ; AVX512VLVBMI2-NEXT: retq
2877 ;
2878 ; XOP-LABEL: splatconstant_funnnel_v4i32:
2879 ; XOP: # %bb.0:
2880 ; XOP-NEXT: vpsrld $28, %xmm1, %xmm1
2881 ; XOP-NEXT: vpslld $4, %xmm0, %xmm0
2882 ; XOP-NEXT: vpor %xmm1, %xmm0, %xmm0
2883 ; XOP-NEXT: retq
2884 ;
2885 ; X32-SSE-LABEL: splatconstant_funnnel_v4i32:
2886 ; X32-SSE: # %bb.0:
2887 ; X32-SSE-NEXT: psrld $28, %xmm1
2888 ; X32-SSE-NEXT: pslld $4, %xmm0
2889 ; X32-SSE-NEXT: por %xmm1, %xmm0
2890 ; X32-SSE-NEXT: retl
2891 %res = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 4, i32 4, i32 4, i32 4>)
2892 ret <4 x i32> %res
2893 }
2895 define <8 x i16> @splatconstant_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
2896 ; SSE-LABEL: splatconstant_funnnel_v8i16:
2897 ; SSE: # %bb.0:
2898 ; SSE-NEXT: psrlw $9, %xmm1
2899 ; SSE-NEXT: psllw $7, %xmm0
2900 ; SSE-NEXT: por %xmm1, %xmm0
2901 ; SSE-NEXT: retq
2902 ;
2903 ; AVX-LABEL: splatconstant_funnnel_v8i16:
2904 ; AVX: # %bb.0:
2905 ; AVX-NEXT: vpsrlw $9, %xmm1, %xmm1
2906 ; AVX-NEXT: vpsllw $7, %xmm0, %xmm0
2907 ; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0
2908 ; AVX-NEXT: retq
2909 ;
2910 ; AVX512F-LABEL: splatconstant_funnnel_v8i16:
2911 ; AVX512F: # %bb.0:
2912 ; AVX512F-NEXT: vpsrlw $9, %xmm1, %xmm1
2913 ; AVX512F-NEXT: vpsllw $7, %xmm0, %xmm0
2914 ; AVX512F-NEXT: vpor %xmm1, %xmm0, %xmm0
2915 ; AVX512F-NEXT: retq
2916 ;
2917 ; AVX512VL-LABEL: splatconstant_funnnel_v8i16:
2918 ; AVX512VL: # %bb.0:
2919 ; AVX512VL-NEXT: vpsrlw $9, %xmm1, %xmm1
2920 ; AVX512VL-NEXT: vpsllw $7, %xmm0, %xmm0
2921 ; AVX512VL-NEXT: vpor %xmm1, %xmm0, %xmm0
2922 ; AVX512VL-NEXT: retq
2923 ;
2924 ; AVX512BW-LABEL: splatconstant_funnnel_v8i16:
2925 ; AVX512BW: # %bb.0:
2926 ; AVX512BW-NEXT: vpsrlw $9, %xmm1, %xmm1
2927 ; AVX512BW-NEXT: vpsllw $7, %xmm0, %xmm0
2928 ; AVX512BW-NEXT: vpor %xmm1, %xmm0, %xmm0
2929 ; AVX512BW-NEXT: retq
2930 ;
2931 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i16:
2932 ; AVX512VBMI2: # %bb.0:
2933 ; AVX512VBMI2-NEXT: vpsrlw $9, %xmm1, %xmm1
2934 ; AVX512VBMI2-NEXT: vpsllw $7, %xmm0, %xmm0
2935 ; AVX512VBMI2-NEXT: vpor %xmm1, %xmm0, %xmm0
2936 ; AVX512VBMI2-NEXT: retq
2937 ;
2938 ; AVX512VLBW-LABEL: splatconstant_funnnel_v8i16:
2939 ; AVX512VLBW: # %bb.0:
2940 ; AVX512VLBW-NEXT: vpsrlw $9, %xmm1, %xmm1
2941 ; AVX512VLBW-NEXT: vpsllw $7, %xmm0, %xmm0
2942 ; AVX512VLBW-NEXT: vpor %xmm1, %xmm0, %xmm0
2943 ; AVX512VLBW-NEXT: retq
2944 ;
2945 ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v8i16:
2946 ; AVX512VLVBMI2: # %bb.0:
2947 ; AVX512VLVBMI2-NEXT: vpshldw $7, %xmm1, %xmm0, %xmm0
2948 ; AVX512VLVBMI2-NEXT: retq
2949 ;
2950 ; XOP-LABEL: splatconstant_funnnel_v8i16:
2951 ; XOP: # %bb.0:
2952 ; XOP-NEXT: vpsrlw $9, %xmm1, %xmm1
2953 ; XOP-NEXT: vpsllw $7, %xmm0, %xmm0
2954 ; XOP-NEXT: vpor %xmm1, %xmm0, %xmm0
2955 ; XOP-NEXT: retq
2956 ;
2957 ; X32-SSE-LABEL: splatconstant_funnnel_v8i16:
2958 ; X32-SSE: # %bb.0:
2959 ; X32-SSE-NEXT: psrlw $9, %xmm1
2960 ; X32-SSE-NEXT: psllw $7, %xmm0
2961 ; X32-SSE-NEXT: por %xmm1, %xmm0
2962 ; X32-SSE-NEXT: retl
2963 %res = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>)
2964 ret <8 x i16> %res
2965 }
2967 define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
2968 ; SSE-LABEL: splatconstant_funnnel_v16i8:
2969 ; SSE: # %bb.0:
2970 ; SSE-NEXT: psrlw $4, %xmm1
2971 ; SSE-NEXT: pand {{.*}}(%rip), %xmm1
2972 ; SSE-NEXT: psllw $4, %xmm0
2973 ; SSE-NEXT: pand {{.*}}(%rip), %xmm0
2974 ; SSE-NEXT: por %xmm1, %xmm0
2975 ; SSE-NEXT: retq
2976 ;
2977 ; AVX-LABEL: splatconstant_funnnel_v16i8:
2978 ; AVX: # %bb.0:
2979 ; AVX-NEXT: vpsrlw $4, %xmm1, %xmm1
2980 ; AVX-NEXT: vpand {{.*}}(%rip), %xmm1, %xmm1
2981 ; AVX-NEXT: vpsllw $4, %xmm0, %xmm0
2982 ; AVX-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0
2983 ; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0
2984 ; AVX-NEXT: retq
2985 ;
2986 ; AVX512-LABEL: splatconstant_funnnel_v16i8:
2987 ; AVX512: # %bb.0:
2988 ; AVX512-NEXT: vpsrlw $4, %xmm1, %xmm1
2989 ; AVX512-NEXT: vpand {{.*}}(%rip), %xmm1, %xmm1
2990 ; AVX512-NEXT: vpsllw $4, %xmm0, %xmm0
2991 ; AVX512-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0
2992 ; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0
2993 ; AVX512-NEXT: retq
2994 ;
2995 ; XOP-LABEL: splatconstant_funnnel_v16i8:
2996 ; XOP: # %bb.0:
2997 ; XOP-NEXT: vpshlb {{.*}}(%rip), %xmm1, %xmm1
2998 ; XOP-NEXT: vpshlb {{.*}}(%rip), %xmm0, %xmm0
2999 ; XOP-NEXT: vpor %xmm1, %xmm0, %xmm0
3000 ; XOP-NEXT: retq
3001 ;
3002 ; X32-SSE-LABEL: splatconstant_funnnel_v16i8:
3003 ; X32-SSE: # %bb.0:
3004 ; X32-SSE-NEXT: psrlw $4, %xmm1
3005 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm1
3006 ; X32-SSE-NEXT: psllw $4, %xmm0
3007 ; X32-SSE-NEXT: pand {{\.LCPI.*}}, %xmm0
3008 ; X32-SSE-NEXT: por %xmm1, %xmm0
3009 ; X32-SSE-NEXT: retl
3010 %res = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
3011 ret <16 x i8> %res
3012 }