| blob | 839ea71da9ea8f827328aa8e4cb868628891d56a |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefix=CHECK
4 define <4 x i32> @mask_shuffle_v4i32_1234(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passthru, i8 %mask) {
5 ; CHECK-LABEL: mask_shuffle_v4i32_1234:
6 ; CHECK: # BB#0:
7 ; CHECK-NEXT: kmovd %edi, %k1
8 ; CHECK-NEXT: valignd {{.*#+}} xmm2 {%k1} = xmm0[1,2,3],xmm1[0]
9 ; CHECK-NEXT: vmovdqa %xmm2, %xmm0
10 ; CHECK-NEXT: retq
11 %shuffle = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
12 %mask.cast = bitcast i8 %mask to <8 x i1>
13 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
14 %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> %passthru
15 ret <4 x i32> %res
16 }
18 define <4 x i32> @maskz_shuffle_v4i32_1234(<4 x i32> %a, <4 x i32> %b, i8 %mask) {
19 ; CHECK-LABEL: maskz_shuffle_v4i32_1234:
20 ; CHECK: # BB#0:
21 ; CHECK-NEXT: kmovd %edi, %k1
22 ; CHECK-NEXT: valignd {{.*#+}} xmm0 {%k1} {z} = xmm0[1,2,3],xmm1[0]
23 ; CHECK-NEXT: retq
24 %shuffle = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
25 %mask.cast = bitcast i8 %mask to <8 x i1>
26 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
27 %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> zeroinitializer
28 ret <4 x i32> %res
29 }
31 define <4 x i32> @mask_shuffle_v4i32_2345(<4 x i32> %a, <4 x i32> %b, <4 x i32> %passthru, i8 %mask) {
32 ; CHECK-LABEL: mask_shuffle_v4i32_2345:
33 ; CHECK: # BB#0:
34 ; CHECK-NEXT: kmovd %edi, %k1
35 ; CHECK-NEXT: valignd {{.*#+}} xmm2 {%k1} = xmm0[2,3],xmm1[0,1]
36 ; CHECK-NEXT: vmovdqa %xmm2, %xmm0
37 ; CHECK-NEXT: retq
38 %shuffle = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
39 %mask.cast = bitcast i8 %mask to <8 x i1>
40 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
41 %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> %passthru
42 ret <4 x i32> %res
43 }
45 define <4 x i32> @maskz_shuffle_v4i32_2345(<4 x i32> %a, <4 x i32> %b, i8 %mask) {
46 ; CHECK-LABEL: maskz_shuffle_v4i32_2345:
47 ; CHECK: # BB#0:
48 ; CHECK-NEXT: kmovd %edi, %k1
49 ; CHECK-NEXT: valignd {{.*#+}} xmm0 {%k1} {z} = xmm0[2,3],xmm1[0,1]
50 ; CHECK-NEXT: retq
51 %shuffle = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
52 %mask.cast = bitcast i8 %mask to <8 x i1>
53 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
54 %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> zeroinitializer
55 ret <4 x i32> %res
56 }
58 define <2 x i64> @mask_shuffle_v2i64_12(<2 x i64> %a, <2 x i64> %b, <2 x i64> %passthru, i8 %mask) {
59 ; CHECK-LABEL: mask_shuffle_v2i64_12:
60 ; CHECK: # BB#0:
61 ; CHECK-NEXT: kmovd %edi, %k1
62 ; CHECK-NEXT: valignq {{.*#+}} xmm2 {%k1} = xmm0[1],xmm1[0]
63 ; CHECK-NEXT: vmovdqa %xmm2, %xmm0
64 ; CHECK-NEXT: retq
65 %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> <i32 1, i32 2>
66 %mask.cast = bitcast i8 %mask to <8 x i1>
67 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
68 %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> %passthru
69 ret <2 x i64> %res
70 }
72 define <2 x i64> @maskz_shuffle_v2i64_12(<2 x i64> %a, <2 x i64> %b, i8 %mask) {
73 ; CHECK-LABEL: maskz_shuffle_v2i64_12:
74 ; CHECK: # BB#0:
75 ; CHECK-NEXT: kmovd %edi, %k1
76 ; CHECK-NEXT: valignq {{.*#+}} xmm0 {%k1} {z} = xmm0[1],xmm1[0]
77 ; CHECK-NEXT: retq
78 %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> <i32 1, i32 2>
79 %mask.cast = bitcast i8 %mask to <8 x i1>
80 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
81 %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> zeroinitializer
82 ret <2 x i64> %res
83 }
85 define <4 x i64> @mask_shuffle_v4i64_1234(<4 x i64> %a, <4 x i64> %b, <4 x i64> %passthru, i8 %mask) {
86 ; CHECK-LABEL: mask_shuffle_v4i64_1234:
87 ; CHECK: # BB#0:
88 ; CHECK-NEXT: kmovd %edi, %k1
89 ; CHECK-NEXT: valignq {{.*#+}} ymm2 {%k1} = ymm0[1,2,3],ymm1[0]
90 ; CHECK-NEXT: vmovdqa %ymm2, %ymm0
91 ; CHECK-NEXT: retq
92 %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
93 %mask.cast = bitcast i8 %mask to <8 x i1>
94 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
95 %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle, <4 x i64> %passthru
96 ret <4 x i64> %res
97 }
99 define <4 x i64> @maskz_shuffle_v4i64_1234(<4 x i64> %a, <4 x i64> %b, i8 %mask) {
100 ; CHECK-LABEL: maskz_shuffle_v4i64_1234:
101 ; CHECK: # BB#0:
102 ; CHECK-NEXT: kmovd %edi, %k1
103 ; CHECK-NEXT: valignq {{.*#+}} ymm0 {%k1} {z} = ymm0[1,2,3],ymm1[0]
104 ; CHECK-NEXT: retq
105 %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
106 %mask.cast = bitcast i8 %mask to <8 x i1>
107 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
108 %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle, <4 x i64> zeroinitializer
109 ret <4 x i64> %res
110 }
112 define <4 x i64> @mask_shuffle_v4i64_1230(<4 x i64> %a, <4 x i64> %passthru, i8 %mask) {
113 ; CHECK-LABEL: mask_shuffle_v4i64_1230:
114 ; CHECK: # BB#0:
115 ; CHECK-NEXT: kmovd %edi, %k1
116 ; CHECK-NEXT: vpermq {{.*#+}} ymm1 {%k1} = ymm0[1,2,3,0]
117 ; CHECK-NEXT: vmovdqa %ymm1, %ymm0
118 ; CHECK-NEXT: retq
119 %shuffle = shufflevector <4 x i64> %a, <4 x i64> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
120 %mask.cast = bitcast i8 %mask to <8 x i1>
121 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
122 %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle, <4 x i64> %passthru
123 ret <4 x i64> %res
124 }
126 define <4 x i64> @maskz_shuffle_v4i64_1230(<4 x i64> %a, i8 %mask) {
127 ; CHECK-LABEL: maskz_shuffle_v4i64_1230:
128 ; CHECK: # BB#0:
129 ; CHECK-NEXT: kmovd %edi, %k1
130 ; CHECK-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[1,2,3,0]
131 ; CHECK-NEXT: retq
132 %shuffle = shufflevector <4 x i64> %a, <4 x i64> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
133 %mask.cast = bitcast i8 %mask to <8 x i1>
134 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
135 %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle, <4 x i64> zeroinitializer
136 ret <4 x i64> %res
137 }
139 define <8 x i32> @mask_shuffle_v8i32_12345678(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passthru, i8 %mask) {
140 ; CHECK-LABEL: mask_shuffle_v8i32_12345678:
141 ; CHECK: # BB#0:
142 ; CHECK-NEXT: kmovd %edi, %k1
143 ; CHECK-NEXT: valignd {{.*#+}} ymm2 {%k1} = ymm0[1,2,3,4,5,6,7],ymm1[0]
144 ; CHECK-NEXT: vmovdqa %ymm2, %ymm0
145 ; CHECK-NEXT: retq
146 %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
147 %mask.cast = bitcast i8 %mask to <8 x i1>
148 %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle, <8 x i32> %passthru
149 ret <8 x i32> %res
150 }
152 define <8 x i32> @maskz_shuffle_v8i32_12345678(<8 x i32> %a, <8 x i32> %b, i8 %mask) {
153 ; CHECK-LABEL: maskz_shuffle_v8i32_12345678:
154 ; CHECK: # BB#0:
155 ; CHECK-NEXT: kmovd %edi, %k1
156 ; CHECK-NEXT: valignd {{.*#+}} ymm0 {%k1} {z} = ymm0[1,2,3,4,5,6,7],ymm1[0]
157 ; CHECK-NEXT: retq
158 %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
159 %mask.cast = bitcast i8 %mask to <8 x i1>
160 %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle, <8 x i32> zeroinitializer
161 ret <8 x i32> %res
162 }
164 define <8 x i32> @mask_shuffle_v8i32_23456789(<8 x i32> %a, <8 x i32> %b, <8 x i32> %passthru, i8 %mask) {
165 ; CHECK-LABEL: mask_shuffle_v8i32_23456789:
166 ; CHECK: # BB#0:
167 ; CHECK-NEXT: kmovd %edi, %k1
168 ; CHECK-NEXT: valignd {{.*#+}} ymm2 {%k1} = ymm0[2,3,4,5,6,7],ymm1[0,1]
169 ; CHECK-NEXT: vmovdqa %ymm2, %ymm0
170 ; CHECK-NEXT: retq
171 %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
172 %mask.cast = bitcast i8 %mask to <8 x i1>
173 %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle, <8 x i32> %passthru
174 ret <8 x i32> %res
175 }
177 define <8 x i32> @maskz_shuffle_v8i32_23456789(<8 x i32> %a, <8 x i32> %b, i8 %mask) {
178 ; CHECK-LABEL: maskz_shuffle_v8i32_23456789:
179 ; CHECK: # BB#0:
180 ; CHECK-NEXT: kmovd %edi, %k1
181 ; CHECK-NEXT: valignd {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3,4,5,6,7],ymm1[0,1]
182 ; CHECK-NEXT: retq
183 %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
184 %mask.cast = bitcast i8 %mask to <8 x i1>
185 %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle, <8 x i32> zeroinitializer
186 ret <8 x i32> %res
187 }
189 define <8 x i32> @mask_shuffle_v8i32_12345670(<8 x i32> %a, <8 x i32> %passthru, i8 %mask) {
190 ; CHECK-LABEL: mask_shuffle_v8i32_12345670:
191 ; CHECK: # BB#0:
192 ; CHECK-NEXT: kmovd %edi, %k1
193 ; CHECK-NEXT: valignd {{.*#+}} ymm1 {%k1} = ymm0[1,2,3,4,5,6,7,0]
194 ; CHECK-NEXT: vmovdqa %ymm1, %ymm0
195 ; CHECK-NEXT: retq
196 %shuffle = shufflevector <8 x i32> %a, <8 x i32> undef, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0>
197 %mask.cast = bitcast i8 %mask to <8 x i1>
198 %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle, <8 x i32> %passthru
199 ret <8 x i32> %res
200 }
202 define <8 x i32> @maskz_shuffle_v8i32_12345670(<8 x i32> %a, i8 %mask) {
203 ; CHECK-LABEL: maskz_shuffle_v8i32_12345670:
204 ; CHECK: # BB#0:
205 ; CHECK-NEXT: kmovd %edi, %k1
206 ; CHECK-NEXT: valignd {{.*#+}} ymm0 {%k1} {z} = ymm0[1,2,3,4,5,6,7,0]
207 ; CHECK-NEXT: retq
208 %shuffle = shufflevector <8 x i32> %a, <8 x i32> undef, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0>
209 %mask.cast = bitcast i8 %mask to <8 x i1>
210 %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle, <8 x i32> zeroinitializer
211 ret <8 x i32> %res
212 }
214 define <8 x i32> @mask_shuffle_v8i32_23456701(<8 x i32> %a, <8 x i32> %passthru, i8 %mask) {
215 ; CHECK-LABEL: mask_shuffle_v8i32_23456701:
216 ; CHECK: # BB#0:
217 ; CHECK-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,2,3,0]
218 ; CHECK-NEXT: kmovd %edi, %k1
219 ; CHECK-NEXT: vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
220 ; CHECK-NEXT: retq
221 %shuffle = shufflevector <8 x i32> %a, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1>
222 %mask.cast = bitcast i8 %mask to <8 x i1>
223 %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle, <8 x i32> %passthru
224 ret <8 x i32> %res
225 }
227 define <8 x i32> @maskz_shuffle_v8i32_23456701(<8 x i32> %a, i8 %mask) {
228 ; CHECK-LABEL: maskz_shuffle_v8i32_23456701:
229 ; CHECK: # BB#0:
230 ; CHECK-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,2,3,0]
231 ; CHECK-NEXT: kmovd %edi, %k1
232 ; CHECK-NEXT: vmovdqa32 %ymm0, %ymm0 {%k1} {z}
233 ; CHECK-NEXT: retq
234 %shuffle = shufflevector <8 x i32> %a, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1>
235 %mask.cast = bitcast i8 %mask to <8 x i1>
236 %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle, <8 x i32> zeroinitializer
237 ret <8 x i32> %res
238 }
240 define <4 x i32> @mask_extract_v8i32_v4i32_0(<8 x i32> %a, <4 x i32> %passthru, i8 %mask) {
241 ; CHECK-LABEL: mask_extract_v8i32_v4i32_0:
242 ; CHECK: # BB#0:
243 ; CHECK-NEXT: kmovd %edi, %k1
244 ; CHECK-NEXT: vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
245 ; CHECK-NEXT: vzeroupper
246 ; CHECK-NEXT: retq
247 %shuffle = shufflevector <8 x i32> %a, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
248 %mask.cast = bitcast i8 %mask to <8 x i1>
249 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
250 %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> %passthru
251 ret <4 x i32> %res
252 }
254 define <4 x i32> @mask_extract_v8i32_v4i32_0_z(<8 x i32> %a, i8 %mask) {
255 ; CHECK-LABEL: mask_extract_v8i32_v4i32_0_z:
256 ; CHECK: # BB#0:
257 ; CHECK-NEXT: kmovd %edi, %k1
258 ; CHECK-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z}
259 ; CHECK-NEXT: vzeroupper
260 ; CHECK-NEXT: retq
261 %shuffle = shufflevector <8 x i32> %a, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
262 %mask.cast = bitcast i8 %mask to <8 x i1>
263 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
264 %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> zeroinitializer
265 ret <4 x i32> %res
266 }
268 define <4 x i32> @mask_extract_v8i32_v4i32_1(<8 x i32> %a, <4 x i32> %passthru, i8 %mask) {
269 ; CHECK-LABEL: mask_extract_v8i32_v4i32_1:
270 ; CHECK: # BB#0:
271 ; CHECK-NEXT: kmovd %edi, %k1
272 ; CHECK-NEXT: vextracti32x4 $1, %ymm0, %xmm1 {%k1}
273 ; CHECK-NEXT: vmovdqa %xmm1, %xmm0
274 ; CHECK-NEXT: vzeroupper
275 ; CHECK-NEXT: retq
276 %shuffle = shufflevector <8 x i32> %a, <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
277 %mask.cast = bitcast i8 %mask to <8 x i1>
278 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
279 %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> %passthru
280 ret <4 x i32> %res
281 }
283 define <4 x i32> @mask_extract_v8i32_v4i32_1_z(<8 x i32> %a, i8 %mask) {
284 ; CHECK-LABEL: mask_extract_v8i32_v4i32_1_z:
285 ; CHECK: # BB#0:
286 ; CHECK-NEXT: kmovd %edi, %k1
287 ; CHECK-NEXT: vextracti32x4 $1, %ymm0, %xmm0 {%k1} {z}
288 ; CHECK-NEXT: vzeroupper
289 ; CHECK-NEXT: retq
290 %shuffle = shufflevector <8 x i32> %a, <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
291 %mask.cast = bitcast i8 %mask to <8 x i1>
292 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
293 %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> zeroinitializer
294 ret <4 x i32> %res
295 }
297 define <4 x float> @mask_extract_v8f32_v4f32_0(<8 x float> %a, <4 x float> %passthru, i8 %mask) {
298 ; CHECK-LABEL: mask_extract_v8f32_v4f32_0:
299 ; CHECK: # BB#0:
300 ; CHECK-NEXT: kmovd %edi, %k1
301 ; CHECK-NEXT: vblendmps %xmm0, %xmm1, %xmm0 {%k1}
302 ; CHECK-NEXT: vzeroupper
303 ; CHECK-NEXT: retq
304 %shuffle = shufflevector <8 x float> %a, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
305 %mask.cast = bitcast i8 %mask to <8 x i1>
306 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
307 %res = select <4 x i1> %mask.extract, <4 x float> %shuffle, <4 x float> %passthru
308 ret <4 x float> %res
309 }
311 define <4 x float> @mask_extract_v8f32_v4f32_0_z(<8 x float> %a, i8 %mask) {
312 ; CHECK-LABEL: mask_extract_v8f32_v4f32_0_z:
313 ; CHECK: # BB#0:
314 ; CHECK-NEXT: kmovd %edi, %k1
315 ; CHECK-NEXT: vmovaps %xmm0, %xmm0 {%k1} {z}
316 ; CHECK-NEXT: vzeroupper
317 ; CHECK-NEXT: retq
318 %shuffle = shufflevector <8 x float> %a, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
319 %mask.cast = bitcast i8 %mask to <8 x i1>
320 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
321 %res = select <4 x i1> %mask.extract, <4 x float> %shuffle, <4 x float> zeroinitializer
322 ret <4 x float> %res
323 }
325 define <4 x float> @mask_extract_v8f32_v4f32_1(<8 x float> %a, <4 x float> %passthru, i8 %mask) {
326 ; CHECK-LABEL: mask_extract_v8f32_v4f32_1:
327 ; CHECK: # BB#0:
328 ; CHECK-NEXT: kmovd %edi, %k1
329 ; CHECK-NEXT: vextractf32x4 $1, %ymm0, %xmm1 {%k1}
330 ; CHECK-NEXT: vmovaps %xmm1, %xmm0
331 ; CHECK-NEXT: vzeroupper
332 ; CHECK-NEXT: retq
333 %shuffle = shufflevector <8 x float> %a, <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
334 %mask.cast = bitcast i8 %mask to <8 x i1>
335 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
336 %res = select <4 x i1> %mask.extract, <4 x float> %shuffle, <4 x float> %passthru
337 ret <4 x float> %res
338 }
340 define <4 x float> @mask_extract_v8f32_v4f32_1_z(<8 x float> %a, i8 %mask) {
341 ; CHECK-LABEL: mask_extract_v8f32_v4f32_1_z:
342 ; CHECK: # BB#0:
343 ; CHECK-NEXT: kmovd %edi, %k1
344 ; CHECK-NEXT: vextractf32x4 $1, %ymm0, %xmm0 {%k1} {z}
345 ; CHECK-NEXT: vzeroupper
346 ; CHECK-NEXT: retq
347 %shuffle = shufflevector <8 x float> %a, <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
348 %mask.cast = bitcast i8 %mask to <8 x i1>
349 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
350 %res = select <4 x i1> %mask.extract, <4 x float> %shuffle, <4 x float> zeroinitializer
351 ret <4 x float> %res
352 }
354 define <2 x i64> @mask_extract_v4i64_v2i64_0(<4 x i64> %a, <2 x i64> %passthru, i8 %mask) {
355 ; CHECK-LABEL: mask_extract_v4i64_v2i64_0:
356 ; CHECK: # BB#0:
357 ; CHECK-NEXT: kmovd %edi, %k1
358 ; CHECK-NEXT: vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
359 ; CHECK-NEXT: vzeroupper
360 ; CHECK-NEXT: retq
361 %shuffle = shufflevector <4 x i64> %a, <4 x i64> undef, <2 x i32> <i32 0, i32 1>
362 %mask.cast = bitcast i8 %mask to <8 x i1>
363 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
364 %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> %passthru
365 ret <2 x i64> %res
366 }
368 define <2 x i64> @mask_extract_v4i64_v2i64_0_z(<4 x i64> %a, i8 %mask) {
369 ; CHECK-LABEL: mask_extract_v4i64_v2i64_0_z:
370 ; CHECK: # BB#0:
371 ; CHECK-NEXT: kmovd %edi, %k1
372 ; CHECK-NEXT: vmovdqa64 %xmm0, %xmm0 {%k1} {z}
373 ; CHECK-NEXT: vzeroupper
374 ; CHECK-NEXT: retq
375 %shuffle = shufflevector <4 x i64> %a, <4 x i64> undef, <2 x i32> <i32 0, i32 1>
376 %mask.cast = bitcast i8 %mask to <8 x i1>
377 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
378 %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> zeroinitializer
379 ret <2 x i64> %res
380 }
382 define <2 x i64> @mask_extract_v4i64_v2i64_1(<4 x i64> %a, <2 x i64> %passthru, i8 %mask) {
383 ; CHECK-LABEL: mask_extract_v4i64_v2i64_1:
384 ; CHECK: # BB#0:
385 ; CHECK-NEXT: kmovd %edi, %k1
386 ; CHECK-NEXT: vextracti64x2 $1, %ymm0, %xmm1 {%k1}
387 ; CHECK-NEXT: vmovdqa %xmm1, %xmm0
388 ; CHECK-NEXT: vzeroupper
389 ; CHECK-NEXT: retq
390 %shuffle = shufflevector <4 x i64> %a, <4 x i64> undef, <2 x i32> <i32 2, i32 3>
391 %mask.cast = bitcast i8 %mask to <8 x i1>
392 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
393 %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> %passthru
394 ret <2 x i64> %res
395 }
397 define <2 x i64> @mask_extract_v4i64_v2i64_1_z(<4 x i64> %a, i8 %mask) {
398 ; CHECK-LABEL: mask_extract_v4i64_v2i64_1_z:
399 ; CHECK: # BB#0:
400 ; CHECK-NEXT: kmovd %edi, %k1
401 ; CHECK-NEXT: vextracti64x2 $1, %ymm0, %xmm0 {%k1} {z}
402 ; CHECK-NEXT: vzeroupper
403 ; CHECK-NEXT: retq
404 %shuffle = shufflevector <4 x i64> %a, <4 x i64> undef, <2 x i32> <i32 2, i32 3>
405 %mask.cast = bitcast i8 %mask to <8 x i1>
406 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
407 %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> zeroinitializer
408 ret <2 x i64> %res
409 }
411 define <2 x double> @mask_extract_v4f64_v2f64_0(<4 x double> %a, <2 x double> %passthru, i8 %mask) {
412 ; CHECK-LABEL: mask_extract_v4f64_v2f64_0:
413 ; CHECK: # BB#0:
414 ; CHECK-NEXT: kmovd %edi, %k1
415 ; CHECK-NEXT: vblendmpd %xmm0, %xmm1, %xmm0 {%k1}
416 ; CHECK-NEXT: vzeroupper
417 ; CHECK-NEXT: retq
418 %shuffle = shufflevector <4 x double> %a, <4 x double> undef, <2 x i32> <i32 0, i32 1>
419 %mask.cast = bitcast i8 %mask to <8 x i1>
420 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
421 %res = select <2 x i1> %mask.extract, <2 x double> %shuffle, <2 x double> %passthru
422 ret <2 x double> %res
423 }
425 define <2 x double> @mask_extract_v4f64_v2f64_0_z(<4 x double> %a, i8 %mask) {
426 ; CHECK-LABEL: mask_extract_v4f64_v2f64_0_z:
427 ; CHECK: # BB#0:
428 ; CHECK-NEXT: kmovd %edi, %k1
429 ; CHECK-NEXT: vmovapd %xmm0, %xmm0 {%k1} {z}
430 ; CHECK-NEXT: vzeroupper
431 ; CHECK-NEXT: retq
432 %shuffle = shufflevector <4 x double> %a, <4 x double> undef, <2 x i32> <i32 0, i32 1>
433 %mask.cast = bitcast i8 %mask to <8 x i1>
434 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
435 %res = select <2 x i1> %mask.extract, <2 x double> %shuffle, <2 x double> zeroinitializer
436 ret <2 x double> %res
437 }
439 define <2 x double> @mask_extract_v4f64_v2f64_1(<4 x double> %a, <2 x double> %passthru, i8 %mask) {
440 ; CHECK-LABEL: mask_extract_v4f64_v2f64_1:
441 ; CHECK: # BB#0:
442 ; CHECK-NEXT: kmovd %edi, %k1
443 ; CHECK-NEXT: vextractf64x2 $1, %ymm0, %xmm1 {%k1}
444 ; CHECK-NEXT: vmovapd %xmm1, %xmm0
445 ; CHECK-NEXT: vzeroupper
446 ; CHECK-NEXT: retq
447 %shuffle = shufflevector <4 x double> %a, <4 x double> undef, <2 x i32> <i32 2, i32 3>
448 %mask.cast = bitcast i8 %mask to <8 x i1>
449 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
450 %res = select <2 x i1> %mask.extract, <2 x double> %shuffle, <2 x double> %passthru
451 ret <2 x double> %res
452 }
454 define <2 x double> @mask_extract_v4f64_v2f64_1_z(<4 x double> %a, i8 %mask) {
455 ; CHECK-LABEL: mask_extract_v4f64_v2f64_1_z:
456 ; CHECK: # BB#0:
457 ; CHECK-NEXT: kmovd %edi, %k1
458 ; CHECK-NEXT: vextractf64x2 $1, %ymm0, %xmm0 {%k1} {z}
459 ; CHECK-NEXT: vzeroupper
460 ; CHECK-NEXT: retq
461 %shuffle = shufflevector <4 x double> %a, <4 x double> undef, <2 x i32> <i32 2, i32 3>
462 %mask.cast = bitcast i8 %mask to <8 x i1>
463 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
464 %res = select <2 x i1> %mask.extract, <2 x double> %shuffle, <2 x double> zeroinitializer
465 ret <2 x double> %res
466 }
468 define <4 x i32> @mask_extract_v16i32_v4i32_0(<16 x i32> %a, <4 x i32> %passthru, i8 %mask) {
469 ; CHECK-LABEL: mask_extract_v16i32_v4i32_0:
470 ; CHECK: # BB#0:
471 ; CHECK-NEXT: kmovd %edi, %k1
472 ; CHECK-NEXT: vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
473 ; CHECK-NEXT: vzeroupper
474 ; CHECK-NEXT: retq
475 %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
476 %mask.cast = bitcast i8 %mask to <8 x i1>
477 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
478 %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> %passthru
479 ret <4 x i32> %res
480 }
482 define <4 x i32> @mask_extract_v16i32_v4i32_0_z(<16 x i32> %a, i8 %mask) {
483 ; CHECK-LABEL: mask_extract_v16i32_v4i32_0_z:
484 ; CHECK: # BB#0:
485 ; CHECK-NEXT: kmovd %edi, %k1
486 ; CHECK-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z}
487 ; CHECK-NEXT: vzeroupper
488 ; CHECK-NEXT: retq
489 %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
490 %mask.cast = bitcast i8 %mask to <8 x i1>
491 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
492 %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> zeroinitializer
493 ret <4 x i32> %res
494 }
496 define <4 x i32> @mask_extract_v16i32_v4i32_1(<16 x i32> %a, <4 x i32> %passthru, i8 %mask) {
497 ; CHECK-LABEL: mask_extract_v16i32_v4i32_1:
498 ; CHECK: # BB#0:
499 ; CHECK-NEXT: kmovd %edi, %k1
500 ; CHECK-NEXT: vextracti32x4 $1, %zmm0, %xmm1 {%k1}
501 ; CHECK-NEXT: vmovdqa %xmm1, %xmm0
502 ; CHECK-NEXT: vzeroupper
503 ; CHECK-NEXT: retq
504 %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
505 %mask.cast = bitcast i8 %mask to <8 x i1>
506 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
507 %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> %passthru
508 ret <4 x i32> %res
509 }
511 define <4 x i32> @mask_extract_v16i32_v4i32_1_z(<16 x i32> %a, i8 %mask) {
512 ; CHECK-LABEL: mask_extract_v16i32_v4i32_1_z:
513 ; CHECK: # BB#0:
514 ; CHECK-NEXT: kmovd %edi, %k1
515 ; CHECK-NEXT: vextracti32x4 $1, %zmm0, %xmm0 {%k1} {z}
516 ; CHECK-NEXT: vzeroupper
517 ; CHECK-NEXT: retq
518 %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
519 %mask.cast = bitcast i8 %mask to <8 x i1>
520 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
521 %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> zeroinitializer
522 ret <4 x i32> %res
523 }
525 define <4 x i32> @mask_extract_v16i32_v4i32_2(<16 x i32> %a, <4 x i32> %passthru, i8 %mask) {
526 ; CHECK-LABEL: mask_extract_v16i32_v4i32_2:
527 ; CHECK: # BB#0:
528 ; CHECK-NEXT: kmovd %edi, %k1
529 ; CHECK-NEXT: vextracti32x4 $2, %zmm0, %xmm1 {%k1}
530 ; CHECK-NEXT: vmovdqa %xmm1, %xmm0
531 ; CHECK-NEXT: vzeroupper
532 ; CHECK-NEXT: retq
533 %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
534 %mask.cast = bitcast i8 %mask to <8 x i1>
535 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
536 %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> %passthru
537 ret <4 x i32> %res
538 }
540 define <4 x i32> @mask_extract_v16i32_v4i32_3(<16 x i32> %a, <4 x i32> %passthru, i8 %mask) {
541 ; CHECK-LABEL: mask_extract_v16i32_v4i32_3:
542 ; CHECK: # BB#0:
543 ; CHECK-NEXT: kmovd %edi, %k1
544 ; CHECK-NEXT: vextracti32x4 $3, %zmm0, %xmm1 {%k1}
545 ; CHECK-NEXT: vmovdqa %xmm1, %xmm0
546 ; CHECK-NEXT: vzeroupper
547 ; CHECK-NEXT: retq
548 %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
549 %mask.cast = bitcast i8 %mask to <8 x i1>
550 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
551 %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle, <4 x i32> %passthru
552 ret <4 x i32> %res
553 }
555 define <4 x float> @mask_extract_v16f32_v4f32_0(<16 x float> %a, <4 x float> %passthru, i8 %mask) {
556 ; CHECK-LABEL: mask_extract_v16f32_v4f32_0:
557 ; CHECK: # BB#0:
558 ; CHECK-NEXT: kmovd %edi, %k1
559 ; CHECK-NEXT: vblendmps %xmm0, %xmm1, %xmm0 {%k1}
560 ; CHECK-NEXT: vzeroupper
561 ; CHECK-NEXT: retq
562 %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
563 %mask.cast = bitcast i8 %mask to <8 x i1>
564 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
565 %res = select <4 x i1> %mask.extract, <4 x float> %shuffle, <4 x float> %passthru
566 ret <4 x float> %res
567 }
569 define <4 x float> @mask_extract_v16f32_v4f32_0_z(<16 x float> %a, i8 %mask) {
570 ; CHECK-LABEL: mask_extract_v16f32_v4f32_0_z:
571 ; CHECK: # BB#0:
572 ; CHECK-NEXT: kmovd %edi, %k1
573 ; CHECK-NEXT: vmovaps %xmm0, %xmm0 {%k1} {z}
574 ; CHECK-NEXT: vzeroupper
575 ; CHECK-NEXT: retq
576 %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
577 %mask.cast = bitcast i8 %mask to <8 x i1>
578 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
579 %res = select <4 x i1> %mask.extract, <4 x float> %shuffle, <4 x float> zeroinitializer
580 ret <4 x float> %res
581 }
583 define <4 x float> @mask_extract_v16f32_v4f32_1(<16 x float> %a, <4 x float> %passthru, i8 %mask) {
584 ; CHECK-LABEL: mask_extract_v16f32_v4f32_1:
585 ; CHECK: # BB#0:
586 ; CHECK-NEXT: kmovd %edi, %k1
587 ; CHECK-NEXT: vextractf32x4 $1, %zmm0, %xmm1 {%k1}
588 ; CHECK-NEXT: vmovaps %xmm1, %xmm0
589 ; CHECK-NEXT: vzeroupper
590 ; CHECK-NEXT: retq
591 %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
592 %mask.cast = bitcast i8 %mask to <8 x i1>
593 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
594 %res = select <4 x i1> %mask.extract, <4 x float> %shuffle, <4 x float> %passthru
595 ret <4 x float> %res
596 }
598 define <4 x float> @mask_extract_v16f32_v4f32_1_z(<16 x float> %a, i8 %mask) {
599 ; CHECK-LABEL: mask_extract_v16f32_v4f32_1_z:
600 ; CHECK: # BB#0:
601 ; CHECK-NEXT: kmovd %edi, %k1
602 ; CHECK-NEXT: vextractf32x4 $1, %zmm0, %xmm0 {%k1} {z}
603 ; CHECK-NEXT: vzeroupper
604 ; CHECK-NEXT: retq
605 %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
606 %mask.cast = bitcast i8 %mask to <8 x i1>
607 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
608 %res = select <4 x i1> %mask.extract, <4 x float> %shuffle, <4 x float> zeroinitializer
609 ret <4 x float> %res
610 }
612 define <4 x float> @mask_extract_v16f32_v4f32_2(<16 x float> %a, <4 x float> %passthru, i8 %mask) {
613 ; CHECK-LABEL: mask_extract_v16f32_v4f32_2:
614 ; CHECK: # BB#0:
615 ; CHECK-NEXT: kmovd %edi, %k1
616 ; CHECK-NEXT: vextractf32x4 $2, %zmm0, %xmm1 {%k1}
617 ; CHECK-NEXT: vmovaps %xmm1, %xmm0
618 ; CHECK-NEXT: vzeroupper
619 ; CHECK-NEXT: retq
620 %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
621 %mask.cast = bitcast i8 %mask to <8 x i1>
622 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
623 %res = select <4 x i1> %mask.extract, <4 x float> %shuffle, <4 x float> %passthru
624 ret <4 x float> %res
625 }
627 define <4 x float> @mask_extract_v16f32_v4f32_3(<16 x float> %a, <4 x float> %passthru, i8 %mask) {
628 ; CHECK-LABEL: mask_extract_v16f32_v4f32_3:
629 ; CHECK: # BB#0:
630 ; CHECK-NEXT: kmovd %edi, %k1
631 ; CHECK-NEXT: vextractf32x4 $3, %zmm0, %xmm1 {%k1}
632 ; CHECK-NEXT: vmovaps %xmm1, %xmm0
633 ; CHECK-NEXT: vzeroupper
634 ; CHECK-NEXT: retq
635 %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
636 %mask.cast = bitcast i8 %mask to <8 x i1>
637 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
638 %res = select <4 x i1> %mask.extract, <4 x float> %shuffle, <4 x float> %passthru
639 ret <4 x float> %res
640 }
642 define <8 x i32> @mask_extract_v16i32_v8i32_0(<16 x i32> %a, <8 x i32> %passthru, i8 %mask) {
643 ; CHECK-LABEL: mask_extract_v16i32_v8i32_0:
644 ; CHECK: # BB#0:
645 ; CHECK-NEXT: kmovd %edi, %k1
646 ; CHECK-NEXT: vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
647 ; CHECK-NEXT: retq
648 %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
649 %mask.cast = bitcast i8 %mask to <8 x i1>
650 %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle, <8 x i32> %passthru
651 ret <8 x i32> %res
652 }
654 define <8 x i32> @mask_extract_v16i32_v8i32_0_z(<16 x i32> %a, i8 %mask) {
655 ; CHECK-LABEL: mask_extract_v16i32_v8i32_0_z:
656 ; CHECK: # BB#0:
657 ; CHECK-NEXT: kmovd %edi, %k1
658 ; CHECK-NEXT: vmovdqa32 %ymm0, %ymm0 {%k1} {z}
659 ; CHECK-NEXT: retq
660 %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
661 %mask.cast = bitcast i8 %mask to <8 x i1>
662 %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle, <8 x i32> zeroinitializer
663 ret <8 x i32> %res
664 }
666 define <8 x i32> @mask_extract_v16i32_v8i32_1(<16 x i32> %a, <8 x i32> %passthru, i8 %mask) {
667 ; CHECK-LABEL: mask_extract_v16i32_v8i32_1:
668 ; CHECK: # BB#0:
669 ; CHECK-NEXT: kmovd %edi, %k1
670 ; CHECK-NEXT: vextracti32x8 $1, %zmm0, %ymm1 {%k1}
671 ; CHECK-NEXT: vmovdqa %ymm1, %ymm0
672 ; CHECK-NEXT: retq
673 %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
674 %mask.cast = bitcast i8 %mask to <8 x i1>
675 %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle, <8 x i32> %passthru
676 ret <8 x i32> %res
677 }
679 define <8 x i32> @mask_extract_v16i32_v8i32_1_z(<16 x i32> %a, i8 %mask) {
680 ; CHECK-LABEL: mask_extract_v16i32_v8i32_1_z:
681 ; CHECK: # BB#0:
682 ; CHECK-NEXT: kmovd %edi, %k1
683 ; CHECK-NEXT: vextracti32x8 $1, %zmm0, %ymm0 {%k1} {z}
684 ; CHECK-NEXT: retq
685 %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
686 %mask.cast = bitcast i8 %mask to <8 x i1>
687 %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle, <8 x i32> zeroinitializer
688 ret <8 x i32> %res
689 }
691 define <8 x float> @mask_extract_v16f32_v8f32_0(<16 x float> %a, <8 x float> %passthru, i8 %mask) {
692 ; CHECK-LABEL: mask_extract_v16f32_v8f32_0:
693 ; CHECK: # BB#0:
694 ; CHECK-NEXT: kmovd %edi, %k1
695 ; CHECK-NEXT: vblendmps %ymm0, %ymm1, %ymm0 {%k1}
696 ; CHECK-NEXT: retq
697 %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
698 %mask.cast = bitcast i8 %mask to <8 x i1>
699 %res = select <8 x i1> %mask.cast, <8 x float> %shuffle, <8 x float> %passthru
700 ret <8 x float> %res
701 }
703 define <8 x float> @mask_extract_v16f32_v8f32_0_z(<16 x float> %a, i8 %mask) {
704 ; CHECK-LABEL: mask_extract_v16f32_v8f32_0_z:
705 ; CHECK: # BB#0:
706 ; CHECK-NEXT: kmovd %edi, %k1
707 ; CHECK-NEXT: vmovaps %ymm0, %ymm0 {%k1} {z}
708 ; CHECK-NEXT: retq
709 %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
710 %mask.cast = bitcast i8 %mask to <8 x i1>
711 %res = select <8 x i1> %mask.cast, <8 x float> %shuffle, <8 x float> zeroinitializer
712 ret <8 x float> %res
713 }
715 define <8 x float> @mask_extract_v16f32_v8f32_1(<16 x float> %a, <8 x float> %passthru, i8 %mask) {
716 ; CHECK-LABEL: mask_extract_v16f32_v8f32_1:
717 ; CHECK: # BB#0:
718 ; CHECK-NEXT: kmovd %edi, %k1
719 ; CHECK-NEXT: vextractf32x8 $1, %zmm0, %ymm1 {%k1}
720 ; CHECK-NEXT: vmovaps %ymm1, %ymm0
721 ; CHECK-NEXT: retq
722 %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
723 %mask.cast = bitcast i8 %mask to <8 x i1>
724 %res = select <8 x i1> %mask.cast, <8 x float> %shuffle, <8 x float> %passthru
725 ret <8 x float> %res
726 }
728 define <8 x float> @mask_extract_v16f32_v8f32_1_z(<16 x float> %a, i8 %mask) {
729 ; CHECK-LABEL: mask_extract_v16f32_v8f32_1_z:
730 ; CHECK: # BB#0:
731 ; CHECK-NEXT: kmovd %edi, %k1
732 ; CHECK-NEXT: vextractf32x8 $1, %zmm0, %ymm0 {%k1} {z}
733 ; CHECK-NEXT: retq
734 %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
735 %mask.cast = bitcast i8 %mask to <8 x i1>
736 %res = select <8 x i1> %mask.cast, <8 x float> %shuffle, <8 x float> zeroinitializer
737 ret <8 x float> %res
738 }
740 define <2 x i64> @mask_extract_v8i64_v2i64_0(<8 x i64> %a, <2 x i64> %passthru, i8 %mask) {
741 ; CHECK-LABEL: mask_extract_v8i64_v2i64_0:
742 ; CHECK: # BB#0:
743 ; CHECK-NEXT: kmovd %edi, %k1
744 ; CHECK-NEXT: vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
745 ; CHECK-NEXT: vzeroupper
746 ; CHECK-NEXT: retq
747 %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <2 x i32> <i32 0, i32 1>
748 %mask.cast = bitcast i8 %mask to <8 x i1>
749 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
750 %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> %passthru
751 ret <2 x i64> %res
752 }
754 define <2 x i64> @mask_extract_v8i64_v2i64_0_z(<8 x i64> %a, i8 %mask) {
755 ; CHECK-LABEL: mask_extract_v8i64_v2i64_0_z:
756 ; CHECK: # BB#0:
757 ; CHECK-NEXT: kmovd %edi, %k1
758 ; CHECK-NEXT: vmovdqa64 %xmm0, %xmm0 {%k1} {z}
759 ; CHECK-NEXT: vzeroupper
760 ; CHECK-NEXT: retq
761 %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <2 x i32> <i32 0, i32 1>
762 %mask.cast = bitcast i8 %mask to <8 x i1>
763 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
764 %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> zeroinitializer
765 ret <2 x i64> %res
766 }
768 define <2 x i64> @mask_extract_v8i64_v2i64_1(<8 x i64> %a, <2 x i64> %passthru, i8 %mask) {
769 ; CHECK-LABEL: mask_extract_v8i64_v2i64_1:
770 ; CHECK: # BB#0:
771 ; CHECK-NEXT: kmovd %edi, %k1
772 ; CHECK-NEXT: vextracti64x2 $1, %zmm0, %xmm1 {%k1}
773 ; CHECK-NEXT: vmovdqa %xmm1, %xmm0
774 ; CHECK-NEXT: vzeroupper
775 ; CHECK-NEXT: retq
776 %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <2 x i32> <i32 2, i32 3>
777 %mask.cast = bitcast i8 %mask to <8 x i1>
778 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
779 %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> %passthru
780 ret <2 x i64> %res
781 }
783 define <2 x i64> @mask_extract_v8i64_v2i64_1_z(<8 x i64> %a, i8 %mask) {
784 ; CHECK-LABEL: mask_extract_v8i64_v2i64_1_z:
785 ; CHECK: # BB#0:
786 ; CHECK-NEXT: kmovd %edi, %k1
787 ; CHECK-NEXT: vextracti64x2 $1, %zmm0, %xmm0 {%k1} {z}
788 ; CHECK-NEXT: vzeroupper
789 ; CHECK-NEXT: retq
790 %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <2 x i32> <i32 2, i32 3>
791 %mask.cast = bitcast i8 %mask to <8 x i1>
792 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
793 %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> zeroinitializer
794 ret <2 x i64> %res
795 }
797 define <2 x i64> @mask_extract_v8i64_v2i64_2(<8 x i64> %a, <2 x i64> %passthru, i8 %mask) {
798 ; CHECK-LABEL: mask_extract_v8i64_v2i64_2:
799 ; CHECK: # BB#0:
800 ; CHECK-NEXT: kmovd %edi, %k1
801 ; CHECK-NEXT: vextracti64x2 $2, %zmm0, %xmm1 {%k1}
802 ; CHECK-NEXT: vmovdqa %xmm1, %xmm0
803 ; CHECK-NEXT: vzeroupper
804 ; CHECK-NEXT: retq
805 %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <2 x i32> <i32 4, i32 5>
806 %mask.cast = bitcast i8 %mask to <8 x i1>
807 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
808 %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> %passthru
809 ret <2 x i64> %res
810 }
812 define <2 x i64> @mask_extract_v8i64_v2i64_3(<8 x i64> %a, <2 x i64> %passthru, i8 %mask) {
813 ; CHECK-LABEL: mask_extract_v8i64_v2i64_3:
814 ; CHECK: # BB#0:
815 ; CHECK-NEXT: kmovd %edi, %k1
816 ; CHECK-NEXT: vextracti64x2 $3, %zmm0, %xmm1 {%k1}
817 ; CHECK-NEXT: vmovdqa %xmm1, %xmm0
818 ; CHECK-NEXT: vzeroupper
819 ; CHECK-NEXT: retq
820 %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <2 x i32> <i32 6, i32 7>
821 %mask.cast = bitcast i8 %mask to <8 x i1>
822 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
823 %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle, <2 x i64> %passthru
824 ret <2 x i64> %res
825 }
827 define <2 x double> @mask_extract_v8f64_v2f64_0(<8 x double> %a, <2 x double> %passthru, i8 %mask) {
828 ; CHECK-LABEL: mask_extract_v8f64_v2f64_0:
829 ; CHECK: # BB#0:
830 ; CHECK-NEXT: kmovd %edi, %k1
831 ; CHECK-NEXT: vblendmpd %xmm0, %xmm1, %xmm0 {%k1}
832 ; CHECK-NEXT: vzeroupper
833 ; CHECK-NEXT: retq
834 %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <2 x i32> <i32 0, i32 1>
835 %mask.cast = bitcast i8 %mask to <8 x i1>
836 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
837 %res = select <2 x i1> %mask.extract, <2 x double> %shuffle, <2 x double> %passthru
838 ret <2 x double> %res
839 }
841 define <2 x double> @mask_extract_v8f64_v2f64_0_z(<8 x double> %a, i8 %mask) {
842 ; CHECK-LABEL: mask_extract_v8f64_v2f64_0_z:
843 ; CHECK: # BB#0:
844 ; CHECK-NEXT: kmovd %edi, %k1
845 ; CHECK-NEXT: vmovapd %xmm0, %xmm0 {%k1} {z}
846 ; CHECK-NEXT: vzeroupper
847 ; CHECK-NEXT: retq
848 %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <2 x i32> <i32 0, i32 1>
849 %mask.cast = bitcast i8 %mask to <8 x i1>
850 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
851 %res = select <2 x i1> %mask.extract, <2 x double> %shuffle, <2 x double> zeroinitializer
852 ret <2 x double> %res
853 }
855 define <2 x double> @mask_extract_v8f64_v2f64_1(<8 x double> %a, <2 x double> %passthru, i8 %mask) {
856 ; CHECK-LABEL: mask_extract_v8f64_v2f64_1:
857 ; CHECK: # BB#0:
858 ; CHECK-NEXT: kmovd %edi, %k1
859 ; CHECK-NEXT: vextractf64x2 $1, %zmm0, %xmm1 {%k1}
860 ; CHECK-NEXT: vmovapd %xmm1, %xmm0
861 ; CHECK-NEXT: vzeroupper
862 ; CHECK-NEXT: retq
863 %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <2 x i32> <i32 2, i32 3>
864 %mask.cast = bitcast i8 %mask to <8 x i1>
865 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
866 %res = select <2 x i1> %mask.extract, <2 x double> %shuffle, <2 x double> %passthru
867 ret <2 x double> %res
868 }
870 define <2 x double> @mask_extract_v8f64_v2f64_1_z(<8 x double> %a, i8 %mask) {
871 ; CHECK-LABEL: mask_extract_v8f64_v2f64_1_z:
872 ; CHECK: # BB#0:
873 ; CHECK-NEXT: kmovd %edi, %k1
874 ; CHECK-NEXT: vextractf64x2 $1, %zmm0, %xmm0 {%k1} {z}
875 ; CHECK-NEXT: vzeroupper
876 ; CHECK-NEXT: retq
877 %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <2 x i32> <i32 2, i32 3>
878 %mask.cast = bitcast i8 %mask to <8 x i1>
879 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
880 %res = select <2 x i1> %mask.extract, <2 x double> %shuffle, <2 x double> zeroinitializer
881 ret <2 x double> %res
882 }
884 define <2 x double> @mask_extract_v8f64_v2f64_2(<8 x double> %a, <2 x double> %passthru, i8 %mask) {
885 ; CHECK-LABEL: mask_extract_v8f64_v2f64_2:
886 ; CHECK: # BB#0:
887 ; CHECK-NEXT: kmovd %edi, %k1
888 ; CHECK-NEXT: vextractf64x2 $2, %zmm0, %xmm1 {%k1}
889 ; CHECK-NEXT: vmovapd %xmm1, %xmm0
890 ; CHECK-NEXT: vzeroupper
891 ; CHECK-NEXT: retq
892 %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <2 x i32> <i32 4, i32 5>
893 %mask.cast = bitcast i8 %mask to <8 x i1>
894 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
895 %res = select <2 x i1> %mask.extract, <2 x double> %shuffle, <2 x double> %passthru
896 ret <2 x double> %res
897 }
899 define <2 x double> @mask_extract_v8f64_v2f64_3(<8 x double> %a, <2 x double> %passthru, i8 %mask) {
900 ; CHECK-LABEL: mask_extract_v8f64_v2f64_3:
901 ; CHECK: # BB#0:
902 ; CHECK-NEXT: kmovd %edi, %k1
903 ; CHECK-NEXT: vextractf64x2 $3, %zmm0, %xmm1 {%k1}
904 ; CHECK-NEXT: vmovapd %xmm1, %xmm0
905 ; CHECK-NEXT: vzeroupper
906 ; CHECK-NEXT: retq
907 %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <2 x i32> <i32 6, i32 7>
908 %mask.cast = bitcast i8 %mask to <8 x i1>
909 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
910 %res = select <2 x i1> %mask.extract, <2 x double> %shuffle, <2 x double> %passthru
911 ret <2 x double> %res
912 }
914 define <4 x i64> @mask_extract_v8i64_v4i64_0(<8 x i64> %a, <4 x i64> %passthru, i8 %mask) {
915 ; CHECK-LABEL: mask_extract_v8i64_v4i64_0:
916 ; CHECK: # BB#0:
917 ; CHECK-NEXT: kmovd %edi, %k1
918 ; CHECK-NEXT: vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
919 ; CHECK-NEXT: retq
920 %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
921 %mask.cast = bitcast i8 %mask to <8 x i1>
922 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
923 %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle, <4 x i64> %passthru
924 ret <4 x i64> %res
925 }
927 define <4 x i64> @mask_extract_v8i64_v4i64_0_z(<8 x i64> %a, i8 %mask) {
928 ; CHECK-LABEL: mask_extract_v8i64_v4i64_0_z:
929 ; CHECK: # BB#0:
930 ; CHECK-NEXT: kmovd %edi, %k1
931 ; CHECK-NEXT: vmovdqa64 %ymm0, %ymm0 {%k1} {z}
932 ; CHECK-NEXT: retq
933 %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
934 %mask.cast = bitcast i8 %mask to <8 x i1>
935 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
936 %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle, <4 x i64> zeroinitializer
937 ret <4 x i64> %res
938 }
940 define <4 x i64> @mask_extract_v8i64_v4i64_1(<8 x i64> %a, <4 x i64> %passthru, i8 %mask) {
941 ; CHECK-LABEL: mask_extract_v8i64_v4i64_1:
942 ; CHECK: # BB#0:
943 ; CHECK-NEXT: kmovd %edi, %k1
944 ; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm1 {%k1}
945 ; CHECK-NEXT: vmovdqa %ymm1, %ymm0
946 ; CHECK-NEXT: retq
947 %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
948 %mask.cast = bitcast i8 %mask to <8 x i1>
949 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
950 %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle, <4 x i64> %passthru
951 ret <4 x i64> %res
952 }
954 define <4 x i64> @mask_extract_v8i64_v4i64_1_z(<8 x i64> %a, i8 %mask) {
955 ; CHECK-LABEL: mask_extract_v8i64_v4i64_1_z:
956 ; CHECK: # BB#0:
957 ; CHECK-NEXT: kmovd %edi, %k1
958 ; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm0 {%k1} {z}
959 ; CHECK-NEXT: retq
960 %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
961 %mask.cast = bitcast i8 %mask to <8 x i1>
962 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
963 %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle, <4 x i64> zeroinitializer
964 ret <4 x i64> %res
965 }
967 define <4 x double> @mask_extract_v8f64_v4f64_0(<8 x double> %a, <4 x double> %passthru, i8 %mask) {
968 ; CHECK-LABEL: mask_extract_v8f64_v4f64_0:
969 ; CHECK: # BB#0:
970 ; CHECK-NEXT: kmovd %edi, %k1
971 ; CHECK-NEXT: vblendmpd %ymm0, %ymm1, %ymm0 {%k1}
972 ; CHECK-NEXT: retq
973 %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
974 %mask.cast = bitcast i8 %mask to <8 x i1>
975 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
976 %res = select <4 x i1> %mask.extract, <4 x double> %shuffle, <4 x double> %passthru
977 ret <4 x double> %res
978 }
980 define <4 x double> @mask_extract_v8f64_v4f64_0_z(<8 x double> %a, i8 %mask) {
981 ; CHECK-LABEL: mask_extract_v8f64_v4f64_0_z:
982 ; CHECK: # BB#0:
983 ; CHECK-NEXT: kmovd %edi, %k1
984 ; CHECK-NEXT: vmovapd %ymm0, %ymm0 {%k1} {z}
985 ; CHECK-NEXT: retq
986 %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
987 %mask.cast = bitcast i8 %mask to <8 x i1>
988 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
989 %res = select <4 x i1> %mask.extract, <4 x double> %shuffle, <4 x double> zeroinitializer
990 ret <4 x double> %res
991 }
993 define <4 x double> @mask_extract_v8f64_v4f64_1(<8 x double> %a, <4 x double> %passthru, i8 %mask) {
994 ; CHECK-LABEL: mask_extract_v8f64_v4f64_1:
995 ; CHECK: # BB#0:
996 ; CHECK-NEXT: kmovd %edi, %k1
997 ; CHECK-NEXT: vextractf64x4 $1, %zmm0, %ymm1 {%k1}
998 ; CHECK-NEXT: vmovapd %ymm1, %ymm0
999 ; CHECK-NEXT: retq
1000 %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
1001 %mask.cast = bitcast i8 %mask to <8 x i1>
1002 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1003 %res = select <4 x i1> %mask.extract, <4 x double> %shuffle, <4 x double> %passthru
1004 ret <4 x double> %res
1005 }
1007 define <4 x double> @mask_extract_v8f64_v4f64_1_z(<8 x double> %a, i8 %mask) {
1008 ; CHECK-LABEL: mask_extract_v8f64_v4f64_1_z:
1009 ; CHECK: # BB#0:
1010 ; CHECK-NEXT: kmovd %edi, %k1
1011 ; CHECK-NEXT: vextractf64x4 $1, %zmm0, %ymm0 {%k1} {z}
1012 ; CHECK-NEXT: retq
1013 %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
1014 %mask.cast = bitcast i8 %mask to <8 x i1>
1015 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1016 %res = select <4 x i1> %mask.extract, <4 x double> %shuffle, <4 x double> zeroinitializer
1017 ret <4 x double> %res
1018 }
1020 define <8 x i32> @mask_cast_extract_v8i64_v8i32_0(<8 x i64> %a, <8 x i32> %passthru, i8 %mask) {
1021 ; CHECK-LABEL: mask_cast_extract_v8i64_v8i32_0:
1022 ; CHECK: # BB#0:
1023 ; CHECK-NEXT: kmovd %edi, %k1
1024 ; CHECK-NEXT: vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
1025 ; CHECK-NEXT: retq
1026 %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1027 %shuffle.cast = bitcast <4 x i64> %shuffle to <8 x i32>
1028 %mask.cast = bitcast i8 %mask to <8 x i1>
1029 %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle.cast, <8 x i32> %passthru
1030 ret <8 x i32> %res
1031 }
1033 define <8 x i32> @mask_cast_extract_v8i64_v8i32_0_z(<8 x i64> %a, i8 %mask) {
1034 ; CHECK-LABEL: mask_cast_extract_v8i64_v8i32_0_z:
1035 ; CHECK: # BB#0:
1036 ; CHECK-NEXT: kmovd %edi, %k1
1037 ; CHECK-NEXT: vmovdqa32 %ymm0, %ymm0 {%k1} {z}
1038 ; CHECK-NEXT: retq
1039 %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1040 %shuffle.cast = bitcast <4 x i64> %shuffle to <8 x i32>
1041 %mask.cast = bitcast i8 %mask to <8 x i1>
1042 %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle.cast, <8 x i32> zeroinitializer
1043 ret <8 x i32> %res
1044 }
1046 define <8 x i32> @mask_cast_extract_v8i64_v8i32_1(<8 x i64> %a, <8 x i32> %passthru, i8 %mask) {
1047 ; CHECK-LABEL: mask_cast_extract_v8i64_v8i32_1:
1048 ; CHECK: # BB#0:
1049 ; CHECK-NEXT: kmovd %edi, %k1
1050 ; CHECK-NEXT: vextracti32x8 $1, %zmm0, %ymm1 {%k1}
1051 ; CHECK-NEXT: vmovdqa %ymm1, %ymm0
1052 ; CHECK-NEXT: retq
1053 %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
1054 %shuffle.cast = bitcast <4 x i64> %shuffle to <8 x i32>
1055 %mask.cast = bitcast i8 %mask to <8 x i1>
1056 %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle.cast, <8 x i32> %passthru
1057 ret <8 x i32> %res
1058 }
1060 define <8 x i32> @mask_cast_extract_v8i64_v8i32_1_z(<8 x i64> %a, i8 %mask) {
1061 ; CHECK-LABEL: mask_cast_extract_v8i64_v8i32_1_z:
1062 ; CHECK: # BB#0:
1063 ; CHECK-NEXT: kmovd %edi, %k1
1064 ; CHECK-NEXT: vextracti32x8 $1, %zmm0, %ymm0 {%k1} {z}
1065 ; CHECK-NEXT: retq
1066 %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
1067 %shuffle.cast = bitcast <4 x i64> %shuffle to <8 x i32>
1068 %mask.cast = bitcast i8 %mask to <8 x i1>
1069 %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle.cast, <8 x i32> zeroinitializer
1070 ret <8 x i32> %res
1071 }
1073 define <8 x float> @mask_cast_extract_v8f64_v8f32_0(<8 x double> %a, <8 x float> %passthru, i8 %mask) {
1074 ; CHECK-LABEL: mask_cast_extract_v8f64_v8f32_0:
1075 ; CHECK: # BB#0:
1076 ; CHECK-NEXT: kmovd %edi, %k1
1077 ; CHECK-NEXT: vblendmps %ymm0, %ymm1, %ymm0 {%k1}
1078 ; CHECK-NEXT: retq
1079 %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1080 %shuffle.cast = bitcast <4 x double> %shuffle to <8 x float>
1081 %mask.cast = bitcast i8 %mask to <8 x i1>
1082 %res = select <8 x i1> %mask.cast, <8 x float> %shuffle.cast, <8 x float> %passthru
1083 ret <8 x float> %res
1084 }
1086 define <8 x float> @mask_cast_extract_v8f64_v8f32_0_z(<8 x double> %a, i8 %mask) {
1087 ; CHECK-LABEL: mask_cast_extract_v8f64_v8f32_0_z:
1088 ; CHECK: # BB#0:
1089 ; CHECK-NEXT: kmovd %edi, %k1
1090 ; CHECK-NEXT: vmovaps %ymm0, %ymm0 {%k1} {z}
1091 ; CHECK-NEXT: retq
1092 %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1093 %shuffle.cast = bitcast <4 x double> %shuffle to <8 x float>
1094 %mask.cast = bitcast i8 %mask to <8 x i1>
1095 %res = select <8 x i1> %mask.cast, <8 x float> %shuffle.cast, <8 x float> zeroinitializer
1096 ret <8 x float> %res
1097 }
1099 define <8 x float> @mask_cast_extract_v8f64_v8f32_1(<8 x double> %a, <8 x float> %passthru, i8 %mask) {
1100 ; CHECK-LABEL: mask_cast_extract_v8f64_v8f32_1:
1101 ; CHECK: # BB#0:
1102 ; CHECK-NEXT: kmovd %edi, %k1
1103 ; CHECK-NEXT: vextractf32x8 $1, %zmm0, %ymm1 {%k1}
1104 ; CHECK-NEXT: vmovaps %ymm1, %ymm0
1105 ; CHECK-NEXT: retq
1106 %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
1107 %shuffle.cast = bitcast <4 x double> %shuffle to <8 x float>
1108 %mask.cast = bitcast i8 %mask to <8 x i1>
1109 %res = select <8 x i1> %mask.cast, <8 x float> %shuffle.cast, <8 x float> %passthru
1110 ret <8 x float> %res
1111 }
1113 define <8 x float> @mask_cast_extract_v8f64_v8f32_1_z(<8 x double> %a, i8 %mask) {
1114 ; CHECK-LABEL: mask_cast_extract_v8f64_v8f32_1_z:
1115 ; CHECK: # BB#0:
1116 ; CHECK-NEXT: kmovd %edi, %k1
1117 ; CHECK-NEXT: vextractf32x8 $1, %zmm0, %ymm0 {%k1} {z}
1118 ; CHECK-NEXT: retq
1119 %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
1120 %shuffle.cast = bitcast <4 x double> %shuffle to <8 x float>
1121 %mask.cast = bitcast i8 %mask to <8 x i1>
1122 %res = select <8 x i1> %mask.cast, <8 x float> %shuffle.cast, <8 x float> zeroinitializer
1123 ret <8 x float> %res
1124 }
1126 define <4 x i32> @mask_cast_extract_v8i64_v4i32_0(<8 x i64> %a, <4 x i32> %passthru, i8 %mask) {
1127 ; CHECK-LABEL: mask_cast_extract_v8i64_v4i32_0:
1128 ; CHECK: # BB#0:
1129 ; CHECK-NEXT: kmovd %edi, %k1
1130 ; CHECK-NEXT: vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
1131 ; CHECK-NEXT: vzeroupper
1132 ; CHECK-NEXT: retq
1133 %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <2 x i32> <i32 0, i32 1>
1134 %shuffle.cast = bitcast <2 x i64> %shuffle to <4 x i32>
1135 %mask.cast = bitcast i8 %mask to <8 x i1>
1136 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1137 %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle.cast, <4 x i32> %passthru
1138 ret <4 x i32> %res
1139 }
1141 define <4 x i32> @mask_cast_extract_v8i64_v4i32_0_z(<8 x i64> %a, i8 %mask) {
1142 ; CHECK-LABEL: mask_cast_extract_v8i64_v4i32_0_z:
1143 ; CHECK: # BB#0:
1144 ; CHECK-NEXT: kmovd %edi, %k1
1145 ; CHECK-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z}
1146 ; CHECK-NEXT: vzeroupper
1147 ; CHECK-NEXT: retq
1148 %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <2 x i32> <i32 0, i32 1>
1149 %shuffle.cast = bitcast <2 x i64> %shuffle to <4 x i32>
1150 %mask.cast = bitcast i8 %mask to <8 x i1>
1151 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1152 %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle.cast, <4 x i32> zeroinitializer
1153 ret <4 x i32> %res
1154 }
1156 define <4 x i32> @mask_cast_extract_v8i64_v4i32_1(<8 x i64> %a, <4 x i32> %passthru, i8 %mask) {
1157 ; CHECK-LABEL: mask_cast_extract_v8i64_v4i32_1:
1158 ; CHECK: # BB#0:
1159 ; CHECK-NEXT: kmovd %edi, %k1
1160 ; CHECK-NEXT: vextracti32x4 $1, %zmm0, %xmm1 {%k1}
1161 ; CHECK-NEXT: vmovdqa %xmm1, %xmm0
1162 ; CHECK-NEXT: vzeroupper
1163 ; CHECK-NEXT: retq
1164 %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <2 x i32> <i32 2, i32 3>
1165 %shuffle.cast = bitcast <2 x i64> %shuffle to <4 x i32>
1166 %mask.cast = bitcast i8 %mask to <8 x i1>
1167 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1168 %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle.cast, <4 x i32> %passthru
1169 ret <4 x i32> %res
1170 }
1172 define <4 x i32> @mask_cast_extract_v8i64_v4i32_1_z(<8 x i64> %a, i8 %mask) {
1173 ; CHECK-LABEL: mask_cast_extract_v8i64_v4i32_1_z:
1174 ; CHECK: # BB#0:
1175 ; CHECK-NEXT: kmovd %edi, %k1
1176 ; CHECK-NEXT: vextracti32x4 $1, %zmm0, %xmm0 {%k1} {z}
1177 ; CHECK-NEXT: vzeroupper
1178 ; CHECK-NEXT: retq
1179 %shuffle = shufflevector <8 x i64> %a, <8 x i64> undef, <2 x i32> <i32 2, i32 3>
1180 %shuffle.cast = bitcast <2 x i64> %shuffle to <4 x i32>
1181 %mask.cast = bitcast i8 %mask to <8 x i1>
1182 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1183 %res = select <4 x i1> %mask.extract, <4 x i32> %shuffle.cast, <4 x i32> zeroinitializer
1184 ret <4 x i32> %res
1185 }
1187 define <4 x float> @mask_cast_extract_v8f64_v4f32_0(<8 x double> %a, <4 x float> %passthru, i8 %mask) {
1188 ; CHECK-LABEL: mask_cast_extract_v8f64_v4f32_0:
1189 ; CHECK: # BB#0:
1190 ; CHECK-NEXT: kmovd %edi, %k1
1191 ; CHECK-NEXT: vblendmps %xmm0, %xmm1, %xmm0 {%k1}
1192 ; CHECK-NEXT: vzeroupper
1193 ; CHECK-NEXT: retq
1194 %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <2 x i32> <i32 0, i32 1>
1195 %shuffle.cast = bitcast <2 x double> %shuffle to <4 x float>
1196 %mask.cast = bitcast i8 %mask to <8 x i1>
1197 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1198 %res = select <4 x i1> %mask.extract, <4 x float> %shuffle.cast, <4 x float> %passthru
1199 ret <4 x float> %res
1200 }
1202 define <4 x float> @mask_cast_extract_v8f64_v4f32_0_z(<8 x double> %a, i8 %mask) {
1203 ; CHECK-LABEL: mask_cast_extract_v8f64_v4f32_0_z:
1204 ; CHECK: # BB#0:
1205 ; CHECK-NEXT: kmovd %edi, %k1
1206 ; CHECK-NEXT: vmovaps %xmm0, %xmm0 {%k1} {z}
1207 ; CHECK-NEXT: vzeroupper
1208 ; CHECK-NEXT: retq
1209 %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <2 x i32> <i32 0, i32 1>
1210 %shuffle.cast = bitcast <2 x double> %shuffle to <4 x float>
1211 %mask.cast = bitcast i8 %mask to <8 x i1>
1212 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1213 %res = select <4 x i1> %mask.extract, <4 x float> %shuffle.cast, <4 x float> zeroinitializer
1214 ret <4 x float> %res
1215 }
1217 define <4 x float> @mask_cast_extract_v8f64_v4f32_1(<8 x double> %a, <4 x float> %passthru, i8 %mask) {
1218 ; CHECK-LABEL: mask_cast_extract_v8f64_v4f32_1:
1219 ; CHECK: # BB#0:
1220 ; CHECK-NEXT: kmovd %edi, %k1
1221 ; CHECK-NEXT: vextractf32x4 $1, %zmm0, %xmm1 {%k1}
1222 ; CHECK-NEXT: vmovaps %xmm1, %xmm0
1223 ; CHECK-NEXT: vzeroupper
1224 ; CHECK-NEXT: retq
1225 %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <2 x i32> <i32 2, i32 3>
1226 %shuffle.cast = bitcast <2 x double> %shuffle to <4 x float>
1227 %mask.cast = bitcast i8 %mask to <8 x i1>
1228 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1229 %res = select <4 x i1> %mask.extract, <4 x float> %shuffle.cast, <4 x float> %passthru
1230 ret <4 x float> %res
1231 }
1233 define <4 x float> @mask_cast_extract_v8f64_v4f32_1_z(<8 x double> %a, i8 %mask) {
1234 ; CHECK-LABEL: mask_cast_extract_v8f64_v4f32_1_z:
1235 ; CHECK: # BB#0:
1236 ; CHECK-NEXT: kmovd %edi, %k1
1237 ; CHECK-NEXT: vextractf32x4 $1, %zmm0, %xmm0 {%k1} {z}
1238 ; CHECK-NEXT: vzeroupper
1239 ; CHECK-NEXT: retq
1240 %shuffle = shufflevector <8 x double> %a, <8 x double> undef, <2 x i32> <i32 2, i32 3>
1241 %shuffle.cast = bitcast <2 x double> %shuffle to <4 x float>
1242 %mask.cast = bitcast i8 %mask to <8 x i1>
1243 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1244 %res = select <4 x i1> %mask.extract, <4 x float> %shuffle.cast, <4 x float> zeroinitializer
1245 ret <4 x float> %res
1246 }
1248 define <4 x i64> @mask_cast_extract_v16i32_v4i64_0(<16 x i32> %a, <4 x i64> %passthru, i8 %mask) {
1249 ; CHECK-LABEL: mask_cast_extract_v16i32_v4i64_0:
1250 ; CHECK: # BB#0:
1251 ; CHECK-NEXT: kmovd %edi, %k1
1252 ; CHECK-NEXT: vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
1253 ; CHECK-NEXT: retq
1254 %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
1255 %shuffle.cast = bitcast <8 x i32> %shuffle to <4 x i64>
1256 %mask.cast = bitcast i8 %mask to <8 x i1>
1257 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1258 %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle.cast, <4 x i64> %passthru
1259 ret <4 x i64> %res
1260 }
1262 define <4 x i64> @mask_cast_extract_v16i32_v4i64_0_z(<16 x i32> %a, i8 %mask) {
1263 ; CHECK-LABEL: mask_cast_extract_v16i32_v4i64_0_z:
1264 ; CHECK: # BB#0:
1265 ; CHECK-NEXT: kmovd %edi, %k1
1266 ; CHECK-NEXT: vmovdqa64 %ymm0, %ymm0 {%k1} {z}
1267 ; CHECK-NEXT: retq
1268 %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
1269 %shuffle.cast = bitcast <8 x i32> %shuffle to <4 x i64>
1270 %mask.cast = bitcast i8 %mask to <8 x i1>
1271 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1272 %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle.cast, <4 x i64> zeroinitializer
1273 ret <4 x i64> %res
1274 }
1276 define <4 x i64> @mask_cast_extract_v16i32_v4i64_1(<16 x i32> %a, <4 x i64> %passthru, i8 %mask) {
1277 ; CHECK-LABEL: mask_cast_extract_v16i32_v4i64_1:
1278 ; CHECK: # BB#0:
1279 ; CHECK-NEXT: kmovd %edi, %k1
1280 ; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm1 {%k1}
1281 ; CHECK-NEXT: vmovdqa %ymm1, %ymm0
1282 ; CHECK-NEXT: retq
1283 %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
1284 %shuffle.cast = bitcast <8 x i32> %shuffle to <4 x i64>
1285 %mask.cast = bitcast i8 %mask to <8 x i1>
1286 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1287 %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle.cast, <4 x i64> %passthru
1288 ret <4 x i64> %res
1289 }
1291 define <4 x i64> @mask_cast_extract_v16i32_v4i64_1_z(<16 x i32> %a, i8 %mask) {
1292 ; CHECK-LABEL: mask_cast_extract_v16i32_v4i64_1_z:
1293 ; CHECK: # BB#0:
1294 ; CHECK-NEXT: kmovd %edi, %k1
1295 ; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm0 {%k1} {z}
1296 ; CHECK-NEXT: retq
1297 %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
1298 %shuffle.cast = bitcast <8 x i32> %shuffle to <4 x i64>
1299 %mask.cast = bitcast i8 %mask to <8 x i1>
1300 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1301 %res = select <4 x i1> %mask.extract, <4 x i64> %shuffle.cast, <4 x i64> zeroinitializer
1302 ret <4 x i64> %res
1303 }
1305 define <4 x double> @mask_cast_extract_v16f32_v4f64_0(<16 x float> %a, <4 x double> %passthru, i8 %mask) {
1306 ; CHECK-LABEL: mask_cast_extract_v16f32_v4f64_0:
1307 ; CHECK: # BB#0:
1308 ; CHECK-NEXT: kmovd %edi, %k1
1309 ; CHECK-NEXT: vblendmpd %ymm0, %ymm1, %ymm0 {%k1}
1310 ; CHECK-NEXT: retq
1311 %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
1312 %shuffle.cast = bitcast <8 x float> %shuffle to <4 x double>
1313 %mask.cast = bitcast i8 %mask to <8 x i1>
1314 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1315 %res = select <4 x i1> %mask.extract, <4 x double> %shuffle.cast, <4 x double> %passthru
1316 ret <4 x double> %res
1317 }
1319 define <4 x double> @mask_cast_extract_v16f32_v4f64_0_z(<16 x float> %a, i8 %mask) {
1320 ; CHECK-LABEL: mask_cast_extract_v16f32_v4f64_0_z:
1321 ; CHECK: # BB#0:
1322 ; CHECK-NEXT: kmovd %edi, %k1
1323 ; CHECK-NEXT: vmovapd %ymm0, %ymm0 {%k1} {z}
1324 ; CHECK-NEXT: retq
1325 %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
1326 %shuffle.cast = bitcast <8 x float> %shuffle to <4 x double>
1327 %mask.cast = bitcast i8 %mask to <8 x i1>
1328 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1329 %res = select <4 x i1> %mask.extract, <4 x double> %shuffle.cast, <4 x double> zeroinitializer
1330 ret <4 x double> %res
1331 }
1333 define <4 x double> @mask_cast_extract_v16f32_v4f64_1(<16 x float> %a, <4 x double> %passthru, i8 %mask) {
1334 ; CHECK-LABEL: mask_cast_extract_v16f32_v4f64_1:
1335 ; CHECK: # BB#0:
1336 ; CHECK-NEXT: kmovd %edi, %k1
1337 ; CHECK-NEXT: vextractf64x4 $1, %zmm0, %ymm1 {%k1}
1338 ; CHECK-NEXT: vmovapd %ymm1, %ymm0
1339 ; CHECK-NEXT: retq
1340 %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
1341 %shuffle.cast = bitcast <8 x float> %shuffle to <4 x double>
1342 %mask.cast = bitcast i8 %mask to <8 x i1>
1343 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1344 %res = select <4 x i1> %mask.extract, <4 x double> %shuffle.cast, <4 x double> %passthru
1345 ret <4 x double> %res
1346 }
1348 define <4 x double> @mask_cast_extract_v16f32_v4f64_1_z(<16 x float> %a, i8 %mask) {
1349 ; CHECK-LABEL: mask_cast_extract_v16f32_v4f64_1_z:
1350 ; CHECK: # BB#0:
1351 ; CHECK-NEXT: kmovd %edi, %k1
1352 ; CHECK-NEXT: vextractf64x4 $1, %zmm0, %ymm0 {%k1} {z}
1353 ; CHECK-NEXT: retq
1354 %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
1355 %shuffle.cast = bitcast <8 x float> %shuffle to <4 x double>
1356 %mask.cast = bitcast i8 %mask to <8 x i1>
1357 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1358 %res = select <4 x i1> %mask.extract, <4 x double> %shuffle.cast, <4 x double> zeroinitializer
1359 ret <4 x double> %res
1360 }
1362 define <2 x i64> @mask_cast_extract_v16i32_v2i64_0(<16 x i32> %a, <2 x i64> %passthru, i8 %mask) {
1363 ; CHECK-LABEL: mask_cast_extract_v16i32_v2i64_0:
1364 ; CHECK: # BB#0:
1365 ; CHECK-NEXT: kmovd %edi, %k1
1366 ; CHECK-NEXT: vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
1367 ; CHECK-NEXT: vzeroupper
1368 ; CHECK-NEXT: retq
1369 %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1370 %shuffle.cast = bitcast <4 x i32> %shuffle to <2 x i64>
1371 %mask.cast = bitcast i8 %mask to <8 x i1>
1372 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
1373 %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle.cast, <2 x i64> %passthru
1374 ret <2 x i64> %res
1375 }
1377 define <2 x i64> @mask_cast_extract_v16i32_v2i64_0_z(<16 x i32> %a, i8 %mask) {
1378 ; CHECK-LABEL: mask_cast_extract_v16i32_v2i64_0_z:
1379 ; CHECK: # BB#0:
1380 ; CHECK-NEXT: kmovd %edi, %k1
1381 ; CHECK-NEXT: vmovdqa64 %xmm0, %xmm0 {%k1} {z}
1382 ; CHECK-NEXT: vzeroupper
1383 ; CHECK-NEXT: retq
1384 %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1385 %shuffle.cast = bitcast <4 x i32> %shuffle to <2 x i64>
1386 %mask.cast = bitcast i8 %mask to <8 x i1>
1387 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
1388 %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle.cast, <2 x i64> zeroinitializer
1389 ret <2 x i64> %res
1390 }
1392 define <2 x i64> @mask_cast_extract_v16i32_v2i64_1(<16 x i32> %a, <2 x i64> %passthru, i8 %mask) {
1393 ; CHECK-LABEL: mask_cast_extract_v16i32_v2i64_1:
1394 ; CHECK: # BB#0:
1395 ; CHECK-NEXT: kmovd %edi, %k1
1396 ; CHECK-NEXT: vextracti64x2 $1, %zmm0, %xmm1 {%k1}
1397 ; CHECK-NEXT: vmovdqa %xmm1, %xmm0
1398 ; CHECK-NEXT: vzeroupper
1399 ; CHECK-NEXT: retq
1400 %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
1401 %shuffle.cast = bitcast <4 x i32> %shuffle to <2 x i64>
1402 %mask.cast = bitcast i8 %mask to <8 x i1>
1403 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
1404 %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle.cast, <2 x i64> %passthru
1405 ret <2 x i64> %res
1406 }
1408 define <2 x i64> @mask_cast_extract_v16i32_v2i64_1_z(<16 x i32> %a, i8 %mask) {
1409 ; CHECK-LABEL: mask_cast_extract_v16i32_v2i64_1_z:
1410 ; CHECK: # BB#0:
1411 ; CHECK-NEXT: kmovd %edi, %k1
1412 ; CHECK-NEXT: vextracti64x2 $1, %zmm0, %xmm0 {%k1} {z}
1413 ; CHECK-NEXT: vzeroupper
1414 ; CHECK-NEXT: retq
1415 %shuffle = shufflevector <16 x i32> %a, <16 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
1416 %shuffle.cast = bitcast <4 x i32> %shuffle to <2 x i64>
1417 %mask.cast = bitcast i8 %mask to <8 x i1>
1418 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
1419 %res = select <2 x i1> %mask.extract, <2 x i64> %shuffle.cast, <2 x i64> zeroinitializer
1420 ret <2 x i64> %res
1421 }
1423 define <2 x double> @mask_cast_extract_v16f32_v2f64_0(<16 x float> %a, <2 x double> %passthru, i8 %mask) {
1424 ; CHECK-LABEL: mask_cast_extract_v16f32_v2f64_0:
1425 ; CHECK: # BB#0:
1426 ; CHECK-NEXT: kmovd %edi, %k1
1427 ; CHECK-NEXT: vblendmpd %xmm0, %xmm1, %xmm0 {%k1}
1428 ; CHECK-NEXT: vzeroupper
1429 ; CHECK-NEXT: retq
1430 %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1431 %shuffle.cast = bitcast <4 x float> %shuffle to <2 x double>
1432 %mask.cast = bitcast i8 %mask to <8 x i1>
1433 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
1434 %res = select <2 x i1> %mask.extract, <2 x double> %shuffle.cast, <2 x double> %passthru
1435 ret <2 x double> %res
1436 }
1438 define <2 x double> @mask_cast_extract_v16f32_v2f64_0_z(<16 x float> %a, i8 %mask) {
1439 ; CHECK-LABEL: mask_cast_extract_v16f32_v2f64_0_z:
1440 ; CHECK: # BB#0:
1441 ; CHECK-NEXT: kmovd %edi, %k1
1442 ; CHECK-NEXT: vmovapd %xmm0, %xmm0 {%k1} {z}
1443 ; CHECK-NEXT: vzeroupper
1444 ; CHECK-NEXT: retq
1445 %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1446 %shuffle.cast = bitcast <4 x float> %shuffle to <2 x double>
1447 %mask.cast = bitcast i8 %mask to <8 x i1>
1448 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
1449 %res = select <2 x i1> %mask.extract, <2 x double> %shuffle.cast, <2 x double> zeroinitializer
1450 ret <2 x double> %res
1451 }
1453 define <2 x double> @mask_cast_extract_v16f32_v2f64_1(<16 x float> %a, <2 x double> %passthru, i8 %mask) {
1454 ; CHECK-LABEL: mask_cast_extract_v16f32_v2f64_1:
1455 ; CHECK: # BB#0:
1456 ; CHECK-NEXT: kmovd %edi, %k1
1457 ; CHECK-NEXT: vextractf64x2 $1, %zmm0, %xmm1 {%k1}
1458 ; CHECK-NEXT: vmovapd %xmm1, %xmm0
1459 ; CHECK-NEXT: vzeroupper
1460 ; CHECK-NEXT: retq
1461 %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
1462 %shuffle.cast = bitcast <4 x float> %shuffle to <2 x double>
1463 %mask.cast = bitcast i8 %mask to <8 x i1>
1464 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
1465 %res = select <2 x i1> %mask.extract, <2 x double> %shuffle.cast, <2 x double> %passthru
1466 ret <2 x double> %res
1467 }
1469 define <2 x double> @mask_cast_extract_v16f32_v2f64_1_z(<16 x float> %a, i8 %mask) {
1470 ; CHECK-LABEL: mask_cast_extract_v16f32_v2f64_1_z:
1471 ; CHECK: # BB#0:
1472 ; CHECK-NEXT: kmovd %edi, %k1
1473 ; CHECK-NEXT: vextractf64x2 $1, %zmm0, %xmm0 {%k1} {z}
1474 ; CHECK-NEXT: vzeroupper
1475 ; CHECK-NEXT: retq
1476 %shuffle = shufflevector <16 x float> %a, <16 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
1477 %shuffle.cast = bitcast <4 x float> %shuffle to <2 x double>
1478 %mask.cast = bitcast i8 %mask to <8 x i1>
1479 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
1480 %res = select <2 x i1> %mask.extract, <2 x double> %shuffle.cast, <2 x double> zeroinitializer
1481 ret <2 x double> %res
1482 }
1484 define <2 x double> @broadcast_v4f32_0101_from_v2f32_mask(double* %x, <2 x double> %passthru, i8 %mask) {
1485 ; CHECK-LABEL: broadcast_v4f32_0101_from_v2f32_mask:
1486 ; CHECK: # BB#0:
1487 ; CHECK-NEXT: kmovd %esi, %k1
1488 ; CHECK-NEXT: vmovddup {{.*#+}} xmm0 {%k1} = mem[0,0]
1489 ; CHECK-NEXT: retq
1490 %q = load double, double* %x, align 1
1491 %vecinit.i = insertelement <2 x double> undef, double %q, i32 0
1492 %vecinit2.i = insertelement <2 x double> %vecinit.i, double %q, i32 1
1493 %mask.cast = bitcast i8 %mask to <8 x i1>
1494 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
1495 %res = select <2 x i1> %mask.extract, <2 x double> %vecinit2.i, <2 x double> %passthru
1496 ret <2 x double> %res
1497 }
1499 define <2 x double> @broadcast_v4f32_0101_from_v2f32_maskz(double* %x, i8 %mask) {
1500 ; CHECK-LABEL: broadcast_v4f32_0101_from_v2f32_maskz:
1501 ; CHECK: # BB#0:
1502 ; CHECK-NEXT: kmovd %esi, %k1
1503 ; CHECK-NEXT: vmovddup {{.*#+}} xmm0 {%k1} {z} = mem[0,0]
1504 ; CHECK-NEXT: retq
1505 %q = load double, double* %x, align 1
1506 %vecinit.i = insertelement <2 x double> undef, double %q, i32 0
1507 %vecinit2.i = insertelement <2 x double> %vecinit.i, double %q, i32 1
1508 %mask.cast = bitcast i8 %mask to <8 x i1>
1509 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
1510 %res = select <2 x i1> %mask.extract, <2 x double> %vecinit2.i, <2 x double> zeroinitializer
1511 ret <2 x double> %res
1512 }
1514 define <8 x float> @test_broadcast_2f64_8f32(<2 x double> *%p, i8 %mask) nounwind {
1515 ; CHECK-LABEL: test_broadcast_2f64_8f32:
1516 ; CHECK: # BB#0:
1517 ; CHECK-NEXT: kmovd %esi, %k1
1518 ; CHECK-NEXT: vbroadcastf32x4 {{.*#+}} ymm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3]
1519 ; CHECK-NEXT: retq
1520 %1 = load <2 x double>, <2 x double> *%p
1521 %2 = shufflevector <2 x double> %1, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
1522 %3 = bitcast <4 x double> %2 to <8 x float>
1523 %mask.cast = bitcast i8 %mask to <8 x i1>
1524 %res = select <8 x i1> %mask.cast, <8 x float> %3, <8 x float> zeroinitializer
1525 ret <8 x float> %res
1526 }
1528 define <8 x i32> @test_broadcast_2i64_8i32(<2 x i64> *%p, i8 %mask) nounwind {
1529 ; CHECK-LABEL: test_broadcast_2i64_8i32:
1530 ; CHECK: # BB#0:
1531 ; CHECK-NEXT: kmovd %esi, %k1
1532 ; CHECK-NEXT: vbroadcasti32x4 {{.*#+}} ymm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3]
1533 ; CHECK-NEXT: retq
1534 %1 = load <2 x i64>, <2 x i64> *%p
1535 %2 = shufflevector <2 x i64> %1, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
1536 %3 = bitcast <4 x i64> %2 to <8 x i32>
1537 %mask.cast = bitcast i8 %mask to <8 x i1>
1538 %res = select <8 x i1> %mask.cast, <8 x i32> %3, <8 x i32> zeroinitializer
1539 ret <8 x i32> %res
1540 }
1542 define <16 x float> @test_broadcast_2f64_16f32(<2 x double> *%p, i16 %mask) nounwind {
1543 ; CHECK-LABEL: test_broadcast_2f64_16f32:
1544 ; CHECK: # BB#0:
1545 ; CHECK-NEXT: kmovd %esi, %k1
1546 ; CHECK-NEXT: vbroadcastf32x4 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
1547 ; CHECK-NEXT: retq
1548 %1 = load <2 x double>, <2 x double> *%p
1549 %2 = shufflevector <2 x double> %1, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
1550 %3 = bitcast <8 x double> %2 to <16 x float>
1551 %mask.cast = bitcast i16 %mask to <16 x i1>
1552 %res = select <16 x i1> %mask.cast, <16 x float> %3, <16 x float> zeroinitializer
1553 ret <16 x float> %res
1554 }
1556 define <16 x i32> @test_broadcast_2i64_16i32(<2 x i64> *%p, i16 %mask) nounwind {
1557 ; CHECK-LABEL: test_broadcast_2i64_16i32:
1558 ; CHECK: # BB#0:
1559 ; CHECK-NEXT: kmovd %esi, %k1
1560 ; CHECK-NEXT: vbroadcasti32x4 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
1561 ; CHECK-NEXT: retq
1562 %1 = load <2 x i64>, <2 x i64> *%p
1563 %2 = shufflevector <2 x i64> %1, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
1564 %3 = bitcast <8 x i64> %2 to <16 x i32>
1565 %mask.cast = bitcast i16 %mask to <16 x i1>
1566 %res = select <16 x i1> %mask.cast, <16 x i32> %3, <16 x i32> zeroinitializer
1567 ret <16 x i32> %res
1568 }
1570 define <16 x float> @test_broadcast_4f64_16f32(<4 x double> *%p, i16 %mask) nounwind {
1571 ; CHECK-LABEL: test_broadcast_4f64_16f32:
1572 ; CHECK: # BB#0:
1573 ; CHECK-NEXT: kmovd %esi, %k1
1574 ; CHECK-NEXT: vbroadcastf32x8 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7]
1575 ; CHECK-NEXT: retq
1576 %1 = load <4 x double>, <4 x double> *%p
1577 %2 = shufflevector <4 x double> %1, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
1578 %3 = bitcast <8 x double> %2 to <16 x float>
1579 %mask.cast = bitcast i16 %mask to <16 x i1>
1580 %res = select <16 x i1> %mask.cast, <16 x float> %3, <16 x float> zeroinitializer
1581 ret <16 x float> %res
1582 }
1584 define <16 x i32> @test_broadcast_4i64_16i32(<4 x i64> *%p, i16 %mask) nounwind {
1585 ; CHECK-LABEL: test_broadcast_4i64_16i32:
1586 ; CHECK: # BB#0:
1587 ; CHECK-NEXT: kmovd %esi, %k1
1588 ; CHECK-NEXT: vbroadcasti32x8 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7]
1589 ; CHECK-NEXT: retq
1590 %1 = load <4 x i64>, <4 x i64> *%p
1591 %2 = shufflevector <4 x i64> %1, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
1592 %3 = bitcast <8 x i64> %2 to <16 x i32>
1593 %mask.cast = bitcast i16 %mask to <16 x i1>
1594 %res = select <16 x i1> %mask.cast, <16 x i32> %3, <16 x i32> zeroinitializer
1595 ret <16 x i32> %res
1596 }
1598 define <4 x double> @test_broadcast_4f32_4f64(<4 x float> *%p, i8 %mask) nounwind {
1599 ; CHECK-LABEL: test_broadcast_4f32_4f64:
1600 ; CHECK: # BB#0:
1601 ; CHECK-NEXT: kmovd %esi, %k1
1602 ; CHECK-NEXT: vbroadcastf64x2 {{.*#+}} ymm0 {%k1} {z} = mem[0,1,0,1]
1603 ; CHECK-NEXT: retq
1604 %1 = load <4 x float>, <4 x float> *%p
1605 %2 = shufflevector <4 x float> %1, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
1606 %3 = bitcast <8 x float> %2 to <4 x double>
1607 %mask.cast = bitcast i8 %mask to <8 x i1>
1608 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1609 %res = select <4 x i1> %mask.extract, <4 x double> %3, <4 x double> zeroinitializer
1610 ret <4 x double> %res
1611 }
1613 define <4 x i64> @test_broadcast_4i32_4i64(<4 x i32> *%p, i8 %mask) nounwind {
1614 ; CHECK-LABEL: test_broadcast_4i32_4i64:
1615 ; CHECK: # BB#0:
1616 ; CHECK-NEXT: kmovd %esi, %k1
1617 ; CHECK-NEXT: vbroadcasti64x2 {{.*#+}} ymm0 {%k1} {z} = mem[0,1,0,1]
1618 ; CHECK-NEXT: retq
1619 %1 = load <4 x i32>, <4 x i32> *%p
1620 %2 = shufflevector <4 x i32> %1, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
1621 %3 = bitcast <8 x i32> %2 to <4 x i64>
1622 %mask.cast = bitcast i8 %mask to <8 x i1>
1623 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1624 %res = select <4 x i1> %mask.extract, <4 x i64> %3, <4 x i64> zeroinitializer
1625 ret <4 x i64> %res
1626 }
1628 define <8 x double> @test_broadcast_4f32_8f64(<4 x float> *%p, i8 %mask) nounwind {
1629 ; CHECK-LABEL: test_broadcast_4f32_8f64:
1630 ; CHECK: # BB#0:
1631 ; CHECK-NEXT: kmovd %esi, %k1
1632 ; CHECK-NEXT: vbroadcastf64x2 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,0,1,0,1,0,1]
1633 ; CHECK-NEXT: retq
1634 %1 = load <4 x float>, <4 x float> *%p
1635 %2 = shufflevector <4 x float> %1, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
1636 %3 = bitcast <16 x float> %2 to <8 x double>
1637 %mask.cast = bitcast i8 %mask to <8 x i1>
1638 %res = select <8 x i1> %mask.cast, <8 x double> %3, <8 x double> zeroinitializer
1639 ret <8 x double> %res
1640 }
1642 define <8 x i64> @test_broadcast_4i32_8i64(<4 x i32> *%p, i8 %mask) nounwind {
1643 ; CHECK-LABEL: test_broadcast_4i32_8i64:
1644 ; CHECK: # BB#0:
1645 ; CHECK-NEXT: kmovd %esi, %k1
1646 ; CHECK-NEXT: vbroadcasti64x2 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,0,1,0,1,0,1]
1647 ; CHECK-NEXT: retq
1648 %1 = load <4 x i32>, <4 x i32> *%p
1649 %2 = shufflevector <4 x i32> %1, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
1650 %3 = bitcast <16 x i32> %2 to <8 x i64>
1651 %mask.cast = bitcast i8 %mask to <8 x i1>
1652 %res = select <8 x i1> %mask.cast, <8 x i64> %3, <8 x i64> zeroinitializer
1653 ret <8 x i64> %res
1654 }
1656 define <8 x double> @test_broadcast_8f32_8f64(<8 x float> *%p, i8 %mask) nounwind {
1657 ; CHECK-LABEL: test_broadcast_8f32_8f64:
1658 ; CHECK: # BB#0:
1659 ; CHECK-NEXT: kmovd %esi, %k1
1660 ; CHECK-NEXT: vbroadcastf64x4 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3]
1661 ; CHECK-NEXT: retq
1662 %1 = load <8 x float>, <8 x float> *%p
1663 %2 = shufflevector <8 x float> %1, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
1664 %3 = bitcast <16 x float> %2 to <8 x double>
1665 %mask.cast = bitcast i8 %mask to <8 x i1>
1666 %res = select <8 x i1> %mask.cast, <8 x double> %3, <8 x double> zeroinitializer
1667 ret <8 x double> %res
1668 }
1670 define <8 x i64> @test_broadcast_8i32_8i64(<8 x i32> *%p, i8 %mask) nounwind {
1671 ; CHECK-LABEL: test_broadcast_8i32_8i64:
1672 ; CHECK: # BB#0:
1673 ; CHECK-NEXT: kmovd %esi, %k1
1674 ; CHECK-NEXT: vbroadcasti64x4 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3]
1675 ; CHECK-NEXT: retq
1676 %1 = load <8 x i32>, <8 x i32> *%p
1677 %2 = shufflevector <8 x i32> %1, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
1678 %3 = bitcast <16 x i32> %2 to <8 x i64>
1679 %mask.cast = bitcast i8 %mask to <8 x i1>
1680 %res = select <8 x i1> %mask.cast, <8 x i64> %3, <8 x i64> zeroinitializer
1681 ret <8 x i64> %res
1682 }
1684 define <4 x float> @test_broadcastf32x2_v4f32(<4 x float> %vec, <4 x float> %passthru, i8 %mask) {
1685 ; CHECK-LABEL: test_broadcastf32x2_v4f32:
1686 ; CHECK: # BB#0:
1687 ; CHECK-NEXT: vmovddup {{.*#+}} xmm0 = xmm0[0,0]
1688 ; CHECK-NEXT: kmovd %edi, %k1
1689 ; CHECK-NEXT: vblendmps %xmm0, %xmm1, %xmm0 {%k1}
1690 ; CHECK-NEXT: retq
1691 %shuf = shufflevector <4 x float> %vec, <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
1692 %mask.cast = bitcast i8 %mask to <8 x i1>
1693 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1694 %res = select <4 x i1> %mask.extract, <4 x float> %shuf, <4 x float> %passthru
1695 ret <4 x float> %res
1696 }
1698 define <4 x float> @test_broadcastf32x2_v4f32_z(<4 x float> %vec, i8 %mask) {
1699 ; CHECK-LABEL: test_broadcastf32x2_v4f32_z:
1700 ; CHECK: # BB#0:
1701 ; CHECK-NEXT: vmovddup {{.*#+}} xmm0 = xmm0[0,0]
1702 ; CHECK-NEXT: kmovd %edi, %k1
1703 ; CHECK-NEXT: vmovaps %xmm0, %xmm0 {%k1} {z}
1704 ; CHECK-NEXT: retq
1705 %shuf = shufflevector <4 x float> %vec, <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
1706 %mask.cast = bitcast i8 %mask to <8 x i1>
1707 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1708 %res = select <4 x i1> %mask.extract, <4 x float> %shuf, <4 x float> zeroinitializer
1709 ret <4 x float> %res
1710 }
1712 define <4 x i32> @test_broadcasti32x2_v4i32(<4 x i32> %vec, <4 x i32> %passthru, i8 %mask) {
1713 ; CHECK-LABEL: test_broadcasti32x2_v4i32:
1714 ; CHECK: # BB#0:
1715 ; CHECK-NEXT: kmovd %edi, %k1
1716 ; CHECK-NEXT: vbroadcasti32x2 {{.*#+}} xmm1 {%k1} = xmm0[0,1,0,1]
1717 ; CHECK-NEXT: vmovdqa %xmm1, %xmm0
1718 ; CHECK-NEXT: retq
1719 %shuf = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
1720 %mask.cast = bitcast i8 %mask to <8 x i1>
1721 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1722 %res = select <4 x i1> %mask.extract, <4 x i32> %shuf, <4 x i32> %passthru
1723 ret <4 x i32> %res
1724 }
1726 define <4 x i32> @test_broadcasti32x2_v4i32_z(<4 x i32> %vec, i8 %mask) {
1727 ; CHECK-LABEL: test_broadcasti32x2_v4i32_z:
1728 ; CHECK: # BB#0:
1729 ; CHECK-NEXT: kmovd %edi, %k1
1730 ; CHECK-NEXT: vbroadcasti32x2 {{.*#+}} xmm0 {%k1} {z} = xmm0[0,1,0,1]
1731 ; CHECK-NEXT: retq
1732 %shuf = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
1733 %mask.cast = bitcast i8 %mask to <8 x i1>
1734 %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1735 %res = select <4 x i1> %mask.extract, <4 x i32> %shuf, <4 x i32> zeroinitializer
1736 ret <4 x i32> %res
1737 }
1739 define <8 x float> @test_broadcastf32x2_v8f32(<8 x float> %vec, <8 x float> %passthru, i8 %mask) {
1740 ; CHECK-LABEL: test_broadcastf32x2_v8f32:
1741 ; CHECK: # BB#0:
1742 ; CHECK-NEXT: kmovd %edi, %k1
1743 ; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} ymm1 {%k1} = xmm0[0,1,0,1,0,1,0,1]
1744 ; CHECK-NEXT: vmovapd %ymm1, %ymm0
1745 ; CHECK-NEXT: retq
1746 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
1747 %mask.cast = bitcast i8 %mask to <8 x i1>
1748 %res = select <8 x i1> %mask.cast, <8 x float> %shuf, <8 x float> %passthru
1749 ret <8 x float> %res
1750 }
1752 define <8 x float> @test_broadcastf32x2_v8f32_z(<8 x float> %vec, i8 %mask) {
1753 ; CHECK-LABEL: test_broadcastf32x2_v8f32_z:
1754 ; CHECK: # BB#0:
1755 ; CHECK-NEXT: kmovd %edi, %k1
1756 ; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} ymm0 {%k1} {z} = xmm0[0,1,0,1,0,1,0,1]
1757 ; CHECK-NEXT: retq
1758 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
1759 %mask.cast = bitcast i8 %mask to <8 x i1>
1760 %res = select <8 x i1> %mask.cast, <8 x float> %shuf, <8 x float> zeroinitializer
1761 ret <8 x float> %res
1762 }
1764 define <8 x i32> @test_broadcasti32x2_v8i32(<8 x i32> %vec, <8 x i32> %passthru, i8 %mask) {
1765 ; CHECK-LABEL: test_broadcasti32x2_v8i32:
1766 ; CHECK: # BB#0:
1767 ; CHECK-NEXT: kmovd %edi, %k1
1768 ; CHECK-NEXT: vbroadcasti32x2 {{.*#+}} ymm1 {%k1} = xmm0[0,1,0,1,0,1,0,1]
1769 ; CHECK-NEXT: vmovdqa %ymm1, %ymm0
1770 ; CHECK-NEXT: retq
1771 %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
1772 %mask.cast = bitcast i8 %mask to <8 x i1>
1773 %res = select <8 x i1> %mask.cast, <8 x i32> %shuf, <8 x i32> %passthru
1774 ret <8 x i32> %res
1775 }
1777 define <8 x i32> @test_broadcasti32x2_v8i32_z(<8 x i32> %vec, i8 %mask) {
1778 ; CHECK-LABEL: test_broadcasti32x2_v8i32_z:
1779 ; CHECK: # BB#0:
1780 ; CHECK-NEXT: kmovd %edi, %k1
1781 ; CHECK-NEXT: vbroadcasti32x2 {{.*#+}} ymm0 {%k1} {z} = xmm0[0,1,0,1,0,1,0,1]
1782 ; CHECK-NEXT: retq
1783 %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
1784 %mask.cast = bitcast i8 %mask to <8 x i1>
1785 %res = select <8 x i1> %mask.cast, <8 x i32> %shuf, <8 x i32> zeroinitializer
1786 ret <8 x i32> %res
1787 }
1789 define <16 x float> @test_broadcastf32x2_v16f32_z(<16 x float> %vec, i16 %mask) {
1790 ; CHECK-LABEL: test_broadcastf32x2_v16f32_z:
1791 ; CHECK: # BB#0:
1792 ; CHECK-NEXT: kmovd %edi, %k1
1793 ; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} zmm0 {%k1} {z} = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]
1794 ; CHECK-NEXT: retq
1795 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
1796 %mask.cast = bitcast i16 %mask to <16 x i1>
1797 %res = select <16 x i1> %mask.cast, <16 x float> %shuf, <16 x float> zeroinitializer
1798 ret <16 x float> %res
1799 }
1801 define <16 x i32> @test_broadcasti32x2_v16i32(<16 x i32> %vec, <16 x i32> %passthru, i16 %mask) {
1802 ; CHECK-LABEL: test_broadcasti32x2_v16i32:
1803 ; CHECK: # BB#0:
1804 ; CHECK-NEXT: kmovd %edi, %k1
1805 ; CHECK-NEXT: vbroadcasti32x2 {{.*#+}} zmm1 {%k1} = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]
1806 ; CHECK-NEXT: vmovdqa64 %zmm1, %zmm0
1807 ; CHECK-NEXT: retq
1808 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
1809 %mask.cast = bitcast i16 %mask to <16 x i1>
1810 %res = select <16 x i1> %mask.cast, <16 x i32> %shuf, <16 x i32> %passthru
1811 ret <16 x i32> %res
1812 }
1814 define <16 x float> @test_broadcastf32x2_v16f32(<16 x float> %vec, <16 x float> %passthru, i16 %mask) {
1815 ; CHECK-LABEL: test_broadcastf32x2_v16f32:
1816 ; CHECK: # BB#0:
1817 ; CHECK-NEXT: kmovd %edi, %k1
1818 ; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} zmm1 {%k1} = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]
1819 ; CHECK-NEXT: vmovapd %zmm1, %zmm0
1820 ; CHECK-NEXT: retq
1821 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
1822 %mask.cast = bitcast i16 %mask to <16 x i1>
1823 %res = select <16 x i1> %mask.cast, <16 x float> %shuf, <16 x float> %passthru
1824 ret <16 x float> %res
1825 }
1827 define <16 x i32> @test_broadcasti32x2_v16i32_z(<16 x i32> %vec, i16 %mask) {
1828 ; CHECK-LABEL: test_broadcasti32x2_v16i32_z:
1829 ; CHECK: # BB#0:
1830 ; CHECK-NEXT: kmovd %edi, %k1
1831 ; CHECK-NEXT: vbroadcasti32x2 {{.*#+}} zmm0 {%k1} {z} = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]
1832 ; CHECK-NEXT: retq
1833 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
1834 %mask.cast = bitcast i16 %mask to <16 x i1>
1835 %res = select <16 x i1> %mask.cast, <16 x i32> %shuf, <16 x i32> zeroinitializer
1836 ret <16 x i32> %res
1837 }
1839 define <16 x i8> @mask_shuffle_v16i8_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15_16(<16 x i8> %a, <16 x i8> %b, <16 x i8> %passthru, i16 %mask) {
1840 ; CHECK-LABEL: mask_shuffle_v16i8_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15_16:
1841 ; CHECK: # BB#0:
1842 ; CHECK-NEXT: kmovd %edi, %k1
1843 ; CHECK-NEXT: vpalignr {{.*#+}} xmm2 {%k1} = xmm0[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm1[0]
1844 ; CHECK-NEXT: vmovdqa %xmm2, %xmm0
1845 ; CHECK-NEXT: retq
1846 %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>
1847 %mask.cast = bitcast i16 %mask to <16 x i1>
1848 %res = select <16 x i1> %mask.cast, <16 x i8> %shuffle, <16 x i8> %passthru
1849 ret <16 x i8> %res
1850 }
1852 define <16 x i8> @maskz_shuffle_v16i8_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15_16(<16 x i8> %a, <16 x i8> %b, i16 %mask) {
1853 ; CHECK-LABEL: maskz_shuffle_v16i8_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15_16:
1854 ; CHECK: # BB#0:
1855 ; CHECK-NEXT: kmovd %edi, %k1
1856 ; CHECK-NEXT: vpalignr {{.*#+}} xmm0 {%k1} {z} = xmm0[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm1[0]
1857 ; CHECK-NEXT: retq
1858 %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>
1859 %mask.cast = bitcast i16 %mask to <16 x i1>
1860 %res = select <16 x i1> %mask.cast, <16 x i8> %shuffle, <16 x i8> zeroinitializer
1861 ret <16 x i8> %res
1862 }
1864 define <16 x i8> @mask_shuffle_v16i8_4_5_6_7_8_9_10_11_12_13_14_15_16_17_18_19(<16 x i8> %a, <16 x i8> %b, <16 x i8> %passthru, i16 %mask) {
1865 ; CHECK-LABEL: mask_shuffle_v16i8_4_5_6_7_8_9_10_11_12_13_14_15_16_17_18_19:
1866 ; CHECK: # BB#0:
1867 ; CHECK-NEXT: kmovd %edi, %k1
1868 ; CHECK-NEXT: vpalignr {{.*#+}} xmm2 {%k1} = xmm0[4,5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3]
1869 ; CHECK-NEXT: vmovdqa %xmm2, %xmm0
1870 ; CHECK-NEXT: retq
1871 %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19>
1872 %mask.cast = bitcast i16 %mask to <16 x i1>
1873 %res = select <16 x i1> %mask.cast, <16 x i8> %shuffle, <16 x i8> %passthru
1874 ret <16 x i8> %res
1875 }
1877 define <16 x i8> @maskz_shuffle_v16i8_4_5_6_7_8_9_10_11_12_13_14_15_16_17_18_19(<16 x i8> %a, <16 x i8> %b, i16 %mask) {
1878 ; CHECK-LABEL: maskz_shuffle_v16i8_4_5_6_7_8_9_10_11_12_13_14_15_16_17_18_19:
1879 ; CHECK: # BB#0:
1880 ; CHECK-NEXT: kmovd %edi, %k1
1881 ; CHECK-NEXT: vpalignr {{.*#+}} xmm0 {%k1} {z} = xmm0[4,5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3]
1882 ; CHECK-NEXT: retq
1883 %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19>
1884 %mask.cast = bitcast i16 %mask to <16 x i1>
1885 %res = select <16 x i1> %mask.cast, <16 x i8> %shuffle, <16 x i8> zeroinitializer
1886 ret <16 x i8> %res
1887 }
1889 define <16 x i8> @mask_shuffle_v16i8_8_9_10_11_12_13_14_15_16_17_18_19_20_21_22_23(<16 x i8> %a, <16 x i8> %b, <16 x i8> %passthru, i16 %mask) {
1890 ; CHECK-LABEL: mask_shuffle_v16i8_8_9_10_11_12_13_14_15_16_17_18_19_20_21_22_23:
1891 ; CHECK: # BB#0:
1892 ; CHECK-NEXT: kmovd %edi, %k1
1893 ; CHECK-NEXT: vpalignr {{.*#+}} xmm2 {%k1} = xmm0[8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7]
1894 ; CHECK-NEXT: vmovdqa %xmm2, %xmm0
1895 ; CHECK-NEXT: retq
1896 %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
1897 %mask.cast = bitcast i16 %mask to <16 x i1>
1898 %res = select <16 x i1> %mask.cast, <16 x i8> %shuffle, <16 x i8> %passthru
1899 ret <16 x i8> %res
1900 }
1902 define <16 x i8> @maskz_shuffle_v16i8_8_9_10_11_12_13_14_15_16_17_18_19_20_21_22_23(<16 x i8> %a, <16 x i8> %b, i16 %mask) {
1903 ; CHECK-LABEL: maskz_shuffle_v16i8_8_9_10_11_12_13_14_15_16_17_18_19_20_21_22_23:
1904 ; CHECK: # BB#0:
1905 ; CHECK-NEXT: kmovd %edi, %k1
1906 ; CHECK-NEXT: vpalignr {{.*#+}} xmm0 {%k1} {z} = xmm0[8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7]
1907 ; CHECK-NEXT: retq
1908 %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
1909 %mask.cast = bitcast i16 %mask to <16 x i1>
1910 %res = select <16 x i1> %mask.cast, <16 x i8> %shuffle, <16 x i8> zeroinitializer
1911 ret <16 x i8> %res
1912 }