| blob | 97b988880fac4c139efd8d98c5a844bc9e9c93be |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512cd | FileCheck %s --check-prefix=AVX512CD
3 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512cd,+avx512bw | FileCheck %s --check-prefix=AVX512CDBW
4 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=AVX512BW
5 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vpopcntdq | FileCheck %s --check-prefix=AVX512VPOPCNTDQ
6 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bitalg | FileCheck %s --check-prefix=BITALG
8 define <8 x i64> @testv8i64(<8 x i64> %in) nounwind {
9 ; AVX512CD-LABEL: testv8i64:
10 ; AVX512CD: # %bb.0:
11 ; AVX512CD-NEXT: vpternlogd {{.*#+}} zmm1 = -1
12 ; AVX512CD-NEXT: vpaddq %zmm1, %zmm0, %zmm1
13 ; AVX512CD-NEXT: vpandnq %zmm1, %zmm0, %zmm0
14 ; AVX512CD-NEXT: vplzcntq %zmm0, %zmm0
15 ; AVX512CD-NEXT: vpbroadcastq {{.*#+}} zmm1 = [64,64,64,64,64,64,64,64]
16 ; AVX512CD-NEXT: vpsubq %zmm0, %zmm1, %zmm0
17 ; AVX512CD-NEXT: retq
18 ;
19 ; AVX512CDBW-LABEL: testv8i64:
20 ; AVX512CDBW: # %bb.0:
21 ; AVX512CDBW-NEXT: vpternlogd {{.*#+}} zmm1 = -1
22 ; AVX512CDBW-NEXT: vpaddq %zmm1, %zmm0, %zmm1
23 ; AVX512CDBW-NEXT: vpandnq %zmm1, %zmm0, %zmm0
24 ; AVX512CDBW-NEXT: vplzcntq %zmm0, %zmm0
25 ; AVX512CDBW-NEXT: vpbroadcastq {{.*#+}} zmm1 = [64,64,64,64,64,64,64,64]
26 ; AVX512CDBW-NEXT: vpsubq %zmm0, %zmm1, %zmm0
27 ; AVX512CDBW-NEXT: retq
28 ;
29 ; AVX512BW-LABEL: testv8i64:
30 ; AVX512BW: # %bb.0:
31 ; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm1 = -1
32 ; AVX512BW-NEXT: vpaddq %zmm1, %zmm0, %zmm1
33 ; AVX512BW-NEXT: vpandnq %zmm1, %zmm0, %zmm0
34 ; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
35 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm2
36 ; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
37 ; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
38 ; AVX512BW-NEXT: vpshufb %zmm2, %zmm3, %zmm2
39 ; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0
40 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm0
41 ; AVX512BW-NEXT: vpshufb %zmm0, %zmm3, %zmm0
42 ; AVX512BW-NEXT: vpaddb %zmm2, %zmm0, %zmm0
43 ; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
44 ; AVX512BW-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
45 ; AVX512BW-NEXT: retq
46 ;
47 ; AVX512VPOPCNTDQ-LABEL: testv8i64:
48 ; AVX512VPOPCNTDQ: # %bb.0:
49 ; AVX512VPOPCNTDQ-NEXT: vpternlogd {{.*#+}} zmm1 = -1
50 ; AVX512VPOPCNTDQ-NEXT: vpaddq %zmm1, %zmm0, %zmm1
51 ; AVX512VPOPCNTDQ-NEXT: vpandnq %zmm1, %zmm0, %zmm0
52 ; AVX512VPOPCNTDQ-NEXT: vpopcntq %zmm0, %zmm0
53 ; AVX512VPOPCNTDQ-NEXT: retq
54 ;
55 ; BITALG-LABEL: testv8i64:
56 ; BITALG: # %bb.0:
57 ; BITALG-NEXT: vpternlogd {{.*#+}} zmm1 = -1
58 ; BITALG-NEXT: vpaddq %zmm1, %zmm0, %zmm1
59 ; BITALG-NEXT: vpandnq %zmm1, %zmm0, %zmm0
60 ; BITALG-NEXT: vpopcntb %zmm0, %zmm0
61 ; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
62 ; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
63 ; BITALG-NEXT: retq
64 %out = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %in, i1 0)
65 ret <8 x i64> %out
66 }
68 define <8 x i64> @testv8i64u(<8 x i64> %in) nounwind {
69 ; AVX512CD-LABEL: testv8i64u:
70 ; AVX512CD: # %bb.0:
71 ; AVX512CD-NEXT: vpternlogd {{.*#+}} zmm1 = -1
72 ; AVX512CD-NEXT: vpaddq %zmm1, %zmm0, %zmm1
73 ; AVX512CD-NEXT: vpandnq %zmm1, %zmm0, %zmm0
74 ; AVX512CD-NEXT: vplzcntq %zmm0, %zmm0
75 ; AVX512CD-NEXT: vpbroadcastq {{.*#+}} zmm1 = [64,64,64,64,64,64,64,64]
76 ; AVX512CD-NEXT: vpsubq %zmm0, %zmm1, %zmm0
77 ; AVX512CD-NEXT: retq
78 ;
79 ; AVX512CDBW-LABEL: testv8i64u:
80 ; AVX512CDBW: # %bb.0:
81 ; AVX512CDBW-NEXT: vpternlogd {{.*#+}} zmm1 = -1
82 ; AVX512CDBW-NEXT: vpaddq %zmm1, %zmm0, %zmm1
83 ; AVX512CDBW-NEXT: vpandnq %zmm1, %zmm0, %zmm0
84 ; AVX512CDBW-NEXT: vplzcntq %zmm0, %zmm0
85 ; AVX512CDBW-NEXT: vpbroadcastq {{.*#+}} zmm1 = [64,64,64,64,64,64,64,64]
86 ; AVX512CDBW-NEXT: vpsubq %zmm0, %zmm1, %zmm0
87 ; AVX512CDBW-NEXT: retq
88 ;
89 ; AVX512BW-LABEL: testv8i64u:
90 ; AVX512BW: # %bb.0:
91 ; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm1 = -1
92 ; AVX512BW-NEXT: vpaddq %zmm1, %zmm0, %zmm1
93 ; AVX512BW-NEXT: vpandnq %zmm1, %zmm0, %zmm0
94 ; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
95 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm2
96 ; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
97 ; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
98 ; AVX512BW-NEXT: vpshufb %zmm2, %zmm3, %zmm2
99 ; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0
100 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm0
101 ; AVX512BW-NEXT: vpshufb %zmm0, %zmm3, %zmm0
102 ; AVX512BW-NEXT: vpaddb %zmm2, %zmm0, %zmm0
103 ; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
104 ; AVX512BW-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
105 ; AVX512BW-NEXT: retq
106 ;
107 ; AVX512VPOPCNTDQ-LABEL: testv8i64u:
108 ; AVX512VPOPCNTDQ: # %bb.0:
109 ; AVX512VPOPCNTDQ-NEXT: vpternlogd {{.*#+}} zmm1 = -1
110 ; AVX512VPOPCNTDQ-NEXT: vpaddq %zmm1, %zmm0, %zmm1
111 ; AVX512VPOPCNTDQ-NEXT: vpandnq %zmm1, %zmm0, %zmm0
112 ; AVX512VPOPCNTDQ-NEXT: vpopcntq %zmm0, %zmm0
113 ; AVX512VPOPCNTDQ-NEXT: retq
114 ;
115 ; BITALG-LABEL: testv8i64u:
116 ; BITALG: # %bb.0:
117 ; BITALG-NEXT: vpternlogd {{.*#+}} zmm1 = -1
118 ; BITALG-NEXT: vpaddq %zmm1, %zmm0, %zmm1
119 ; BITALG-NEXT: vpandnq %zmm1, %zmm0, %zmm0
120 ; BITALG-NEXT: vpopcntb %zmm0, %zmm0
121 ; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
122 ; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
123 ; BITALG-NEXT: retq
124 %out = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %in, i1 -1)
125 ret <8 x i64> %out
126 }
128 define <16 x i32> @testv16i32(<16 x i32> %in) nounwind {
129 ; AVX512CD-LABEL: testv16i32:
130 ; AVX512CD: # %bb.0:
131 ; AVX512CD-NEXT: vpternlogd {{.*#+}} zmm1 = -1
132 ; AVX512CD-NEXT: vpaddd %zmm1, %zmm0, %zmm1
133 ; AVX512CD-NEXT: vpandnd %zmm1, %zmm0, %zmm0
134 ; AVX512CD-NEXT: vplzcntd %zmm0, %zmm0
135 ; AVX512CD-NEXT: vpbroadcastd {{.*#+}} zmm1 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
136 ; AVX512CD-NEXT: vpsubd %zmm0, %zmm1, %zmm0
137 ; AVX512CD-NEXT: retq
138 ;
139 ; AVX512CDBW-LABEL: testv16i32:
140 ; AVX512CDBW: # %bb.0:
141 ; AVX512CDBW-NEXT: vpternlogd {{.*#+}} zmm1 = -1
142 ; AVX512CDBW-NEXT: vpaddd %zmm1, %zmm0, %zmm1
143 ; AVX512CDBW-NEXT: vpandnd %zmm1, %zmm0, %zmm0
144 ; AVX512CDBW-NEXT: vplzcntd %zmm0, %zmm0
145 ; AVX512CDBW-NEXT: vpbroadcastd {{.*#+}} zmm1 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
146 ; AVX512CDBW-NEXT: vpsubd %zmm0, %zmm1, %zmm0
147 ; AVX512CDBW-NEXT: retq
148 ;
149 ; AVX512BW-LABEL: testv16i32:
150 ; AVX512BW: # %bb.0:
151 ; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm1 = -1
152 ; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm1
153 ; AVX512BW-NEXT: vpandnd %zmm1, %zmm0, %zmm0
154 ; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
155 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm2
156 ; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
157 ; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
158 ; AVX512BW-NEXT: vpshufb %zmm2, %zmm3, %zmm2
159 ; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0
160 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm0
161 ; AVX512BW-NEXT: vpshufb %zmm0, %zmm3, %zmm0
162 ; AVX512BW-NEXT: vpaddb %zmm2, %zmm0, %zmm0
163 ; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
164 ; AVX512BW-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
165 ; AVX512BW-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
166 ; AVX512BW-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
167 ; AVX512BW-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
168 ; AVX512BW-NEXT: vpackuswb %zmm2, %zmm0, %zmm0
169 ; AVX512BW-NEXT: retq
170 ;
171 ; AVX512VPOPCNTDQ-LABEL: testv16i32:
172 ; AVX512VPOPCNTDQ: # %bb.0:
173 ; AVX512VPOPCNTDQ-NEXT: vpternlogd {{.*#+}} zmm1 = -1
174 ; AVX512VPOPCNTDQ-NEXT: vpaddd %zmm1, %zmm0, %zmm1
175 ; AVX512VPOPCNTDQ-NEXT: vpandnd %zmm1, %zmm0, %zmm0
176 ; AVX512VPOPCNTDQ-NEXT: vpopcntd %zmm0, %zmm0
177 ; AVX512VPOPCNTDQ-NEXT: retq
178 ;
179 ; BITALG-LABEL: testv16i32:
180 ; BITALG: # %bb.0:
181 ; BITALG-NEXT: vpternlogd {{.*#+}} zmm1 = -1
182 ; BITALG-NEXT: vpaddd %zmm1, %zmm0, %zmm1
183 ; BITALG-NEXT: vpandnd %zmm1, %zmm0, %zmm0
184 ; BITALG-NEXT: vpopcntb %zmm0, %zmm0
185 ; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
186 ; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
187 ; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
188 ; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
189 ; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
190 ; BITALG-NEXT: vpackuswb %zmm2, %zmm0, %zmm0
191 ; BITALG-NEXT: retq
192 %out = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %in, i1 0)
193 ret <16 x i32> %out
194 }
196 define <16 x i32> @testv16i32u(<16 x i32> %in) nounwind {
197 ; AVX512CD-LABEL: testv16i32u:
198 ; AVX512CD: # %bb.0:
199 ; AVX512CD-NEXT: vpternlogd {{.*#+}} zmm1 = -1
200 ; AVX512CD-NEXT: vpaddd %zmm1, %zmm0, %zmm1
201 ; AVX512CD-NEXT: vpandnd %zmm1, %zmm0, %zmm0
202 ; AVX512CD-NEXT: vplzcntd %zmm0, %zmm0
203 ; AVX512CD-NEXT: vpbroadcastd {{.*#+}} zmm1 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
204 ; AVX512CD-NEXT: vpsubd %zmm0, %zmm1, %zmm0
205 ; AVX512CD-NEXT: retq
206 ;
207 ; AVX512CDBW-LABEL: testv16i32u:
208 ; AVX512CDBW: # %bb.0:
209 ; AVX512CDBW-NEXT: vpternlogd {{.*#+}} zmm1 = -1
210 ; AVX512CDBW-NEXT: vpaddd %zmm1, %zmm0, %zmm1
211 ; AVX512CDBW-NEXT: vpandnd %zmm1, %zmm0, %zmm0
212 ; AVX512CDBW-NEXT: vplzcntd %zmm0, %zmm0
213 ; AVX512CDBW-NEXT: vpbroadcastd {{.*#+}} zmm1 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
214 ; AVX512CDBW-NEXT: vpsubd %zmm0, %zmm1, %zmm0
215 ; AVX512CDBW-NEXT: retq
216 ;
217 ; AVX512BW-LABEL: testv16i32u:
218 ; AVX512BW: # %bb.0:
219 ; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm1 = -1
220 ; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm1
221 ; AVX512BW-NEXT: vpandnd %zmm1, %zmm0, %zmm0
222 ; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
223 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm2
224 ; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
225 ; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
226 ; AVX512BW-NEXT: vpshufb %zmm2, %zmm3, %zmm2
227 ; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0
228 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm0
229 ; AVX512BW-NEXT: vpshufb %zmm0, %zmm3, %zmm0
230 ; AVX512BW-NEXT: vpaddb %zmm2, %zmm0, %zmm0
231 ; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
232 ; AVX512BW-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
233 ; AVX512BW-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
234 ; AVX512BW-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
235 ; AVX512BW-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
236 ; AVX512BW-NEXT: vpackuswb %zmm2, %zmm0, %zmm0
237 ; AVX512BW-NEXT: retq
238 ;
239 ; AVX512VPOPCNTDQ-LABEL: testv16i32u:
240 ; AVX512VPOPCNTDQ: # %bb.0:
241 ; AVX512VPOPCNTDQ-NEXT: vpternlogd {{.*#+}} zmm1 = -1
242 ; AVX512VPOPCNTDQ-NEXT: vpaddd %zmm1, %zmm0, %zmm1
243 ; AVX512VPOPCNTDQ-NEXT: vpandnd %zmm1, %zmm0, %zmm0
244 ; AVX512VPOPCNTDQ-NEXT: vpopcntd %zmm0, %zmm0
245 ; AVX512VPOPCNTDQ-NEXT: retq
246 ;
247 ; BITALG-LABEL: testv16i32u:
248 ; BITALG: # %bb.0:
249 ; BITALG-NEXT: vpternlogd {{.*#+}} zmm1 = -1
250 ; BITALG-NEXT: vpaddd %zmm1, %zmm0, %zmm1
251 ; BITALG-NEXT: vpandnd %zmm1, %zmm0, %zmm0
252 ; BITALG-NEXT: vpopcntb %zmm0, %zmm0
253 ; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
254 ; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
255 ; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
256 ; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
257 ; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
258 ; BITALG-NEXT: vpackuswb %zmm2, %zmm0, %zmm0
259 ; BITALG-NEXT: retq
260 %out = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %in, i1 -1)
261 ret <16 x i32> %out
262 }
264 define <32 x i16> @testv32i16(<32 x i16> %in) nounwind {
265 ; AVX512CD-LABEL: testv32i16:
266 ; AVX512CD: # %bb.0:
267 ; AVX512CD-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
268 ; AVX512CD-NEXT: vpaddw %ymm1, %ymm0, %ymm2
269 ; AVX512CD-NEXT: vpandn %ymm2, %ymm0, %ymm2
270 ; AVX512CD-NEXT: vpbroadcastb {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
271 ; AVX512CD-NEXT: vpand %ymm3, %ymm2, %ymm4
272 ; AVX512CD-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
273 ; AVX512CD-NEXT: # ymm5 = mem[0,1,0,1]
274 ; AVX512CD-NEXT: vpshufb %ymm4, %ymm5, %ymm4
275 ; AVX512CD-NEXT: vpsrlw $4, %ymm2, %ymm2
276 ; AVX512CD-NEXT: vpand %ymm3, %ymm2, %ymm2
277 ; AVX512CD-NEXT: vpshufb %ymm2, %ymm5, %ymm2
278 ; AVX512CD-NEXT: vpaddb %ymm4, %ymm2, %ymm2
279 ; AVX512CD-NEXT: vpsllw $8, %ymm2, %ymm4
280 ; AVX512CD-NEXT: vpaddb %ymm2, %ymm4, %ymm2
281 ; AVX512CD-NEXT: vpsrlw $8, %ymm2, %ymm2
282 ; AVX512CD-NEXT: vextracti64x4 $1, %zmm0, %ymm0
283 ; AVX512CD-NEXT: vpaddw %ymm1, %ymm0, %ymm1
284 ; AVX512CD-NEXT: vpandn %ymm1, %ymm0, %ymm0
285 ; AVX512CD-NEXT: vpand %ymm3, %ymm0, %ymm1
286 ; AVX512CD-NEXT: vpshufb %ymm1, %ymm5, %ymm1
287 ; AVX512CD-NEXT: vpsrlw $4, %ymm0, %ymm0
288 ; AVX512CD-NEXT: vpand %ymm3, %ymm0, %ymm0
289 ; AVX512CD-NEXT: vpshufb %ymm0, %ymm5, %ymm0
290 ; AVX512CD-NEXT: vpaddb %ymm1, %ymm0, %ymm0
291 ; AVX512CD-NEXT: vpsllw $8, %ymm0, %ymm1
292 ; AVX512CD-NEXT: vpaddb %ymm0, %ymm1, %ymm0
293 ; AVX512CD-NEXT: vpsrlw $8, %ymm0, %ymm0
294 ; AVX512CD-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0
295 ; AVX512CD-NEXT: retq
296 ;
297 ; AVX512CDBW-LABEL: testv32i16:
298 ; AVX512CDBW: # %bb.0:
299 ; AVX512CDBW-NEXT: vpternlogd {{.*#+}} zmm1 = -1
300 ; AVX512CDBW-NEXT: vpaddw %zmm1, %zmm0, %zmm1
301 ; AVX512CDBW-NEXT: vpandnq %zmm1, %zmm0, %zmm0
302 ; AVX512CDBW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
303 ; AVX512CDBW-NEXT: vpandq %zmm1, %zmm0, %zmm2
304 ; AVX512CDBW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
305 ; AVX512CDBW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
306 ; AVX512CDBW-NEXT: vpshufb %zmm2, %zmm3, %zmm2
307 ; AVX512CDBW-NEXT: vpsrlw $4, %zmm0, %zmm0
308 ; AVX512CDBW-NEXT: vpandq %zmm1, %zmm0, %zmm0
309 ; AVX512CDBW-NEXT: vpshufb %zmm0, %zmm3, %zmm0
310 ; AVX512CDBW-NEXT: vpaddb %zmm2, %zmm0, %zmm0
311 ; AVX512CDBW-NEXT: vpsllw $8, %zmm0, %zmm1
312 ; AVX512CDBW-NEXT: vpaddb %zmm0, %zmm1, %zmm0
313 ; AVX512CDBW-NEXT: vpsrlw $8, %zmm0, %zmm0
314 ; AVX512CDBW-NEXT: retq
315 ;
316 ; AVX512BW-LABEL: testv32i16:
317 ; AVX512BW: # %bb.0:
318 ; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm1 = -1
319 ; AVX512BW-NEXT: vpaddw %zmm1, %zmm0, %zmm1
320 ; AVX512BW-NEXT: vpandnq %zmm1, %zmm0, %zmm0
321 ; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
322 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm2
323 ; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
324 ; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
325 ; AVX512BW-NEXT: vpshufb %zmm2, %zmm3, %zmm2
326 ; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0
327 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm0
328 ; AVX512BW-NEXT: vpshufb %zmm0, %zmm3, %zmm0
329 ; AVX512BW-NEXT: vpaddb %zmm2, %zmm0, %zmm0
330 ; AVX512BW-NEXT: vpsllw $8, %zmm0, %zmm1
331 ; AVX512BW-NEXT: vpaddb %zmm0, %zmm1, %zmm0
332 ; AVX512BW-NEXT: vpsrlw $8, %zmm0, %zmm0
333 ; AVX512BW-NEXT: retq
334 ;
335 ; AVX512VPOPCNTDQ-LABEL: testv32i16:
336 ; AVX512VPOPCNTDQ: # %bb.0:
337 ; AVX512VPOPCNTDQ-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
338 ; AVX512VPOPCNTDQ-NEXT: vpaddw %ymm1, %ymm0, %ymm2
339 ; AVX512VPOPCNTDQ-NEXT: vpandn %ymm2, %ymm0, %ymm2
340 ; AVX512VPOPCNTDQ-NEXT: vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
341 ; AVX512VPOPCNTDQ-NEXT: vpopcntd %zmm2, %zmm2
342 ; AVX512VPOPCNTDQ-NEXT: vpmovdw %zmm2, %ymm2
343 ; AVX512VPOPCNTDQ-NEXT: vextracti64x4 $1, %zmm0, %ymm0
344 ; AVX512VPOPCNTDQ-NEXT: vpaddw %ymm1, %ymm0, %ymm1
345 ; AVX512VPOPCNTDQ-NEXT: vpandn %ymm1, %ymm0, %ymm0
346 ; AVX512VPOPCNTDQ-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
347 ; AVX512VPOPCNTDQ-NEXT: vpopcntd %zmm0, %zmm0
348 ; AVX512VPOPCNTDQ-NEXT: vpmovdw %zmm0, %ymm0
349 ; AVX512VPOPCNTDQ-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0
350 ; AVX512VPOPCNTDQ-NEXT: retq
351 ;
352 ; BITALG-LABEL: testv32i16:
353 ; BITALG: # %bb.0:
354 ; BITALG-NEXT: vpternlogd {{.*#+}} zmm1 = -1
355 ; BITALG-NEXT: vpaddw %zmm1, %zmm0, %zmm1
356 ; BITALG-NEXT: vpandnq %zmm1, %zmm0, %zmm0
357 ; BITALG-NEXT: vpopcntw %zmm0, %zmm0
358 ; BITALG-NEXT: retq
359 %out = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %in, i1 0)
360 ret <32 x i16> %out
361 }
363 define <32 x i16> @testv32i16u(<32 x i16> %in) nounwind {
364 ; AVX512CD-LABEL: testv32i16u:
365 ; AVX512CD: # %bb.0:
366 ; AVX512CD-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
367 ; AVX512CD-NEXT: vpaddw %ymm1, %ymm0, %ymm2
368 ; AVX512CD-NEXT: vpandn %ymm2, %ymm0, %ymm2
369 ; AVX512CD-NEXT: vpbroadcastb {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
370 ; AVX512CD-NEXT: vpand %ymm3, %ymm2, %ymm4
371 ; AVX512CD-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
372 ; AVX512CD-NEXT: # ymm5 = mem[0,1,0,1]
373 ; AVX512CD-NEXT: vpshufb %ymm4, %ymm5, %ymm4
374 ; AVX512CD-NEXT: vpsrlw $4, %ymm2, %ymm2
375 ; AVX512CD-NEXT: vpand %ymm3, %ymm2, %ymm2
376 ; AVX512CD-NEXT: vpshufb %ymm2, %ymm5, %ymm2
377 ; AVX512CD-NEXT: vpaddb %ymm4, %ymm2, %ymm2
378 ; AVX512CD-NEXT: vpsllw $8, %ymm2, %ymm4
379 ; AVX512CD-NEXT: vpaddb %ymm2, %ymm4, %ymm2
380 ; AVX512CD-NEXT: vpsrlw $8, %ymm2, %ymm2
381 ; AVX512CD-NEXT: vextracti64x4 $1, %zmm0, %ymm0
382 ; AVX512CD-NEXT: vpaddw %ymm1, %ymm0, %ymm1
383 ; AVX512CD-NEXT: vpandn %ymm1, %ymm0, %ymm0
384 ; AVX512CD-NEXT: vpand %ymm3, %ymm0, %ymm1
385 ; AVX512CD-NEXT: vpshufb %ymm1, %ymm5, %ymm1
386 ; AVX512CD-NEXT: vpsrlw $4, %ymm0, %ymm0
387 ; AVX512CD-NEXT: vpand %ymm3, %ymm0, %ymm0
388 ; AVX512CD-NEXT: vpshufb %ymm0, %ymm5, %ymm0
389 ; AVX512CD-NEXT: vpaddb %ymm1, %ymm0, %ymm0
390 ; AVX512CD-NEXT: vpsllw $8, %ymm0, %ymm1
391 ; AVX512CD-NEXT: vpaddb %ymm0, %ymm1, %ymm0
392 ; AVX512CD-NEXT: vpsrlw $8, %ymm0, %ymm0
393 ; AVX512CD-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0
394 ; AVX512CD-NEXT: retq
395 ;
396 ; AVX512CDBW-LABEL: testv32i16u:
397 ; AVX512CDBW: # %bb.0:
398 ; AVX512CDBW-NEXT: vpternlogd {{.*#+}} zmm1 = -1
399 ; AVX512CDBW-NEXT: vpaddw %zmm1, %zmm0, %zmm1
400 ; AVX512CDBW-NEXT: vpandnq %zmm1, %zmm0, %zmm0
401 ; AVX512CDBW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
402 ; AVX512CDBW-NEXT: vpandq %zmm1, %zmm0, %zmm2
403 ; AVX512CDBW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
404 ; AVX512CDBW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
405 ; AVX512CDBW-NEXT: vpshufb %zmm2, %zmm3, %zmm2
406 ; AVX512CDBW-NEXT: vpsrlw $4, %zmm0, %zmm0
407 ; AVX512CDBW-NEXT: vpandq %zmm1, %zmm0, %zmm0
408 ; AVX512CDBW-NEXT: vpshufb %zmm0, %zmm3, %zmm0
409 ; AVX512CDBW-NEXT: vpaddb %zmm2, %zmm0, %zmm0
410 ; AVX512CDBW-NEXT: vpsllw $8, %zmm0, %zmm1
411 ; AVX512CDBW-NEXT: vpaddb %zmm0, %zmm1, %zmm0
412 ; AVX512CDBW-NEXT: vpsrlw $8, %zmm0, %zmm0
413 ; AVX512CDBW-NEXT: retq
414 ;
415 ; AVX512BW-LABEL: testv32i16u:
416 ; AVX512BW: # %bb.0:
417 ; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm1 = -1
418 ; AVX512BW-NEXT: vpaddw %zmm1, %zmm0, %zmm1
419 ; AVX512BW-NEXT: vpandnq %zmm1, %zmm0, %zmm0
420 ; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
421 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm2
422 ; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
423 ; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
424 ; AVX512BW-NEXT: vpshufb %zmm2, %zmm3, %zmm2
425 ; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0
426 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm0
427 ; AVX512BW-NEXT: vpshufb %zmm0, %zmm3, %zmm0
428 ; AVX512BW-NEXT: vpaddb %zmm2, %zmm0, %zmm0
429 ; AVX512BW-NEXT: vpsllw $8, %zmm0, %zmm1
430 ; AVX512BW-NEXT: vpaddb %zmm0, %zmm1, %zmm0
431 ; AVX512BW-NEXT: vpsrlw $8, %zmm0, %zmm0
432 ; AVX512BW-NEXT: retq
433 ;
434 ; AVX512VPOPCNTDQ-LABEL: testv32i16u:
435 ; AVX512VPOPCNTDQ: # %bb.0:
436 ; AVX512VPOPCNTDQ-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
437 ; AVX512VPOPCNTDQ-NEXT: vpaddw %ymm1, %ymm0, %ymm2
438 ; AVX512VPOPCNTDQ-NEXT: vpandn %ymm2, %ymm0, %ymm2
439 ; AVX512VPOPCNTDQ-NEXT: vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
440 ; AVX512VPOPCNTDQ-NEXT: vpopcntd %zmm2, %zmm2
441 ; AVX512VPOPCNTDQ-NEXT: vpmovdw %zmm2, %ymm2
442 ; AVX512VPOPCNTDQ-NEXT: vextracti64x4 $1, %zmm0, %ymm0
443 ; AVX512VPOPCNTDQ-NEXT: vpaddw %ymm1, %ymm0, %ymm1
444 ; AVX512VPOPCNTDQ-NEXT: vpandn %ymm1, %ymm0, %ymm0
445 ; AVX512VPOPCNTDQ-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
446 ; AVX512VPOPCNTDQ-NEXT: vpopcntd %zmm0, %zmm0
447 ; AVX512VPOPCNTDQ-NEXT: vpmovdw %zmm0, %ymm0
448 ; AVX512VPOPCNTDQ-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0
449 ; AVX512VPOPCNTDQ-NEXT: retq
450 ;
451 ; BITALG-LABEL: testv32i16u:
452 ; BITALG: # %bb.0:
453 ; BITALG-NEXT: vpternlogd {{.*#+}} zmm1 = -1
454 ; BITALG-NEXT: vpaddw %zmm1, %zmm0, %zmm1
455 ; BITALG-NEXT: vpandnq %zmm1, %zmm0, %zmm0
456 ; BITALG-NEXT: vpopcntw %zmm0, %zmm0
457 ; BITALG-NEXT: retq
458 %out = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %in, i1 -1)
459 ret <32 x i16> %out
460 }
462 define <64 x i8> @testv64i8(<64 x i8> %in) nounwind {
463 ; AVX512CD-LABEL: testv64i8:
464 ; AVX512CD: # %bb.0:
465 ; AVX512CD-NEXT: vextracti64x4 $1, %zmm0, %ymm1
466 ; AVX512CD-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
467 ; AVX512CD-NEXT: vpaddb %ymm2, %ymm1, %ymm3
468 ; AVX512CD-NEXT: vpandn %ymm3, %ymm1, %ymm1
469 ; AVX512CD-NEXT: vpbroadcastb {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
470 ; AVX512CD-NEXT: vpand %ymm3, %ymm1, %ymm4
471 ; AVX512CD-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
472 ; AVX512CD-NEXT: # ymm5 = mem[0,1,0,1]
473 ; AVX512CD-NEXT: vpshufb %ymm4, %ymm5, %ymm4
474 ; AVX512CD-NEXT: vpsrlw $4, %ymm1, %ymm1
475 ; AVX512CD-NEXT: vpand %ymm3, %ymm1, %ymm1
476 ; AVX512CD-NEXT: vpshufb %ymm1, %ymm5, %ymm1
477 ; AVX512CD-NEXT: vpaddb %ymm4, %ymm1, %ymm1
478 ; AVX512CD-NEXT: vpaddb %ymm2, %ymm0, %ymm2
479 ; AVX512CD-NEXT: vpandn %ymm2, %ymm0, %ymm0
480 ; AVX512CD-NEXT: vpand %ymm3, %ymm0, %ymm2
481 ; AVX512CD-NEXT: vpshufb %ymm2, %ymm5, %ymm2
482 ; AVX512CD-NEXT: vpsrlw $4, %ymm0, %ymm0
483 ; AVX512CD-NEXT: vpand %ymm3, %ymm0, %ymm0
484 ; AVX512CD-NEXT: vpshufb %ymm0, %ymm5, %ymm0
485 ; AVX512CD-NEXT: vpaddb %ymm2, %ymm0, %ymm0
486 ; AVX512CD-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
487 ; AVX512CD-NEXT: retq
488 ;
489 ; AVX512CDBW-LABEL: testv64i8:
490 ; AVX512CDBW: # %bb.0:
491 ; AVX512CDBW-NEXT: vpternlogd {{.*#+}} zmm1 = -1
492 ; AVX512CDBW-NEXT: vpaddb %zmm1, %zmm0, %zmm1
493 ; AVX512CDBW-NEXT: vpandnq %zmm1, %zmm0, %zmm0
494 ; AVX512CDBW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
495 ; AVX512CDBW-NEXT: vpandq %zmm1, %zmm0, %zmm2
496 ; AVX512CDBW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
497 ; AVX512CDBW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
498 ; AVX512CDBW-NEXT: vpshufb %zmm2, %zmm3, %zmm2
499 ; AVX512CDBW-NEXT: vpsrlw $4, %zmm0, %zmm0
500 ; AVX512CDBW-NEXT: vpandq %zmm1, %zmm0, %zmm0
501 ; AVX512CDBW-NEXT: vpshufb %zmm0, %zmm3, %zmm0
502 ; AVX512CDBW-NEXT: vpaddb %zmm2, %zmm0, %zmm0
503 ; AVX512CDBW-NEXT: retq
504 ;
505 ; AVX512BW-LABEL: testv64i8:
506 ; AVX512BW: # %bb.0:
507 ; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm1 = -1
508 ; AVX512BW-NEXT: vpaddb %zmm1, %zmm0, %zmm1
509 ; AVX512BW-NEXT: vpandnq %zmm1, %zmm0, %zmm0
510 ; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
511 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm2
512 ; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
513 ; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
514 ; AVX512BW-NEXT: vpshufb %zmm2, %zmm3, %zmm2
515 ; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0
516 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm0
517 ; AVX512BW-NEXT: vpshufb %zmm0, %zmm3, %zmm0
518 ; AVX512BW-NEXT: vpaddb %zmm2, %zmm0, %zmm0
519 ; AVX512BW-NEXT: retq
520 ;
521 ; AVX512VPOPCNTDQ-LABEL: testv64i8:
522 ; AVX512VPOPCNTDQ: # %bb.0:
523 ; AVX512VPOPCNTDQ-NEXT: vextracti64x4 $1, %zmm0, %ymm1
524 ; AVX512VPOPCNTDQ-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
525 ; AVX512VPOPCNTDQ-NEXT: vpaddb %ymm2, %ymm1, %ymm3
526 ; AVX512VPOPCNTDQ-NEXT: vpandn %ymm3, %ymm1, %ymm1
527 ; AVX512VPOPCNTDQ-NEXT: vpbroadcastb {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
528 ; AVX512VPOPCNTDQ-NEXT: vpand %ymm3, %ymm1, %ymm4
529 ; AVX512VPOPCNTDQ-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
530 ; AVX512VPOPCNTDQ-NEXT: # ymm5 = mem[0,1,0,1]
531 ; AVX512VPOPCNTDQ-NEXT: vpshufb %ymm4, %ymm5, %ymm4
532 ; AVX512VPOPCNTDQ-NEXT: vpsrlw $4, %ymm1, %ymm1
533 ; AVX512VPOPCNTDQ-NEXT: vpand %ymm3, %ymm1, %ymm1
534 ; AVX512VPOPCNTDQ-NEXT: vpshufb %ymm1, %ymm5, %ymm1
535 ; AVX512VPOPCNTDQ-NEXT: vpaddb %ymm4, %ymm1, %ymm1
536 ; AVX512VPOPCNTDQ-NEXT: vpaddb %ymm2, %ymm0, %ymm2
537 ; AVX512VPOPCNTDQ-NEXT: vpandn %ymm2, %ymm0, %ymm0
538 ; AVX512VPOPCNTDQ-NEXT: vpand %ymm3, %ymm0, %ymm2
539 ; AVX512VPOPCNTDQ-NEXT: vpshufb %ymm2, %ymm5, %ymm2
540 ; AVX512VPOPCNTDQ-NEXT: vpsrlw $4, %ymm0, %ymm0
541 ; AVX512VPOPCNTDQ-NEXT: vpand %ymm3, %ymm0, %ymm0
542 ; AVX512VPOPCNTDQ-NEXT: vpshufb %ymm0, %ymm5, %ymm0
543 ; AVX512VPOPCNTDQ-NEXT: vpaddb %ymm2, %ymm0, %ymm0
544 ; AVX512VPOPCNTDQ-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
545 ; AVX512VPOPCNTDQ-NEXT: retq
546 ;
547 ; BITALG-LABEL: testv64i8:
548 ; BITALG: # %bb.0:
549 ; BITALG-NEXT: vpternlogd {{.*#+}} zmm1 = -1
550 ; BITALG-NEXT: vpaddb %zmm1, %zmm0, %zmm1
551 ; BITALG-NEXT: vpandnq %zmm1, %zmm0, %zmm0
552 ; BITALG-NEXT: vpopcntb %zmm0, %zmm0
553 ; BITALG-NEXT: retq
554 %out = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %in, i1 0)
555 ret <64 x i8> %out
556 }
558 define <64 x i8> @testv64i8u(<64 x i8> %in) nounwind {
559 ; AVX512CD-LABEL: testv64i8u:
560 ; AVX512CD: # %bb.0:
561 ; AVX512CD-NEXT: vextracti64x4 $1, %zmm0, %ymm1
562 ; AVX512CD-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
563 ; AVX512CD-NEXT: vpaddb %ymm2, %ymm1, %ymm3
564 ; AVX512CD-NEXT: vpandn %ymm3, %ymm1, %ymm1
565 ; AVX512CD-NEXT: vpbroadcastb {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
566 ; AVX512CD-NEXT: vpand %ymm3, %ymm1, %ymm4
567 ; AVX512CD-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
568 ; AVX512CD-NEXT: # ymm5 = mem[0,1,0,1]
569 ; AVX512CD-NEXT: vpshufb %ymm4, %ymm5, %ymm4
570 ; AVX512CD-NEXT: vpsrlw $4, %ymm1, %ymm1
571 ; AVX512CD-NEXT: vpand %ymm3, %ymm1, %ymm1
572 ; AVX512CD-NEXT: vpshufb %ymm1, %ymm5, %ymm1
573 ; AVX512CD-NEXT: vpaddb %ymm4, %ymm1, %ymm1
574 ; AVX512CD-NEXT: vpaddb %ymm2, %ymm0, %ymm2
575 ; AVX512CD-NEXT: vpandn %ymm2, %ymm0, %ymm0
576 ; AVX512CD-NEXT: vpand %ymm3, %ymm0, %ymm2
577 ; AVX512CD-NEXT: vpshufb %ymm2, %ymm5, %ymm2
578 ; AVX512CD-NEXT: vpsrlw $4, %ymm0, %ymm0
579 ; AVX512CD-NEXT: vpand %ymm3, %ymm0, %ymm0
580 ; AVX512CD-NEXT: vpshufb %ymm0, %ymm5, %ymm0
581 ; AVX512CD-NEXT: vpaddb %ymm2, %ymm0, %ymm0
582 ; AVX512CD-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
583 ; AVX512CD-NEXT: retq
584 ;
585 ; AVX512CDBW-LABEL: testv64i8u:
586 ; AVX512CDBW: # %bb.0:
587 ; AVX512CDBW-NEXT: vpternlogd {{.*#+}} zmm1 = -1
588 ; AVX512CDBW-NEXT: vpaddb %zmm1, %zmm0, %zmm1
589 ; AVX512CDBW-NEXT: vpandnq %zmm1, %zmm0, %zmm0
590 ; AVX512CDBW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
591 ; AVX512CDBW-NEXT: vpandq %zmm1, %zmm0, %zmm2
592 ; AVX512CDBW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
593 ; AVX512CDBW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
594 ; AVX512CDBW-NEXT: vpshufb %zmm2, %zmm3, %zmm2
595 ; AVX512CDBW-NEXT: vpsrlw $4, %zmm0, %zmm0
596 ; AVX512CDBW-NEXT: vpandq %zmm1, %zmm0, %zmm0
597 ; AVX512CDBW-NEXT: vpshufb %zmm0, %zmm3, %zmm0
598 ; AVX512CDBW-NEXT: vpaddb %zmm2, %zmm0, %zmm0
599 ; AVX512CDBW-NEXT: retq
600 ;
601 ; AVX512BW-LABEL: testv64i8u:
602 ; AVX512BW: # %bb.0:
603 ; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm1 = -1
604 ; AVX512BW-NEXT: vpaddb %zmm1, %zmm0, %zmm1
605 ; AVX512BW-NEXT: vpandnq %zmm1, %zmm0, %zmm0
606 ; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
607 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm2
608 ; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
609 ; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
610 ; AVX512BW-NEXT: vpshufb %zmm2, %zmm3, %zmm2
611 ; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0
612 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm0
613 ; AVX512BW-NEXT: vpshufb %zmm0, %zmm3, %zmm0
614 ; AVX512BW-NEXT: vpaddb %zmm2, %zmm0, %zmm0
615 ; AVX512BW-NEXT: retq
616 ;
617 ; AVX512VPOPCNTDQ-LABEL: testv64i8u:
618 ; AVX512VPOPCNTDQ: # %bb.0:
619 ; AVX512VPOPCNTDQ-NEXT: vextracti64x4 $1, %zmm0, %ymm1
620 ; AVX512VPOPCNTDQ-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
621 ; AVX512VPOPCNTDQ-NEXT: vpaddb %ymm2, %ymm1, %ymm3
622 ; AVX512VPOPCNTDQ-NEXT: vpandn %ymm3, %ymm1, %ymm1
623 ; AVX512VPOPCNTDQ-NEXT: vpbroadcastb {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
624 ; AVX512VPOPCNTDQ-NEXT: vpand %ymm3, %ymm1, %ymm4
625 ; AVX512VPOPCNTDQ-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
626 ; AVX512VPOPCNTDQ-NEXT: # ymm5 = mem[0,1,0,1]
627 ; AVX512VPOPCNTDQ-NEXT: vpshufb %ymm4, %ymm5, %ymm4
628 ; AVX512VPOPCNTDQ-NEXT: vpsrlw $4, %ymm1, %ymm1
629 ; AVX512VPOPCNTDQ-NEXT: vpand %ymm3, %ymm1, %ymm1
630 ; AVX512VPOPCNTDQ-NEXT: vpshufb %ymm1, %ymm5, %ymm1
631 ; AVX512VPOPCNTDQ-NEXT: vpaddb %ymm4, %ymm1, %ymm1
632 ; AVX512VPOPCNTDQ-NEXT: vpaddb %ymm2, %ymm0, %ymm2
633 ; AVX512VPOPCNTDQ-NEXT: vpandn %ymm2, %ymm0, %ymm0
634 ; AVX512VPOPCNTDQ-NEXT: vpand %ymm3, %ymm0, %ymm2
635 ; AVX512VPOPCNTDQ-NEXT: vpshufb %ymm2, %ymm5, %ymm2
636 ; AVX512VPOPCNTDQ-NEXT: vpsrlw $4, %ymm0, %ymm0
637 ; AVX512VPOPCNTDQ-NEXT: vpand %ymm3, %ymm0, %ymm0
638 ; AVX512VPOPCNTDQ-NEXT: vpshufb %ymm0, %ymm5, %ymm0
639 ; AVX512VPOPCNTDQ-NEXT: vpaddb %ymm2, %ymm0, %ymm0
640 ; AVX512VPOPCNTDQ-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
641 ; AVX512VPOPCNTDQ-NEXT: retq
642 ;
643 ; BITALG-LABEL: testv64i8u:
644 ; BITALG: # %bb.0:
645 ; BITALG-NEXT: vpternlogd {{.*#+}} zmm1 = -1
646 ; BITALG-NEXT: vpaddb %zmm1, %zmm0, %zmm1
647 ; BITALG-NEXT: vpandnq %zmm1, %zmm0, %zmm0
648 ; BITALG-NEXT: vpopcntb %zmm0, %zmm0
649 ; BITALG-NEXT: retq
650 %out = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %in, i1 -1)
651 ret <64 x i8> %out
652 }
654 declare <8 x i64> @llvm.cttz.v8i64(<8 x i64>, i1)
655 declare <16 x i32> @llvm.cttz.v16i32(<16 x i32>, i1)
656 declare <32 x i16> @llvm.cttz.v32i16(<32 x i16>, i1)
657 declare <64 x i8> @llvm.cttz.v64i8(<64 x i8>, i1)