| blob | 1c1caf8ee468130afbc991f306a4f031f4f38f51 |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX512F
3 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=AVX512BW
4 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vpopcntdq | FileCheck %s --check-prefixes=AVX512VPOPCNTDQ,AVX512VPOPCNTDQ-NOBW
5 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vpopcntdq,+avx512bw | FileCheck %s --check-prefixes=AVX512VPOPCNTDQ,AVX512VPOPCNTDQ-BW
6 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bitalg | FileCheck %s --check-prefix=BITALG
8 define <8 x i64> @testv8i64(<8 x i64> %in) nounwind {
9 ; AVX512F-LABEL: testv8i64:
10 ; AVX512F: # %bb.0:
11 ; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
12 ; AVX512F-NEXT: vpbroadcastb {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
13 ; AVX512F-NEXT: vpand %ymm2, %ymm1, %ymm3
14 ; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
15 ; AVX512F-NEXT: # ymm4 = mem[0,1,0,1]
16 ; AVX512F-NEXT: vpshufb %ymm3, %ymm4, %ymm3
17 ; AVX512F-NEXT: vpsrlw $4, %ymm1, %ymm1
18 ; AVX512F-NEXT: vpand %ymm2, %ymm1, %ymm1
19 ; AVX512F-NEXT: vpshufb %ymm1, %ymm4, %ymm1
20 ; AVX512F-NEXT: vpaddb %ymm3, %ymm1, %ymm1
21 ; AVX512F-NEXT: vpxor %xmm3, %xmm3, %xmm3
22 ; AVX512F-NEXT: vpsadbw %ymm3, %ymm1, %ymm1
23 ; AVX512F-NEXT: vpand %ymm2, %ymm0, %ymm5
24 ; AVX512F-NEXT: vpshufb %ymm5, %ymm4, %ymm5
25 ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm0
26 ; AVX512F-NEXT: vpand %ymm2, %ymm0, %ymm0
27 ; AVX512F-NEXT: vpshufb %ymm0, %ymm4, %ymm0
28 ; AVX512F-NEXT: vpaddb %ymm5, %ymm0, %ymm0
29 ; AVX512F-NEXT: vpsadbw %ymm3, %ymm0, %ymm0
30 ; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
31 ; AVX512F-NEXT: retq
32 ;
33 ; AVX512BW-LABEL: testv8i64:
34 ; AVX512BW: # %bb.0:
35 ; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
36 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm2
37 ; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
38 ; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
39 ; AVX512BW-NEXT: vpshufb %zmm2, %zmm3, %zmm2
40 ; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0
41 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm0
42 ; AVX512BW-NEXT: vpshufb %zmm0, %zmm3, %zmm0
43 ; AVX512BW-NEXT: vpaddb %zmm2, %zmm0, %zmm0
44 ; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
45 ; AVX512BW-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
46 ; AVX512BW-NEXT: retq
47 ;
48 ; AVX512VPOPCNTDQ-LABEL: testv8i64:
49 ; AVX512VPOPCNTDQ: # %bb.0:
50 ; AVX512VPOPCNTDQ-NEXT: vpopcntq %zmm0, %zmm0
51 ; AVX512VPOPCNTDQ-NEXT: retq
52 ;
53 ; BITALG-LABEL: testv8i64:
54 ; BITALG: # %bb.0:
55 ; BITALG-NEXT: vpopcntb %zmm0, %zmm0
56 ; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
57 ; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
58 ; BITALG-NEXT: retq
59 %out = call <8 x i64> @llvm.ctpop.v8i64(<8 x i64> %in)
60 ret <8 x i64> %out
61 }
63 define <16 x i32> @testv16i32(<16 x i32> %in) nounwind {
64 ; AVX512F-LABEL: testv16i32:
65 ; AVX512F: # %bb.0:
66 ; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
67 ; AVX512F-NEXT: vpbroadcastb {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
68 ; AVX512F-NEXT: vpand %ymm2, %ymm1, %ymm3
69 ; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
70 ; AVX512F-NEXT: # ymm4 = mem[0,1,0,1]
71 ; AVX512F-NEXT: vpshufb %ymm3, %ymm4, %ymm3
72 ; AVX512F-NEXT: vpsrlw $4, %ymm1, %ymm1
73 ; AVX512F-NEXT: vpand %ymm2, %ymm1, %ymm1
74 ; AVX512F-NEXT: vpshufb %ymm1, %ymm4, %ymm1
75 ; AVX512F-NEXT: vpaddb %ymm3, %ymm1, %ymm1
76 ; AVX512F-NEXT: vpxor %xmm3, %xmm3, %xmm3
77 ; AVX512F-NEXT: vpunpckhdq {{.*#+}} ymm5 = ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[6],ymm3[6],ymm1[7],ymm3[7]
78 ; AVX512F-NEXT: vpsadbw %ymm3, %ymm5, %ymm5
79 ; AVX512F-NEXT: vpunpckldq {{.*#+}} ymm1 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[4],ymm3[4],ymm1[5],ymm3[5]
80 ; AVX512F-NEXT: vpsadbw %ymm3, %ymm1, %ymm1
81 ; AVX512F-NEXT: vpackuswb %ymm5, %ymm1, %ymm1
82 ; AVX512F-NEXT: vpand %ymm2, %ymm0, %ymm5
83 ; AVX512F-NEXT: vpshufb %ymm5, %ymm4, %ymm5
84 ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm0
85 ; AVX512F-NEXT: vpand %ymm2, %ymm0, %ymm0
86 ; AVX512F-NEXT: vpshufb %ymm0, %ymm4, %ymm0
87 ; AVX512F-NEXT: vpaddb %ymm5, %ymm0, %ymm0
88 ; AVX512F-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm3[2],ymm0[3],ymm3[3],ymm0[6],ymm3[6],ymm0[7],ymm3[7]
89 ; AVX512F-NEXT: vpsadbw %ymm3, %ymm2, %ymm2
90 ; AVX512F-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[4],ymm3[4],ymm0[5],ymm3[5]
91 ; AVX512F-NEXT: vpsadbw %ymm3, %ymm0, %ymm0
92 ; AVX512F-NEXT: vpackuswb %ymm2, %ymm0, %ymm0
93 ; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
94 ; AVX512F-NEXT: retq
95 ;
96 ; AVX512BW-LABEL: testv16i32:
97 ; AVX512BW: # %bb.0:
98 ; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
99 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm2
100 ; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
101 ; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
102 ; AVX512BW-NEXT: vpshufb %zmm2, %zmm3, %zmm2
103 ; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0
104 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm0
105 ; AVX512BW-NEXT: vpshufb %zmm0, %zmm3, %zmm0
106 ; AVX512BW-NEXT: vpaddb %zmm2, %zmm0, %zmm0
107 ; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
108 ; AVX512BW-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
109 ; AVX512BW-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
110 ; AVX512BW-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
111 ; AVX512BW-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
112 ; AVX512BW-NEXT: vpackuswb %zmm2, %zmm0, %zmm0
113 ; AVX512BW-NEXT: retq
114 ;
115 ; AVX512VPOPCNTDQ-LABEL: testv16i32:
116 ; AVX512VPOPCNTDQ: # %bb.0:
117 ; AVX512VPOPCNTDQ-NEXT: vpopcntd %zmm0, %zmm0
118 ; AVX512VPOPCNTDQ-NEXT: retq
119 ;
120 ; BITALG-LABEL: testv16i32:
121 ; BITALG: # %bb.0:
122 ; BITALG-NEXT: vpopcntb %zmm0, %zmm0
123 ; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
124 ; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
125 ; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
126 ; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
127 ; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
128 ; BITALG-NEXT: vpackuswb %zmm2, %zmm0, %zmm0
129 ; BITALG-NEXT: retq
130 %out = call <16 x i32> @llvm.ctpop.v16i32(<16 x i32> %in)
131 ret <16 x i32> %out
132 }
134 define <32 x i16> @testv32i16(<32 x i16> %in) nounwind {
135 ; AVX512F-LABEL: testv32i16:
136 ; AVX512F: # %bb.0:
137 ; AVX512F-NEXT: vpbroadcastb {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
138 ; AVX512F-NEXT: vpand %ymm1, %ymm0, %ymm2
139 ; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
140 ; AVX512F-NEXT: # ymm3 = mem[0,1,0,1]
141 ; AVX512F-NEXT: vpshufb %ymm2, %ymm3, %ymm2
142 ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm4
143 ; AVX512F-NEXT: vpand %ymm1, %ymm4, %ymm4
144 ; AVX512F-NEXT: vpshufb %ymm4, %ymm3, %ymm4
145 ; AVX512F-NEXT: vpaddb %ymm2, %ymm4, %ymm2
146 ; AVX512F-NEXT: vpsllw $8, %ymm2, %ymm4
147 ; AVX512F-NEXT: vpaddb %ymm2, %ymm4, %ymm2
148 ; AVX512F-NEXT: vpsrlw $8, %ymm2, %ymm2
149 ; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm0
150 ; AVX512F-NEXT: vpand %ymm1, %ymm0, %ymm4
151 ; AVX512F-NEXT: vpshufb %ymm4, %ymm3, %ymm4
152 ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm0
153 ; AVX512F-NEXT: vpand %ymm1, %ymm0, %ymm0
154 ; AVX512F-NEXT: vpshufb %ymm0, %ymm3, %ymm0
155 ; AVX512F-NEXT: vpaddb %ymm4, %ymm0, %ymm0
156 ; AVX512F-NEXT: vpsllw $8, %ymm0, %ymm1
157 ; AVX512F-NEXT: vpaddb %ymm0, %ymm1, %ymm0
158 ; AVX512F-NEXT: vpsrlw $8, %ymm0, %ymm0
159 ; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0
160 ; AVX512F-NEXT: retq
161 ;
162 ; AVX512BW-LABEL: testv32i16:
163 ; AVX512BW: # %bb.0:
164 ; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
165 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm2
166 ; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
167 ; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
168 ; AVX512BW-NEXT: vpshufb %zmm2, %zmm3, %zmm2
169 ; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0
170 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm0
171 ; AVX512BW-NEXT: vpshufb %zmm0, %zmm3, %zmm0
172 ; AVX512BW-NEXT: vpaddb %zmm2, %zmm0, %zmm0
173 ; AVX512BW-NEXT: vpsllw $8, %zmm0, %zmm1
174 ; AVX512BW-NEXT: vpaddb %zmm0, %zmm1, %zmm0
175 ; AVX512BW-NEXT: vpsrlw $8, %zmm0, %zmm0
176 ; AVX512BW-NEXT: retq
177 ;
178 ; AVX512VPOPCNTDQ-NOBW-LABEL: testv32i16:
179 ; AVX512VPOPCNTDQ-NOBW: # %bb.0:
180 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
181 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpopcntd %zmm1, %zmm1
182 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpmovdw %zmm1, %ymm1
183 ; AVX512VPOPCNTDQ-NOBW-NEXT: vextracti64x4 $1, %zmm0, %ymm0
184 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
185 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpopcntd %zmm0, %zmm0
186 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpmovdw %zmm0, %ymm0
187 ; AVX512VPOPCNTDQ-NOBW-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
188 ; AVX512VPOPCNTDQ-NOBW-NEXT: retq
189 ;
190 ; AVX512VPOPCNTDQ-BW-LABEL: testv32i16:
191 ; AVX512VPOPCNTDQ-BW: # %bb.0:
192 ; AVX512VPOPCNTDQ-BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
193 ; AVX512VPOPCNTDQ-BW-NEXT: vpandq %zmm1, %zmm0, %zmm2
194 ; AVX512VPOPCNTDQ-BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
195 ; AVX512VPOPCNTDQ-BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
196 ; AVX512VPOPCNTDQ-BW-NEXT: vpshufb %zmm2, %zmm3, %zmm2
197 ; AVX512VPOPCNTDQ-BW-NEXT: vpsrlw $4, %zmm0, %zmm0
198 ; AVX512VPOPCNTDQ-BW-NEXT: vpandq %zmm1, %zmm0, %zmm0
199 ; AVX512VPOPCNTDQ-BW-NEXT: vpshufb %zmm0, %zmm3, %zmm0
200 ; AVX512VPOPCNTDQ-BW-NEXT: vpaddb %zmm2, %zmm0, %zmm0
201 ; AVX512VPOPCNTDQ-BW-NEXT: vpsllw $8, %zmm0, %zmm1
202 ; AVX512VPOPCNTDQ-BW-NEXT: vpaddb %zmm0, %zmm1, %zmm0
203 ; AVX512VPOPCNTDQ-BW-NEXT: vpsrlw $8, %zmm0, %zmm0
204 ; AVX512VPOPCNTDQ-BW-NEXT: retq
205 ;
206 ; BITALG-LABEL: testv32i16:
207 ; BITALG: # %bb.0:
208 ; BITALG-NEXT: vpopcntw %zmm0, %zmm0
209 ; BITALG-NEXT: retq
210 %out = call <32 x i16> @llvm.ctpop.v32i16(<32 x i16> %in)
211 ret <32 x i16> %out
212 }
214 define <64 x i8> @testv64i8(<64 x i8> %in) nounwind {
215 ; AVX512F-LABEL: testv64i8:
216 ; AVX512F: # %bb.0:
217 ; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
218 ; AVX512F-NEXT: vpbroadcastb {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
219 ; AVX512F-NEXT: vpand %ymm2, %ymm1, %ymm3
220 ; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
221 ; AVX512F-NEXT: # ymm4 = mem[0,1,0,1]
222 ; AVX512F-NEXT: vpshufb %ymm3, %ymm4, %ymm3
223 ; AVX512F-NEXT: vpsrlw $4, %ymm1, %ymm1
224 ; AVX512F-NEXT: vpand %ymm2, %ymm1, %ymm1
225 ; AVX512F-NEXT: vpshufb %ymm1, %ymm4, %ymm1
226 ; AVX512F-NEXT: vpaddb %ymm3, %ymm1, %ymm1
227 ; AVX512F-NEXT: vpand %ymm2, %ymm0, %ymm3
228 ; AVX512F-NEXT: vpshufb %ymm3, %ymm4, %ymm3
229 ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm0
230 ; AVX512F-NEXT: vpand %ymm2, %ymm0, %ymm0
231 ; AVX512F-NEXT: vpshufb %ymm0, %ymm4, %ymm0
232 ; AVX512F-NEXT: vpaddb %ymm3, %ymm0, %ymm0
233 ; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
234 ; AVX512F-NEXT: retq
235 ;
236 ; AVX512BW-LABEL: testv64i8:
237 ; AVX512BW: # %bb.0:
238 ; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
239 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm2
240 ; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
241 ; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
242 ; AVX512BW-NEXT: vpshufb %zmm2, %zmm3, %zmm2
243 ; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0
244 ; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm0
245 ; AVX512BW-NEXT: vpshufb %zmm0, %zmm3, %zmm0
246 ; AVX512BW-NEXT: vpaddb %zmm2, %zmm0, %zmm0
247 ; AVX512BW-NEXT: retq
248 ;
249 ; AVX512VPOPCNTDQ-NOBW-LABEL: testv64i8:
250 ; AVX512VPOPCNTDQ-NOBW: # %bb.0:
251 ; AVX512VPOPCNTDQ-NOBW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
252 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpbroadcastb {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
253 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpand %ymm2, %ymm1, %ymm3
254 ; AVX512VPOPCNTDQ-NOBW-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
255 ; AVX512VPOPCNTDQ-NOBW-NEXT: # ymm4 = mem[0,1,0,1]
256 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpshufb %ymm3, %ymm4, %ymm3
257 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpsrlw $4, %ymm1, %ymm1
258 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpand %ymm2, %ymm1, %ymm1
259 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpshufb %ymm1, %ymm4, %ymm1
260 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpaddb %ymm3, %ymm1, %ymm1
261 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpand %ymm2, %ymm0, %ymm3
262 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpshufb %ymm3, %ymm4, %ymm3
263 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpsrlw $4, %ymm0, %ymm0
264 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpand %ymm2, %ymm0, %ymm0
265 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpshufb %ymm0, %ymm4, %ymm0
266 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpaddb %ymm3, %ymm0, %ymm0
267 ; AVX512VPOPCNTDQ-NOBW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
268 ; AVX512VPOPCNTDQ-NOBW-NEXT: retq
269 ;
270 ; AVX512VPOPCNTDQ-BW-LABEL: testv64i8:
271 ; AVX512VPOPCNTDQ-BW: # %bb.0:
272 ; AVX512VPOPCNTDQ-BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
273 ; AVX512VPOPCNTDQ-BW-NEXT: vpandq %zmm1, %zmm0, %zmm2
274 ; AVX512VPOPCNTDQ-BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
275 ; AVX512VPOPCNTDQ-BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
276 ; AVX512VPOPCNTDQ-BW-NEXT: vpshufb %zmm2, %zmm3, %zmm2
277 ; AVX512VPOPCNTDQ-BW-NEXT: vpsrlw $4, %zmm0, %zmm0
278 ; AVX512VPOPCNTDQ-BW-NEXT: vpandq %zmm1, %zmm0, %zmm0
279 ; AVX512VPOPCNTDQ-BW-NEXT: vpshufb %zmm0, %zmm3, %zmm0
280 ; AVX512VPOPCNTDQ-BW-NEXT: vpaddb %zmm2, %zmm0, %zmm0
281 ; AVX512VPOPCNTDQ-BW-NEXT: retq
282 ;
283 ; BITALG-LABEL: testv64i8:
284 ; BITALG: # %bb.0:
285 ; BITALG-NEXT: vpopcntb %zmm0, %zmm0
286 ; BITALG-NEXT: retq
287 %out = call <64 x i8> @llvm.ctpop.v64i8(<64 x i8> %in)
288 ret <64 x i8> %out
289 }
291 define <8 x i64> @eq_1_v8i64(<8 x i64> %0) {
292 ; AVX512F-LABEL: eq_1_v8i64:
293 ; AVX512F: # %bb.0:
294 ; AVX512F-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1
295 ; AVX512F-NEXT: vpaddq %zmm1, %zmm0, %zmm1
296 ; AVX512F-NEXT: vptestnmq %zmm1, %zmm0, %k1
297 ; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1 {%k1}
298 ; AVX512F-NEXT: vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
299 ; AVX512F-NEXT: retq
300 ;
301 ; AVX512BW-LABEL: eq_1_v8i64:
302 ; AVX512BW: # %bb.0:
303 ; AVX512BW-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1
304 ; AVX512BW-NEXT: vpaddq %zmm1, %zmm0, %zmm1
305 ; AVX512BW-NEXT: vptestnmq %zmm1, %zmm0, %k1
306 ; AVX512BW-NEXT: vptestmq %zmm0, %zmm0, %k1 {%k1}
307 ; AVX512BW-NEXT: vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
308 ; AVX512BW-NEXT: retq
309 ;
310 ; AVX512VPOPCNTDQ-LABEL: eq_1_v8i64:
311 ; AVX512VPOPCNTDQ: # %bb.0:
312 ; AVX512VPOPCNTDQ-NEXT: vpopcntq %zmm0, %zmm0
313 ; AVX512VPOPCNTDQ-NEXT: vpcmpeqq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
314 ; AVX512VPOPCNTDQ-NEXT: vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
315 ; AVX512VPOPCNTDQ-NEXT: retq
316 ;
317 ; BITALG-LABEL: eq_1_v8i64:
318 ; BITALG: # %bb.0:
319 ; BITALG-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1
320 ; BITALG-NEXT: vpaddq %zmm1, %zmm0, %zmm1
321 ; BITALG-NEXT: vptestnmq %zmm1, %zmm0, %k1
322 ; BITALG-NEXT: vptestmq %zmm0, %zmm0, %k1 {%k1}
323 ; BITALG-NEXT: vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
324 ; BITALG-NEXT: retq
325 %2 = tail call <8 x i64> @llvm.ctpop.v8i64(<8 x i64> %0)
326 %3 = icmp eq <8 x i64> %2, <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>
327 %4 = sext <8 x i1> %3 to <8 x i64>
328 ret <8 x i64> %4
329 }
331 define <8 x i64> @ne_1_v8i64(<8 x i64> %0) {
332 ; AVX512F-LABEL: ne_1_v8i64:
333 ; AVX512F: # %bb.0:
334 ; AVX512F-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1
335 ; AVX512F-NEXT: vpaddq %zmm1, %zmm0, %zmm1
336 ; AVX512F-NEXT: vptestmq %zmm1, %zmm0, %k0
337 ; AVX512F-NEXT: vptestnmq %zmm0, %zmm0, %k1
338 ; AVX512F-NEXT: korw %k0, %k1, %k1
339 ; AVX512F-NEXT: vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
340 ; AVX512F-NEXT: retq
341 ;
342 ; AVX512BW-LABEL: ne_1_v8i64:
343 ; AVX512BW: # %bb.0:
344 ; AVX512BW-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1
345 ; AVX512BW-NEXT: vpaddq %zmm1, %zmm0, %zmm1
346 ; AVX512BW-NEXT: vptestmq %zmm1, %zmm0, %k0
347 ; AVX512BW-NEXT: vptestnmq %zmm0, %zmm0, %k1
348 ; AVX512BW-NEXT: korw %k0, %k1, %k1
349 ; AVX512BW-NEXT: vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
350 ; AVX512BW-NEXT: retq
351 ;
352 ; AVX512VPOPCNTDQ-LABEL: ne_1_v8i64:
353 ; AVX512VPOPCNTDQ: # %bb.0:
354 ; AVX512VPOPCNTDQ-NEXT: vpopcntq %zmm0, %zmm0
355 ; AVX512VPOPCNTDQ-NEXT: vpcmpneqq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
356 ; AVX512VPOPCNTDQ-NEXT: vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
357 ; AVX512VPOPCNTDQ-NEXT: retq
358 ;
359 ; BITALG-LABEL: ne_1_v8i64:
360 ; BITALG: # %bb.0:
361 ; BITALG-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1
362 ; BITALG-NEXT: vpaddq %zmm1, %zmm0, %zmm1
363 ; BITALG-NEXT: vptestmq %zmm1, %zmm0, %k0
364 ; BITALG-NEXT: vptestnmq %zmm0, %zmm0, %k1
365 ; BITALG-NEXT: korw %k0, %k1, %k1
366 ; BITALG-NEXT: vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
367 ; BITALG-NEXT: retq
368 %2 = tail call <8 x i64> @llvm.ctpop.v8i64(<8 x i64> %0)
369 %3 = icmp ne <8 x i64> %2, <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>
370 %4 = sext <8 x i1> %3 to <8 x i64>
371 ret <8 x i64> %4
372 }
374 define <16 x i32> @eq_1_v16i32(<16 x i32> %0) {
375 ; AVX512F-LABEL: eq_1_v16i32:
376 ; AVX512F: # %bb.0:
377 ; AVX512F-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1
378 ; AVX512F-NEXT: vpaddd %zmm1, %zmm0, %zmm1
379 ; AVX512F-NEXT: vptestnmd %zmm1, %zmm0, %k1
380 ; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k1 {%k1}
381 ; AVX512F-NEXT: vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
382 ; AVX512F-NEXT: retq
383 ;
384 ; AVX512BW-LABEL: eq_1_v16i32:
385 ; AVX512BW: # %bb.0:
386 ; AVX512BW-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1
387 ; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm1
388 ; AVX512BW-NEXT: vptestnmd %zmm1, %zmm0, %k1
389 ; AVX512BW-NEXT: vptestmd %zmm0, %zmm0, %k1 {%k1}
390 ; AVX512BW-NEXT: vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
391 ; AVX512BW-NEXT: retq
392 ;
393 ; AVX512VPOPCNTDQ-LABEL: eq_1_v16i32:
394 ; AVX512VPOPCNTDQ: # %bb.0:
395 ; AVX512VPOPCNTDQ-NEXT: vpopcntd %zmm0, %zmm0
396 ; AVX512VPOPCNTDQ-NEXT: vpcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %k1
397 ; AVX512VPOPCNTDQ-NEXT: vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
398 ; AVX512VPOPCNTDQ-NEXT: retq
399 ;
400 ; BITALG-LABEL: eq_1_v16i32:
401 ; BITALG: # %bb.0:
402 ; BITALG-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1
403 ; BITALG-NEXT: vpaddd %zmm1, %zmm0, %zmm1
404 ; BITALG-NEXT: vptestnmd %zmm1, %zmm0, %k1
405 ; BITALG-NEXT: vptestmd %zmm0, %zmm0, %k1 {%k1}
406 ; BITALG-NEXT: vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
407 ; BITALG-NEXT: retq
408 %2 = tail call <16 x i32> @llvm.ctpop.v16i32(<16 x i32> %0)
409 %3 = icmp eq <16 x i32> %2, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
410 %4 = sext <16 x i1> %3 to <16 x i32>
411 ret <16 x i32> %4
412 }
414 define <16 x i32> @ne_1_v16i32(<16 x i32> %0) {
415 ; AVX512F-LABEL: ne_1_v16i32:
416 ; AVX512F: # %bb.0:
417 ; AVX512F-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1
418 ; AVX512F-NEXT: vpaddd %zmm1, %zmm0, %zmm1
419 ; AVX512F-NEXT: vptestmd %zmm1, %zmm0, %k0
420 ; AVX512F-NEXT: vptestnmd %zmm0, %zmm0, %k1
421 ; AVX512F-NEXT: korw %k0, %k1, %k1
422 ; AVX512F-NEXT: vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
423 ; AVX512F-NEXT: retq
424 ;
425 ; AVX512BW-LABEL: ne_1_v16i32:
426 ; AVX512BW: # %bb.0:
427 ; AVX512BW-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1
428 ; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm1
429 ; AVX512BW-NEXT: vptestmd %zmm1, %zmm0, %k0
430 ; AVX512BW-NEXT: vptestnmd %zmm0, %zmm0, %k1
431 ; AVX512BW-NEXT: korw %k0, %k1, %k1
432 ; AVX512BW-NEXT: vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
433 ; AVX512BW-NEXT: retq
434 ;
435 ; AVX512VPOPCNTDQ-LABEL: ne_1_v16i32:
436 ; AVX512VPOPCNTDQ: # %bb.0:
437 ; AVX512VPOPCNTDQ-NEXT: vpopcntd %zmm0, %zmm0
438 ; AVX512VPOPCNTDQ-NEXT: vpcmpneqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %k1
439 ; AVX512VPOPCNTDQ-NEXT: vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
440 ; AVX512VPOPCNTDQ-NEXT: retq
441 ;
442 ; BITALG-LABEL: ne_1_v16i32:
443 ; BITALG: # %bb.0:
444 ; BITALG-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1
445 ; BITALG-NEXT: vpaddd %zmm1, %zmm0, %zmm1
446 ; BITALG-NEXT: vptestmd %zmm1, %zmm0, %k0
447 ; BITALG-NEXT: vptestnmd %zmm0, %zmm0, %k1
448 ; BITALG-NEXT: korw %k0, %k1, %k1
449 ; BITALG-NEXT: vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
450 ; BITALG-NEXT: retq
451 %2 = tail call <16 x i32> @llvm.ctpop.v16i32(<16 x i32> %0)
452 %3 = icmp ne <16 x i32> %2, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
453 %4 = sext <16 x i1> %3 to <16 x i32>
454 ret <16 x i32> %4
455 }
457 define <32 x i16> @eq_1_v32i16(<32 x i16> %0) {
458 ; AVX512F-LABEL: eq_1_v32i16:
459 ; AVX512F: # %bb.0:
460 ; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
461 ; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
462 ; AVX512F-NEXT: vpcmpeqw %ymm2, %ymm1, %ymm3
463 ; AVX512F-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm4
464 ; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm3
465 ; AVX512F-NEXT: vpcmpeqd %ymm4, %ymm4, %ymm4
466 ; AVX512F-NEXT: vpaddw %ymm4, %ymm1, %ymm5
467 ; AVX512F-NEXT: vpand %ymm5, %ymm1, %ymm1
468 ; AVX512F-NEXT: vpcmpeqw %ymm2, %ymm1, %ymm1
469 ; AVX512F-NEXT: vpaddw %ymm4, %ymm0, %ymm4
470 ; AVX512F-NEXT: vpand %ymm4, %ymm0, %ymm0
471 ; AVX512F-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm0
472 ; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
473 ; AVX512F-NEXT: vpandnq %zmm0, %zmm3, %zmm0
474 ; AVX512F-NEXT: retq
475 ;
476 ; AVX512BW-LABEL: eq_1_v32i16:
477 ; AVX512BW: # %bb.0:
478 ; AVX512BW-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1
479 ; AVX512BW-NEXT: vpaddw %zmm1, %zmm0, %zmm1
480 ; AVX512BW-NEXT: vptestnmw %zmm1, %zmm0, %k1
481 ; AVX512BW-NEXT: vptestmw %zmm0, %zmm0, %k0 {%k1}
482 ; AVX512BW-NEXT: vpmovm2w %k0, %zmm0
483 ; AVX512BW-NEXT: retq
484 ;
485 ; AVX512VPOPCNTDQ-NOBW-LABEL: eq_1_v32i16:
486 ; AVX512VPOPCNTDQ-NOBW: # %bb.0:
487 ; AVX512VPOPCNTDQ-NOBW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
488 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
489 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpcmpeqw %ymm2, %ymm1, %ymm3
490 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm4
491 ; AVX512VPOPCNTDQ-NOBW-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm3
492 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpcmpeqd %ymm4, %ymm4, %ymm4
493 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpaddw %ymm4, %ymm1, %ymm5
494 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpand %ymm5, %ymm1, %ymm1
495 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpcmpeqw %ymm2, %ymm1, %ymm1
496 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpaddw %ymm4, %ymm0, %ymm4
497 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpand %ymm4, %ymm0, %ymm0
498 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm0
499 ; AVX512VPOPCNTDQ-NOBW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
500 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpandnq %zmm0, %zmm3, %zmm0
501 ; AVX512VPOPCNTDQ-NOBW-NEXT: retq
502 ;
503 ; AVX512VPOPCNTDQ-BW-LABEL: eq_1_v32i16:
504 ; AVX512VPOPCNTDQ-BW: # %bb.0:
505 ; AVX512VPOPCNTDQ-BW-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1
506 ; AVX512VPOPCNTDQ-BW-NEXT: vpaddw %zmm1, %zmm0, %zmm1
507 ; AVX512VPOPCNTDQ-BW-NEXT: vptestnmw %zmm1, %zmm0, %k1
508 ; AVX512VPOPCNTDQ-BW-NEXT: vptestmw %zmm0, %zmm0, %k0 {%k1}
509 ; AVX512VPOPCNTDQ-BW-NEXT: vpmovm2w %k0, %zmm0
510 ; AVX512VPOPCNTDQ-BW-NEXT: retq
511 ;
512 ; BITALG-LABEL: eq_1_v32i16:
513 ; BITALG: # %bb.0:
514 ; BITALG-NEXT: vpopcntw %zmm0, %zmm0
515 ; BITALG-NEXT: vpcmpeqw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k0
516 ; BITALG-NEXT: vpmovm2w %k0, %zmm0
517 ; BITALG-NEXT: retq
518 %2 = tail call <32 x i16> @llvm.ctpop.v32i16(<32 x i16> %0)
519 %3 = icmp eq <32 x i16> %2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
520 %4 = sext <32 x i1> %3 to <32 x i16>
521 ret <32 x i16> %4
522 }
524 define <32 x i16> @ne_1_v32i16(<32 x i16> %0) {
525 ; AVX512F-LABEL: ne_1_v32i16:
526 ; AVX512F: # %bb.0:
527 ; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
528 ; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
529 ; AVX512F-NEXT: vpcmpeqw %ymm2, %ymm1, %ymm3
530 ; AVX512F-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm4
531 ; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm3
532 ; AVX512F-NEXT: vpcmpeqd %ymm4, %ymm4, %ymm4
533 ; AVX512F-NEXT: vpaddw %ymm4, %ymm1, %ymm5
534 ; AVX512F-NEXT: vpand %ymm5, %ymm1, %ymm1
535 ; AVX512F-NEXT: vpcmpeqw %ymm2, %ymm1, %ymm1
536 ; AVX512F-NEXT: vpaddw %ymm4, %ymm0, %ymm4
537 ; AVX512F-NEXT: vpand %ymm4, %ymm0, %ymm0
538 ; AVX512F-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm0
539 ; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm1
540 ; AVX512F-NEXT: vpternlogd $255, %zmm0, %zmm0, %zmm0
541 ; AVX512F-NEXT: vpternlogq $222, %zmm1, %zmm3, %zmm0
542 ; AVX512F-NEXT: retq
543 ;
544 ; AVX512BW-LABEL: ne_1_v32i16:
545 ; AVX512BW: # %bb.0:
546 ; AVX512BW-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1
547 ; AVX512BW-NEXT: vpaddw %zmm1, %zmm0, %zmm1
548 ; AVX512BW-NEXT: vptestmw %zmm1, %zmm0, %k0
549 ; AVX512BW-NEXT: vptestnmw %zmm0, %zmm0, %k1
550 ; AVX512BW-NEXT: kord %k0, %k1, %k0
551 ; AVX512BW-NEXT: vpmovm2w %k0, %zmm0
552 ; AVX512BW-NEXT: retq
553 ;
554 ; AVX512VPOPCNTDQ-NOBW-LABEL: ne_1_v32i16:
555 ; AVX512VPOPCNTDQ-NOBW: # %bb.0:
556 ; AVX512VPOPCNTDQ-NOBW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
557 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
558 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpcmpeqw %ymm2, %ymm1, %ymm3
559 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm4
560 ; AVX512VPOPCNTDQ-NOBW-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm3
561 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpcmpeqd %ymm4, %ymm4, %ymm4
562 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpaddw %ymm4, %ymm1, %ymm5
563 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpand %ymm5, %ymm1, %ymm1
564 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpcmpeqw %ymm2, %ymm1, %ymm1
565 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpaddw %ymm4, %ymm0, %ymm4
566 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpand %ymm4, %ymm0, %ymm0
567 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm0
568 ; AVX512VPOPCNTDQ-NOBW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm1
569 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpternlogd $255, %zmm0, %zmm0, %zmm0
570 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpternlogq $222, %zmm1, %zmm3, %zmm0
571 ; AVX512VPOPCNTDQ-NOBW-NEXT: retq
572 ;
573 ; AVX512VPOPCNTDQ-BW-LABEL: ne_1_v32i16:
574 ; AVX512VPOPCNTDQ-BW: # %bb.0:
575 ; AVX512VPOPCNTDQ-BW-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1
576 ; AVX512VPOPCNTDQ-BW-NEXT: vpaddw %zmm1, %zmm0, %zmm1
577 ; AVX512VPOPCNTDQ-BW-NEXT: vptestmw %zmm1, %zmm0, %k0
578 ; AVX512VPOPCNTDQ-BW-NEXT: vptestnmw %zmm0, %zmm0, %k1
579 ; AVX512VPOPCNTDQ-BW-NEXT: kord %k0, %k1, %k0
580 ; AVX512VPOPCNTDQ-BW-NEXT: vpmovm2w %k0, %zmm0
581 ; AVX512VPOPCNTDQ-BW-NEXT: retq
582 ;
583 ; BITALG-LABEL: ne_1_v32i16:
584 ; BITALG: # %bb.0:
585 ; BITALG-NEXT: vpopcntw %zmm0, %zmm0
586 ; BITALG-NEXT: vpcmpneqw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k0
587 ; BITALG-NEXT: vpmovm2w %k0, %zmm0
588 ; BITALG-NEXT: retq
589 %2 = tail call <32 x i16> @llvm.ctpop.v32i16(<32 x i16> %0)
590 %3 = icmp ne <32 x i16> %2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
591 %4 = sext <32 x i1> %3 to <32 x i16>
592 ret <32 x i16> %4
593 }
595 define <64 x i8> @eq_1_v64i8(<64 x i8> %0) {
596 ; AVX512F-LABEL: eq_1_v64i8:
597 ; AVX512F: # %bb.0:
598 ; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
599 ; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
600 ; AVX512F-NEXT: vpcmpeqb %ymm2, %ymm1, %ymm3
601 ; AVX512F-NEXT: vpcmpeqb %ymm2, %ymm0, %ymm4
602 ; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm3
603 ; AVX512F-NEXT: vpcmpeqd %ymm4, %ymm4, %ymm4
604 ; AVX512F-NEXT: vpaddb %ymm4, %ymm1, %ymm5
605 ; AVX512F-NEXT: vpand %ymm5, %ymm1, %ymm1
606 ; AVX512F-NEXT: vpcmpeqb %ymm2, %ymm1, %ymm1
607 ; AVX512F-NEXT: vpaddb %ymm4, %ymm0, %ymm4
608 ; AVX512F-NEXT: vpand %ymm4, %ymm0, %ymm0
609 ; AVX512F-NEXT: vpcmpeqb %ymm2, %ymm0, %ymm0
610 ; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
611 ; AVX512F-NEXT: vpandnq %zmm0, %zmm3, %zmm0
612 ; AVX512F-NEXT: retq
613 ;
614 ; AVX512BW-LABEL: eq_1_v64i8:
615 ; AVX512BW: # %bb.0:
616 ; AVX512BW-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1
617 ; AVX512BW-NEXT: vpaddb %zmm1, %zmm0, %zmm1
618 ; AVX512BW-NEXT: vptestnmb %zmm1, %zmm0, %k1
619 ; AVX512BW-NEXT: vptestmb %zmm0, %zmm0, %k0 {%k1}
620 ; AVX512BW-NEXT: vpmovm2b %k0, %zmm0
621 ; AVX512BW-NEXT: retq
622 ;
623 ; AVX512VPOPCNTDQ-NOBW-LABEL: eq_1_v64i8:
624 ; AVX512VPOPCNTDQ-NOBW: # %bb.0:
625 ; AVX512VPOPCNTDQ-NOBW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
626 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
627 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpcmpeqb %ymm2, %ymm1, %ymm3
628 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpcmpeqb %ymm2, %ymm0, %ymm4
629 ; AVX512VPOPCNTDQ-NOBW-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm3
630 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpcmpeqd %ymm4, %ymm4, %ymm4
631 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpaddb %ymm4, %ymm1, %ymm5
632 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpand %ymm5, %ymm1, %ymm1
633 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpcmpeqb %ymm2, %ymm1, %ymm1
634 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpaddb %ymm4, %ymm0, %ymm4
635 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpand %ymm4, %ymm0, %ymm0
636 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpcmpeqb %ymm2, %ymm0, %ymm0
637 ; AVX512VPOPCNTDQ-NOBW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
638 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpandnq %zmm0, %zmm3, %zmm0
639 ; AVX512VPOPCNTDQ-NOBW-NEXT: retq
640 ;
641 ; AVX512VPOPCNTDQ-BW-LABEL: eq_1_v64i8:
642 ; AVX512VPOPCNTDQ-BW: # %bb.0:
643 ; AVX512VPOPCNTDQ-BW-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1
644 ; AVX512VPOPCNTDQ-BW-NEXT: vpaddb %zmm1, %zmm0, %zmm1
645 ; AVX512VPOPCNTDQ-BW-NEXT: vptestnmb %zmm1, %zmm0, %k1
646 ; AVX512VPOPCNTDQ-BW-NEXT: vptestmb %zmm0, %zmm0, %k0 {%k1}
647 ; AVX512VPOPCNTDQ-BW-NEXT: vpmovm2b %k0, %zmm0
648 ; AVX512VPOPCNTDQ-BW-NEXT: retq
649 ;
650 ; BITALG-LABEL: eq_1_v64i8:
651 ; BITALG: # %bb.0:
652 ; BITALG-NEXT: vpopcntb %zmm0, %zmm0
653 ; BITALG-NEXT: vpcmpeqb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k0
654 ; BITALG-NEXT: vpmovm2b %k0, %zmm0
655 ; BITALG-NEXT: retq
656 %2 = tail call <64 x i8> @llvm.ctpop.v64i8(<64 x i8> %0)
657 %3 = icmp eq <64 x i8> %2, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
658 %4 = sext <64 x i1> %3 to <64 x i8>
659 ret <64 x i8> %4
660 }
662 define <64 x i8> @ne_1_v64i8(<64 x i8> %0) {
663 ; AVX512F-LABEL: ne_1_v64i8:
664 ; AVX512F: # %bb.0:
665 ; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
666 ; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
667 ; AVX512F-NEXT: vpcmpeqb %ymm2, %ymm1, %ymm3
668 ; AVX512F-NEXT: vpcmpeqb %ymm2, %ymm0, %ymm4
669 ; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm3
670 ; AVX512F-NEXT: vpcmpeqd %ymm4, %ymm4, %ymm4
671 ; AVX512F-NEXT: vpaddb %ymm4, %ymm1, %ymm5
672 ; AVX512F-NEXT: vpand %ymm5, %ymm1, %ymm1
673 ; AVX512F-NEXT: vpcmpeqb %ymm2, %ymm1, %ymm1
674 ; AVX512F-NEXT: vpaddb %ymm4, %ymm0, %ymm4
675 ; AVX512F-NEXT: vpand %ymm4, %ymm0, %ymm0
676 ; AVX512F-NEXT: vpcmpeqb %ymm2, %ymm0, %ymm0
677 ; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm1
678 ; AVX512F-NEXT: vpternlogd $255, %zmm0, %zmm0, %zmm0
679 ; AVX512F-NEXT: vpternlogq $222, %zmm1, %zmm3, %zmm0
680 ; AVX512F-NEXT: retq
681 ;
682 ; AVX512BW-LABEL: ne_1_v64i8:
683 ; AVX512BW: # %bb.0:
684 ; AVX512BW-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1
685 ; AVX512BW-NEXT: vpaddb %zmm1, %zmm0, %zmm1
686 ; AVX512BW-NEXT: vptestmb %zmm1, %zmm0, %k0
687 ; AVX512BW-NEXT: vptestnmb %zmm0, %zmm0, %k1
688 ; AVX512BW-NEXT: korq %k0, %k1, %k0
689 ; AVX512BW-NEXT: vpmovm2b %k0, %zmm0
690 ; AVX512BW-NEXT: retq
691 ;
692 ; AVX512VPOPCNTDQ-NOBW-LABEL: ne_1_v64i8:
693 ; AVX512VPOPCNTDQ-NOBW: # %bb.0:
694 ; AVX512VPOPCNTDQ-NOBW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
695 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
696 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpcmpeqb %ymm2, %ymm1, %ymm3
697 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpcmpeqb %ymm2, %ymm0, %ymm4
698 ; AVX512VPOPCNTDQ-NOBW-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm3
699 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpcmpeqd %ymm4, %ymm4, %ymm4
700 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpaddb %ymm4, %ymm1, %ymm5
701 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpand %ymm5, %ymm1, %ymm1
702 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpcmpeqb %ymm2, %ymm1, %ymm1
703 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpaddb %ymm4, %ymm0, %ymm4
704 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpand %ymm4, %ymm0, %ymm0
705 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpcmpeqb %ymm2, %ymm0, %ymm0
706 ; AVX512VPOPCNTDQ-NOBW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm1
707 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpternlogd $255, %zmm0, %zmm0, %zmm0
708 ; AVX512VPOPCNTDQ-NOBW-NEXT: vpternlogq $222, %zmm1, %zmm3, %zmm0
709 ; AVX512VPOPCNTDQ-NOBW-NEXT: retq
710 ;
711 ; AVX512VPOPCNTDQ-BW-LABEL: ne_1_v64i8:
712 ; AVX512VPOPCNTDQ-BW: # %bb.0:
713 ; AVX512VPOPCNTDQ-BW-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1
714 ; AVX512VPOPCNTDQ-BW-NEXT: vpaddb %zmm1, %zmm0, %zmm1
715 ; AVX512VPOPCNTDQ-BW-NEXT: vptestmb %zmm1, %zmm0, %k0
716 ; AVX512VPOPCNTDQ-BW-NEXT: vptestnmb %zmm0, %zmm0, %k1
717 ; AVX512VPOPCNTDQ-BW-NEXT: korq %k0, %k1, %k0
718 ; AVX512VPOPCNTDQ-BW-NEXT: vpmovm2b %k0, %zmm0
719 ; AVX512VPOPCNTDQ-BW-NEXT: retq
720 ;
721 ; BITALG-LABEL: ne_1_v64i8:
722 ; BITALG: # %bb.0:
723 ; BITALG-NEXT: vpopcntb %zmm0, %zmm0
724 ; BITALG-NEXT: vpcmpneqb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k0
725 ; BITALG-NEXT: vpmovm2b %k0, %zmm0
726 ; BITALG-NEXT: retq
727 %2 = tail call <64 x i8> @llvm.ctpop.v64i8(<64 x i8> %0)
728 %3 = icmp ne <64 x i8> %2, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
729 %4 = sext <64 x i1> %3 to <64 x i8>
730 ret <64 x i8> %4
731 }
733 declare <8 x i64> @llvm.ctpop.v8i64(<8 x i64>)
734 declare <16 x i32> @llvm.ctpop.v16i32(<16 x i32>)
735 declare <32 x i16> @llvm.ctpop.v32i16(<32 x i16>)
736 declare <64 x i8> @llvm.ctpop.v64i8(<64 x i8>)