| blob | 9ed0178c0df42b014cb95e416ecd22482f40dda6 |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX1-FALLBACK
3 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX2
4 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop | FileCheck %s --check-prefix=XOP-FALLBACK
5 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefix=XOPAVX1
6 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefix=AVX2
7 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
8 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL,AVX512VL-FALLBACK
9 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW-FALLBACK
10 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512VL,AVX512VLBW
12 ; These test cases are inspired by C++2a std::midpoint().
13 ; See https://bugs.llvm.org/show_bug.cgi?id=40965
15 ; Using 256-bit vector regs.
17 ; ---------------------------------------------------------------------------- ;
18 ; 32-bit width. 256 / 32 = 8 elts.
19 ; ---------------------------------------------------------------------------- ;
21 ; Values come from regs
23 define <8 x i32> @vec256_i32_signed_reg_reg(<8 x i32> %a1, <8 x i32> %a2) nounwind {
24 ; AVX1-FALLBACK-LABEL: vec256_i32_signed_reg_reg:
25 ; AVX1-FALLBACK: # %bb.0:
26 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm1, %xmm2
27 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm3
28 ; AVX1-FALLBACK-NEXT: vpminsd %xmm2, %xmm3, %xmm4
29 ; AVX1-FALLBACK-NEXT: vpminsd %xmm1, %xmm0, %xmm5
30 ; AVX1-FALLBACK-NEXT: vpmaxsd %xmm2, %xmm3, %xmm2
31 ; AVX1-FALLBACK-NEXT: vpsubd %xmm4, %xmm2, %xmm2
32 ; AVX1-FALLBACK-NEXT: vpmaxsd %xmm1, %xmm0, %xmm1
33 ; AVX1-FALLBACK-NEXT: vpsubd %xmm5, %xmm1, %xmm1
34 ; AVX1-FALLBACK-NEXT: vpsrld $1, %xmm2, %xmm2
35 ; AVX1-FALLBACK-NEXT: vpsrld $1, %xmm1, %xmm1
36 ; AVX1-FALLBACK-NEXT: vpmulld %xmm1, %xmm1, %xmm1
37 ; AVX1-FALLBACK-NEXT: vpmulld %xmm2, %xmm2, %xmm2
38 ; AVX1-FALLBACK-NEXT: vpaddd %xmm3, %xmm2, %xmm2
39 ; AVX1-FALLBACK-NEXT: vpaddd %xmm0, %xmm1, %xmm0
40 ; AVX1-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
41 ; AVX1-FALLBACK-NEXT: retq
42 ;
43 ; AVX2-LABEL: vec256_i32_signed_reg_reg:
44 ; AVX2: # %bb.0:
45 ; AVX2-NEXT: vpminsd %ymm1, %ymm0, %ymm2
46 ; AVX2-NEXT: vpmaxsd %ymm1, %ymm0, %ymm1
47 ; AVX2-NEXT: vpsubd %ymm2, %ymm1, %ymm1
48 ; AVX2-NEXT: vpsrld $1, %ymm1, %ymm1
49 ; AVX2-NEXT: vpmulld %ymm1, %ymm1, %ymm1
50 ; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
51 ; AVX2-NEXT: retq
52 ;
53 ; XOP-FALLBACK-LABEL: vec256_i32_signed_reg_reg:
54 ; XOP-FALLBACK: # %bb.0:
55 ; XOP-FALLBACK-NEXT: vpminsd %xmm1, %xmm0, %xmm2
56 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm1, %xmm3
57 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm4
58 ; XOP-FALLBACK-NEXT: vpminsd %xmm3, %xmm4, %xmm5
59 ; XOP-FALLBACK-NEXT: vpmaxsd %xmm1, %xmm0, %xmm1
60 ; XOP-FALLBACK-NEXT: vpsubd %xmm2, %xmm1, %xmm1
61 ; XOP-FALLBACK-NEXT: vpmaxsd %xmm3, %xmm4, %xmm2
62 ; XOP-FALLBACK-NEXT: vpsubd %xmm5, %xmm2, %xmm2
63 ; XOP-FALLBACK-NEXT: vpsrld $1, %xmm1, %xmm1
64 ; XOP-FALLBACK-NEXT: vpsrld $1, %xmm2, %xmm2
65 ; XOP-FALLBACK-NEXT: vpmacsdd %xmm4, %xmm2, %xmm2, %xmm2
66 ; XOP-FALLBACK-NEXT: vpmacsdd %xmm0, %xmm1, %xmm1, %xmm0
67 ; XOP-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
68 ; XOP-FALLBACK-NEXT: retq
69 ;
70 ; XOPAVX1-LABEL: vec256_i32_signed_reg_reg:
71 ; XOPAVX1: # %bb.0:
72 ; XOPAVX1-NEXT: vpminsd %xmm1, %xmm0, %xmm2
73 ; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
74 ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
75 ; XOPAVX1-NEXT: vpminsd %xmm3, %xmm4, %xmm5
76 ; XOPAVX1-NEXT: vpmaxsd %xmm1, %xmm0, %xmm1
77 ; XOPAVX1-NEXT: vpsubd %xmm2, %xmm1, %xmm1
78 ; XOPAVX1-NEXT: vpmaxsd %xmm3, %xmm4, %xmm2
79 ; XOPAVX1-NEXT: vpsubd %xmm5, %xmm2, %xmm2
80 ; XOPAVX1-NEXT: vpsrld $1, %xmm1, %xmm1
81 ; XOPAVX1-NEXT: vpsrld $1, %xmm2, %xmm2
82 ; XOPAVX1-NEXT: vpmacsdd %xmm4, %xmm2, %xmm2, %xmm2
83 ; XOPAVX1-NEXT: vpmacsdd %xmm0, %xmm1, %xmm1, %xmm0
84 ; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
85 ; XOPAVX1-NEXT: retq
86 ;
87 ; AVX512-LABEL: vec256_i32_signed_reg_reg:
88 ; AVX512: # %bb.0:
89 ; AVX512-NEXT: vpminsd %ymm1, %ymm0, %ymm2
90 ; AVX512-NEXT: vpmaxsd %ymm1, %ymm0, %ymm1
91 ; AVX512-NEXT: vpsubd %ymm2, %ymm1, %ymm1
92 ; AVX512-NEXT: vpsrld $1, %ymm1, %ymm1
93 ; AVX512-NEXT: vpmulld %ymm1, %ymm1, %ymm1
94 ; AVX512-NEXT: vpaddd %ymm0, %ymm1, %ymm0
95 ; AVX512-NEXT: retq
96 %t3 = icmp sgt <8 x i32> %a1, %a2 ; signed
97 %t4 = select <8 x i1> %t3, <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
98 %t5 = select <8 x i1> %t3, <8 x i32> %a2, <8 x i32> %a1
99 %t6 = select <8 x i1> %t3, <8 x i32> %a1, <8 x i32> %a2
100 %t7 = sub <8 x i32> %t6, %t5
101 %t8 = lshr <8 x i32> %t7, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
102 %t9 = mul nsw <8 x i32> %t8, %t8 ; signed
103 %a10 = add nsw <8 x i32> %t9, %a1 ; signed
104 ret <8 x i32> %a10
105 }
107 define <8 x i32> @vec256_i32_unsigned_reg_reg(<8 x i32> %a1, <8 x i32> %a2) nounwind {
108 ; AVX1-FALLBACK-LABEL: vec256_i32_unsigned_reg_reg:
109 ; AVX1-FALLBACK: # %bb.0:
110 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm1, %xmm2
111 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm3
112 ; AVX1-FALLBACK-NEXT: vpminud %xmm2, %xmm3, %xmm4
113 ; AVX1-FALLBACK-NEXT: vpminud %xmm1, %xmm0, %xmm5
114 ; AVX1-FALLBACK-NEXT: vpmaxud %xmm2, %xmm3, %xmm2
115 ; AVX1-FALLBACK-NEXT: vpsubd %xmm4, %xmm2, %xmm2
116 ; AVX1-FALLBACK-NEXT: vpmaxud %xmm1, %xmm0, %xmm1
117 ; AVX1-FALLBACK-NEXT: vpsubd %xmm5, %xmm1, %xmm1
118 ; AVX1-FALLBACK-NEXT: vpsrld $1, %xmm2, %xmm2
119 ; AVX1-FALLBACK-NEXT: vpsrld $1, %xmm1, %xmm1
120 ; AVX1-FALLBACK-NEXT: vpmulld %xmm1, %xmm1, %xmm1
121 ; AVX1-FALLBACK-NEXT: vpmulld %xmm2, %xmm2, %xmm2
122 ; AVX1-FALLBACK-NEXT: vpaddd %xmm3, %xmm2, %xmm2
123 ; AVX1-FALLBACK-NEXT: vpaddd %xmm0, %xmm1, %xmm0
124 ; AVX1-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
125 ; AVX1-FALLBACK-NEXT: retq
126 ;
127 ; AVX2-LABEL: vec256_i32_unsigned_reg_reg:
128 ; AVX2: # %bb.0:
129 ; AVX2-NEXT: vpminud %ymm1, %ymm0, %ymm2
130 ; AVX2-NEXT: vpmaxud %ymm1, %ymm0, %ymm1
131 ; AVX2-NEXT: vpsubd %ymm2, %ymm1, %ymm1
132 ; AVX2-NEXT: vpsrld $1, %ymm1, %ymm1
133 ; AVX2-NEXT: vpmulld %ymm1, %ymm1, %ymm1
134 ; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
135 ; AVX2-NEXT: retq
136 ;
137 ; XOP-FALLBACK-LABEL: vec256_i32_unsigned_reg_reg:
138 ; XOP-FALLBACK: # %bb.0:
139 ; XOP-FALLBACK-NEXT: vpminud %xmm1, %xmm0, %xmm2
140 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm1, %xmm3
141 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm4
142 ; XOP-FALLBACK-NEXT: vpminud %xmm3, %xmm4, %xmm5
143 ; XOP-FALLBACK-NEXT: vpmaxud %xmm1, %xmm0, %xmm1
144 ; XOP-FALLBACK-NEXT: vpsubd %xmm2, %xmm1, %xmm1
145 ; XOP-FALLBACK-NEXT: vpmaxud %xmm3, %xmm4, %xmm2
146 ; XOP-FALLBACK-NEXT: vpsubd %xmm5, %xmm2, %xmm2
147 ; XOP-FALLBACK-NEXT: vpsrld $1, %xmm1, %xmm1
148 ; XOP-FALLBACK-NEXT: vpsrld $1, %xmm2, %xmm2
149 ; XOP-FALLBACK-NEXT: vpmacsdd %xmm4, %xmm2, %xmm2, %xmm2
150 ; XOP-FALLBACK-NEXT: vpmacsdd %xmm0, %xmm1, %xmm1, %xmm0
151 ; XOP-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
152 ; XOP-FALLBACK-NEXT: retq
153 ;
154 ; XOPAVX1-LABEL: vec256_i32_unsigned_reg_reg:
155 ; XOPAVX1: # %bb.0:
156 ; XOPAVX1-NEXT: vpminud %xmm1, %xmm0, %xmm2
157 ; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
158 ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
159 ; XOPAVX1-NEXT: vpminud %xmm3, %xmm4, %xmm5
160 ; XOPAVX1-NEXT: vpmaxud %xmm1, %xmm0, %xmm1
161 ; XOPAVX1-NEXT: vpsubd %xmm2, %xmm1, %xmm1
162 ; XOPAVX1-NEXT: vpmaxud %xmm3, %xmm4, %xmm2
163 ; XOPAVX1-NEXT: vpsubd %xmm5, %xmm2, %xmm2
164 ; XOPAVX1-NEXT: vpsrld $1, %xmm1, %xmm1
165 ; XOPAVX1-NEXT: vpsrld $1, %xmm2, %xmm2
166 ; XOPAVX1-NEXT: vpmacsdd %xmm4, %xmm2, %xmm2, %xmm2
167 ; XOPAVX1-NEXT: vpmacsdd %xmm0, %xmm1, %xmm1, %xmm0
168 ; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
169 ; XOPAVX1-NEXT: retq
170 ;
171 ; AVX512-LABEL: vec256_i32_unsigned_reg_reg:
172 ; AVX512: # %bb.0:
173 ; AVX512-NEXT: vpminud %ymm1, %ymm0, %ymm2
174 ; AVX512-NEXT: vpmaxud %ymm1, %ymm0, %ymm1
175 ; AVX512-NEXT: vpsubd %ymm2, %ymm1, %ymm1
176 ; AVX512-NEXT: vpsrld $1, %ymm1, %ymm1
177 ; AVX512-NEXT: vpmulld %ymm1, %ymm1, %ymm1
178 ; AVX512-NEXT: vpaddd %ymm0, %ymm1, %ymm0
179 ; AVX512-NEXT: retq
180 %t3 = icmp ugt <8 x i32> %a1, %a2
181 %t4 = select <8 x i1> %t3, <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
182 %t5 = select <8 x i1> %t3, <8 x i32> %a2, <8 x i32> %a1
183 %t6 = select <8 x i1> %t3, <8 x i32> %a1, <8 x i32> %a2
184 %t7 = sub <8 x i32> %t6, %t5
185 %t8 = lshr <8 x i32> %t7, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
186 %t9 = mul <8 x i32> %t8, %t8
187 %a10 = add <8 x i32> %t9, %a1
188 ret <8 x i32> %a10
189 }
191 ; Values are loaded. Only check signed case.
193 define <8 x i32> @vec256_i32_signed_mem_reg(ptr %a1_addr, <8 x i32> %a2) nounwind {
194 ; AVX1-FALLBACK-LABEL: vec256_i32_signed_mem_reg:
195 ; AVX1-FALLBACK: # %bb.0:
196 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm1
197 ; AVX1-FALLBACK-NEXT: vmovdqa (%rdi), %xmm2
198 ; AVX1-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm3
199 ; AVX1-FALLBACK-NEXT: vpminsd %xmm1, %xmm3, %xmm4
200 ; AVX1-FALLBACK-NEXT: vpminsd %xmm0, %xmm2, %xmm5
201 ; AVX1-FALLBACK-NEXT: vpmaxsd %xmm1, %xmm3, %xmm1
202 ; AVX1-FALLBACK-NEXT: vpsubd %xmm4, %xmm1, %xmm1
203 ; AVX1-FALLBACK-NEXT: vpmaxsd %xmm0, %xmm2, %xmm0
204 ; AVX1-FALLBACK-NEXT: vpsubd %xmm5, %xmm0, %xmm0
205 ; AVX1-FALLBACK-NEXT: vpsrld $1, %xmm1, %xmm1
206 ; AVX1-FALLBACK-NEXT: vpsrld $1, %xmm0, %xmm0
207 ; AVX1-FALLBACK-NEXT: vpmulld %xmm0, %xmm0, %xmm0
208 ; AVX1-FALLBACK-NEXT: vpmulld %xmm1, %xmm1, %xmm1
209 ; AVX1-FALLBACK-NEXT: vpaddd %xmm3, %xmm1, %xmm1
210 ; AVX1-FALLBACK-NEXT: vpaddd %xmm2, %xmm0, %xmm0
211 ; AVX1-FALLBACK-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
212 ; AVX1-FALLBACK-NEXT: retq
213 ;
214 ; AVX2-LABEL: vec256_i32_signed_mem_reg:
215 ; AVX2: # %bb.0:
216 ; AVX2-NEXT: vmovdqa (%rdi), %ymm1
217 ; AVX2-NEXT: vpminsd %ymm0, %ymm1, %ymm2
218 ; AVX2-NEXT: vpmaxsd %ymm0, %ymm1, %ymm0
219 ; AVX2-NEXT: vpsubd %ymm2, %ymm0, %ymm0
220 ; AVX2-NEXT: vpsrld $1, %ymm0, %ymm0
221 ; AVX2-NEXT: vpmulld %ymm0, %ymm0, %ymm0
222 ; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0
223 ; AVX2-NEXT: retq
224 ;
225 ; XOP-FALLBACK-LABEL: vec256_i32_signed_mem_reg:
226 ; XOP-FALLBACK: # %bb.0:
227 ; XOP-FALLBACK-NEXT: vmovdqa (%rdi), %xmm1
228 ; XOP-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm2
229 ; XOP-FALLBACK-NEXT: vpminsd %xmm0, %xmm1, %xmm3
230 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm4
231 ; XOP-FALLBACK-NEXT: vpminsd %xmm4, %xmm2, %xmm5
232 ; XOP-FALLBACK-NEXT: vpmaxsd %xmm0, %xmm1, %xmm0
233 ; XOP-FALLBACK-NEXT: vpsubd %xmm3, %xmm0, %xmm0
234 ; XOP-FALLBACK-NEXT: vpmaxsd %xmm4, %xmm2, %xmm3
235 ; XOP-FALLBACK-NEXT: vpsubd %xmm5, %xmm3, %xmm3
236 ; XOP-FALLBACK-NEXT: vpsrld $1, %xmm0, %xmm0
237 ; XOP-FALLBACK-NEXT: vpsrld $1, %xmm3, %xmm3
238 ; XOP-FALLBACK-NEXT: vpmacsdd %xmm2, %xmm3, %xmm3, %xmm2
239 ; XOP-FALLBACK-NEXT: vpmacsdd %xmm1, %xmm0, %xmm0, %xmm0
240 ; XOP-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
241 ; XOP-FALLBACK-NEXT: retq
242 ;
243 ; XOPAVX1-LABEL: vec256_i32_signed_mem_reg:
244 ; XOPAVX1: # %bb.0:
245 ; XOPAVX1-NEXT: vmovdqa (%rdi), %xmm1
246 ; XOPAVX1-NEXT: vmovdqa 16(%rdi), %xmm2
247 ; XOPAVX1-NEXT: vpminsd %xmm0, %xmm1, %xmm3
248 ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
249 ; XOPAVX1-NEXT: vpminsd %xmm4, %xmm2, %xmm5
250 ; XOPAVX1-NEXT: vpmaxsd %xmm0, %xmm1, %xmm0
251 ; XOPAVX1-NEXT: vpsubd %xmm3, %xmm0, %xmm0
252 ; XOPAVX1-NEXT: vpmaxsd %xmm4, %xmm2, %xmm3
253 ; XOPAVX1-NEXT: vpsubd %xmm5, %xmm3, %xmm3
254 ; XOPAVX1-NEXT: vpsrld $1, %xmm0, %xmm0
255 ; XOPAVX1-NEXT: vpsrld $1, %xmm3, %xmm3
256 ; XOPAVX1-NEXT: vpmacsdd %xmm2, %xmm3, %xmm3, %xmm2
257 ; XOPAVX1-NEXT: vpmacsdd %xmm1, %xmm0, %xmm0, %xmm0
258 ; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
259 ; XOPAVX1-NEXT: retq
260 ;
261 ; AVX512-LABEL: vec256_i32_signed_mem_reg:
262 ; AVX512: # %bb.0:
263 ; AVX512-NEXT: vmovdqa (%rdi), %ymm1
264 ; AVX512-NEXT: vpminsd %ymm0, %ymm1, %ymm2
265 ; AVX512-NEXT: vpmaxsd %ymm0, %ymm1, %ymm0
266 ; AVX512-NEXT: vpsubd %ymm2, %ymm0, %ymm0
267 ; AVX512-NEXT: vpsrld $1, %ymm0, %ymm0
268 ; AVX512-NEXT: vpmulld %ymm0, %ymm0, %ymm0
269 ; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
270 ; AVX512-NEXT: retq
271 %a1 = load <8 x i32>, ptr %a1_addr
272 %t3 = icmp sgt <8 x i32> %a1, %a2 ; signed
273 %t4 = select <8 x i1> %t3, <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
274 %t5 = select <8 x i1> %t3, <8 x i32> %a2, <8 x i32> %a1
275 %t6 = select <8 x i1> %t3, <8 x i32> %a1, <8 x i32> %a2
276 %t7 = sub <8 x i32> %t6, %t5
277 %t8 = lshr <8 x i32> %t7, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
278 %t9 = mul nsw <8 x i32> %t8, %t8 ; signed
279 %a10 = add nsw <8 x i32> %t9, %a1 ; signed
280 ret <8 x i32> %a10
281 }
283 define <8 x i32> @vec256_i32_signed_reg_mem(<8 x i32> %a1, ptr %a2_addr) nounwind {
284 ; AVX1-FALLBACK-LABEL: vec256_i32_signed_reg_mem:
285 ; AVX1-FALLBACK: # %bb.0:
286 ; AVX1-FALLBACK-NEXT: vmovdqa (%rdi), %xmm1
287 ; AVX1-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm2
288 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm3
289 ; AVX1-FALLBACK-NEXT: vpminsd %xmm2, %xmm3, %xmm4
290 ; AVX1-FALLBACK-NEXT: vpminsd %xmm1, %xmm0, %xmm5
291 ; AVX1-FALLBACK-NEXT: vpmaxsd %xmm2, %xmm3, %xmm2
292 ; AVX1-FALLBACK-NEXT: vpsubd %xmm4, %xmm2, %xmm2
293 ; AVX1-FALLBACK-NEXT: vpmaxsd %xmm1, %xmm0, %xmm1
294 ; AVX1-FALLBACK-NEXT: vpsubd %xmm5, %xmm1, %xmm1
295 ; AVX1-FALLBACK-NEXT: vpsrld $1, %xmm2, %xmm2
296 ; AVX1-FALLBACK-NEXT: vpsrld $1, %xmm1, %xmm1
297 ; AVX1-FALLBACK-NEXT: vpmulld %xmm1, %xmm1, %xmm1
298 ; AVX1-FALLBACK-NEXT: vpmulld %xmm2, %xmm2, %xmm2
299 ; AVX1-FALLBACK-NEXT: vpaddd %xmm3, %xmm2, %xmm2
300 ; AVX1-FALLBACK-NEXT: vpaddd %xmm0, %xmm1, %xmm0
301 ; AVX1-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
302 ; AVX1-FALLBACK-NEXT: retq
303 ;
304 ; AVX2-LABEL: vec256_i32_signed_reg_mem:
305 ; AVX2: # %bb.0:
306 ; AVX2-NEXT: vmovdqa (%rdi), %ymm1
307 ; AVX2-NEXT: vpminsd %ymm1, %ymm0, %ymm2
308 ; AVX2-NEXT: vpmaxsd %ymm1, %ymm0, %ymm1
309 ; AVX2-NEXT: vpsubd %ymm2, %ymm1, %ymm1
310 ; AVX2-NEXT: vpsrld $1, %ymm1, %ymm1
311 ; AVX2-NEXT: vpmulld %ymm1, %ymm1, %ymm1
312 ; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
313 ; AVX2-NEXT: retq
314 ;
315 ; XOP-FALLBACK-LABEL: vec256_i32_signed_reg_mem:
316 ; XOP-FALLBACK: # %bb.0:
317 ; XOP-FALLBACK-NEXT: vmovdqa (%rdi), %xmm1
318 ; XOP-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm2
319 ; XOP-FALLBACK-NEXT: vpminsd %xmm1, %xmm0, %xmm3
320 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm4
321 ; XOP-FALLBACK-NEXT: vpminsd %xmm2, %xmm4, %xmm5
322 ; XOP-FALLBACK-NEXT: vpmaxsd %xmm1, %xmm0, %xmm1
323 ; XOP-FALLBACK-NEXT: vpsubd %xmm3, %xmm1, %xmm1
324 ; XOP-FALLBACK-NEXT: vpmaxsd %xmm2, %xmm4, %xmm2
325 ; XOP-FALLBACK-NEXT: vpsubd %xmm5, %xmm2, %xmm2
326 ; XOP-FALLBACK-NEXT: vpsrld $1, %xmm1, %xmm1
327 ; XOP-FALLBACK-NEXT: vpsrld $1, %xmm2, %xmm2
328 ; XOP-FALLBACK-NEXT: vpmacsdd %xmm4, %xmm2, %xmm2, %xmm2
329 ; XOP-FALLBACK-NEXT: vpmacsdd %xmm0, %xmm1, %xmm1, %xmm0
330 ; XOP-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
331 ; XOP-FALLBACK-NEXT: retq
332 ;
333 ; XOPAVX1-LABEL: vec256_i32_signed_reg_mem:
334 ; XOPAVX1: # %bb.0:
335 ; XOPAVX1-NEXT: vmovdqa (%rdi), %xmm1
336 ; XOPAVX1-NEXT: vmovdqa 16(%rdi), %xmm2
337 ; XOPAVX1-NEXT: vpminsd %xmm1, %xmm0, %xmm3
338 ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
339 ; XOPAVX1-NEXT: vpminsd %xmm2, %xmm4, %xmm5
340 ; XOPAVX1-NEXT: vpmaxsd %xmm1, %xmm0, %xmm1
341 ; XOPAVX1-NEXT: vpsubd %xmm3, %xmm1, %xmm1
342 ; XOPAVX1-NEXT: vpmaxsd %xmm2, %xmm4, %xmm2
343 ; XOPAVX1-NEXT: vpsubd %xmm5, %xmm2, %xmm2
344 ; XOPAVX1-NEXT: vpsrld $1, %xmm1, %xmm1
345 ; XOPAVX1-NEXT: vpsrld $1, %xmm2, %xmm2
346 ; XOPAVX1-NEXT: vpmacsdd %xmm4, %xmm2, %xmm2, %xmm2
347 ; XOPAVX1-NEXT: vpmacsdd %xmm0, %xmm1, %xmm1, %xmm0
348 ; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
349 ; XOPAVX1-NEXT: retq
350 ;
351 ; AVX512-LABEL: vec256_i32_signed_reg_mem:
352 ; AVX512: # %bb.0:
353 ; AVX512-NEXT: vmovdqa (%rdi), %ymm1
354 ; AVX512-NEXT: vpminsd %ymm1, %ymm0, %ymm2
355 ; AVX512-NEXT: vpmaxsd %ymm1, %ymm0, %ymm1
356 ; AVX512-NEXT: vpsubd %ymm2, %ymm1, %ymm1
357 ; AVX512-NEXT: vpsrld $1, %ymm1, %ymm1
358 ; AVX512-NEXT: vpmulld %ymm1, %ymm1, %ymm1
359 ; AVX512-NEXT: vpaddd %ymm0, %ymm1, %ymm0
360 ; AVX512-NEXT: retq
361 %a2 = load <8 x i32>, ptr %a2_addr
362 %t3 = icmp sgt <8 x i32> %a1, %a2 ; signed
363 %t4 = select <8 x i1> %t3, <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
364 %t5 = select <8 x i1> %t3, <8 x i32> %a2, <8 x i32> %a1
365 %t6 = select <8 x i1> %t3, <8 x i32> %a1, <8 x i32> %a2
366 %t7 = sub <8 x i32> %t6, %t5
367 %t8 = lshr <8 x i32> %t7, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
368 %t9 = mul nsw <8 x i32> %t8, %t8 ; signed
369 %a10 = add nsw <8 x i32> %t9, %a1 ; signed
370 ret <8 x i32> %a10
371 }
373 define <8 x i32> @vec256_i32_signed_mem_mem(ptr %a1_addr, ptr %a2_addr) nounwind {
374 ; AVX1-FALLBACK-LABEL: vec256_i32_signed_mem_mem:
375 ; AVX1-FALLBACK: # %bb.0:
376 ; AVX1-FALLBACK-NEXT: vmovdqa (%rsi), %xmm0
377 ; AVX1-FALLBACK-NEXT: vmovdqa 16(%rsi), %xmm1
378 ; AVX1-FALLBACK-NEXT: vmovdqa (%rdi), %xmm2
379 ; AVX1-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm3
380 ; AVX1-FALLBACK-NEXT: vpminsd %xmm1, %xmm3, %xmm4
381 ; AVX1-FALLBACK-NEXT: vpminsd %xmm0, %xmm2, %xmm5
382 ; AVX1-FALLBACK-NEXT: vpmaxsd %xmm1, %xmm3, %xmm1
383 ; AVX1-FALLBACK-NEXT: vpsubd %xmm4, %xmm1, %xmm1
384 ; AVX1-FALLBACK-NEXT: vpmaxsd %xmm0, %xmm2, %xmm0
385 ; AVX1-FALLBACK-NEXT: vpsubd %xmm5, %xmm0, %xmm0
386 ; AVX1-FALLBACK-NEXT: vpsrld $1, %xmm1, %xmm1
387 ; AVX1-FALLBACK-NEXT: vpsrld $1, %xmm0, %xmm0
388 ; AVX1-FALLBACK-NEXT: vpmulld %xmm0, %xmm0, %xmm0
389 ; AVX1-FALLBACK-NEXT: vpmulld %xmm1, %xmm1, %xmm1
390 ; AVX1-FALLBACK-NEXT: vpaddd %xmm3, %xmm1, %xmm1
391 ; AVX1-FALLBACK-NEXT: vpaddd %xmm2, %xmm0, %xmm0
392 ; AVX1-FALLBACK-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
393 ; AVX1-FALLBACK-NEXT: retq
394 ;
395 ; AVX2-LABEL: vec256_i32_signed_mem_mem:
396 ; AVX2: # %bb.0:
397 ; AVX2-NEXT: vmovdqa (%rdi), %ymm0
398 ; AVX2-NEXT: vmovdqa (%rsi), %ymm1
399 ; AVX2-NEXT: vpminsd %ymm1, %ymm0, %ymm2
400 ; AVX2-NEXT: vpmaxsd %ymm1, %ymm0, %ymm1
401 ; AVX2-NEXT: vpsubd %ymm2, %ymm1, %ymm1
402 ; AVX2-NEXT: vpsrld $1, %ymm1, %ymm1
403 ; AVX2-NEXT: vpmulld %ymm1, %ymm1, %ymm1
404 ; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
405 ; AVX2-NEXT: retq
406 ;
407 ; XOP-FALLBACK-LABEL: vec256_i32_signed_mem_mem:
408 ; XOP-FALLBACK: # %bb.0:
409 ; XOP-FALLBACK-NEXT: vmovdqa (%rsi), %xmm0
410 ; XOP-FALLBACK-NEXT: vmovdqa 16(%rsi), %xmm1
411 ; XOP-FALLBACK-NEXT: vmovdqa (%rdi), %xmm2
412 ; XOP-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm3
413 ; XOP-FALLBACK-NEXT: vpminsd %xmm0, %xmm2, %xmm4
414 ; XOP-FALLBACK-NEXT: vpminsd %xmm1, %xmm3, %xmm5
415 ; XOP-FALLBACK-NEXT: vpmaxsd %xmm0, %xmm2, %xmm0
416 ; XOP-FALLBACK-NEXT: vpsubd %xmm4, %xmm0, %xmm0
417 ; XOP-FALLBACK-NEXT: vpmaxsd %xmm1, %xmm3, %xmm1
418 ; XOP-FALLBACK-NEXT: vpsubd %xmm5, %xmm1, %xmm1
419 ; XOP-FALLBACK-NEXT: vpsrld $1, %xmm0, %xmm0
420 ; XOP-FALLBACK-NEXT: vpsrld $1, %xmm1, %xmm1
421 ; XOP-FALLBACK-NEXT: vpmacsdd %xmm3, %xmm1, %xmm1, %xmm1
422 ; XOP-FALLBACK-NEXT: vpmacsdd %xmm2, %xmm0, %xmm0, %xmm0
423 ; XOP-FALLBACK-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
424 ; XOP-FALLBACK-NEXT: retq
425 ;
426 ; XOPAVX1-LABEL: vec256_i32_signed_mem_mem:
427 ; XOPAVX1: # %bb.0:
428 ; XOPAVX1-NEXT: vmovdqa (%rsi), %xmm0
429 ; XOPAVX1-NEXT: vmovdqa 16(%rsi), %xmm1
430 ; XOPAVX1-NEXT: vmovdqa (%rdi), %xmm2
431 ; XOPAVX1-NEXT: vmovdqa 16(%rdi), %xmm3
432 ; XOPAVX1-NEXT: vpminsd %xmm0, %xmm2, %xmm4
433 ; XOPAVX1-NEXT: vpminsd %xmm1, %xmm3, %xmm5
434 ; XOPAVX1-NEXT: vpmaxsd %xmm0, %xmm2, %xmm0
435 ; XOPAVX1-NEXT: vpsubd %xmm4, %xmm0, %xmm0
436 ; XOPAVX1-NEXT: vpmaxsd %xmm1, %xmm3, %xmm1
437 ; XOPAVX1-NEXT: vpsubd %xmm5, %xmm1, %xmm1
438 ; XOPAVX1-NEXT: vpsrld $1, %xmm0, %xmm0
439 ; XOPAVX1-NEXT: vpsrld $1, %xmm1, %xmm1
440 ; XOPAVX1-NEXT: vpmacsdd %xmm3, %xmm1, %xmm1, %xmm1
441 ; XOPAVX1-NEXT: vpmacsdd %xmm2, %xmm0, %xmm0, %xmm0
442 ; XOPAVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
443 ; XOPAVX1-NEXT: retq
444 ;
445 ; AVX512-LABEL: vec256_i32_signed_mem_mem:
446 ; AVX512: # %bb.0:
447 ; AVX512-NEXT: vmovdqa (%rdi), %ymm0
448 ; AVX512-NEXT: vmovdqa (%rsi), %ymm1
449 ; AVX512-NEXT: vpminsd %ymm1, %ymm0, %ymm2
450 ; AVX512-NEXT: vpmaxsd %ymm1, %ymm0, %ymm1
451 ; AVX512-NEXT: vpsubd %ymm2, %ymm1, %ymm1
452 ; AVX512-NEXT: vpsrld $1, %ymm1, %ymm1
453 ; AVX512-NEXT: vpmulld %ymm1, %ymm1, %ymm1
454 ; AVX512-NEXT: vpaddd %ymm0, %ymm1, %ymm0
455 ; AVX512-NEXT: retq
456 %a1 = load <8 x i32>, ptr %a1_addr
457 %a2 = load <8 x i32>, ptr %a2_addr
458 %t3 = icmp sgt <8 x i32> %a1, %a2 ; signed
459 %t4 = select <8 x i1> %t3, <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
460 %t5 = select <8 x i1> %t3, <8 x i32> %a2, <8 x i32> %a1
461 %t6 = select <8 x i1> %t3, <8 x i32> %a1, <8 x i32> %a2
462 %t7 = sub <8 x i32> %t6, %t5
463 %t8 = lshr <8 x i32> %t7, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
464 %t9 = mul nsw <8 x i32> %t8, %t8 ; signed
465 %a10 = add nsw <8 x i32> %t9, %a1 ; signed
466 ret <8 x i32> %a10
467 }
469 ; ---------------------------------------------------------------------------- ;
470 ; 64-bit width. 256 / 64 = 4 elts.
471 ; ---------------------------------------------------------------------------- ;
473 ; Values come from regs
475 define <4 x i64> @vec256_i64_signed_reg_reg(<4 x i64> %a1, <4 x i64> %a2) nounwind {
476 ; AVX1-FALLBACK-LABEL: vec256_i64_signed_reg_reg:
477 ; AVX1-FALLBACK: # %bb.0:
478 ; AVX1-FALLBACK-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm2
479 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm1, %xmm3
480 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm4
481 ; AVX1-FALLBACK-NEXT: vpcmpgtq %xmm3, %xmm4, %xmm5
482 ; AVX1-FALLBACK-NEXT: vblendvpd %xmm5, %xmm3, %xmm4, %xmm6
483 ; AVX1-FALLBACK-NEXT: vblendvpd %xmm2, %xmm1, %xmm0, %xmm7
484 ; AVX1-FALLBACK-NEXT: vblendvpd %xmm5, %xmm4, %xmm3, %xmm3
485 ; AVX1-FALLBACK-NEXT: vpsubq %xmm6, %xmm3, %xmm3
486 ; AVX1-FALLBACK-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm1
487 ; AVX1-FALLBACK-NEXT: vpsubq %xmm7, %xmm1, %xmm1
488 ; AVX1-FALLBACK-NEXT: vpsrlq $1, %xmm3, %xmm9
489 ; AVX1-FALLBACK-NEXT: vpsrlq $1, %xmm1, %xmm7
490 ; AVX1-FALLBACK-NEXT: vpsrlq $33, %xmm1, %xmm1
491 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm8 = [1,1]
492 ; AVX1-FALLBACK-NEXT: vpor %xmm2, %xmm8, %xmm2
493 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm2, %xmm1, %xmm1
494 ; AVX1-FALLBACK-NEXT: vpsrlq $32, %xmm2, %xmm6
495 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm6, %xmm7, %xmm6
496 ; AVX1-FALLBACK-NEXT: vpaddq %xmm1, %xmm6, %xmm1
497 ; AVX1-FALLBACK-NEXT: vpsllq $32, %xmm1, %xmm1
498 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm2, %xmm7, %xmm2
499 ; AVX1-FALLBACK-NEXT: vpsrlq $33, %xmm3, %xmm3
500 ; AVX1-FALLBACK-NEXT: vpor %xmm5, %xmm8, %xmm5
501 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm5, %xmm3, %xmm3
502 ; AVX1-FALLBACK-NEXT: vpsrlq $32, %xmm5, %xmm6
503 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm6, %xmm9, %xmm6
504 ; AVX1-FALLBACK-NEXT: vpaddq %xmm3, %xmm6, %xmm3
505 ; AVX1-FALLBACK-NEXT: vpsllq $32, %xmm3, %xmm3
506 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm5, %xmm9, %xmm5
507 ; AVX1-FALLBACK-NEXT: vpaddq %xmm4, %xmm3, %xmm3
508 ; AVX1-FALLBACK-NEXT: vpaddq %xmm3, %xmm5, %xmm3
509 ; AVX1-FALLBACK-NEXT: vpaddq %xmm0, %xmm1, %xmm0
510 ; AVX1-FALLBACK-NEXT: vpaddq %xmm0, %xmm2, %xmm0
511 ; AVX1-FALLBACK-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0
512 ; AVX1-FALLBACK-NEXT: retq
513 ;
514 ; AVX2-LABEL: vec256_i64_signed_reg_reg:
515 ; AVX2: # %bb.0:
516 ; AVX2-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm2
517 ; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
518 ; AVX2-NEXT: vpor %ymm3, %ymm2, %ymm3
519 ; AVX2-NEXT: vblendvpd %ymm2, %ymm1, %ymm0, %ymm4
520 ; AVX2-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm1
521 ; AVX2-NEXT: vpsubq %ymm4, %ymm1, %ymm1
522 ; AVX2-NEXT: vpsrlq $1, %ymm1, %ymm2
523 ; AVX2-NEXT: vpsrlq $33, %ymm1, %ymm1
524 ; AVX2-NEXT: vpmuludq %ymm3, %ymm1, %ymm1
525 ; AVX2-NEXT: vpsrlq $32, %ymm3, %ymm4
526 ; AVX2-NEXT: vpmuludq %ymm4, %ymm2, %ymm4
527 ; AVX2-NEXT: vpaddq %ymm1, %ymm4, %ymm1
528 ; AVX2-NEXT: vpsllq $32, %ymm1, %ymm1
529 ; AVX2-NEXT: vpmuludq %ymm3, %ymm2, %ymm2
530 ; AVX2-NEXT: vpaddq %ymm0, %ymm1, %ymm0
531 ; AVX2-NEXT: vpaddq %ymm0, %ymm2, %ymm0
532 ; AVX2-NEXT: retq
533 ;
534 ; XOP-FALLBACK-LABEL: vec256_i64_signed_reg_reg:
535 ; XOP-FALLBACK: # %bb.0:
536 ; XOP-FALLBACK-NEXT: vpcomgtq %xmm1, %xmm0, %xmm2
537 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm1, %xmm3
538 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm4
539 ; XOP-FALLBACK-NEXT: vpcomgtq %xmm3, %xmm4, %xmm9
540 ; XOP-FALLBACK-NEXT: vpcomltq %xmm3, %xmm4, %xmm6
541 ; XOP-FALLBACK-NEXT: vblendvpd %xmm6, %xmm4, %xmm3, %xmm6
542 ; XOP-FALLBACK-NEXT: vpcomltq %xmm1, %xmm0, %xmm7
543 ; XOP-FALLBACK-NEXT: vblendvpd %xmm7, %xmm0, %xmm1, %xmm7
544 ; XOP-FALLBACK-NEXT: vblendvpd %xmm9, %xmm4, %xmm3, %xmm3
545 ; XOP-FALLBACK-NEXT: vpsubq %xmm6, %xmm3, %xmm3
546 ; XOP-FALLBACK-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm1
547 ; XOP-FALLBACK-NEXT: vpsubq %xmm7, %xmm1, %xmm1
548 ; XOP-FALLBACK-NEXT: vpsrlq $1, %xmm3, %xmm6
549 ; XOP-FALLBACK-NEXT: vpsrlq $1, %xmm1, %xmm7
550 ; XOP-FALLBACK-NEXT: vpsrlq $33, %xmm1, %xmm1
551 ; XOP-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm8 = [1,1]
552 ; XOP-FALLBACK-NEXT: vpor %xmm2, %xmm8, %xmm2
553 ; XOP-FALLBACK-NEXT: vpmuludq %xmm2, %xmm1, %xmm1
554 ; XOP-FALLBACK-NEXT: vpsrlq $32, %xmm2, %xmm5
555 ; XOP-FALLBACK-NEXT: vpmuludq %xmm5, %xmm7, %xmm5
556 ; XOP-FALLBACK-NEXT: vpaddq %xmm1, %xmm5, %xmm1
557 ; XOP-FALLBACK-NEXT: vpsllq $32, %xmm1, %xmm1
558 ; XOP-FALLBACK-NEXT: vpmuludq %xmm2, %xmm7, %xmm2
559 ; XOP-FALLBACK-NEXT: vpsrlq $33, %xmm3, %xmm3
560 ; XOP-FALLBACK-NEXT: vpor %xmm8, %xmm9, %xmm5
561 ; XOP-FALLBACK-NEXT: vpmuludq %xmm5, %xmm3, %xmm3
562 ; XOP-FALLBACK-NEXT: vpsrlq $32, %xmm5, %xmm7
563 ; XOP-FALLBACK-NEXT: vpmuludq %xmm7, %xmm6, %xmm7
564 ; XOP-FALLBACK-NEXT: vpaddq %xmm3, %xmm7, %xmm3
565 ; XOP-FALLBACK-NEXT: vpsllq $32, %xmm3, %xmm3
566 ; XOP-FALLBACK-NEXT: vpmuludq %xmm5, %xmm6, %xmm5
567 ; XOP-FALLBACK-NEXT: vpaddq %xmm4, %xmm3, %xmm3
568 ; XOP-FALLBACK-NEXT: vpaddq %xmm3, %xmm5, %xmm3
569 ; XOP-FALLBACK-NEXT: vpaddq %xmm0, %xmm1, %xmm0
570 ; XOP-FALLBACK-NEXT: vpaddq %xmm0, %xmm2, %xmm0
571 ; XOP-FALLBACK-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0
572 ; XOP-FALLBACK-NEXT: retq
573 ;
574 ; XOPAVX1-LABEL: vec256_i64_signed_reg_reg:
575 ; XOPAVX1: # %bb.0:
576 ; XOPAVX1-NEXT: vpcomgtq %xmm1, %xmm0, %xmm2
577 ; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
578 ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
579 ; XOPAVX1-NEXT: vpcomgtq %xmm3, %xmm4, %xmm9
580 ; XOPAVX1-NEXT: vpcomltq %xmm3, %xmm4, %xmm6
581 ; XOPAVX1-NEXT: vblendvpd %xmm6, %xmm4, %xmm3, %xmm6
582 ; XOPAVX1-NEXT: vpcomltq %xmm1, %xmm0, %xmm7
583 ; XOPAVX1-NEXT: vblendvpd %xmm7, %xmm0, %xmm1, %xmm7
584 ; XOPAVX1-NEXT: vblendvpd %xmm9, %xmm4, %xmm3, %xmm3
585 ; XOPAVX1-NEXT: vpsubq %xmm6, %xmm3, %xmm3
586 ; XOPAVX1-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm1
587 ; XOPAVX1-NEXT: vpsubq %xmm7, %xmm1, %xmm1
588 ; XOPAVX1-NEXT: vpsrlq $1, %xmm3, %xmm6
589 ; XOPAVX1-NEXT: vpsrlq $1, %xmm1, %xmm7
590 ; XOPAVX1-NEXT: vpsrlq $33, %xmm1, %xmm1
591 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm8 = [1,1]
592 ; XOPAVX1-NEXT: vpor %xmm2, %xmm8, %xmm2
593 ; XOPAVX1-NEXT: vpmuludq %xmm2, %xmm1, %xmm1
594 ; XOPAVX1-NEXT: vpsrlq $32, %xmm2, %xmm5
595 ; XOPAVX1-NEXT: vpmuludq %xmm5, %xmm7, %xmm5
596 ; XOPAVX1-NEXT: vpaddq %xmm1, %xmm5, %xmm1
597 ; XOPAVX1-NEXT: vpsllq $32, %xmm1, %xmm1
598 ; XOPAVX1-NEXT: vpmuludq %xmm2, %xmm7, %xmm2
599 ; XOPAVX1-NEXT: vpsrlq $33, %xmm3, %xmm3
600 ; XOPAVX1-NEXT: vpor %xmm8, %xmm9, %xmm5
601 ; XOPAVX1-NEXT: vpmuludq %xmm5, %xmm3, %xmm3
602 ; XOPAVX1-NEXT: vpsrlq $32, %xmm5, %xmm7
603 ; XOPAVX1-NEXT: vpmuludq %xmm7, %xmm6, %xmm7
604 ; XOPAVX1-NEXT: vpaddq %xmm3, %xmm7, %xmm3
605 ; XOPAVX1-NEXT: vpsllq $32, %xmm3, %xmm3
606 ; XOPAVX1-NEXT: vpmuludq %xmm5, %xmm6, %xmm5
607 ; XOPAVX1-NEXT: vpaddq %xmm4, %xmm3, %xmm3
608 ; XOPAVX1-NEXT: vpaddq %xmm3, %xmm5, %xmm3
609 ; XOPAVX1-NEXT: vpaddq %xmm0, %xmm1, %xmm0
610 ; XOPAVX1-NEXT: vpaddq %xmm0, %xmm2, %xmm0
611 ; XOPAVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0
612 ; XOPAVX1-NEXT: retq
613 ;
614 ; AVX512F-LABEL: vec256_i64_signed_reg_reg:
615 ; AVX512F: # %bb.0:
616 ; AVX512F-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
617 ; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
618 ; AVX512F-NEXT: vpcmpgtq %zmm1, %zmm0, %k1
619 ; AVX512F-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
620 ; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
621 ; AVX512F-NEXT: vmovdqa64 %zmm2, %zmm3 {%k1}
622 ; AVX512F-NEXT: vpminsq %zmm1, %zmm0, %zmm2
623 ; AVX512F-NEXT: vpmaxsq %zmm1, %zmm0, %zmm1
624 ; AVX512F-NEXT: vpsubq %ymm2, %ymm1, %ymm1
625 ; AVX512F-NEXT: vpsrlq $1, %ymm1, %ymm2
626 ; AVX512F-NEXT: vpsrlq $33, %ymm1, %ymm1
627 ; AVX512F-NEXT: vpmuludq %ymm3, %ymm1, %ymm1
628 ; AVX512F-NEXT: vpsrlq $32, %ymm3, %ymm4
629 ; AVX512F-NEXT: vpmuludq %ymm4, %ymm2, %ymm4
630 ; AVX512F-NEXT: vpaddq %ymm1, %ymm4, %ymm1
631 ; AVX512F-NEXT: vpsllq $32, %ymm1, %ymm1
632 ; AVX512F-NEXT: vpmuludq %ymm3, %ymm2, %ymm2
633 ; AVX512F-NEXT: vpaddq %ymm0, %ymm1, %ymm0
634 ; AVX512F-NEXT: vpaddq %ymm0, %ymm2, %ymm0
635 ; AVX512F-NEXT: retq
636 ;
637 ; AVX512VL-LABEL: vec256_i64_signed_reg_reg:
638 ; AVX512VL: # %bb.0:
639 ; AVX512VL-NEXT: vpcmpgtq %ymm1, %ymm0, %k1
640 ; AVX512VL-NEXT: vpminsq %ymm1, %ymm0, %ymm2
641 ; AVX512VL-NEXT: vpmaxsq %ymm1, %ymm0, %ymm1
642 ; AVX512VL-NEXT: vpsubq %ymm2, %ymm1, %ymm1
643 ; AVX512VL-NEXT: vpsrlq $1, %ymm1, %ymm1
644 ; AVX512VL-NEXT: vpxor %xmm2, %xmm2, %xmm2
645 ; AVX512VL-NEXT: vpsubq %ymm1, %ymm2, %ymm1 {%k1}
646 ; AVX512VL-NEXT: vpaddq %ymm0, %ymm1, %ymm0
647 ; AVX512VL-NEXT: retq
648 ;
649 ; AVX512BW-FALLBACK-LABEL: vec256_i64_signed_reg_reg:
650 ; AVX512BW-FALLBACK: # %bb.0:
651 ; AVX512BW-FALLBACK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
652 ; AVX512BW-FALLBACK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
653 ; AVX512BW-FALLBACK-NEXT: vpcmpgtq %zmm1, %zmm0, %k1
654 ; AVX512BW-FALLBACK-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
655 ; AVX512BW-FALLBACK-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
656 ; AVX512BW-FALLBACK-NEXT: vmovdqa64 %zmm2, %zmm3 {%k1}
657 ; AVX512BW-FALLBACK-NEXT: vpminsq %zmm1, %zmm0, %zmm2
658 ; AVX512BW-FALLBACK-NEXT: vpmaxsq %zmm1, %zmm0, %zmm1
659 ; AVX512BW-FALLBACK-NEXT: vpsubq %ymm2, %ymm1, %ymm1
660 ; AVX512BW-FALLBACK-NEXT: vpsrlq $1, %ymm1, %ymm2
661 ; AVX512BW-FALLBACK-NEXT: vpsrlq $33, %ymm1, %ymm1
662 ; AVX512BW-FALLBACK-NEXT: vpmuludq %ymm3, %ymm1, %ymm1
663 ; AVX512BW-FALLBACK-NEXT: vpsrlq $32, %ymm3, %ymm4
664 ; AVX512BW-FALLBACK-NEXT: vpmuludq %ymm4, %ymm2, %ymm4
665 ; AVX512BW-FALLBACK-NEXT: vpaddq %ymm1, %ymm4, %ymm1
666 ; AVX512BW-FALLBACK-NEXT: vpsllq $32, %ymm1, %ymm1
667 ; AVX512BW-FALLBACK-NEXT: vpmuludq %ymm3, %ymm2, %ymm2
668 ; AVX512BW-FALLBACK-NEXT: vpaddq %ymm0, %ymm1, %ymm0
669 ; AVX512BW-FALLBACK-NEXT: vpaddq %ymm0, %ymm2, %ymm0
670 ; AVX512BW-FALLBACK-NEXT: retq
671 %t3 = icmp sgt <4 x i64> %a1, %a2 ; signed
672 %t4 = select <4 x i1> %t3, <4 x i64> <i64 -1, i64 -1, i64 -1, i64 -1>, <4 x i64> <i64 1, i64 1, i64 1, i64 1>
673 %t5 = select <4 x i1> %t3, <4 x i64> %a2, <4 x i64> %a1
674 %t6 = select <4 x i1> %t3, <4 x i64> %a1, <4 x i64> %a2
675 %t7 = sub <4 x i64> %t6, %t5
676 %t8 = lshr <4 x i64> %t7, <i64 1, i64 1, i64 1, i64 1>
677 %t9 = mul nsw <4 x i64> %t8, %t4 ; signed
678 %a10 = add nsw <4 x i64> %t9, %a1 ; signed
679 ret <4 x i64> %a10
680 }
682 define <4 x i64> @vec256_i64_unsigned_reg_reg(<4 x i64> %a1, <4 x i64> %a2) nounwind {
683 ; AVX1-FALLBACK-LABEL: vec256_i64_unsigned_reg_reg:
684 ; AVX1-FALLBACK: # %bb.0:
685 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
686 ; AVX1-FALLBACK-NEXT: vpxor %xmm2, %xmm1, %xmm3
687 ; AVX1-FALLBACK-NEXT: vpxor %xmm2, %xmm0, %xmm4
688 ; AVX1-FALLBACK-NEXT: vpcmpgtq %xmm3, %xmm4, %xmm8
689 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm1, %xmm6
690 ; AVX1-FALLBACK-NEXT: vpxor %xmm2, %xmm6, %xmm7
691 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm5
692 ; AVX1-FALLBACK-NEXT: vpxor %xmm2, %xmm5, %xmm2
693 ; AVX1-FALLBACK-NEXT: vpcmpgtq %xmm7, %xmm2, %xmm9
694 ; AVX1-FALLBACK-NEXT: vpcmpgtq %xmm2, %xmm7, %xmm2
695 ; AVX1-FALLBACK-NEXT: vblendvpd %xmm2, %xmm5, %xmm6, %xmm2
696 ; AVX1-FALLBACK-NEXT: vpcmpgtq %xmm4, %xmm3, %xmm3
697 ; AVX1-FALLBACK-NEXT: vblendvpd %xmm3, %xmm0, %xmm1, %xmm3
698 ; AVX1-FALLBACK-NEXT: vblendvpd %xmm9, %xmm5, %xmm6, %xmm4
699 ; AVX1-FALLBACK-NEXT: vpsubq %xmm2, %xmm4, %xmm2
700 ; AVX1-FALLBACK-NEXT: vblendvpd %xmm8, %xmm0, %xmm1, %xmm1
701 ; AVX1-FALLBACK-NEXT: vpsubq %xmm3, %xmm1, %xmm1
702 ; AVX1-FALLBACK-NEXT: vpsrlq $1, %xmm2, %xmm3
703 ; AVX1-FALLBACK-NEXT: vpsrlq $1, %xmm1, %xmm4
704 ; AVX1-FALLBACK-NEXT: vpsrlq $33, %xmm1, %xmm1
705 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm10 = [1,1]
706 ; AVX1-FALLBACK-NEXT: vpor %xmm10, %xmm8, %xmm7
707 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm7, %xmm1, %xmm1
708 ; AVX1-FALLBACK-NEXT: vpsrlq $32, %xmm7, %xmm6
709 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm6, %xmm4, %xmm6
710 ; AVX1-FALLBACK-NEXT: vpaddq %xmm1, %xmm6, %xmm1
711 ; AVX1-FALLBACK-NEXT: vpsllq $32, %xmm1, %xmm1
712 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm7, %xmm4, %xmm4
713 ; AVX1-FALLBACK-NEXT: vpsrlq $33, %xmm2, %xmm2
714 ; AVX1-FALLBACK-NEXT: vpor %xmm10, %xmm9, %xmm6
715 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm6, %xmm2, %xmm2
716 ; AVX1-FALLBACK-NEXT: vpsrlq $32, %xmm6, %xmm7
717 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm7, %xmm3, %xmm7
718 ; AVX1-FALLBACK-NEXT: vpaddq %xmm2, %xmm7, %xmm2
719 ; AVX1-FALLBACK-NEXT: vpsllq $32, %xmm2, %xmm2
720 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm6, %xmm3, %xmm3
721 ; AVX1-FALLBACK-NEXT: vpaddq %xmm5, %xmm2, %xmm2
722 ; AVX1-FALLBACK-NEXT: vpaddq %xmm2, %xmm3, %xmm2
723 ; AVX1-FALLBACK-NEXT: vpaddq %xmm0, %xmm1, %xmm0
724 ; AVX1-FALLBACK-NEXT: vpaddq %xmm0, %xmm4, %xmm0
725 ; AVX1-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
726 ; AVX1-FALLBACK-NEXT: retq
727 ;
728 ; AVX2-LABEL: vec256_i64_unsigned_reg_reg:
729 ; AVX2: # %bb.0:
730 ; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm2 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
731 ; AVX2-NEXT: vpxor %ymm2, %ymm1, %ymm3
732 ; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm2
733 ; AVX2-NEXT: vpcmpgtq %ymm3, %ymm2, %ymm4
734 ; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm5 = [1,1,1,1]
735 ; AVX2-NEXT: vpor %ymm5, %ymm4, %ymm5
736 ; AVX2-NEXT: vpcmpgtq %ymm2, %ymm3, %ymm2
737 ; AVX2-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm2
738 ; AVX2-NEXT: vblendvpd %ymm4, %ymm0, %ymm1, %ymm1
739 ; AVX2-NEXT: vpsubq %ymm2, %ymm1, %ymm1
740 ; AVX2-NEXT: vpsrlq $1, %ymm1, %ymm2
741 ; AVX2-NEXT: vpsrlq $33, %ymm1, %ymm1
742 ; AVX2-NEXT: vpmuludq %ymm5, %ymm1, %ymm1
743 ; AVX2-NEXT: vpsrlq $32, %ymm5, %ymm3
744 ; AVX2-NEXT: vpmuludq %ymm3, %ymm2, %ymm3
745 ; AVX2-NEXT: vpaddq %ymm1, %ymm3, %ymm1
746 ; AVX2-NEXT: vpsllq $32, %ymm1, %ymm1
747 ; AVX2-NEXT: vpmuludq %ymm5, %ymm2, %ymm2
748 ; AVX2-NEXT: vpaddq %ymm0, %ymm1, %ymm0
749 ; AVX2-NEXT: vpaddq %ymm0, %ymm2, %ymm0
750 ; AVX2-NEXT: retq
751 ;
752 ; XOP-FALLBACK-LABEL: vec256_i64_unsigned_reg_reg:
753 ; XOP-FALLBACK: # %bb.0:
754 ; XOP-FALLBACK-NEXT: vpcomgtuq %xmm1, %xmm0, %xmm2
755 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm1, %xmm3
756 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm4
757 ; XOP-FALLBACK-NEXT: vpcomgtuq %xmm3, %xmm4, %xmm9
758 ; XOP-FALLBACK-NEXT: vpcomltuq %xmm3, %xmm4, %xmm6
759 ; XOP-FALLBACK-NEXT: vblendvpd %xmm6, %xmm4, %xmm3, %xmm6
760 ; XOP-FALLBACK-NEXT: vpcomltuq %xmm1, %xmm0, %xmm7
761 ; XOP-FALLBACK-NEXT: vblendvpd %xmm7, %xmm0, %xmm1, %xmm7
762 ; XOP-FALLBACK-NEXT: vblendvpd %xmm9, %xmm4, %xmm3, %xmm3
763 ; XOP-FALLBACK-NEXT: vpsubq %xmm6, %xmm3, %xmm3
764 ; XOP-FALLBACK-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm1
765 ; XOP-FALLBACK-NEXT: vpsubq %xmm7, %xmm1, %xmm1
766 ; XOP-FALLBACK-NEXT: vpsrlq $1, %xmm3, %xmm6
767 ; XOP-FALLBACK-NEXT: vpsrlq $1, %xmm1, %xmm7
768 ; XOP-FALLBACK-NEXT: vpsrlq $33, %xmm1, %xmm1
769 ; XOP-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm8 = [1,1]
770 ; XOP-FALLBACK-NEXT: vpor %xmm2, %xmm8, %xmm2
771 ; XOP-FALLBACK-NEXT: vpmuludq %xmm2, %xmm1, %xmm1
772 ; XOP-FALLBACK-NEXT: vpsrlq $32, %xmm2, %xmm5
773 ; XOP-FALLBACK-NEXT: vpmuludq %xmm5, %xmm7, %xmm5
774 ; XOP-FALLBACK-NEXT: vpaddq %xmm1, %xmm5, %xmm1
775 ; XOP-FALLBACK-NEXT: vpsllq $32, %xmm1, %xmm1
776 ; XOP-FALLBACK-NEXT: vpmuludq %xmm2, %xmm7, %xmm2
777 ; XOP-FALLBACK-NEXT: vpsrlq $33, %xmm3, %xmm3
778 ; XOP-FALLBACK-NEXT: vpor %xmm8, %xmm9, %xmm5
779 ; XOP-FALLBACK-NEXT: vpmuludq %xmm5, %xmm3, %xmm3
780 ; XOP-FALLBACK-NEXT: vpsrlq $32, %xmm5, %xmm7
781 ; XOP-FALLBACK-NEXT: vpmuludq %xmm7, %xmm6, %xmm7
782 ; XOP-FALLBACK-NEXT: vpaddq %xmm3, %xmm7, %xmm3
783 ; XOP-FALLBACK-NEXT: vpsllq $32, %xmm3, %xmm3
784 ; XOP-FALLBACK-NEXT: vpmuludq %xmm5, %xmm6, %xmm5
785 ; XOP-FALLBACK-NEXT: vpaddq %xmm4, %xmm3, %xmm3
786 ; XOP-FALLBACK-NEXT: vpaddq %xmm3, %xmm5, %xmm3
787 ; XOP-FALLBACK-NEXT: vpaddq %xmm0, %xmm1, %xmm0
788 ; XOP-FALLBACK-NEXT: vpaddq %xmm0, %xmm2, %xmm0
789 ; XOP-FALLBACK-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0
790 ; XOP-FALLBACK-NEXT: retq
791 ;
792 ; XOPAVX1-LABEL: vec256_i64_unsigned_reg_reg:
793 ; XOPAVX1: # %bb.0:
794 ; XOPAVX1-NEXT: vpcomgtuq %xmm1, %xmm0, %xmm2
795 ; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
796 ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
797 ; XOPAVX1-NEXT: vpcomgtuq %xmm3, %xmm4, %xmm9
798 ; XOPAVX1-NEXT: vpcomltuq %xmm3, %xmm4, %xmm6
799 ; XOPAVX1-NEXT: vblendvpd %xmm6, %xmm4, %xmm3, %xmm6
800 ; XOPAVX1-NEXT: vpcomltuq %xmm1, %xmm0, %xmm7
801 ; XOPAVX1-NEXT: vblendvpd %xmm7, %xmm0, %xmm1, %xmm7
802 ; XOPAVX1-NEXT: vblendvpd %xmm9, %xmm4, %xmm3, %xmm3
803 ; XOPAVX1-NEXT: vpsubq %xmm6, %xmm3, %xmm3
804 ; XOPAVX1-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm1
805 ; XOPAVX1-NEXT: vpsubq %xmm7, %xmm1, %xmm1
806 ; XOPAVX1-NEXT: vpsrlq $1, %xmm3, %xmm6
807 ; XOPAVX1-NEXT: vpsrlq $1, %xmm1, %xmm7
808 ; XOPAVX1-NEXT: vpsrlq $33, %xmm1, %xmm1
809 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm8 = [1,1]
810 ; XOPAVX1-NEXT: vpor %xmm2, %xmm8, %xmm2
811 ; XOPAVX1-NEXT: vpmuludq %xmm2, %xmm1, %xmm1
812 ; XOPAVX1-NEXT: vpsrlq $32, %xmm2, %xmm5
813 ; XOPAVX1-NEXT: vpmuludq %xmm5, %xmm7, %xmm5
814 ; XOPAVX1-NEXT: vpaddq %xmm1, %xmm5, %xmm1
815 ; XOPAVX1-NEXT: vpsllq $32, %xmm1, %xmm1
816 ; XOPAVX1-NEXT: vpmuludq %xmm2, %xmm7, %xmm2
817 ; XOPAVX1-NEXT: vpsrlq $33, %xmm3, %xmm3
818 ; XOPAVX1-NEXT: vpor %xmm8, %xmm9, %xmm5
819 ; XOPAVX1-NEXT: vpmuludq %xmm5, %xmm3, %xmm3
820 ; XOPAVX1-NEXT: vpsrlq $32, %xmm5, %xmm7
821 ; XOPAVX1-NEXT: vpmuludq %xmm7, %xmm6, %xmm7
822 ; XOPAVX1-NEXT: vpaddq %xmm3, %xmm7, %xmm3
823 ; XOPAVX1-NEXT: vpsllq $32, %xmm3, %xmm3
824 ; XOPAVX1-NEXT: vpmuludq %xmm5, %xmm6, %xmm5
825 ; XOPAVX1-NEXT: vpaddq %xmm4, %xmm3, %xmm3
826 ; XOPAVX1-NEXT: vpaddq %xmm3, %xmm5, %xmm3
827 ; XOPAVX1-NEXT: vpaddq %xmm0, %xmm1, %xmm0
828 ; XOPAVX1-NEXT: vpaddq %xmm0, %xmm2, %xmm0
829 ; XOPAVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0
830 ; XOPAVX1-NEXT: retq
831 ;
832 ; AVX512F-LABEL: vec256_i64_unsigned_reg_reg:
833 ; AVX512F: # %bb.0:
834 ; AVX512F-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
835 ; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
836 ; AVX512F-NEXT: vpcmpnleuq %zmm1, %zmm0, %k1
837 ; AVX512F-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
838 ; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
839 ; AVX512F-NEXT: vmovdqa64 %zmm2, %zmm3 {%k1}
840 ; AVX512F-NEXT: vpminuq %zmm1, %zmm0, %zmm2
841 ; AVX512F-NEXT: vpmaxuq %zmm1, %zmm0, %zmm1
842 ; AVX512F-NEXT: vpsubq %ymm2, %ymm1, %ymm1
843 ; AVX512F-NEXT: vpsrlq $1, %ymm1, %ymm2
844 ; AVX512F-NEXT: vpsrlq $33, %ymm1, %ymm1
845 ; AVX512F-NEXT: vpmuludq %ymm3, %ymm1, %ymm1
846 ; AVX512F-NEXT: vpsrlq $32, %ymm3, %ymm4
847 ; AVX512F-NEXT: vpmuludq %ymm4, %ymm2, %ymm4
848 ; AVX512F-NEXT: vpaddq %ymm1, %ymm4, %ymm1
849 ; AVX512F-NEXT: vpsllq $32, %ymm1, %ymm1
850 ; AVX512F-NEXT: vpmuludq %ymm3, %ymm2, %ymm2
851 ; AVX512F-NEXT: vpaddq %ymm0, %ymm1, %ymm0
852 ; AVX512F-NEXT: vpaddq %ymm0, %ymm2, %ymm0
853 ; AVX512F-NEXT: retq
854 ;
855 ; AVX512VL-LABEL: vec256_i64_unsigned_reg_reg:
856 ; AVX512VL: # %bb.0:
857 ; AVX512VL-NEXT: vpcmpnleuq %ymm1, %ymm0, %k1
858 ; AVX512VL-NEXT: vpminuq %ymm1, %ymm0, %ymm2
859 ; AVX512VL-NEXT: vpmaxuq %ymm1, %ymm0, %ymm1
860 ; AVX512VL-NEXT: vpsubq %ymm2, %ymm1, %ymm1
861 ; AVX512VL-NEXT: vpsrlq $1, %ymm1, %ymm1
862 ; AVX512VL-NEXT: vpxor %xmm2, %xmm2, %xmm2
863 ; AVX512VL-NEXT: vpsubq %ymm1, %ymm2, %ymm1 {%k1}
864 ; AVX512VL-NEXT: vpaddq %ymm0, %ymm1, %ymm0
865 ; AVX512VL-NEXT: retq
866 ;
867 ; AVX512BW-FALLBACK-LABEL: vec256_i64_unsigned_reg_reg:
868 ; AVX512BW-FALLBACK: # %bb.0:
869 ; AVX512BW-FALLBACK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
870 ; AVX512BW-FALLBACK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
871 ; AVX512BW-FALLBACK-NEXT: vpcmpnleuq %zmm1, %zmm0, %k1
872 ; AVX512BW-FALLBACK-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
873 ; AVX512BW-FALLBACK-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
874 ; AVX512BW-FALLBACK-NEXT: vmovdqa64 %zmm2, %zmm3 {%k1}
875 ; AVX512BW-FALLBACK-NEXT: vpminuq %zmm1, %zmm0, %zmm2
876 ; AVX512BW-FALLBACK-NEXT: vpmaxuq %zmm1, %zmm0, %zmm1
877 ; AVX512BW-FALLBACK-NEXT: vpsubq %ymm2, %ymm1, %ymm1
878 ; AVX512BW-FALLBACK-NEXT: vpsrlq $1, %ymm1, %ymm2
879 ; AVX512BW-FALLBACK-NEXT: vpsrlq $33, %ymm1, %ymm1
880 ; AVX512BW-FALLBACK-NEXT: vpmuludq %ymm3, %ymm1, %ymm1
881 ; AVX512BW-FALLBACK-NEXT: vpsrlq $32, %ymm3, %ymm4
882 ; AVX512BW-FALLBACK-NEXT: vpmuludq %ymm4, %ymm2, %ymm4
883 ; AVX512BW-FALLBACK-NEXT: vpaddq %ymm1, %ymm4, %ymm1
884 ; AVX512BW-FALLBACK-NEXT: vpsllq $32, %ymm1, %ymm1
885 ; AVX512BW-FALLBACK-NEXT: vpmuludq %ymm3, %ymm2, %ymm2
886 ; AVX512BW-FALLBACK-NEXT: vpaddq %ymm0, %ymm1, %ymm0
887 ; AVX512BW-FALLBACK-NEXT: vpaddq %ymm0, %ymm2, %ymm0
888 ; AVX512BW-FALLBACK-NEXT: retq
889 %t3 = icmp ugt <4 x i64> %a1, %a2
890 %t4 = select <4 x i1> %t3, <4 x i64> <i64 -1, i64 -1, i64 -1, i64 -1>, <4 x i64> <i64 1, i64 1, i64 1, i64 1>
891 %t5 = select <4 x i1> %t3, <4 x i64> %a2, <4 x i64> %a1
892 %t6 = select <4 x i1> %t3, <4 x i64> %a1, <4 x i64> %a2
893 %t7 = sub <4 x i64> %t6, %t5
894 %t8 = lshr <4 x i64> %t7, <i64 1, i64 1, i64 1, i64 1>
895 %t9 = mul <4 x i64> %t8, %t4
896 %a10 = add <4 x i64> %t9, %a1
897 ret <4 x i64> %a10
898 }
900 ; Values are loaded. Only check signed case.
902 define <4 x i64> @vec256_i64_signed_mem_reg(ptr %a1_addr, <4 x i64> %a2) nounwind {
903 ; AVX1-FALLBACK-LABEL: vec256_i64_signed_mem_reg:
904 ; AVX1-FALLBACK: # %bb.0:
905 ; AVX1-FALLBACK-NEXT: vmovdqa (%rdi), %xmm1
906 ; AVX1-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm2
907 ; AVX1-FALLBACK-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm3
908 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm4
909 ; AVX1-FALLBACK-NEXT: vpcmpgtq %xmm4, %xmm2, %xmm5
910 ; AVX1-FALLBACK-NEXT: vblendvpd %xmm5, %xmm4, %xmm2, %xmm6
911 ; AVX1-FALLBACK-NEXT: vblendvpd %xmm3, %xmm0, %xmm1, %xmm7
912 ; AVX1-FALLBACK-NEXT: vblendvpd %xmm5, %xmm2, %xmm4, %xmm4
913 ; AVX1-FALLBACK-NEXT: vpsubq %xmm6, %xmm4, %xmm4
914 ; AVX1-FALLBACK-NEXT: vblendvpd %xmm3, %xmm1, %xmm0, %xmm0
915 ; AVX1-FALLBACK-NEXT: vpsubq %xmm7, %xmm0, %xmm0
916 ; AVX1-FALLBACK-NEXT: vpsrlq $1, %xmm4, %xmm9
917 ; AVX1-FALLBACK-NEXT: vpsrlq $1, %xmm0, %xmm7
918 ; AVX1-FALLBACK-NEXT: vpsrlq $33, %xmm0, %xmm0
919 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm8 = [1,1]
920 ; AVX1-FALLBACK-NEXT: vpor %xmm3, %xmm8, %xmm3
921 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm3, %xmm0, %xmm0
922 ; AVX1-FALLBACK-NEXT: vpsrlq $32, %xmm3, %xmm6
923 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm6, %xmm7, %xmm6
924 ; AVX1-FALLBACK-NEXT: vpaddq %xmm0, %xmm6, %xmm0
925 ; AVX1-FALLBACK-NEXT: vpsllq $32, %xmm0, %xmm0
926 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm3, %xmm7, %xmm3
927 ; AVX1-FALLBACK-NEXT: vpsrlq $33, %xmm4, %xmm4
928 ; AVX1-FALLBACK-NEXT: vpor %xmm5, %xmm8, %xmm5
929 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm5, %xmm4, %xmm4
930 ; AVX1-FALLBACK-NEXT: vpsrlq $32, %xmm5, %xmm6
931 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm6, %xmm9, %xmm6
932 ; AVX1-FALLBACK-NEXT: vpaddq %xmm4, %xmm6, %xmm4
933 ; AVX1-FALLBACK-NEXT: vpsllq $32, %xmm4, %xmm4
934 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm5, %xmm9, %xmm5
935 ; AVX1-FALLBACK-NEXT: vpaddq %xmm2, %xmm4, %xmm2
936 ; AVX1-FALLBACK-NEXT: vpaddq %xmm2, %xmm5, %xmm2
937 ; AVX1-FALLBACK-NEXT: vpaddq %xmm1, %xmm0, %xmm0
938 ; AVX1-FALLBACK-NEXT: vpaddq %xmm0, %xmm3, %xmm0
939 ; AVX1-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
940 ; AVX1-FALLBACK-NEXT: retq
941 ;
942 ; AVX2-LABEL: vec256_i64_signed_mem_reg:
943 ; AVX2: # %bb.0:
944 ; AVX2-NEXT: vmovdqa (%rdi), %ymm1
945 ; AVX2-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm2
946 ; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
947 ; AVX2-NEXT: vpor %ymm3, %ymm2, %ymm3
948 ; AVX2-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm4
949 ; AVX2-NEXT: vblendvpd %ymm2, %ymm1, %ymm0, %ymm0
950 ; AVX2-NEXT: vpsubq %ymm4, %ymm0, %ymm0
951 ; AVX2-NEXT: vpsrlq $1, %ymm0, %ymm2
952 ; AVX2-NEXT: vpsrlq $33, %ymm0, %ymm0
953 ; AVX2-NEXT: vpmuludq %ymm3, %ymm0, %ymm0
954 ; AVX2-NEXT: vpsrlq $32, %ymm3, %ymm4
955 ; AVX2-NEXT: vpmuludq %ymm4, %ymm2, %ymm4
956 ; AVX2-NEXT: vpaddq %ymm0, %ymm4, %ymm0
957 ; AVX2-NEXT: vpsllq $32, %ymm0, %ymm0
958 ; AVX2-NEXT: vpmuludq %ymm3, %ymm2, %ymm2
959 ; AVX2-NEXT: vpaddq %ymm1, %ymm0, %ymm0
960 ; AVX2-NEXT: vpaddq %ymm0, %ymm2, %ymm0
961 ; AVX2-NEXT: retq
962 ;
963 ; XOP-FALLBACK-LABEL: vec256_i64_signed_mem_reg:
964 ; XOP-FALLBACK: # %bb.0:
965 ; XOP-FALLBACK-NEXT: vmovdqa (%rdi), %xmm1
966 ; XOP-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm2
967 ; XOP-FALLBACK-NEXT: vpcomgtq %xmm0, %xmm1, %xmm3
968 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm4
969 ; XOP-FALLBACK-NEXT: vpcomgtq %xmm4, %xmm2, %xmm9
970 ; XOP-FALLBACK-NEXT: vpcomltq %xmm4, %xmm2, %xmm6
971 ; XOP-FALLBACK-NEXT: vblendvpd %xmm6, %xmm2, %xmm4, %xmm6
972 ; XOP-FALLBACK-NEXT: vpcomltq %xmm0, %xmm1, %xmm7
973 ; XOP-FALLBACK-NEXT: vblendvpd %xmm7, %xmm1, %xmm0, %xmm7
974 ; XOP-FALLBACK-NEXT: vblendvpd %xmm9, %xmm2, %xmm4, %xmm4
975 ; XOP-FALLBACK-NEXT: vpsubq %xmm6, %xmm4, %xmm4
976 ; XOP-FALLBACK-NEXT: vblendvpd %xmm3, %xmm1, %xmm0, %xmm0
977 ; XOP-FALLBACK-NEXT: vpsubq %xmm7, %xmm0, %xmm0
978 ; XOP-FALLBACK-NEXT: vpsrlq $1, %xmm4, %xmm6
979 ; XOP-FALLBACK-NEXT: vpsrlq $1, %xmm0, %xmm7
980 ; XOP-FALLBACK-NEXT: vpsrlq $33, %xmm0, %xmm0
981 ; XOP-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm8 = [1,1]
982 ; XOP-FALLBACK-NEXT: vpor %xmm3, %xmm8, %xmm3
983 ; XOP-FALLBACK-NEXT: vpmuludq %xmm3, %xmm0, %xmm0
984 ; XOP-FALLBACK-NEXT: vpsrlq $32, %xmm3, %xmm5
985 ; XOP-FALLBACK-NEXT: vpmuludq %xmm5, %xmm7, %xmm5
986 ; XOP-FALLBACK-NEXT: vpaddq %xmm0, %xmm5, %xmm0
987 ; XOP-FALLBACK-NEXT: vpsllq $32, %xmm0, %xmm0
988 ; XOP-FALLBACK-NEXT: vpmuludq %xmm3, %xmm7, %xmm3
989 ; XOP-FALLBACK-NEXT: vpsrlq $33, %xmm4, %xmm4
990 ; XOP-FALLBACK-NEXT: vpor %xmm8, %xmm9, %xmm5
991 ; XOP-FALLBACK-NEXT: vpmuludq %xmm5, %xmm4, %xmm4
992 ; XOP-FALLBACK-NEXT: vpsrlq $32, %xmm5, %xmm7
993 ; XOP-FALLBACK-NEXT: vpmuludq %xmm7, %xmm6, %xmm7
994 ; XOP-FALLBACK-NEXT: vpaddq %xmm4, %xmm7, %xmm4
995 ; XOP-FALLBACK-NEXT: vpsllq $32, %xmm4, %xmm4
996 ; XOP-FALLBACK-NEXT: vpmuludq %xmm5, %xmm6, %xmm5
997 ; XOP-FALLBACK-NEXT: vpaddq %xmm2, %xmm4, %xmm2
998 ; XOP-FALLBACK-NEXT: vpaddq %xmm2, %xmm5, %xmm2
999 ; XOP-FALLBACK-NEXT: vpaddq %xmm1, %xmm0, %xmm0
1000 ; XOP-FALLBACK-NEXT: vpaddq %xmm0, %xmm3, %xmm0
1001 ; XOP-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
1002 ; XOP-FALLBACK-NEXT: retq
1003 ;
1004 ; XOPAVX1-LABEL: vec256_i64_signed_mem_reg:
1005 ; XOPAVX1: # %bb.0:
1006 ; XOPAVX1-NEXT: vmovdqa (%rdi), %xmm1
1007 ; XOPAVX1-NEXT: vmovdqa 16(%rdi), %xmm2
1008 ; XOPAVX1-NEXT: vpcomgtq %xmm0, %xmm1, %xmm3
1009 ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
1010 ; XOPAVX1-NEXT: vpcomgtq %xmm4, %xmm2, %xmm9
1011 ; XOPAVX1-NEXT: vpcomltq %xmm4, %xmm2, %xmm6
1012 ; XOPAVX1-NEXT: vblendvpd %xmm6, %xmm2, %xmm4, %xmm6
1013 ; XOPAVX1-NEXT: vpcomltq %xmm0, %xmm1, %xmm7
1014 ; XOPAVX1-NEXT: vblendvpd %xmm7, %xmm1, %xmm0, %xmm7
1015 ; XOPAVX1-NEXT: vblendvpd %xmm9, %xmm2, %xmm4, %xmm4
1016 ; XOPAVX1-NEXT: vpsubq %xmm6, %xmm4, %xmm4
1017 ; XOPAVX1-NEXT: vblendvpd %xmm3, %xmm1, %xmm0, %xmm0
1018 ; XOPAVX1-NEXT: vpsubq %xmm7, %xmm0, %xmm0
1019 ; XOPAVX1-NEXT: vpsrlq $1, %xmm4, %xmm6
1020 ; XOPAVX1-NEXT: vpsrlq $1, %xmm0, %xmm7
1021 ; XOPAVX1-NEXT: vpsrlq $33, %xmm0, %xmm0
1022 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm8 = [1,1]
1023 ; XOPAVX1-NEXT: vpor %xmm3, %xmm8, %xmm3
1024 ; XOPAVX1-NEXT: vpmuludq %xmm3, %xmm0, %xmm0
1025 ; XOPAVX1-NEXT: vpsrlq $32, %xmm3, %xmm5
1026 ; XOPAVX1-NEXT: vpmuludq %xmm5, %xmm7, %xmm5
1027 ; XOPAVX1-NEXT: vpaddq %xmm0, %xmm5, %xmm0
1028 ; XOPAVX1-NEXT: vpsllq $32, %xmm0, %xmm0
1029 ; XOPAVX1-NEXT: vpmuludq %xmm3, %xmm7, %xmm3
1030 ; XOPAVX1-NEXT: vpsrlq $33, %xmm4, %xmm4
1031 ; XOPAVX1-NEXT: vpor %xmm8, %xmm9, %xmm5
1032 ; XOPAVX1-NEXT: vpmuludq %xmm5, %xmm4, %xmm4
1033 ; XOPAVX1-NEXT: vpsrlq $32, %xmm5, %xmm7
1034 ; XOPAVX1-NEXT: vpmuludq %xmm7, %xmm6, %xmm7
1035 ; XOPAVX1-NEXT: vpaddq %xmm4, %xmm7, %xmm4
1036 ; XOPAVX1-NEXT: vpsllq $32, %xmm4, %xmm4
1037 ; XOPAVX1-NEXT: vpmuludq %xmm5, %xmm6, %xmm5
1038 ; XOPAVX1-NEXT: vpaddq %xmm2, %xmm4, %xmm2
1039 ; XOPAVX1-NEXT: vpaddq %xmm2, %xmm5, %xmm2
1040 ; XOPAVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
1041 ; XOPAVX1-NEXT: vpaddq %xmm0, %xmm3, %xmm0
1042 ; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
1043 ; XOPAVX1-NEXT: retq
1044 ;
1045 ; AVX512F-LABEL: vec256_i64_signed_mem_reg:
1046 ; AVX512F: # %bb.0:
1047 ; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
1048 ; AVX512F-NEXT: vmovdqa (%rdi), %ymm1
1049 ; AVX512F-NEXT: vpcmpgtq %zmm0, %zmm1, %k1
1050 ; AVX512F-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
1051 ; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
1052 ; AVX512F-NEXT: vmovdqa64 %zmm2, %zmm3 {%k1}
1053 ; AVX512F-NEXT: vpminsq %zmm0, %zmm1, %zmm2
1054 ; AVX512F-NEXT: vpmaxsq %zmm0, %zmm1, %zmm0
1055 ; AVX512F-NEXT: vpsubq %ymm2, %ymm0, %ymm0
1056 ; AVX512F-NEXT: vpsrlq $1, %ymm0, %ymm2
1057 ; AVX512F-NEXT: vpsrlq $33, %ymm0, %ymm0
1058 ; AVX512F-NEXT: vpmuludq %ymm3, %ymm0, %ymm0
1059 ; AVX512F-NEXT: vpsrlq $32, %ymm3, %ymm4
1060 ; AVX512F-NEXT: vpmuludq %ymm4, %ymm2, %ymm4
1061 ; AVX512F-NEXT: vpaddq %ymm0, %ymm4, %ymm0
1062 ; AVX512F-NEXT: vpsllq $32, %ymm0, %ymm0
1063 ; AVX512F-NEXT: vpmuludq %ymm3, %ymm2, %ymm2
1064 ; AVX512F-NEXT: vpaddq %ymm1, %ymm0, %ymm0
1065 ; AVX512F-NEXT: vpaddq %ymm0, %ymm2, %ymm0
1066 ; AVX512F-NEXT: retq
1067 ;
1068 ; AVX512VL-LABEL: vec256_i64_signed_mem_reg:
1069 ; AVX512VL: # %bb.0:
1070 ; AVX512VL-NEXT: vmovdqa (%rdi), %ymm1
1071 ; AVX512VL-NEXT: vpcmpgtq %ymm0, %ymm1, %k1
1072 ; AVX512VL-NEXT: vpminsq %ymm0, %ymm1, %ymm2
1073 ; AVX512VL-NEXT: vpmaxsq %ymm0, %ymm1, %ymm0
1074 ; AVX512VL-NEXT: vpsubq %ymm2, %ymm0, %ymm0
1075 ; AVX512VL-NEXT: vpsrlq $1, %ymm0, %ymm0
1076 ; AVX512VL-NEXT: vpxor %xmm2, %xmm2, %xmm2
1077 ; AVX512VL-NEXT: vpsubq %ymm0, %ymm2, %ymm0 {%k1}
1078 ; AVX512VL-NEXT: vpaddq %ymm1, %ymm0, %ymm0
1079 ; AVX512VL-NEXT: retq
1080 ;
1081 ; AVX512BW-FALLBACK-LABEL: vec256_i64_signed_mem_reg:
1082 ; AVX512BW-FALLBACK: # %bb.0:
1083 ; AVX512BW-FALLBACK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
1084 ; AVX512BW-FALLBACK-NEXT: vmovdqa (%rdi), %ymm1
1085 ; AVX512BW-FALLBACK-NEXT: vpcmpgtq %zmm0, %zmm1, %k1
1086 ; AVX512BW-FALLBACK-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
1087 ; AVX512BW-FALLBACK-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
1088 ; AVX512BW-FALLBACK-NEXT: vmovdqa64 %zmm2, %zmm3 {%k1}
1089 ; AVX512BW-FALLBACK-NEXT: vpminsq %zmm0, %zmm1, %zmm2
1090 ; AVX512BW-FALLBACK-NEXT: vpmaxsq %zmm0, %zmm1, %zmm0
1091 ; AVX512BW-FALLBACK-NEXT: vpsubq %ymm2, %ymm0, %ymm0
1092 ; AVX512BW-FALLBACK-NEXT: vpsrlq $1, %ymm0, %ymm2
1093 ; AVX512BW-FALLBACK-NEXT: vpsrlq $33, %ymm0, %ymm0
1094 ; AVX512BW-FALLBACK-NEXT: vpmuludq %ymm3, %ymm0, %ymm0
1095 ; AVX512BW-FALLBACK-NEXT: vpsrlq $32, %ymm3, %ymm4
1096 ; AVX512BW-FALLBACK-NEXT: vpmuludq %ymm4, %ymm2, %ymm4
1097 ; AVX512BW-FALLBACK-NEXT: vpaddq %ymm0, %ymm4, %ymm0
1098 ; AVX512BW-FALLBACK-NEXT: vpsllq $32, %ymm0, %ymm0
1099 ; AVX512BW-FALLBACK-NEXT: vpmuludq %ymm3, %ymm2, %ymm2
1100 ; AVX512BW-FALLBACK-NEXT: vpaddq %ymm1, %ymm0, %ymm0
1101 ; AVX512BW-FALLBACK-NEXT: vpaddq %ymm0, %ymm2, %ymm0
1102 ; AVX512BW-FALLBACK-NEXT: retq
1103 %a1 = load <4 x i64>, ptr %a1_addr
1104 %t3 = icmp sgt <4 x i64> %a1, %a2 ; signed
1105 %t4 = select <4 x i1> %t3, <4 x i64> <i64 -1, i64 -1, i64 -1, i64 -1>, <4 x i64> <i64 1, i64 1, i64 1, i64 1>
1106 %t5 = select <4 x i1> %t3, <4 x i64> %a2, <4 x i64> %a1
1107 %t6 = select <4 x i1> %t3, <4 x i64> %a1, <4 x i64> %a2
1108 %t7 = sub <4 x i64> %t6, %t5
1109 %t8 = lshr <4 x i64> %t7, <i64 1, i64 1, i64 1, i64 1>
1110 %t9 = mul nsw <4 x i64> %t8, %t4 ; signed
1111 %a10 = add nsw <4 x i64> %t9, %a1 ; signed
1112 ret <4 x i64> %a10
1113 }
1115 define <4 x i64> @vec256_i64_signed_reg_mem(<4 x i64> %a1, ptr %a2_addr) nounwind {
1116 ; AVX1-FALLBACK-LABEL: vec256_i64_signed_reg_mem:
1117 ; AVX1-FALLBACK: # %bb.0:
1118 ; AVX1-FALLBACK-NEXT: vmovdqa (%rdi), %xmm1
1119 ; AVX1-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm2
1120 ; AVX1-FALLBACK-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm3
1121 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm4
1122 ; AVX1-FALLBACK-NEXT: vpcmpgtq %xmm2, %xmm4, %xmm5
1123 ; AVX1-FALLBACK-NEXT: vblendvpd %xmm5, %xmm2, %xmm4, %xmm6
1124 ; AVX1-FALLBACK-NEXT: vblendvpd %xmm3, %xmm1, %xmm0, %xmm7
1125 ; AVX1-FALLBACK-NEXT: vblendvpd %xmm5, %xmm4, %xmm2, %xmm2
1126 ; AVX1-FALLBACK-NEXT: vpsubq %xmm6, %xmm2, %xmm2
1127 ; AVX1-FALLBACK-NEXT: vblendvpd %xmm3, %xmm0, %xmm1, %xmm1
1128 ; AVX1-FALLBACK-NEXT: vpsubq %xmm7, %xmm1, %xmm1
1129 ; AVX1-FALLBACK-NEXT: vpsrlq $1, %xmm2, %xmm9
1130 ; AVX1-FALLBACK-NEXT: vpsrlq $1, %xmm1, %xmm7
1131 ; AVX1-FALLBACK-NEXT: vpsrlq $33, %xmm1, %xmm1
1132 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm8 = [1,1]
1133 ; AVX1-FALLBACK-NEXT: vpor %xmm3, %xmm8, %xmm3
1134 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm3, %xmm1, %xmm1
1135 ; AVX1-FALLBACK-NEXT: vpsrlq $32, %xmm3, %xmm6
1136 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm6, %xmm7, %xmm6
1137 ; AVX1-FALLBACK-NEXT: vpaddq %xmm1, %xmm6, %xmm1
1138 ; AVX1-FALLBACK-NEXT: vpsllq $32, %xmm1, %xmm1
1139 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm3, %xmm7, %xmm3
1140 ; AVX1-FALLBACK-NEXT: vpsrlq $33, %xmm2, %xmm2
1141 ; AVX1-FALLBACK-NEXT: vpor %xmm5, %xmm8, %xmm5
1142 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm5, %xmm2, %xmm2
1143 ; AVX1-FALLBACK-NEXT: vpsrlq $32, %xmm5, %xmm6
1144 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm6, %xmm9, %xmm6
1145 ; AVX1-FALLBACK-NEXT: vpaddq %xmm2, %xmm6, %xmm2
1146 ; AVX1-FALLBACK-NEXT: vpsllq $32, %xmm2, %xmm2
1147 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm5, %xmm9, %xmm5
1148 ; AVX1-FALLBACK-NEXT: vpaddq %xmm4, %xmm2, %xmm2
1149 ; AVX1-FALLBACK-NEXT: vpaddq %xmm2, %xmm5, %xmm2
1150 ; AVX1-FALLBACK-NEXT: vpaddq %xmm0, %xmm1, %xmm0
1151 ; AVX1-FALLBACK-NEXT: vpaddq %xmm0, %xmm3, %xmm0
1152 ; AVX1-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
1153 ; AVX1-FALLBACK-NEXT: retq
1154 ;
1155 ; AVX2-LABEL: vec256_i64_signed_reg_mem:
1156 ; AVX2: # %bb.0:
1157 ; AVX2-NEXT: vmovdqa (%rdi), %ymm1
1158 ; AVX2-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm2
1159 ; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
1160 ; AVX2-NEXT: vpor %ymm3, %ymm2, %ymm3
1161 ; AVX2-NEXT: vblendvpd %ymm2, %ymm1, %ymm0, %ymm4
1162 ; AVX2-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm1
1163 ; AVX2-NEXT: vpsubq %ymm4, %ymm1, %ymm1
1164 ; AVX2-NEXT: vpsrlq $1, %ymm1, %ymm2
1165 ; AVX2-NEXT: vpsrlq $33, %ymm1, %ymm1
1166 ; AVX2-NEXT: vpmuludq %ymm3, %ymm1, %ymm1
1167 ; AVX2-NEXT: vpsrlq $32, %ymm3, %ymm4
1168 ; AVX2-NEXT: vpmuludq %ymm4, %ymm2, %ymm4
1169 ; AVX2-NEXT: vpaddq %ymm1, %ymm4, %ymm1
1170 ; AVX2-NEXT: vpsllq $32, %ymm1, %ymm1
1171 ; AVX2-NEXT: vpmuludq %ymm3, %ymm2, %ymm2
1172 ; AVX2-NEXT: vpaddq %ymm0, %ymm1, %ymm0
1173 ; AVX2-NEXT: vpaddq %ymm0, %ymm2, %ymm0
1174 ; AVX2-NEXT: retq
1175 ;
1176 ; XOP-FALLBACK-LABEL: vec256_i64_signed_reg_mem:
1177 ; XOP-FALLBACK: # %bb.0:
1178 ; XOP-FALLBACK-NEXT: vmovdqa (%rdi), %xmm1
1179 ; XOP-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm2
1180 ; XOP-FALLBACK-NEXT: vpcomgtq %xmm1, %xmm0, %xmm3
1181 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm4
1182 ; XOP-FALLBACK-NEXT: vpcomgtq %xmm2, %xmm4, %xmm9
1183 ; XOP-FALLBACK-NEXT: vpcomltq %xmm2, %xmm4, %xmm6
1184 ; XOP-FALLBACK-NEXT: vblendvpd %xmm6, %xmm4, %xmm2, %xmm6
1185 ; XOP-FALLBACK-NEXT: vpcomltq %xmm1, %xmm0, %xmm7
1186 ; XOP-FALLBACK-NEXT: vblendvpd %xmm7, %xmm0, %xmm1, %xmm7
1187 ; XOP-FALLBACK-NEXT: vblendvpd %xmm9, %xmm4, %xmm2, %xmm2
1188 ; XOP-FALLBACK-NEXT: vpsubq %xmm6, %xmm2, %xmm2
1189 ; XOP-FALLBACK-NEXT: vblendvpd %xmm3, %xmm0, %xmm1, %xmm1
1190 ; XOP-FALLBACK-NEXT: vpsubq %xmm7, %xmm1, %xmm1
1191 ; XOP-FALLBACK-NEXT: vpsrlq $1, %xmm2, %xmm6
1192 ; XOP-FALLBACK-NEXT: vpsrlq $1, %xmm1, %xmm7
1193 ; XOP-FALLBACK-NEXT: vpsrlq $33, %xmm1, %xmm1
1194 ; XOP-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm8 = [1,1]
1195 ; XOP-FALLBACK-NEXT: vpor %xmm3, %xmm8, %xmm3
1196 ; XOP-FALLBACK-NEXT: vpmuludq %xmm3, %xmm1, %xmm1
1197 ; XOP-FALLBACK-NEXT: vpsrlq $32, %xmm3, %xmm5
1198 ; XOP-FALLBACK-NEXT: vpmuludq %xmm5, %xmm7, %xmm5
1199 ; XOP-FALLBACK-NEXT: vpaddq %xmm1, %xmm5, %xmm1
1200 ; XOP-FALLBACK-NEXT: vpsllq $32, %xmm1, %xmm1
1201 ; XOP-FALLBACK-NEXT: vpmuludq %xmm3, %xmm7, %xmm3
1202 ; XOP-FALLBACK-NEXT: vpsrlq $33, %xmm2, %xmm2
1203 ; XOP-FALLBACK-NEXT: vpor %xmm8, %xmm9, %xmm5
1204 ; XOP-FALLBACK-NEXT: vpmuludq %xmm5, %xmm2, %xmm2
1205 ; XOP-FALLBACK-NEXT: vpsrlq $32, %xmm5, %xmm7
1206 ; XOP-FALLBACK-NEXT: vpmuludq %xmm7, %xmm6, %xmm7
1207 ; XOP-FALLBACK-NEXT: vpaddq %xmm2, %xmm7, %xmm2
1208 ; XOP-FALLBACK-NEXT: vpsllq $32, %xmm2, %xmm2
1209 ; XOP-FALLBACK-NEXT: vpmuludq %xmm5, %xmm6, %xmm5
1210 ; XOP-FALLBACK-NEXT: vpaddq %xmm4, %xmm2, %xmm2
1211 ; XOP-FALLBACK-NEXT: vpaddq %xmm2, %xmm5, %xmm2
1212 ; XOP-FALLBACK-NEXT: vpaddq %xmm0, %xmm1, %xmm0
1213 ; XOP-FALLBACK-NEXT: vpaddq %xmm0, %xmm3, %xmm0
1214 ; XOP-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
1215 ; XOP-FALLBACK-NEXT: retq
1216 ;
1217 ; XOPAVX1-LABEL: vec256_i64_signed_reg_mem:
1218 ; XOPAVX1: # %bb.0:
1219 ; XOPAVX1-NEXT: vmovdqa (%rdi), %xmm1
1220 ; XOPAVX1-NEXT: vmovdqa 16(%rdi), %xmm2
1221 ; XOPAVX1-NEXT: vpcomgtq %xmm1, %xmm0, %xmm3
1222 ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
1223 ; XOPAVX1-NEXT: vpcomgtq %xmm2, %xmm4, %xmm9
1224 ; XOPAVX1-NEXT: vpcomltq %xmm2, %xmm4, %xmm6
1225 ; XOPAVX1-NEXT: vblendvpd %xmm6, %xmm4, %xmm2, %xmm6
1226 ; XOPAVX1-NEXT: vpcomltq %xmm1, %xmm0, %xmm7
1227 ; XOPAVX1-NEXT: vblendvpd %xmm7, %xmm0, %xmm1, %xmm7
1228 ; XOPAVX1-NEXT: vblendvpd %xmm9, %xmm4, %xmm2, %xmm2
1229 ; XOPAVX1-NEXT: vpsubq %xmm6, %xmm2, %xmm2
1230 ; XOPAVX1-NEXT: vblendvpd %xmm3, %xmm0, %xmm1, %xmm1
1231 ; XOPAVX1-NEXT: vpsubq %xmm7, %xmm1, %xmm1
1232 ; XOPAVX1-NEXT: vpsrlq $1, %xmm2, %xmm6
1233 ; XOPAVX1-NEXT: vpsrlq $1, %xmm1, %xmm7
1234 ; XOPAVX1-NEXT: vpsrlq $33, %xmm1, %xmm1
1235 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm8 = [1,1]
1236 ; XOPAVX1-NEXT: vpor %xmm3, %xmm8, %xmm3
1237 ; XOPAVX1-NEXT: vpmuludq %xmm3, %xmm1, %xmm1
1238 ; XOPAVX1-NEXT: vpsrlq $32, %xmm3, %xmm5
1239 ; XOPAVX1-NEXT: vpmuludq %xmm5, %xmm7, %xmm5
1240 ; XOPAVX1-NEXT: vpaddq %xmm1, %xmm5, %xmm1
1241 ; XOPAVX1-NEXT: vpsllq $32, %xmm1, %xmm1
1242 ; XOPAVX1-NEXT: vpmuludq %xmm3, %xmm7, %xmm3
1243 ; XOPAVX1-NEXT: vpsrlq $33, %xmm2, %xmm2
1244 ; XOPAVX1-NEXT: vpor %xmm8, %xmm9, %xmm5
1245 ; XOPAVX1-NEXT: vpmuludq %xmm5, %xmm2, %xmm2
1246 ; XOPAVX1-NEXT: vpsrlq $32, %xmm5, %xmm7
1247 ; XOPAVX1-NEXT: vpmuludq %xmm7, %xmm6, %xmm7
1248 ; XOPAVX1-NEXT: vpaddq %xmm2, %xmm7, %xmm2
1249 ; XOPAVX1-NEXT: vpsllq $32, %xmm2, %xmm2
1250 ; XOPAVX1-NEXT: vpmuludq %xmm5, %xmm6, %xmm5
1251 ; XOPAVX1-NEXT: vpaddq %xmm4, %xmm2, %xmm2
1252 ; XOPAVX1-NEXT: vpaddq %xmm2, %xmm5, %xmm2
1253 ; XOPAVX1-NEXT: vpaddq %xmm0, %xmm1, %xmm0
1254 ; XOPAVX1-NEXT: vpaddq %xmm0, %xmm3, %xmm0
1255 ; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
1256 ; XOPAVX1-NEXT: retq
1257 ;
1258 ; AVX512F-LABEL: vec256_i64_signed_reg_mem:
1259 ; AVX512F: # %bb.0:
1260 ; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
1261 ; AVX512F-NEXT: vmovdqa (%rdi), %ymm1
1262 ; AVX512F-NEXT: vpcmpgtq %zmm1, %zmm0, %k1
1263 ; AVX512F-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
1264 ; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
1265 ; AVX512F-NEXT: vmovdqa64 %zmm2, %zmm3 {%k1}
1266 ; AVX512F-NEXT: vpminsq %zmm1, %zmm0, %zmm2
1267 ; AVX512F-NEXT: vpmaxsq %zmm1, %zmm0, %zmm1
1268 ; AVX512F-NEXT: vpsubq %ymm2, %ymm1, %ymm1
1269 ; AVX512F-NEXT: vpsrlq $1, %ymm1, %ymm2
1270 ; AVX512F-NEXT: vpsrlq $33, %ymm1, %ymm1
1271 ; AVX512F-NEXT: vpmuludq %ymm3, %ymm1, %ymm1
1272 ; AVX512F-NEXT: vpsrlq $32, %ymm3, %ymm4
1273 ; AVX512F-NEXT: vpmuludq %ymm4, %ymm2, %ymm4
1274 ; AVX512F-NEXT: vpaddq %ymm1, %ymm4, %ymm1
1275 ; AVX512F-NEXT: vpsllq $32, %ymm1, %ymm1
1276 ; AVX512F-NEXT: vpmuludq %ymm3, %ymm2, %ymm2
1277 ; AVX512F-NEXT: vpaddq %ymm0, %ymm1, %ymm0
1278 ; AVX512F-NEXT: vpaddq %ymm0, %ymm2, %ymm0
1279 ; AVX512F-NEXT: retq
1280 ;
1281 ; AVX512VL-LABEL: vec256_i64_signed_reg_mem:
1282 ; AVX512VL: # %bb.0:
1283 ; AVX512VL-NEXT: vmovdqa (%rdi), %ymm1
1284 ; AVX512VL-NEXT: vpcmpgtq %ymm1, %ymm0, %k1
1285 ; AVX512VL-NEXT: vpminsq %ymm1, %ymm0, %ymm2
1286 ; AVX512VL-NEXT: vpmaxsq %ymm1, %ymm0, %ymm1
1287 ; AVX512VL-NEXT: vpsubq %ymm2, %ymm1, %ymm1
1288 ; AVX512VL-NEXT: vpsrlq $1, %ymm1, %ymm1
1289 ; AVX512VL-NEXT: vpxor %xmm2, %xmm2, %xmm2
1290 ; AVX512VL-NEXT: vpsubq %ymm1, %ymm2, %ymm1 {%k1}
1291 ; AVX512VL-NEXT: vpaddq %ymm0, %ymm1, %ymm0
1292 ; AVX512VL-NEXT: retq
1293 ;
1294 ; AVX512BW-FALLBACK-LABEL: vec256_i64_signed_reg_mem:
1295 ; AVX512BW-FALLBACK: # %bb.0:
1296 ; AVX512BW-FALLBACK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
1297 ; AVX512BW-FALLBACK-NEXT: vmovdqa (%rdi), %ymm1
1298 ; AVX512BW-FALLBACK-NEXT: vpcmpgtq %zmm1, %zmm0, %k1
1299 ; AVX512BW-FALLBACK-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
1300 ; AVX512BW-FALLBACK-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
1301 ; AVX512BW-FALLBACK-NEXT: vmovdqa64 %zmm2, %zmm3 {%k1}
1302 ; AVX512BW-FALLBACK-NEXT: vpminsq %zmm1, %zmm0, %zmm2
1303 ; AVX512BW-FALLBACK-NEXT: vpmaxsq %zmm1, %zmm0, %zmm1
1304 ; AVX512BW-FALLBACK-NEXT: vpsubq %ymm2, %ymm1, %ymm1
1305 ; AVX512BW-FALLBACK-NEXT: vpsrlq $1, %ymm1, %ymm2
1306 ; AVX512BW-FALLBACK-NEXT: vpsrlq $33, %ymm1, %ymm1
1307 ; AVX512BW-FALLBACK-NEXT: vpmuludq %ymm3, %ymm1, %ymm1
1308 ; AVX512BW-FALLBACK-NEXT: vpsrlq $32, %ymm3, %ymm4
1309 ; AVX512BW-FALLBACK-NEXT: vpmuludq %ymm4, %ymm2, %ymm4
1310 ; AVX512BW-FALLBACK-NEXT: vpaddq %ymm1, %ymm4, %ymm1
1311 ; AVX512BW-FALLBACK-NEXT: vpsllq $32, %ymm1, %ymm1
1312 ; AVX512BW-FALLBACK-NEXT: vpmuludq %ymm3, %ymm2, %ymm2
1313 ; AVX512BW-FALLBACK-NEXT: vpaddq %ymm0, %ymm1, %ymm0
1314 ; AVX512BW-FALLBACK-NEXT: vpaddq %ymm0, %ymm2, %ymm0
1315 ; AVX512BW-FALLBACK-NEXT: retq
1316 %a2 = load <4 x i64>, ptr %a2_addr
1317 %t3 = icmp sgt <4 x i64> %a1, %a2 ; signed
1318 %t4 = select <4 x i1> %t3, <4 x i64> <i64 -1, i64 -1, i64 -1, i64 -1>, <4 x i64> <i64 1, i64 1, i64 1, i64 1>
1319 %t5 = select <4 x i1> %t3, <4 x i64> %a2, <4 x i64> %a1
1320 %t6 = select <4 x i1> %t3, <4 x i64> %a1, <4 x i64> %a2
1321 %t7 = sub <4 x i64> %t6, %t5
1322 %t8 = lshr <4 x i64> %t7, <i64 1, i64 1, i64 1, i64 1>
1323 %t9 = mul nsw <4 x i64> %t8, %t4 ; signed
1324 %a10 = add nsw <4 x i64> %t9, %a1 ; signed
1325 ret <4 x i64> %a10
1326 }
1328 define <4 x i64> @vec256_i64_signed_mem_mem(ptr %a1_addr, ptr %a2_addr) nounwind {
1329 ; AVX1-FALLBACK-LABEL: vec256_i64_signed_mem_mem:
1330 ; AVX1-FALLBACK: # %bb.0:
1331 ; AVX1-FALLBACK-NEXT: vmovdqa (%rsi), %xmm0
1332 ; AVX1-FALLBACK-NEXT: vmovdqa 16(%rsi), %xmm1
1333 ; AVX1-FALLBACK-NEXT: vmovdqa (%rdi), %xmm2
1334 ; AVX1-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm3
1335 ; AVX1-FALLBACK-NEXT: vpcmpgtq %xmm0, %xmm2, %xmm4
1336 ; AVX1-FALLBACK-NEXT: vpcmpgtq %xmm1, %xmm3, %xmm5
1337 ; AVX1-FALLBACK-NEXT: vblendvpd %xmm5, %xmm1, %xmm3, %xmm6
1338 ; AVX1-FALLBACK-NEXT: vblendvpd %xmm4, %xmm0, %xmm2, %xmm7
1339 ; AVX1-FALLBACK-NEXT: vblendvpd %xmm5, %xmm3, %xmm1, %xmm1
1340 ; AVX1-FALLBACK-NEXT: vpsubq %xmm6, %xmm1, %xmm1
1341 ; AVX1-FALLBACK-NEXT: vblendvpd %xmm4, %xmm2, %xmm0, %xmm0
1342 ; AVX1-FALLBACK-NEXT: vpsubq %xmm7, %xmm0, %xmm0
1343 ; AVX1-FALLBACK-NEXT: vpsrlq $1, %xmm1, %xmm9
1344 ; AVX1-FALLBACK-NEXT: vpsrlq $1, %xmm0, %xmm7
1345 ; AVX1-FALLBACK-NEXT: vpsrlq $33, %xmm0, %xmm0
1346 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm8 = [1,1]
1347 ; AVX1-FALLBACK-NEXT: vpor %xmm4, %xmm8, %xmm4
1348 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm4, %xmm0, %xmm0
1349 ; AVX1-FALLBACK-NEXT: vpsrlq $32, %xmm4, %xmm6
1350 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm6, %xmm7, %xmm6
1351 ; AVX1-FALLBACK-NEXT: vpaddq %xmm0, %xmm6, %xmm0
1352 ; AVX1-FALLBACK-NEXT: vpsllq $32, %xmm0, %xmm0
1353 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm4, %xmm7, %xmm4
1354 ; AVX1-FALLBACK-NEXT: vpsrlq $33, %xmm1, %xmm1
1355 ; AVX1-FALLBACK-NEXT: vpor %xmm5, %xmm8, %xmm5
1356 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm5, %xmm1, %xmm1
1357 ; AVX1-FALLBACK-NEXT: vpsrlq $32, %xmm5, %xmm6
1358 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm6, %xmm9, %xmm6
1359 ; AVX1-FALLBACK-NEXT: vpaddq %xmm1, %xmm6, %xmm1
1360 ; AVX1-FALLBACK-NEXT: vpsllq $32, %xmm1, %xmm1
1361 ; AVX1-FALLBACK-NEXT: vpmuludq %xmm5, %xmm9, %xmm5
1362 ; AVX1-FALLBACK-NEXT: vpaddq %xmm3, %xmm1, %xmm1
1363 ; AVX1-FALLBACK-NEXT: vpaddq %xmm1, %xmm5, %xmm1
1364 ; AVX1-FALLBACK-NEXT: vpaddq %xmm2, %xmm0, %xmm0
1365 ; AVX1-FALLBACK-NEXT: vpaddq %xmm0, %xmm4, %xmm0
1366 ; AVX1-FALLBACK-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
1367 ; AVX1-FALLBACK-NEXT: retq
1368 ;
1369 ; AVX2-LABEL: vec256_i64_signed_mem_mem:
1370 ; AVX2: # %bb.0:
1371 ; AVX2-NEXT: vmovdqa (%rdi), %ymm0
1372 ; AVX2-NEXT: vmovdqa (%rsi), %ymm1
1373 ; AVX2-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm2
1374 ; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
1375 ; AVX2-NEXT: vpor %ymm3, %ymm2, %ymm3
1376 ; AVX2-NEXT: vblendvpd %ymm2, %ymm1, %ymm0, %ymm4
1377 ; AVX2-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm1
1378 ; AVX2-NEXT: vpsubq %ymm4, %ymm1, %ymm1
1379 ; AVX2-NEXT: vpsrlq $1, %ymm1, %ymm2
1380 ; AVX2-NEXT: vpsrlq $33, %ymm1, %ymm1
1381 ; AVX2-NEXT: vpmuludq %ymm3, %ymm1, %ymm1
1382 ; AVX2-NEXT: vpsrlq $32, %ymm3, %ymm4
1383 ; AVX2-NEXT: vpmuludq %ymm4, %ymm2, %ymm4
1384 ; AVX2-NEXT: vpaddq %ymm1, %ymm4, %ymm1
1385 ; AVX2-NEXT: vpsllq $32, %ymm1, %ymm1
1386 ; AVX2-NEXT: vpmuludq %ymm3, %ymm2, %ymm2
1387 ; AVX2-NEXT: vpaddq %ymm0, %ymm1, %ymm0
1388 ; AVX2-NEXT: vpaddq %ymm0, %ymm2, %ymm0
1389 ; AVX2-NEXT: retq
1390 ;
1391 ; XOP-FALLBACK-LABEL: vec256_i64_signed_mem_mem:
1392 ; XOP-FALLBACK: # %bb.0:
1393 ; XOP-FALLBACK-NEXT: vmovdqa (%rsi), %xmm1
1394 ; XOP-FALLBACK-NEXT: vmovdqa 16(%rsi), %xmm2
1395 ; XOP-FALLBACK-NEXT: vmovdqa (%rdi), %xmm0
1396 ; XOP-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm3
1397 ; XOP-FALLBACK-NEXT: vpcomgtq %xmm1, %xmm0, %xmm4
1398 ; XOP-FALLBACK-NEXT: vpcomgtq %xmm2, %xmm3, %xmm9
1399 ; XOP-FALLBACK-NEXT: vpcomltq %xmm2, %xmm3, %xmm6
1400 ; XOP-FALLBACK-NEXT: vblendvpd %xmm6, %xmm3, %xmm2, %xmm6
1401 ; XOP-FALLBACK-NEXT: vpcomltq %xmm1, %xmm0, %xmm7
1402 ; XOP-FALLBACK-NEXT: vblendvpd %xmm7, %xmm0, %xmm1, %xmm7
1403 ; XOP-FALLBACK-NEXT: vblendvpd %xmm9, %xmm3, %xmm2, %xmm2
1404 ; XOP-FALLBACK-NEXT: vpsubq %xmm6, %xmm2, %xmm2
1405 ; XOP-FALLBACK-NEXT: vblendvpd %xmm4, %xmm0, %xmm1, %xmm1
1406 ; XOP-FALLBACK-NEXT: vpsubq %xmm7, %xmm1, %xmm1
1407 ; XOP-FALLBACK-NEXT: vpsrlq $1, %xmm2, %xmm6
1408 ; XOP-FALLBACK-NEXT: vpsrlq $1, %xmm1, %xmm7
1409 ; XOP-FALLBACK-NEXT: vpsrlq $33, %xmm1, %xmm1
1410 ; XOP-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm8 = [1,1]
1411 ; XOP-FALLBACK-NEXT: vpor %xmm4, %xmm8, %xmm4
1412 ; XOP-FALLBACK-NEXT: vpmuludq %xmm4, %xmm1, %xmm1
1413 ; XOP-FALLBACK-NEXT: vpsrlq $32, %xmm4, %xmm5
1414 ; XOP-FALLBACK-NEXT: vpmuludq %xmm5, %xmm7, %xmm5
1415 ; XOP-FALLBACK-NEXT: vpaddq %xmm1, %xmm5, %xmm1
1416 ; XOP-FALLBACK-NEXT: vpsllq $32, %xmm1, %xmm1
1417 ; XOP-FALLBACK-NEXT: vpmuludq %xmm4, %xmm7, %xmm4
1418 ; XOP-FALLBACK-NEXT: vpsrlq $33, %xmm2, %xmm2
1419 ; XOP-FALLBACK-NEXT: vpor %xmm8, %xmm9, %xmm5
1420 ; XOP-FALLBACK-NEXT: vpmuludq %xmm5, %xmm2, %xmm2
1421 ; XOP-FALLBACK-NEXT: vpsrlq $32, %xmm5, %xmm7
1422 ; XOP-FALLBACK-NEXT: vpmuludq %xmm7, %xmm6, %xmm7
1423 ; XOP-FALLBACK-NEXT: vpaddq %xmm2, %xmm7, %xmm2
1424 ; XOP-FALLBACK-NEXT: vpsllq $32, %xmm2, %xmm2
1425 ; XOP-FALLBACK-NEXT: vpmuludq %xmm5, %xmm6, %xmm5
1426 ; XOP-FALLBACK-NEXT: vpaddq %xmm3, %xmm2, %xmm2
1427 ; XOP-FALLBACK-NEXT: vpaddq %xmm2, %xmm5, %xmm2
1428 ; XOP-FALLBACK-NEXT: vpaddq %xmm0, %xmm1, %xmm0
1429 ; XOP-FALLBACK-NEXT: vpaddq %xmm0, %xmm4, %xmm0
1430 ; XOP-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
1431 ; XOP-FALLBACK-NEXT: retq
1432 ;
1433 ; XOPAVX1-LABEL: vec256_i64_signed_mem_mem:
1434 ; XOPAVX1: # %bb.0:
1435 ; XOPAVX1-NEXT: vmovdqa (%rsi), %xmm1
1436 ; XOPAVX1-NEXT: vmovdqa 16(%rsi), %xmm2
1437 ; XOPAVX1-NEXT: vmovdqa (%rdi), %xmm0
1438 ; XOPAVX1-NEXT: vmovdqa 16(%rdi), %xmm3
1439 ; XOPAVX1-NEXT: vpcomgtq %xmm1, %xmm0, %xmm4
1440 ; XOPAVX1-NEXT: vpcomgtq %xmm2, %xmm3, %xmm9
1441 ; XOPAVX1-NEXT: vpcomltq %xmm2, %xmm3, %xmm6
1442 ; XOPAVX1-NEXT: vblendvpd %xmm6, %xmm3, %xmm2, %xmm6
1443 ; XOPAVX1-NEXT: vpcomltq %xmm1, %xmm0, %xmm7
1444 ; XOPAVX1-NEXT: vblendvpd %xmm7, %xmm0, %xmm1, %xmm7
1445 ; XOPAVX1-NEXT: vblendvpd %xmm9, %xmm3, %xmm2, %xmm2
1446 ; XOPAVX1-NEXT: vpsubq %xmm6, %xmm2, %xmm2
1447 ; XOPAVX1-NEXT: vblendvpd %xmm4, %xmm0, %xmm1, %xmm1
1448 ; XOPAVX1-NEXT: vpsubq %xmm7, %xmm1, %xmm1
1449 ; XOPAVX1-NEXT: vpsrlq $1, %xmm2, %xmm6
1450 ; XOPAVX1-NEXT: vpsrlq $1, %xmm1, %xmm7
1451 ; XOPAVX1-NEXT: vpsrlq $33, %xmm1, %xmm1
1452 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm8 = [1,1]
1453 ; XOPAVX1-NEXT: vpor %xmm4, %xmm8, %xmm4
1454 ; XOPAVX1-NEXT: vpmuludq %xmm4, %xmm1, %xmm1
1455 ; XOPAVX1-NEXT: vpsrlq $32, %xmm4, %xmm5
1456 ; XOPAVX1-NEXT: vpmuludq %xmm5, %xmm7, %xmm5
1457 ; XOPAVX1-NEXT: vpaddq %xmm1, %xmm5, %xmm1
1458 ; XOPAVX1-NEXT: vpsllq $32, %xmm1, %xmm1
1459 ; XOPAVX1-NEXT: vpmuludq %xmm4, %xmm7, %xmm4
1460 ; XOPAVX1-NEXT: vpsrlq $33, %xmm2, %xmm2
1461 ; XOPAVX1-NEXT: vpor %xmm8, %xmm9, %xmm5
1462 ; XOPAVX1-NEXT: vpmuludq %xmm5, %xmm2, %xmm2
1463 ; XOPAVX1-NEXT: vpsrlq $32, %xmm5, %xmm7
1464 ; XOPAVX1-NEXT: vpmuludq %xmm7, %xmm6, %xmm7
1465 ; XOPAVX1-NEXT: vpaddq %xmm2, %xmm7, %xmm2
1466 ; XOPAVX1-NEXT: vpsllq $32, %xmm2, %xmm2
1467 ; XOPAVX1-NEXT: vpmuludq %xmm5, %xmm6, %xmm5
1468 ; XOPAVX1-NEXT: vpaddq %xmm3, %xmm2, %xmm2
1469 ; XOPAVX1-NEXT: vpaddq %xmm2, %xmm5, %xmm2
1470 ; XOPAVX1-NEXT: vpaddq %xmm0, %xmm1, %xmm0
1471 ; XOPAVX1-NEXT: vpaddq %xmm0, %xmm4, %xmm0
1472 ; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
1473 ; XOPAVX1-NEXT: retq
1474 ;
1475 ; AVX512F-LABEL: vec256_i64_signed_mem_mem:
1476 ; AVX512F: # %bb.0:
1477 ; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
1478 ; AVX512F-NEXT: vmovdqa (%rsi), %ymm1
1479 ; AVX512F-NEXT: vpcmpgtq %zmm1, %zmm0, %k1
1480 ; AVX512F-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
1481 ; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
1482 ; AVX512F-NEXT: vmovdqa64 %zmm2, %zmm3 {%k1}
1483 ; AVX512F-NEXT: vpminsq %zmm1, %zmm0, %zmm2
1484 ; AVX512F-NEXT: vpmaxsq %zmm1, %zmm0, %zmm1
1485 ; AVX512F-NEXT: vpsubq %ymm2, %ymm1, %ymm1
1486 ; AVX512F-NEXT: vpsrlq $1, %ymm1, %ymm2
1487 ; AVX512F-NEXT: vpsrlq $33, %ymm1, %ymm1
1488 ; AVX512F-NEXT: vpmuludq %ymm3, %ymm1, %ymm1
1489 ; AVX512F-NEXT: vpsrlq $32, %ymm3, %ymm4
1490 ; AVX512F-NEXT: vpmuludq %ymm4, %ymm2, %ymm4
1491 ; AVX512F-NEXT: vpaddq %ymm1, %ymm4, %ymm1
1492 ; AVX512F-NEXT: vpsllq $32, %ymm1, %ymm1
1493 ; AVX512F-NEXT: vpmuludq %ymm3, %ymm2, %ymm2
1494 ; AVX512F-NEXT: vpaddq %ymm0, %ymm1, %ymm0
1495 ; AVX512F-NEXT: vpaddq %ymm0, %ymm2, %ymm0
1496 ; AVX512F-NEXT: retq
1497 ;
1498 ; AVX512VL-LABEL: vec256_i64_signed_mem_mem:
1499 ; AVX512VL: # %bb.0:
1500 ; AVX512VL-NEXT: vmovdqa (%rdi), %ymm0
1501 ; AVX512VL-NEXT: vmovdqa (%rsi), %ymm1
1502 ; AVX512VL-NEXT: vpcmpgtq %ymm1, %ymm0, %k1
1503 ; AVX512VL-NEXT: vpminsq %ymm1, %ymm0, %ymm2
1504 ; AVX512VL-NEXT: vpmaxsq %ymm1, %ymm0, %ymm1
1505 ; AVX512VL-NEXT: vpsubq %ymm2, %ymm1, %ymm1
1506 ; AVX512VL-NEXT: vpsrlq $1, %ymm1, %ymm1
1507 ; AVX512VL-NEXT: vpxor %xmm2, %xmm2, %xmm2
1508 ; AVX512VL-NEXT: vpsubq %ymm1, %ymm2, %ymm1 {%k1}
1509 ; AVX512VL-NEXT: vpaddq %ymm0, %ymm1, %ymm0
1510 ; AVX512VL-NEXT: retq
1511 ;
1512 ; AVX512BW-FALLBACK-LABEL: vec256_i64_signed_mem_mem:
1513 ; AVX512BW-FALLBACK: # %bb.0:
1514 ; AVX512BW-FALLBACK-NEXT: vmovdqa (%rdi), %ymm0
1515 ; AVX512BW-FALLBACK-NEXT: vmovdqa (%rsi), %ymm1
1516 ; AVX512BW-FALLBACK-NEXT: vpcmpgtq %zmm1, %zmm0, %k1
1517 ; AVX512BW-FALLBACK-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
1518 ; AVX512BW-FALLBACK-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
1519 ; AVX512BW-FALLBACK-NEXT: vmovdqa64 %zmm2, %zmm3 {%k1}
1520 ; AVX512BW-FALLBACK-NEXT: vpminsq %zmm1, %zmm0, %zmm2
1521 ; AVX512BW-FALLBACK-NEXT: vpmaxsq %zmm1, %zmm0, %zmm1
1522 ; AVX512BW-FALLBACK-NEXT: vpsubq %ymm2, %ymm1, %ymm1
1523 ; AVX512BW-FALLBACK-NEXT: vpsrlq $1, %ymm1, %ymm2
1524 ; AVX512BW-FALLBACK-NEXT: vpsrlq $33, %ymm1, %ymm1
1525 ; AVX512BW-FALLBACK-NEXT: vpmuludq %ymm3, %ymm1, %ymm1
1526 ; AVX512BW-FALLBACK-NEXT: vpsrlq $32, %ymm3, %ymm4
1527 ; AVX512BW-FALLBACK-NEXT: vpmuludq %ymm4, %ymm2, %ymm4
1528 ; AVX512BW-FALLBACK-NEXT: vpaddq %ymm1, %ymm4, %ymm1
1529 ; AVX512BW-FALLBACK-NEXT: vpsllq $32, %ymm1, %ymm1
1530 ; AVX512BW-FALLBACK-NEXT: vpmuludq %ymm3, %ymm2, %ymm2
1531 ; AVX512BW-FALLBACK-NEXT: vpaddq %ymm0, %ymm1, %ymm0
1532 ; AVX512BW-FALLBACK-NEXT: vpaddq %ymm0, %ymm2, %ymm0
1533 ; AVX512BW-FALLBACK-NEXT: retq
1534 %a1 = load <4 x i64>, ptr %a1_addr
1535 %a2 = load <4 x i64>, ptr %a2_addr
1536 %t3 = icmp sgt <4 x i64> %a1, %a2 ; signed
1537 %t4 = select <4 x i1> %t3, <4 x i64> <i64 -1, i64 -1, i64 -1, i64 -1>, <4 x i64> <i64 1, i64 1, i64 1, i64 1>
1538 %t5 = select <4 x i1> %t3, <4 x i64> %a2, <4 x i64> %a1
1539 %t6 = select <4 x i1> %t3, <4 x i64> %a1, <4 x i64> %a2
1540 %t7 = sub <4 x i64> %t6, %t5
1541 %t8 = lshr <4 x i64> %t7, <i64 1, i64 1, i64 1, i64 1>
1542 %t9 = mul nsw <4 x i64> %t8, %t4 ; signed
1543 %a10 = add nsw <4 x i64> %t9, %a1 ; signed
1544 ret <4 x i64> %a10
1545 }
1547 ; ---------------------------------------------------------------------------- ;
1548 ; 16-bit width. 256 / 16 = 16 elts.
1549 ; ---------------------------------------------------------------------------- ;
1551 ; Values come from regs
1553 define <16 x i16> @vec256_i16_signed_reg_reg(<16 x i16> %a1, <16 x i16> %a2) nounwind {
1554 ; AVX1-FALLBACK-LABEL: vec256_i16_signed_reg_reg:
1555 ; AVX1-FALLBACK: # %bb.0:
1556 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm1, %xmm2
1557 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm3
1558 ; AVX1-FALLBACK-NEXT: vpcmpgtw %xmm2, %xmm3, %xmm4
1559 ; AVX1-FALLBACK-NEXT: vpcmpgtw %xmm1, %xmm0, %xmm5
1560 ; AVX1-FALLBACK-NEXT: vpminsw %xmm2, %xmm3, %xmm6
1561 ; AVX1-FALLBACK-NEXT: vpminsw %xmm1, %xmm0, %xmm7
1562 ; AVX1-FALLBACK-NEXT: vpmaxsw %xmm2, %xmm3, %xmm2
1563 ; AVX1-FALLBACK-NEXT: vpsubw %xmm6, %xmm2, %xmm2
1564 ; AVX1-FALLBACK-NEXT: vpmaxsw %xmm1, %xmm0, %xmm1
1565 ; AVX1-FALLBACK-NEXT: vpsubw %xmm7, %xmm1, %xmm1
1566 ; AVX1-FALLBACK-NEXT: vpsrlw $1, %xmm2, %xmm2
1567 ; AVX1-FALLBACK-NEXT: vpsrlw $1, %xmm1, %xmm1
1568 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm6 = [1,1,1,1,1,1,1,1]
1569 ; AVX1-FALLBACK-NEXT: vpor %xmm6, %xmm5, %xmm5
1570 ; AVX1-FALLBACK-NEXT: vpmullw %xmm5, %xmm1, %xmm1
1571 ; AVX1-FALLBACK-NEXT: vpor %xmm6, %xmm4, %xmm4
1572 ; AVX1-FALLBACK-NEXT: vpmullw %xmm4, %xmm2, %xmm2
1573 ; AVX1-FALLBACK-NEXT: vpaddw %xmm3, %xmm2, %xmm2
1574 ; AVX1-FALLBACK-NEXT: vpaddw %xmm0, %xmm1, %xmm0
1575 ; AVX1-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
1576 ; AVX1-FALLBACK-NEXT: retq
1577 ;
1578 ; AVX2-LABEL: vec256_i16_signed_reg_reg:
1579 ; AVX2: # %bb.0:
1580 ; AVX2-NEXT: vpcmpgtw %ymm1, %ymm0, %ymm2
1581 ; AVX2-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
1582 ; AVX2-NEXT: vpminsw %ymm1, %ymm0, %ymm3
1583 ; AVX2-NEXT: vpmaxsw %ymm1, %ymm0, %ymm1
1584 ; AVX2-NEXT: vpsubw %ymm3, %ymm1, %ymm1
1585 ; AVX2-NEXT: vpsrlw $1, %ymm1, %ymm1
1586 ; AVX2-NEXT: vpmullw %ymm2, %ymm1, %ymm1
1587 ; AVX2-NEXT: vpaddw %ymm0, %ymm1, %ymm0
1588 ; AVX2-NEXT: retq
1589 ;
1590 ; XOP-FALLBACK-LABEL: vec256_i16_signed_reg_reg:
1591 ; XOP-FALLBACK: # %bb.0:
1592 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm1, %xmm2
1593 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm3
1594 ; XOP-FALLBACK-NEXT: vpcomgtw %xmm2, %xmm3, %xmm4
1595 ; XOP-FALLBACK-NEXT: vpcomgtw %xmm1, %xmm0, %xmm5
1596 ; XOP-FALLBACK-NEXT: vpminsw %xmm1, %xmm0, %xmm6
1597 ; XOP-FALLBACK-NEXT: vpminsw %xmm2, %xmm3, %xmm7
1598 ; XOP-FALLBACK-NEXT: vpmaxsw %xmm1, %xmm0, %xmm1
1599 ; XOP-FALLBACK-NEXT: vpsubw %xmm6, %xmm1, %xmm1
1600 ; XOP-FALLBACK-NEXT: vpmaxsw %xmm2, %xmm3, %xmm2
1601 ; XOP-FALLBACK-NEXT: vpsubw %xmm7, %xmm2, %xmm2
1602 ; XOP-FALLBACK-NEXT: vpsrlw $1, %xmm1, %xmm1
1603 ; XOP-FALLBACK-NEXT: vpsrlw $1, %xmm2, %xmm2
1604 ; XOP-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm6 = [1,1,1,1,1,1,1,1]
1605 ; XOP-FALLBACK-NEXT: vpor %xmm6, %xmm5, %xmm5
1606 ; XOP-FALLBACK-NEXT: vpor %xmm6, %xmm4, %xmm4
1607 ; XOP-FALLBACK-NEXT: vpmacsww %xmm3, %xmm4, %xmm2, %xmm2
1608 ; XOP-FALLBACK-NEXT: vpmacsww %xmm0, %xmm5, %xmm1, %xmm0
1609 ; XOP-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
1610 ; XOP-FALLBACK-NEXT: retq
1611 ;
1612 ; XOPAVX1-LABEL: vec256_i16_signed_reg_reg:
1613 ; XOPAVX1: # %bb.0:
1614 ; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
1615 ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
1616 ; XOPAVX1-NEXT: vpcomgtw %xmm2, %xmm3, %xmm4
1617 ; XOPAVX1-NEXT: vpcomgtw %xmm1, %xmm0, %xmm5
1618 ; XOPAVX1-NEXT: vpminsw %xmm1, %xmm0, %xmm6
1619 ; XOPAVX1-NEXT: vpminsw %xmm2, %xmm3, %xmm7
1620 ; XOPAVX1-NEXT: vpmaxsw %xmm1, %xmm0, %xmm1
1621 ; XOPAVX1-NEXT: vpsubw %xmm6, %xmm1, %xmm1
1622 ; XOPAVX1-NEXT: vpmaxsw %xmm2, %xmm3, %xmm2
1623 ; XOPAVX1-NEXT: vpsubw %xmm7, %xmm2, %xmm2
1624 ; XOPAVX1-NEXT: vpsrlw $1, %xmm1, %xmm1
1625 ; XOPAVX1-NEXT: vpsrlw $1, %xmm2, %xmm2
1626 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm6 = [1,1,1,1,1,1,1,1]
1627 ; XOPAVX1-NEXT: vpor %xmm6, %xmm5, %xmm5
1628 ; XOPAVX1-NEXT: vpor %xmm6, %xmm4, %xmm4
1629 ; XOPAVX1-NEXT: vpmacsww %xmm3, %xmm4, %xmm2, %xmm2
1630 ; XOPAVX1-NEXT: vpmacsww %xmm0, %xmm5, %xmm1, %xmm0
1631 ; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
1632 ; XOPAVX1-NEXT: retq
1633 ;
1634 ; AVX512F-LABEL: vec256_i16_signed_reg_reg:
1635 ; AVX512F: # %bb.0:
1636 ; AVX512F-NEXT: vpcmpgtw %ymm1, %ymm0, %ymm2
1637 ; AVX512F-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
1638 ; AVX512F-NEXT: vpminsw %ymm1, %ymm0, %ymm3
1639 ; AVX512F-NEXT: vpmaxsw %ymm1, %ymm0, %ymm1
1640 ; AVX512F-NEXT: vpsubw %ymm3, %ymm1, %ymm1
1641 ; AVX512F-NEXT: vpsrlw $1, %ymm1, %ymm1
1642 ; AVX512F-NEXT: vpmullw %ymm2, %ymm1, %ymm1
1643 ; AVX512F-NEXT: vpaddw %ymm0, %ymm1, %ymm0
1644 ; AVX512F-NEXT: retq
1645 ;
1646 ; AVX512VL-FALLBACK-LABEL: vec256_i16_signed_reg_reg:
1647 ; AVX512VL-FALLBACK: # %bb.0:
1648 ; AVX512VL-FALLBACK-NEXT: vpminsw %ymm1, %ymm0, %ymm2
1649 ; AVX512VL-FALLBACK-NEXT: vpmaxsw %ymm1, %ymm0, %ymm3
1650 ; AVX512VL-FALLBACK-NEXT: vpsubw %ymm2, %ymm3, %ymm2
1651 ; AVX512VL-FALLBACK-NEXT: vpsrlw $1, %ymm2, %ymm2
1652 ; AVX512VL-FALLBACK-NEXT: vpcmpgtw %ymm1, %ymm0, %ymm1
1653 ; AVX512VL-FALLBACK-NEXT: vpxor %ymm1, %ymm2, %ymm2
1654 ; AVX512VL-FALLBACK-NEXT: vpsubw %ymm1, %ymm2, %ymm1
1655 ; AVX512VL-FALLBACK-NEXT: vpaddw %ymm0, %ymm1, %ymm0
1656 ; AVX512VL-FALLBACK-NEXT: retq
1657 ;
1658 ; AVX512BW-FALLBACK-LABEL: vec256_i16_signed_reg_reg:
1659 ; AVX512BW-FALLBACK: # %bb.0:
1660 ; AVX512BW-FALLBACK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
1661 ; AVX512BW-FALLBACK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
1662 ; AVX512BW-FALLBACK-NEXT: vpcmpgtw %zmm1, %zmm0, %k1
1663 ; AVX512BW-FALLBACK-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
1664 ; AVX512BW-FALLBACK-NEXT: vmovdqa {{.*#+}} ymm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
1665 ; AVX512BW-FALLBACK-NEXT: vmovdqu16 %zmm2, %zmm3 {%k1}
1666 ; AVX512BW-FALLBACK-NEXT: vpminsw %ymm1, %ymm0, %ymm2
1667 ; AVX512BW-FALLBACK-NEXT: vpmaxsw %ymm1, %ymm0, %ymm1
1668 ; AVX512BW-FALLBACK-NEXT: vpsubw %ymm2, %ymm1, %ymm1
1669 ; AVX512BW-FALLBACK-NEXT: vpsrlw $1, %ymm1, %ymm1
1670 ; AVX512BW-FALLBACK-NEXT: vpmullw %ymm3, %ymm1, %ymm1
1671 ; AVX512BW-FALLBACK-NEXT: vpaddw %ymm0, %ymm1, %ymm0
1672 ; AVX512BW-FALLBACK-NEXT: retq
1673 ;
1674 ; AVX512VLBW-LABEL: vec256_i16_signed_reg_reg:
1675 ; AVX512VLBW: # %bb.0:
1676 ; AVX512VLBW-NEXT: vpcmpgtw %ymm1, %ymm0, %k1
1677 ; AVX512VLBW-NEXT: vpminsw %ymm1, %ymm0, %ymm2
1678 ; AVX512VLBW-NEXT: vpmaxsw %ymm1, %ymm0, %ymm1
1679 ; AVX512VLBW-NEXT: vpsubw %ymm2, %ymm1, %ymm1
1680 ; AVX512VLBW-NEXT: vpsrlw $1, %ymm1, %ymm1
1681 ; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
1682 ; AVX512VLBW-NEXT: vpsubw %ymm1, %ymm2, %ymm1 {%k1}
1683 ; AVX512VLBW-NEXT: vpaddw %ymm0, %ymm1, %ymm0
1684 ; AVX512VLBW-NEXT: retq
1685 %t3 = icmp sgt <16 x i16> %a1, %a2 ; signed
1686 %t4 = select <16 x i1> %t3, <16 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <16 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
1687 %t5 = select <16 x i1> %t3, <16 x i16> %a2, <16 x i16> %a1
1688 %t6 = select <16 x i1> %t3, <16 x i16> %a1, <16 x i16> %a2
1689 %t7 = sub <16 x i16> %t6, %t5
1690 %t16 = lshr <16 x i16> %t7, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
1691 %t9 = mul nsw <16 x i16> %t16, %t4 ; signed
1692 %a10 = add nsw <16 x i16> %t9, %a1 ; signed
1693 ret <16 x i16> %a10
1694 }
1696 define <16 x i16> @vec256_i16_unsigned_reg_reg(<16 x i16> %a1, <16 x i16> %a2) nounwind {
1697 ; AVX1-FALLBACK-LABEL: vec256_i16_unsigned_reg_reg:
1698 ; AVX1-FALLBACK: # %bb.0:
1699 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm1, %xmm2
1700 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm3
1701 ; AVX1-FALLBACK-NEXT: vpminuw %xmm2, %xmm3, %xmm4
1702 ; AVX1-FALLBACK-NEXT: vpcmpeqw %xmm4, %xmm3, %xmm5
1703 ; AVX1-FALLBACK-NEXT: vpcmpeqd %xmm8, %xmm8, %xmm8
1704 ; AVX1-FALLBACK-NEXT: vpxor %xmm5, %xmm8, %xmm5
1705 ; AVX1-FALLBACK-NEXT: vpminuw %xmm1, %xmm0, %xmm7
1706 ; AVX1-FALLBACK-NEXT: vpcmpeqw %xmm7, %xmm0, %xmm6
1707 ; AVX1-FALLBACK-NEXT: vpxor %xmm6, %xmm8, %xmm6
1708 ; AVX1-FALLBACK-NEXT: vpmaxuw %xmm2, %xmm3, %xmm2
1709 ; AVX1-FALLBACK-NEXT: vpmaxuw %xmm1, %xmm0, %xmm1
1710 ; AVX1-FALLBACK-NEXT: vpsubw %xmm7, %xmm1, %xmm1
1711 ; AVX1-FALLBACK-NEXT: vpsubw %xmm4, %xmm2, %xmm2
1712 ; AVX1-FALLBACK-NEXT: vpsrlw $1, %xmm2, %xmm2
1713 ; AVX1-FALLBACK-NEXT: vpsrlw $1, %xmm1, %xmm1
1714 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm4 = [1,1,1,1,1,1,1,1]
1715 ; AVX1-FALLBACK-NEXT: vpor %xmm4, %xmm6, %xmm6
1716 ; AVX1-FALLBACK-NEXT: vpmullw %xmm6, %xmm1, %xmm1
1717 ; AVX1-FALLBACK-NEXT: vpor %xmm4, %xmm5, %xmm4
1718 ; AVX1-FALLBACK-NEXT: vpmullw %xmm4, %xmm2, %xmm2
1719 ; AVX1-FALLBACK-NEXT: vpaddw %xmm3, %xmm2, %xmm2
1720 ; AVX1-FALLBACK-NEXT: vpaddw %xmm0, %xmm1, %xmm0
1721 ; AVX1-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
1722 ; AVX1-FALLBACK-NEXT: retq
1723 ;
1724 ; AVX2-LABEL: vec256_i16_unsigned_reg_reg:
1725 ; AVX2: # %bb.0:
1726 ; AVX2-NEXT: vpminuw %ymm1, %ymm0, %ymm2
1727 ; AVX2-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm3
1728 ; AVX2-NEXT: vpcmpeqd %ymm4, %ymm4, %ymm4
1729 ; AVX2-NEXT: vpxor %ymm4, %ymm3, %ymm3
1730 ; AVX2-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
1731 ; AVX2-NEXT: vpmaxuw %ymm1, %ymm0, %ymm1
1732 ; AVX2-NEXT: vpsubw %ymm2, %ymm1, %ymm1
1733 ; AVX2-NEXT: vpsrlw $1, %ymm1, %ymm1
1734 ; AVX2-NEXT: vpmullw %ymm3, %ymm1, %ymm1
1735 ; AVX2-NEXT: vpaddw %ymm0, %ymm1, %ymm0
1736 ; AVX2-NEXT: retq
1737 ;
1738 ; XOP-FALLBACK-LABEL: vec256_i16_unsigned_reg_reg:
1739 ; XOP-FALLBACK: # %bb.0:
1740 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm1, %xmm2
1741 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm3
1742 ; XOP-FALLBACK-NEXT: vpcomgtuw %xmm2, %xmm3, %xmm4
1743 ; XOP-FALLBACK-NEXT: vpcomgtuw %xmm1, %xmm0, %xmm5
1744 ; XOP-FALLBACK-NEXT: vpminuw %xmm1, %xmm0, %xmm6
1745 ; XOP-FALLBACK-NEXT: vpminuw %xmm2, %xmm3, %xmm7
1746 ; XOP-FALLBACK-NEXT: vpmaxuw %xmm1, %xmm0, %xmm1
1747 ; XOP-FALLBACK-NEXT: vpsubw %xmm6, %xmm1, %xmm1
1748 ; XOP-FALLBACK-NEXT: vpmaxuw %xmm2, %xmm3, %xmm2
1749 ; XOP-FALLBACK-NEXT: vpsubw %xmm7, %xmm2, %xmm2
1750 ; XOP-FALLBACK-NEXT: vpsrlw $1, %xmm1, %xmm1
1751 ; XOP-FALLBACK-NEXT: vpsrlw $1, %xmm2, %xmm2
1752 ; XOP-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm6 = [1,1,1,1,1,1,1,1]
1753 ; XOP-FALLBACK-NEXT: vpor %xmm6, %xmm5, %xmm5
1754 ; XOP-FALLBACK-NEXT: vpor %xmm6, %xmm4, %xmm4
1755 ; XOP-FALLBACK-NEXT: vpmacsww %xmm3, %xmm4, %xmm2, %xmm2
1756 ; XOP-FALLBACK-NEXT: vpmacsww %xmm0, %xmm5, %xmm1, %xmm0
1757 ; XOP-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
1758 ; XOP-FALLBACK-NEXT: retq
1759 ;
1760 ; XOPAVX1-LABEL: vec256_i16_unsigned_reg_reg:
1761 ; XOPAVX1: # %bb.0:
1762 ; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
1763 ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
1764 ; XOPAVX1-NEXT: vpcomgtuw %xmm2, %xmm3, %xmm4
1765 ; XOPAVX1-NEXT: vpcomgtuw %xmm1, %xmm0, %xmm5
1766 ; XOPAVX1-NEXT: vpminuw %xmm1, %xmm0, %xmm6
1767 ; XOPAVX1-NEXT: vpminuw %xmm2, %xmm3, %xmm7
1768 ; XOPAVX1-NEXT: vpmaxuw %xmm1, %xmm0, %xmm1
1769 ; XOPAVX1-NEXT: vpsubw %xmm6, %xmm1, %xmm1
1770 ; XOPAVX1-NEXT: vpmaxuw %xmm2, %xmm3, %xmm2
1771 ; XOPAVX1-NEXT: vpsubw %xmm7, %xmm2, %xmm2
1772 ; XOPAVX1-NEXT: vpsrlw $1, %xmm1, %xmm1
1773 ; XOPAVX1-NEXT: vpsrlw $1, %xmm2, %xmm2
1774 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm6 = [1,1,1,1,1,1,1,1]
1775 ; XOPAVX1-NEXT: vpor %xmm6, %xmm5, %xmm5
1776 ; XOPAVX1-NEXT: vpor %xmm6, %xmm4, %xmm4
1777 ; XOPAVX1-NEXT: vpmacsww %xmm3, %xmm4, %xmm2, %xmm2
1778 ; XOPAVX1-NEXT: vpmacsww %xmm0, %xmm5, %xmm1, %xmm0
1779 ; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
1780 ; XOPAVX1-NEXT: retq
1781 ;
1782 ; AVX512F-LABEL: vec256_i16_unsigned_reg_reg:
1783 ; AVX512F: # %bb.0:
1784 ; AVX512F-NEXT: vpminuw %ymm1, %ymm0, %ymm2
1785 ; AVX512F-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm3
1786 ; AVX512F-NEXT: vpternlogq $15, %zmm3, %zmm3, %zmm3
1787 ; AVX512F-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
1788 ; AVX512F-NEXT: vpmaxuw %ymm1, %ymm0, %ymm1
1789 ; AVX512F-NEXT: vpsubw %ymm2, %ymm1, %ymm1
1790 ; AVX512F-NEXT: vpsrlw $1, %ymm1, %ymm1
1791 ; AVX512F-NEXT: vpmullw %ymm3, %ymm1, %ymm1
1792 ; AVX512F-NEXT: vpaddw %ymm0, %ymm1, %ymm0
1793 ; AVX512F-NEXT: retq
1794 ;
1795 ; AVX512VL-FALLBACK-LABEL: vec256_i16_unsigned_reg_reg:
1796 ; AVX512VL-FALLBACK: # %bb.0:
1797 ; AVX512VL-FALLBACK-NEXT: vpminuw %ymm1, %ymm0, %ymm2
1798 ; AVX512VL-FALLBACK-NEXT: vpmaxuw %ymm1, %ymm0, %ymm1
1799 ; AVX512VL-FALLBACK-NEXT: vpsubw %ymm2, %ymm1, %ymm1
1800 ; AVX512VL-FALLBACK-NEXT: vpsrlw $1, %ymm1, %ymm1
1801 ; AVX512VL-FALLBACK-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm2
1802 ; AVX512VL-FALLBACK-NEXT: vpternlogq $15, %ymm2, %ymm2, %ymm2
1803 ; AVX512VL-FALLBACK-NEXT: vpxor %ymm2, %ymm1, %ymm1
1804 ; AVX512VL-FALLBACK-NEXT: vpsubw %ymm2, %ymm1, %ymm1
1805 ; AVX512VL-FALLBACK-NEXT: vpaddw %ymm0, %ymm1, %ymm0
1806 ; AVX512VL-FALLBACK-NEXT: retq
1807 ;
1808 ; AVX512BW-FALLBACK-LABEL: vec256_i16_unsigned_reg_reg:
1809 ; AVX512BW-FALLBACK: # %bb.0:
1810 ; AVX512BW-FALLBACK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
1811 ; AVX512BW-FALLBACK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
1812 ; AVX512BW-FALLBACK-NEXT: vpcmpnleuw %zmm1, %zmm0, %k1
1813 ; AVX512BW-FALLBACK-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
1814 ; AVX512BW-FALLBACK-NEXT: vmovdqa {{.*#+}} ymm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
1815 ; AVX512BW-FALLBACK-NEXT: vmovdqu16 %zmm2, %zmm3 {%k1}
1816 ; AVX512BW-FALLBACK-NEXT: vpminuw %ymm1, %ymm0, %ymm2
1817 ; AVX512BW-FALLBACK-NEXT: vpmaxuw %ymm1, %ymm0, %ymm1
1818 ; AVX512BW-FALLBACK-NEXT: vpsubw %ymm2, %ymm1, %ymm1
1819 ; AVX512BW-FALLBACK-NEXT: vpsrlw $1, %ymm1, %ymm1
1820 ; AVX512BW-FALLBACK-NEXT: vpmullw %ymm3, %ymm1, %ymm1
1821 ; AVX512BW-FALLBACK-NEXT: vpaddw %ymm0, %ymm1, %ymm0
1822 ; AVX512BW-FALLBACK-NEXT: retq
1823 ;
1824 ; AVX512VLBW-LABEL: vec256_i16_unsigned_reg_reg:
1825 ; AVX512VLBW: # %bb.0:
1826 ; AVX512VLBW-NEXT: vpcmpnleuw %ymm1, %ymm0, %k1
1827 ; AVX512VLBW-NEXT: vpminuw %ymm1, %ymm0, %ymm2
1828 ; AVX512VLBW-NEXT: vpmaxuw %ymm1, %ymm0, %ymm1
1829 ; AVX512VLBW-NEXT: vpsubw %ymm2, %ymm1, %ymm1
1830 ; AVX512VLBW-NEXT: vpsrlw $1, %ymm1, %ymm1
1831 ; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
1832 ; AVX512VLBW-NEXT: vpsubw %ymm1, %ymm2, %ymm1 {%k1}
1833 ; AVX512VLBW-NEXT: vpaddw %ymm0, %ymm1, %ymm0
1834 ; AVX512VLBW-NEXT: retq
1835 %t3 = icmp ugt <16 x i16> %a1, %a2
1836 %t4 = select <16 x i1> %t3, <16 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <16 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
1837 %t5 = select <16 x i1> %t3, <16 x i16> %a2, <16 x i16> %a1
1838 %t6 = select <16 x i1> %t3, <16 x i16> %a1, <16 x i16> %a2
1839 %t7 = sub <16 x i16> %t6, %t5
1840 %t16 = lshr <16 x i16> %t7, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
1841 %t9 = mul <16 x i16> %t16, %t4
1842 %a10 = add <16 x i16> %t9, %a1
1843 ret <16 x i16> %a10
1844 }
1846 ; Values are loaded. Only check signed case.
1848 define <16 x i16> @vec256_i16_signed_mem_reg(ptr %a1_addr, <16 x i16> %a2) nounwind {
1849 ; AVX1-FALLBACK-LABEL: vec256_i16_signed_mem_reg:
1850 ; AVX1-FALLBACK: # %bb.0:
1851 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm1
1852 ; AVX1-FALLBACK-NEXT: vmovdqa (%rdi), %xmm2
1853 ; AVX1-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm3
1854 ; AVX1-FALLBACK-NEXT: vpcmpgtw %xmm1, %xmm3, %xmm4
1855 ; AVX1-FALLBACK-NEXT: vpcmpgtw %xmm0, %xmm2, %xmm5
1856 ; AVX1-FALLBACK-NEXT: vpminsw %xmm1, %xmm3, %xmm6
1857 ; AVX1-FALLBACK-NEXT: vpminsw %xmm0, %xmm2, %xmm7
1858 ; AVX1-FALLBACK-NEXT: vpmaxsw %xmm1, %xmm3, %xmm1
1859 ; AVX1-FALLBACK-NEXT: vpsubw %xmm6, %xmm1, %xmm1
1860 ; AVX1-FALLBACK-NEXT: vpmaxsw %xmm0, %xmm2, %xmm0
1861 ; AVX1-FALLBACK-NEXT: vpsubw %xmm7, %xmm0, %xmm0
1862 ; AVX1-FALLBACK-NEXT: vpsrlw $1, %xmm1, %xmm1
1863 ; AVX1-FALLBACK-NEXT: vpsrlw $1, %xmm0, %xmm0
1864 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm6 = [1,1,1,1,1,1,1,1]
1865 ; AVX1-FALLBACK-NEXT: vpor %xmm6, %xmm5, %xmm5
1866 ; AVX1-FALLBACK-NEXT: vpmullw %xmm5, %xmm0, %xmm0
1867 ; AVX1-FALLBACK-NEXT: vpor %xmm6, %xmm4, %xmm4
1868 ; AVX1-FALLBACK-NEXT: vpmullw %xmm4, %xmm1, %xmm1
1869 ; AVX1-FALLBACK-NEXT: vpaddw %xmm3, %xmm1, %xmm1
1870 ; AVX1-FALLBACK-NEXT: vpaddw %xmm2, %xmm0, %xmm0
1871 ; AVX1-FALLBACK-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
1872 ; AVX1-FALLBACK-NEXT: retq
1873 ;
1874 ; AVX2-LABEL: vec256_i16_signed_mem_reg:
1875 ; AVX2: # %bb.0:
1876 ; AVX2-NEXT: vmovdqa (%rdi), %ymm1
1877 ; AVX2-NEXT: vpcmpgtw %ymm0, %ymm1, %ymm2
1878 ; AVX2-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
1879 ; AVX2-NEXT: vpminsw %ymm0, %ymm1, %ymm3
1880 ; AVX2-NEXT: vpmaxsw %ymm0, %ymm1, %ymm0
1881 ; AVX2-NEXT: vpsubw %ymm3, %ymm0, %ymm0
1882 ; AVX2-NEXT: vpsrlw $1, %ymm0, %ymm0
1883 ; AVX2-NEXT: vpmullw %ymm2, %ymm0, %ymm0
1884 ; AVX2-NEXT: vpaddw %ymm1, %ymm0, %ymm0
1885 ; AVX2-NEXT: retq
1886 ;
1887 ; XOP-FALLBACK-LABEL: vec256_i16_signed_mem_reg:
1888 ; XOP-FALLBACK: # %bb.0:
1889 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm1
1890 ; XOP-FALLBACK-NEXT: vmovdqa (%rdi), %xmm2
1891 ; XOP-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm3
1892 ; XOP-FALLBACK-NEXT: vpcomgtw %xmm1, %xmm3, %xmm4
1893 ; XOP-FALLBACK-NEXT: vpcomgtw %xmm0, %xmm2, %xmm5
1894 ; XOP-FALLBACK-NEXT: vpminsw %xmm0, %xmm2, %xmm6
1895 ; XOP-FALLBACK-NEXT: vpminsw %xmm1, %xmm3, %xmm7
1896 ; XOP-FALLBACK-NEXT: vpmaxsw %xmm0, %xmm2, %xmm0
1897 ; XOP-FALLBACK-NEXT: vpsubw %xmm6, %xmm0, %xmm0
1898 ; XOP-FALLBACK-NEXT: vpmaxsw %xmm1, %xmm3, %xmm1
1899 ; XOP-FALLBACK-NEXT: vpsubw %xmm7, %xmm1, %xmm1
1900 ; XOP-FALLBACK-NEXT: vpsrlw $1, %xmm0, %xmm0
1901 ; XOP-FALLBACK-NEXT: vpsrlw $1, %xmm1, %xmm1
1902 ; XOP-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm6 = [1,1,1,1,1,1,1,1]
1903 ; XOP-FALLBACK-NEXT: vpor %xmm6, %xmm5, %xmm5
1904 ; XOP-FALLBACK-NEXT: vpor %xmm6, %xmm4, %xmm4
1905 ; XOP-FALLBACK-NEXT: vpmacsww %xmm3, %xmm4, %xmm1, %xmm1
1906 ; XOP-FALLBACK-NEXT: vpmacsww %xmm2, %xmm5, %xmm0, %xmm0
1907 ; XOP-FALLBACK-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
1908 ; XOP-FALLBACK-NEXT: retq
1909 ;
1910 ; XOPAVX1-LABEL: vec256_i16_signed_mem_reg:
1911 ; XOPAVX1: # %bb.0:
1912 ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
1913 ; XOPAVX1-NEXT: vmovdqa (%rdi), %xmm2
1914 ; XOPAVX1-NEXT: vmovdqa 16(%rdi), %xmm3
1915 ; XOPAVX1-NEXT: vpcomgtw %xmm1, %xmm3, %xmm4
1916 ; XOPAVX1-NEXT: vpcomgtw %xmm0, %xmm2, %xmm5
1917 ; XOPAVX1-NEXT: vpminsw %xmm0, %xmm2, %xmm6
1918 ; XOPAVX1-NEXT: vpminsw %xmm1, %xmm3, %xmm7
1919 ; XOPAVX1-NEXT: vpmaxsw %xmm0, %xmm2, %xmm0
1920 ; XOPAVX1-NEXT: vpsubw %xmm6, %xmm0, %xmm0
1921 ; XOPAVX1-NEXT: vpmaxsw %xmm1, %xmm3, %xmm1
1922 ; XOPAVX1-NEXT: vpsubw %xmm7, %xmm1, %xmm1
1923 ; XOPAVX1-NEXT: vpsrlw $1, %xmm0, %xmm0
1924 ; XOPAVX1-NEXT: vpsrlw $1, %xmm1, %xmm1
1925 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm6 = [1,1,1,1,1,1,1,1]
1926 ; XOPAVX1-NEXT: vpor %xmm6, %xmm5, %xmm5
1927 ; XOPAVX1-NEXT: vpor %xmm6, %xmm4, %xmm4
1928 ; XOPAVX1-NEXT: vpmacsww %xmm3, %xmm4, %xmm1, %xmm1
1929 ; XOPAVX1-NEXT: vpmacsww %xmm2, %xmm5, %xmm0, %xmm0
1930 ; XOPAVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
1931 ; XOPAVX1-NEXT: retq
1932 ;
1933 ; AVX512F-LABEL: vec256_i16_signed_mem_reg:
1934 ; AVX512F: # %bb.0:
1935 ; AVX512F-NEXT: vmovdqa (%rdi), %ymm1
1936 ; AVX512F-NEXT: vpcmpgtw %ymm0, %ymm1, %ymm2
1937 ; AVX512F-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
1938 ; AVX512F-NEXT: vpminsw %ymm0, %ymm1, %ymm3
1939 ; AVX512F-NEXT: vpmaxsw %ymm0, %ymm1, %ymm0
1940 ; AVX512F-NEXT: vpsubw %ymm3, %ymm0, %ymm0
1941 ; AVX512F-NEXT: vpsrlw $1, %ymm0, %ymm0
1942 ; AVX512F-NEXT: vpmullw %ymm2, %ymm0, %ymm0
1943 ; AVX512F-NEXT: vpaddw %ymm1, %ymm0, %ymm0
1944 ; AVX512F-NEXT: retq
1945 ;
1946 ; AVX512VL-FALLBACK-LABEL: vec256_i16_signed_mem_reg:
1947 ; AVX512VL-FALLBACK: # %bb.0:
1948 ; AVX512VL-FALLBACK-NEXT: vmovdqa (%rdi), %ymm1
1949 ; AVX512VL-FALLBACK-NEXT: vpminsw %ymm0, %ymm1, %ymm2
1950 ; AVX512VL-FALLBACK-NEXT: vpmaxsw %ymm0, %ymm1, %ymm3
1951 ; AVX512VL-FALLBACK-NEXT: vpsubw %ymm2, %ymm3, %ymm2
1952 ; AVX512VL-FALLBACK-NEXT: vpsrlw $1, %ymm2, %ymm2
1953 ; AVX512VL-FALLBACK-NEXT: vpcmpgtw %ymm0, %ymm1, %ymm0
1954 ; AVX512VL-FALLBACK-NEXT: vpxor %ymm0, %ymm2, %ymm2
1955 ; AVX512VL-FALLBACK-NEXT: vpsubw %ymm0, %ymm2, %ymm0
1956 ; AVX512VL-FALLBACK-NEXT: vpaddw %ymm1, %ymm0, %ymm0
1957 ; AVX512VL-FALLBACK-NEXT: retq
1958 ;
1959 ; AVX512BW-FALLBACK-LABEL: vec256_i16_signed_mem_reg:
1960 ; AVX512BW-FALLBACK: # %bb.0:
1961 ; AVX512BW-FALLBACK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
1962 ; AVX512BW-FALLBACK-NEXT: vmovdqa (%rdi), %ymm1
1963 ; AVX512BW-FALLBACK-NEXT: vpcmpgtw %zmm0, %zmm1, %k1
1964 ; AVX512BW-FALLBACK-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
1965 ; AVX512BW-FALLBACK-NEXT: vmovdqa {{.*#+}} ymm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
1966 ; AVX512BW-FALLBACK-NEXT: vmovdqu16 %zmm2, %zmm3 {%k1}
1967 ; AVX512BW-FALLBACK-NEXT: vpminsw %ymm0, %ymm1, %ymm2
1968 ; AVX512BW-FALLBACK-NEXT: vpmaxsw %ymm0, %ymm1, %ymm0
1969 ; AVX512BW-FALLBACK-NEXT: vpsubw %ymm2, %ymm0, %ymm0
1970 ; AVX512BW-FALLBACK-NEXT: vpsrlw $1, %ymm0, %ymm0
1971 ; AVX512BW-FALLBACK-NEXT: vpmullw %ymm3, %ymm0, %ymm0
1972 ; AVX512BW-FALLBACK-NEXT: vpaddw %ymm1, %ymm0, %ymm0
1973 ; AVX512BW-FALLBACK-NEXT: retq
1974 ;
1975 ; AVX512VLBW-LABEL: vec256_i16_signed_mem_reg:
1976 ; AVX512VLBW: # %bb.0:
1977 ; AVX512VLBW-NEXT: vmovdqa (%rdi), %ymm1
1978 ; AVX512VLBW-NEXT: vpcmpgtw %ymm0, %ymm1, %k1
1979 ; AVX512VLBW-NEXT: vpminsw %ymm0, %ymm1, %ymm2
1980 ; AVX512VLBW-NEXT: vpmaxsw %ymm0, %ymm1, %ymm0
1981 ; AVX512VLBW-NEXT: vpsubw %ymm2, %ymm0, %ymm0
1982 ; AVX512VLBW-NEXT: vpsrlw $1, %ymm0, %ymm0
1983 ; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
1984 ; AVX512VLBW-NEXT: vpsubw %ymm0, %ymm2, %ymm0 {%k1}
1985 ; AVX512VLBW-NEXT: vpaddw %ymm1, %ymm0, %ymm0
1986 ; AVX512VLBW-NEXT: retq
1987 %a1 = load <16 x i16>, ptr %a1_addr
1988 %t3 = icmp sgt <16 x i16> %a1, %a2 ; signed
1989 %t4 = select <16 x i1> %t3, <16 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <16 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
1990 %t5 = select <16 x i1> %t3, <16 x i16> %a2, <16 x i16> %a1
1991 %t6 = select <16 x i1> %t3, <16 x i16> %a1, <16 x i16> %a2
1992 %t7 = sub <16 x i16> %t6, %t5
1993 %t16 = lshr <16 x i16> %t7, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
1994 %t9 = mul nsw <16 x i16> %t16, %t4 ; signed
1995 %a10 = add nsw <16 x i16> %t9, %a1 ; signed
1996 ret <16 x i16> %a10
1997 }
1999 define <16 x i16> @vec256_i16_signed_reg_mem(<16 x i16> %a1, ptr %a2_addr) nounwind {
2000 ; AVX1-FALLBACK-LABEL: vec256_i16_signed_reg_mem:
2001 ; AVX1-FALLBACK: # %bb.0:
2002 ; AVX1-FALLBACK-NEXT: vmovdqa (%rdi), %xmm1
2003 ; AVX1-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm2
2004 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm3
2005 ; AVX1-FALLBACK-NEXT: vpcmpgtw %xmm2, %xmm3, %xmm4
2006 ; AVX1-FALLBACK-NEXT: vpcmpgtw %xmm1, %xmm0, %xmm5
2007 ; AVX1-FALLBACK-NEXT: vpminsw %xmm2, %xmm3, %xmm6
2008 ; AVX1-FALLBACK-NEXT: vpminsw %xmm1, %xmm0, %xmm7
2009 ; AVX1-FALLBACK-NEXT: vpmaxsw %xmm2, %xmm3, %xmm2
2010 ; AVX1-FALLBACK-NEXT: vpsubw %xmm6, %xmm2, %xmm2
2011 ; AVX1-FALLBACK-NEXT: vpmaxsw %xmm1, %xmm0, %xmm1
2012 ; AVX1-FALLBACK-NEXT: vpsubw %xmm7, %xmm1, %xmm1
2013 ; AVX1-FALLBACK-NEXT: vpsrlw $1, %xmm2, %xmm2
2014 ; AVX1-FALLBACK-NEXT: vpsrlw $1, %xmm1, %xmm1
2015 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm6 = [1,1,1,1,1,1,1,1]
2016 ; AVX1-FALLBACK-NEXT: vpor %xmm6, %xmm5, %xmm5
2017 ; AVX1-FALLBACK-NEXT: vpmullw %xmm5, %xmm1, %xmm1
2018 ; AVX1-FALLBACK-NEXT: vpor %xmm6, %xmm4, %xmm4
2019 ; AVX1-FALLBACK-NEXT: vpmullw %xmm4, %xmm2, %xmm2
2020 ; AVX1-FALLBACK-NEXT: vpaddw %xmm3, %xmm2, %xmm2
2021 ; AVX1-FALLBACK-NEXT: vpaddw %xmm0, %xmm1, %xmm0
2022 ; AVX1-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
2023 ; AVX1-FALLBACK-NEXT: retq
2024 ;
2025 ; AVX2-LABEL: vec256_i16_signed_reg_mem:
2026 ; AVX2: # %bb.0:
2027 ; AVX2-NEXT: vmovdqa (%rdi), %ymm1
2028 ; AVX2-NEXT: vpcmpgtw %ymm1, %ymm0, %ymm2
2029 ; AVX2-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
2030 ; AVX2-NEXT: vpminsw %ymm1, %ymm0, %ymm3
2031 ; AVX2-NEXT: vpmaxsw %ymm1, %ymm0, %ymm1
2032 ; AVX2-NEXT: vpsubw %ymm3, %ymm1, %ymm1
2033 ; AVX2-NEXT: vpsrlw $1, %ymm1, %ymm1
2034 ; AVX2-NEXT: vpmullw %ymm2, %ymm1, %ymm1
2035 ; AVX2-NEXT: vpaddw %ymm0, %ymm1, %ymm0
2036 ; AVX2-NEXT: retq
2037 ;
2038 ; XOP-FALLBACK-LABEL: vec256_i16_signed_reg_mem:
2039 ; XOP-FALLBACK: # %bb.0:
2040 ; XOP-FALLBACK-NEXT: vmovdqa (%rdi), %xmm1
2041 ; XOP-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm2
2042 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm3
2043 ; XOP-FALLBACK-NEXT: vpcomgtw %xmm2, %xmm3, %xmm4
2044 ; XOP-FALLBACK-NEXT: vpcomgtw %xmm1, %xmm0, %xmm5
2045 ; XOP-FALLBACK-NEXT: vpminsw %xmm1, %xmm0, %xmm6
2046 ; XOP-FALLBACK-NEXT: vpminsw %xmm2, %xmm3, %xmm7
2047 ; XOP-FALLBACK-NEXT: vpmaxsw %xmm1, %xmm0, %xmm1
2048 ; XOP-FALLBACK-NEXT: vpsubw %xmm6, %xmm1, %xmm1
2049 ; XOP-FALLBACK-NEXT: vpmaxsw %xmm2, %xmm3, %xmm2
2050 ; XOP-FALLBACK-NEXT: vpsubw %xmm7, %xmm2, %xmm2
2051 ; XOP-FALLBACK-NEXT: vpsrlw $1, %xmm1, %xmm1
2052 ; XOP-FALLBACK-NEXT: vpsrlw $1, %xmm2, %xmm2
2053 ; XOP-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm6 = [1,1,1,1,1,1,1,1]
2054 ; XOP-FALLBACK-NEXT: vpor %xmm6, %xmm5, %xmm5
2055 ; XOP-FALLBACK-NEXT: vpor %xmm6, %xmm4, %xmm4
2056 ; XOP-FALLBACK-NEXT: vpmacsww %xmm3, %xmm4, %xmm2, %xmm2
2057 ; XOP-FALLBACK-NEXT: vpmacsww %xmm0, %xmm5, %xmm1, %xmm0
2058 ; XOP-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
2059 ; XOP-FALLBACK-NEXT: retq
2060 ;
2061 ; XOPAVX1-LABEL: vec256_i16_signed_reg_mem:
2062 ; XOPAVX1: # %bb.0:
2063 ; XOPAVX1-NEXT: vmovdqa (%rdi), %xmm1
2064 ; XOPAVX1-NEXT: vmovdqa 16(%rdi), %xmm2
2065 ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
2066 ; XOPAVX1-NEXT: vpcomgtw %xmm2, %xmm3, %xmm4
2067 ; XOPAVX1-NEXT: vpcomgtw %xmm1, %xmm0, %xmm5
2068 ; XOPAVX1-NEXT: vpminsw %xmm1, %xmm0, %xmm6
2069 ; XOPAVX1-NEXT: vpminsw %xmm2, %xmm3, %xmm7
2070 ; XOPAVX1-NEXT: vpmaxsw %xmm1, %xmm0, %xmm1
2071 ; XOPAVX1-NEXT: vpsubw %xmm6, %xmm1, %xmm1
2072 ; XOPAVX1-NEXT: vpmaxsw %xmm2, %xmm3, %xmm2
2073 ; XOPAVX1-NEXT: vpsubw %xmm7, %xmm2, %xmm2
2074 ; XOPAVX1-NEXT: vpsrlw $1, %xmm1, %xmm1
2075 ; XOPAVX1-NEXT: vpsrlw $1, %xmm2, %xmm2
2076 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm6 = [1,1,1,1,1,1,1,1]
2077 ; XOPAVX1-NEXT: vpor %xmm6, %xmm5, %xmm5
2078 ; XOPAVX1-NEXT: vpor %xmm6, %xmm4, %xmm4
2079 ; XOPAVX1-NEXT: vpmacsww %xmm3, %xmm4, %xmm2, %xmm2
2080 ; XOPAVX1-NEXT: vpmacsww %xmm0, %xmm5, %xmm1, %xmm0
2081 ; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
2082 ; XOPAVX1-NEXT: retq
2083 ;
2084 ; AVX512F-LABEL: vec256_i16_signed_reg_mem:
2085 ; AVX512F: # %bb.0:
2086 ; AVX512F-NEXT: vmovdqa (%rdi), %ymm1
2087 ; AVX512F-NEXT: vpcmpgtw %ymm1, %ymm0, %ymm2
2088 ; AVX512F-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
2089 ; AVX512F-NEXT: vpminsw %ymm1, %ymm0, %ymm3
2090 ; AVX512F-NEXT: vpmaxsw %ymm1, %ymm0, %ymm1
2091 ; AVX512F-NEXT: vpsubw %ymm3, %ymm1, %ymm1
2092 ; AVX512F-NEXT: vpsrlw $1, %ymm1, %ymm1
2093 ; AVX512F-NEXT: vpmullw %ymm2, %ymm1, %ymm1
2094 ; AVX512F-NEXT: vpaddw %ymm0, %ymm1, %ymm0
2095 ; AVX512F-NEXT: retq
2096 ;
2097 ; AVX512VL-FALLBACK-LABEL: vec256_i16_signed_reg_mem:
2098 ; AVX512VL-FALLBACK: # %bb.0:
2099 ; AVX512VL-FALLBACK-NEXT: vmovdqa (%rdi), %ymm1
2100 ; AVX512VL-FALLBACK-NEXT: vpminsw %ymm1, %ymm0, %ymm2
2101 ; AVX512VL-FALLBACK-NEXT: vpmaxsw %ymm1, %ymm0, %ymm3
2102 ; AVX512VL-FALLBACK-NEXT: vpsubw %ymm2, %ymm3, %ymm2
2103 ; AVX512VL-FALLBACK-NEXT: vpsrlw $1, %ymm2, %ymm2
2104 ; AVX512VL-FALLBACK-NEXT: vpcmpgtw %ymm1, %ymm0, %ymm1
2105 ; AVX512VL-FALLBACK-NEXT: vpxor %ymm1, %ymm2, %ymm2
2106 ; AVX512VL-FALLBACK-NEXT: vpsubw %ymm1, %ymm2, %ymm1
2107 ; AVX512VL-FALLBACK-NEXT: vpaddw %ymm0, %ymm1, %ymm0
2108 ; AVX512VL-FALLBACK-NEXT: retq
2109 ;
2110 ; AVX512BW-FALLBACK-LABEL: vec256_i16_signed_reg_mem:
2111 ; AVX512BW-FALLBACK: # %bb.0:
2112 ; AVX512BW-FALLBACK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
2113 ; AVX512BW-FALLBACK-NEXT: vmovdqa (%rdi), %ymm1
2114 ; AVX512BW-FALLBACK-NEXT: vpcmpgtw %zmm1, %zmm0, %k1
2115 ; AVX512BW-FALLBACK-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
2116 ; AVX512BW-FALLBACK-NEXT: vmovdqa {{.*#+}} ymm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
2117 ; AVX512BW-FALLBACK-NEXT: vmovdqu16 %zmm2, %zmm3 {%k1}
2118 ; AVX512BW-FALLBACK-NEXT: vpminsw %ymm1, %ymm0, %ymm2
2119 ; AVX512BW-FALLBACK-NEXT: vpmaxsw %ymm1, %ymm0, %ymm1
2120 ; AVX512BW-FALLBACK-NEXT: vpsubw %ymm2, %ymm1, %ymm1
2121 ; AVX512BW-FALLBACK-NEXT: vpsrlw $1, %ymm1, %ymm1
2122 ; AVX512BW-FALLBACK-NEXT: vpmullw %ymm3, %ymm1, %ymm1
2123 ; AVX512BW-FALLBACK-NEXT: vpaddw %ymm0, %ymm1, %ymm0
2124 ; AVX512BW-FALLBACK-NEXT: retq
2125 ;
2126 ; AVX512VLBW-LABEL: vec256_i16_signed_reg_mem:
2127 ; AVX512VLBW: # %bb.0:
2128 ; AVX512VLBW-NEXT: vmovdqa (%rdi), %ymm1
2129 ; AVX512VLBW-NEXT: vpcmpgtw %ymm1, %ymm0, %k1
2130 ; AVX512VLBW-NEXT: vpminsw %ymm1, %ymm0, %ymm2
2131 ; AVX512VLBW-NEXT: vpmaxsw %ymm1, %ymm0, %ymm1
2132 ; AVX512VLBW-NEXT: vpsubw %ymm2, %ymm1, %ymm1
2133 ; AVX512VLBW-NEXT: vpsrlw $1, %ymm1, %ymm1
2134 ; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
2135 ; AVX512VLBW-NEXT: vpsubw %ymm1, %ymm2, %ymm1 {%k1}
2136 ; AVX512VLBW-NEXT: vpaddw %ymm0, %ymm1, %ymm0
2137 ; AVX512VLBW-NEXT: retq
2138 %a2 = load <16 x i16>, ptr %a2_addr
2139 %t3 = icmp sgt <16 x i16> %a1, %a2 ; signed
2140 %t4 = select <16 x i1> %t3, <16 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <16 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
2141 %t5 = select <16 x i1> %t3, <16 x i16> %a2, <16 x i16> %a1
2142 %t6 = select <16 x i1> %t3, <16 x i16> %a1, <16 x i16> %a2
2143 %t7 = sub <16 x i16> %t6, %t5
2144 %t16 = lshr <16 x i16> %t7, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
2145 %t9 = mul nsw <16 x i16> %t16, %t4 ; signed
2146 %a10 = add nsw <16 x i16> %t9, %a1 ; signed
2147 ret <16 x i16> %a10
2148 }
2150 define <16 x i16> @vec256_i16_signed_mem_mem(ptr %a1_addr, ptr %a2_addr) nounwind {
2151 ; AVX1-FALLBACK-LABEL: vec256_i16_signed_mem_mem:
2152 ; AVX1-FALLBACK: # %bb.0:
2153 ; AVX1-FALLBACK-NEXT: vmovdqa (%rsi), %xmm0
2154 ; AVX1-FALLBACK-NEXT: vmovdqa 16(%rsi), %xmm1
2155 ; AVX1-FALLBACK-NEXT: vmovdqa (%rdi), %xmm2
2156 ; AVX1-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm3
2157 ; AVX1-FALLBACK-NEXT: vpcmpgtw %xmm1, %xmm3, %xmm4
2158 ; AVX1-FALLBACK-NEXT: vpcmpgtw %xmm0, %xmm2, %xmm5
2159 ; AVX1-FALLBACK-NEXT: vpminsw %xmm1, %xmm3, %xmm6
2160 ; AVX1-FALLBACK-NEXT: vpminsw %xmm0, %xmm2, %xmm7
2161 ; AVX1-FALLBACK-NEXT: vpmaxsw %xmm1, %xmm3, %xmm1
2162 ; AVX1-FALLBACK-NEXT: vpsubw %xmm6, %xmm1, %xmm1
2163 ; AVX1-FALLBACK-NEXT: vpmaxsw %xmm0, %xmm2, %xmm0
2164 ; AVX1-FALLBACK-NEXT: vpsubw %xmm7, %xmm0, %xmm0
2165 ; AVX1-FALLBACK-NEXT: vpsrlw $1, %xmm1, %xmm1
2166 ; AVX1-FALLBACK-NEXT: vpsrlw $1, %xmm0, %xmm0
2167 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm6 = [1,1,1,1,1,1,1,1]
2168 ; AVX1-FALLBACK-NEXT: vpor %xmm6, %xmm5, %xmm5
2169 ; AVX1-FALLBACK-NEXT: vpmullw %xmm5, %xmm0, %xmm0
2170 ; AVX1-FALLBACK-NEXT: vpor %xmm6, %xmm4, %xmm4
2171 ; AVX1-FALLBACK-NEXT: vpmullw %xmm4, %xmm1, %xmm1
2172 ; AVX1-FALLBACK-NEXT: vpaddw %xmm3, %xmm1, %xmm1
2173 ; AVX1-FALLBACK-NEXT: vpaddw %xmm2, %xmm0, %xmm0
2174 ; AVX1-FALLBACK-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
2175 ; AVX1-FALLBACK-NEXT: retq
2176 ;
2177 ; AVX2-LABEL: vec256_i16_signed_mem_mem:
2178 ; AVX2: # %bb.0:
2179 ; AVX2-NEXT: vmovdqa (%rdi), %ymm0
2180 ; AVX2-NEXT: vmovdqa (%rsi), %ymm1
2181 ; AVX2-NEXT: vpcmpgtw %ymm1, %ymm0, %ymm2
2182 ; AVX2-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
2183 ; AVX2-NEXT: vpminsw %ymm1, %ymm0, %ymm3
2184 ; AVX2-NEXT: vpmaxsw %ymm1, %ymm0, %ymm1
2185 ; AVX2-NEXT: vpsubw %ymm3, %ymm1, %ymm1
2186 ; AVX2-NEXT: vpsrlw $1, %ymm1, %ymm1
2187 ; AVX2-NEXT: vpmullw %ymm2, %ymm1, %ymm1
2188 ; AVX2-NEXT: vpaddw %ymm0, %ymm1, %ymm0
2189 ; AVX2-NEXT: retq
2190 ;
2191 ; XOP-FALLBACK-LABEL: vec256_i16_signed_mem_mem:
2192 ; XOP-FALLBACK: # %bb.0:
2193 ; XOP-FALLBACK-NEXT: vmovdqa (%rsi), %xmm0
2194 ; XOP-FALLBACK-NEXT: vmovdqa 16(%rsi), %xmm1
2195 ; XOP-FALLBACK-NEXT: vmovdqa (%rdi), %xmm2
2196 ; XOP-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm3
2197 ; XOP-FALLBACK-NEXT: vpcomgtw %xmm1, %xmm3, %xmm4
2198 ; XOP-FALLBACK-NEXT: vpcomgtw %xmm0, %xmm2, %xmm5
2199 ; XOP-FALLBACK-NEXT: vpminsw %xmm0, %xmm2, %xmm6
2200 ; XOP-FALLBACK-NEXT: vpminsw %xmm1, %xmm3, %xmm7
2201 ; XOP-FALLBACK-NEXT: vpmaxsw %xmm0, %xmm2, %xmm0
2202 ; XOP-FALLBACK-NEXT: vpsubw %xmm6, %xmm0, %xmm0
2203 ; XOP-FALLBACK-NEXT: vpmaxsw %xmm1, %xmm3, %xmm1
2204 ; XOP-FALLBACK-NEXT: vpsubw %xmm7, %xmm1, %xmm1
2205 ; XOP-FALLBACK-NEXT: vpsrlw $1, %xmm0, %xmm0
2206 ; XOP-FALLBACK-NEXT: vpsrlw $1, %xmm1, %xmm1
2207 ; XOP-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm6 = [1,1,1,1,1,1,1,1]
2208 ; XOP-FALLBACK-NEXT: vpor %xmm6, %xmm5, %xmm5
2209 ; XOP-FALLBACK-NEXT: vpor %xmm6, %xmm4, %xmm4
2210 ; XOP-FALLBACK-NEXT: vpmacsww %xmm3, %xmm4, %xmm1, %xmm1
2211 ; XOP-FALLBACK-NEXT: vpmacsww %xmm2, %xmm5, %xmm0, %xmm0
2212 ; XOP-FALLBACK-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
2213 ; XOP-FALLBACK-NEXT: retq
2214 ;
2215 ; XOPAVX1-LABEL: vec256_i16_signed_mem_mem:
2216 ; XOPAVX1: # %bb.0:
2217 ; XOPAVX1-NEXT: vmovdqa (%rsi), %xmm0
2218 ; XOPAVX1-NEXT: vmovdqa 16(%rsi), %xmm1
2219 ; XOPAVX1-NEXT: vmovdqa (%rdi), %xmm2
2220 ; XOPAVX1-NEXT: vmovdqa 16(%rdi), %xmm3
2221 ; XOPAVX1-NEXT: vpcomgtw %xmm1, %xmm3, %xmm4
2222 ; XOPAVX1-NEXT: vpcomgtw %xmm0, %xmm2, %xmm5
2223 ; XOPAVX1-NEXT: vpminsw %xmm0, %xmm2, %xmm6
2224 ; XOPAVX1-NEXT: vpminsw %xmm1, %xmm3, %xmm7
2225 ; XOPAVX1-NEXT: vpmaxsw %xmm0, %xmm2, %xmm0
2226 ; XOPAVX1-NEXT: vpsubw %xmm6, %xmm0, %xmm0
2227 ; XOPAVX1-NEXT: vpmaxsw %xmm1, %xmm3, %xmm1
2228 ; XOPAVX1-NEXT: vpsubw %xmm7, %xmm1, %xmm1
2229 ; XOPAVX1-NEXT: vpsrlw $1, %xmm0, %xmm0
2230 ; XOPAVX1-NEXT: vpsrlw $1, %xmm1, %xmm1
2231 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm6 = [1,1,1,1,1,1,1,1]
2232 ; XOPAVX1-NEXT: vpor %xmm6, %xmm5, %xmm5
2233 ; XOPAVX1-NEXT: vpor %xmm6, %xmm4, %xmm4
2234 ; XOPAVX1-NEXT: vpmacsww %xmm3, %xmm4, %xmm1, %xmm1
2235 ; XOPAVX1-NEXT: vpmacsww %xmm2, %xmm5, %xmm0, %xmm0
2236 ; XOPAVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
2237 ; XOPAVX1-NEXT: retq
2238 ;
2239 ; AVX512F-LABEL: vec256_i16_signed_mem_mem:
2240 ; AVX512F: # %bb.0:
2241 ; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
2242 ; AVX512F-NEXT: vmovdqa (%rsi), %ymm1
2243 ; AVX512F-NEXT: vpcmpgtw %ymm1, %ymm0, %ymm2
2244 ; AVX512F-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
2245 ; AVX512F-NEXT: vpminsw %ymm1, %ymm0, %ymm3
2246 ; AVX512F-NEXT: vpmaxsw %ymm1, %ymm0, %ymm1
2247 ; AVX512F-NEXT: vpsubw %ymm3, %ymm1, %ymm1
2248 ; AVX512F-NEXT: vpsrlw $1, %ymm1, %ymm1
2249 ; AVX512F-NEXT: vpmullw %ymm2, %ymm1, %ymm1
2250 ; AVX512F-NEXT: vpaddw %ymm0, %ymm1, %ymm0
2251 ; AVX512F-NEXT: retq
2252 ;
2253 ; AVX512VL-FALLBACK-LABEL: vec256_i16_signed_mem_mem:
2254 ; AVX512VL-FALLBACK: # %bb.0:
2255 ; AVX512VL-FALLBACK-NEXT: vmovdqa (%rdi), %ymm0
2256 ; AVX512VL-FALLBACK-NEXT: vmovdqa (%rsi), %ymm1
2257 ; AVX512VL-FALLBACK-NEXT: vpminsw %ymm1, %ymm0, %ymm2
2258 ; AVX512VL-FALLBACK-NEXT: vpmaxsw %ymm1, %ymm0, %ymm3
2259 ; AVX512VL-FALLBACK-NEXT: vpsubw %ymm2, %ymm3, %ymm2
2260 ; AVX512VL-FALLBACK-NEXT: vpsrlw $1, %ymm2, %ymm2
2261 ; AVX512VL-FALLBACK-NEXT: vpcmpgtw %ymm1, %ymm0, %ymm1
2262 ; AVX512VL-FALLBACK-NEXT: vpxor %ymm1, %ymm2, %ymm2
2263 ; AVX512VL-FALLBACK-NEXT: vpsubw %ymm1, %ymm2, %ymm1
2264 ; AVX512VL-FALLBACK-NEXT: vpaddw %ymm0, %ymm1, %ymm0
2265 ; AVX512VL-FALLBACK-NEXT: retq
2266 ;
2267 ; AVX512BW-FALLBACK-LABEL: vec256_i16_signed_mem_mem:
2268 ; AVX512BW-FALLBACK: # %bb.0:
2269 ; AVX512BW-FALLBACK-NEXT: vmovdqa (%rdi), %ymm0
2270 ; AVX512BW-FALLBACK-NEXT: vmovdqa (%rsi), %ymm1
2271 ; AVX512BW-FALLBACK-NEXT: vpcmpgtw %zmm1, %zmm0, %k1
2272 ; AVX512BW-FALLBACK-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
2273 ; AVX512BW-FALLBACK-NEXT: vmovdqa {{.*#+}} ymm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
2274 ; AVX512BW-FALLBACK-NEXT: vmovdqu16 %zmm2, %zmm3 {%k1}
2275 ; AVX512BW-FALLBACK-NEXT: vpminsw %ymm1, %ymm0, %ymm2
2276 ; AVX512BW-FALLBACK-NEXT: vpmaxsw %ymm1, %ymm0, %ymm1
2277 ; AVX512BW-FALLBACK-NEXT: vpsubw %ymm2, %ymm1, %ymm1
2278 ; AVX512BW-FALLBACK-NEXT: vpsrlw $1, %ymm1, %ymm1
2279 ; AVX512BW-FALLBACK-NEXT: vpmullw %ymm3, %ymm1, %ymm1
2280 ; AVX512BW-FALLBACK-NEXT: vpaddw %ymm0, %ymm1, %ymm0
2281 ; AVX512BW-FALLBACK-NEXT: retq
2282 ;
2283 ; AVX512VLBW-LABEL: vec256_i16_signed_mem_mem:
2284 ; AVX512VLBW: # %bb.0:
2285 ; AVX512VLBW-NEXT: vmovdqa (%rdi), %ymm0
2286 ; AVX512VLBW-NEXT: vmovdqa (%rsi), %ymm1
2287 ; AVX512VLBW-NEXT: vpcmpgtw %ymm1, %ymm0, %k1
2288 ; AVX512VLBW-NEXT: vpminsw %ymm1, %ymm0, %ymm2
2289 ; AVX512VLBW-NEXT: vpmaxsw %ymm1, %ymm0, %ymm1
2290 ; AVX512VLBW-NEXT: vpsubw %ymm2, %ymm1, %ymm1
2291 ; AVX512VLBW-NEXT: vpsrlw $1, %ymm1, %ymm1
2292 ; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
2293 ; AVX512VLBW-NEXT: vpsubw %ymm1, %ymm2, %ymm1 {%k1}
2294 ; AVX512VLBW-NEXT: vpaddw %ymm0, %ymm1, %ymm0
2295 ; AVX512VLBW-NEXT: retq
2296 %a1 = load <16 x i16>, ptr %a1_addr
2297 %a2 = load <16 x i16>, ptr %a2_addr
2298 %t3 = icmp sgt <16 x i16> %a1, %a2 ; signed
2299 %t4 = select <16 x i1> %t3, <16 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <16 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
2300 %t5 = select <16 x i1> %t3, <16 x i16> %a2, <16 x i16> %a1
2301 %t6 = select <16 x i1> %t3, <16 x i16> %a1, <16 x i16> %a2
2302 %t7 = sub <16 x i16> %t6, %t5
2303 %t16 = lshr <16 x i16> %t7, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
2304 %t9 = mul nsw <16 x i16> %t16, %t4 ; signed
2305 %a10 = add nsw <16 x i16> %t9, %a1 ; signed
2306 ret <16 x i16> %a10
2307 }
2309 ; ---------------------------------------------------------------------------- ;
2310 ; 8-bit width. 256 / 8 = 32 elts.
2311 ; ---------------------------------------------------------------------------- ;
2313 ; Values come from regs
2315 define <32 x i8> @vec256_i8_signed_reg_reg(<32 x i8> %a1, <32 x i8> %a2) nounwind {
2316 ; AVX1-FALLBACK-LABEL: vec256_i8_signed_reg_reg:
2317 ; AVX1-FALLBACK: # %bb.0:
2318 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm1, %xmm3
2319 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm2
2320 ; AVX1-FALLBACK-NEXT: vpcmpgtb %xmm3, %xmm2, %xmm8
2321 ; AVX1-FALLBACK-NEXT: vpcmpgtb %xmm1, %xmm0, %xmm5
2322 ; AVX1-FALLBACK-NEXT: vpminsb %xmm3, %xmm2, %xmm6
2323 ; AVX1-FALLBACK-NEXT: vpminsb %xmm1, %xmm0, %xmm7
2324 ; AVX1-FALLBACK-NEXT: vpmaxsb %xmm3, %xmm2, %xmm3
2325 ; AVX1-FALLBACK-NEXT: vpsubb %xmm6, %xmm3, %xmm3
2326 ; AVX1-FALLBACK-NEXT: vpmaxsb %xmm1, %xmm0, %xmm1
2327 ; AVX1-FALLBACK-NEXT: vpsubb %xmm7, %xmm1, %xmm1
2328 ; AVX1-FALLBACK-NEXT: vpsrlw $1, %xmm3, %xmm3
2329 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
2330 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm3, %xmm3
2331 ; AVX1-FALLBACK-NEXT: vpsrlw $1, %xmm1, %xmm1
2332 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm1, %xmm1
2333 ; AVX1-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2334 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
2335 ; AVX1-FALLBACK-NEXT: vpor %xmm7, %xmm5, %xmm5
2336 ; AVX1-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2337 ; AVX1-FALLBACK-NEXT: vpmullw %xmm4, %xmm6, %xmm4
2338 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]
2339 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm4, %xmm4
2340 ; AVX1-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2341 ; AVX1-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2342 ; AVX1-FALLBACK-NEXT: vpmullw %xmm5, %xmm1, %xmm1
2343 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm1, %xmm1
2344 ; AVX1-FALLBACK-NEXT: vpackuswb %xmm4, %xmm1, %xmm1
2345 ; AVX1-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2346 ; AVX1-FALLBACK-NEXT: vpor %xmm7, %xmm8, %xmm5
2347 ; AVX1-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2348 ; AVX1-FALLBACK-NEXT: vpmullw %xmm7, %xmm4, %xmm4
2349 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm4, %xmm4
2350 ; AVX1-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
2351 ; AVX1-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2352 ; AVX1-FALLBACK-NEXT: vpmullw %xmm5, %xmm3, %xmm3
2353 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm3, %xmm3
2354 ; AVX1-FALLBACK-NEXT: vpackuswb %xmm4, %xmm3, %xmm3
2355 ; AVX1-FALLBACK-NEXT: vpaddb %xmm2, %xmm3, %xmm2
2356 ; AVX1-FALLBACK-NEXT: vpaddb %xmm0, %xmm1, %xmm0
2357 ; AVX1-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
2358 ; AVX1-FALLBACK-NEXT: retq
2359 ;
2360 ; AVX2-LABEL: vec256_i8_signed_reg_reg:
2361 ; AVX2: # %bb.0:
2362 ; AVX2-NEXT: vpcmpgtb %ymm1, %ymm0, %ymm2
2363 ; AVX2-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
2364 ; AVX2-NEXT: vpminsb %ymm1, %ymm0, %ymm3
2365 ; AVX2-NEXT: vpmaxsb %ymm1, %ymm0, %ymm1
2366 ; AVX2-NEXT: vpsubb %ymm3, %ymm1, %ymm1
2367 ; AVX2-NEXT: vpsrlw $1, %ymm1, %ymm1
2368 ; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
2369 ; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2370 ; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2371 ; AVX2-NEXT: vpmullw %ymm4, %ymm3, %ymm3
2372 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2373 ; AVX2-NEXT: vpand %ymm4, %ymm3, %ymm3
2374 ; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2375 ; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2376 ; AVX2-NEXT: vpmullw %ymm2, %ymm1, %ymm1
2377 ; AVX2-NEXT: vpand %ymm4, %ymm1, %ymm1
2378 ; AVX2-NEXT: vpackuswb %ymm3, %ymm1, %ymm1
2379 ; AVX2-NEXT: vpaddb %ymm0, %ymm1, %ymm0
2380 ; AVX2-NEXT: retq
2381 ;
2382 ; XOP-FALLBACK-LABEL: vec256_i8_signed_reg_reg:
2383 ; XOP-FALLBACK: # %bb.0:
2384 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm1, %xmm2
2385 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm3
2386 ; XOP-FALLBACK-NEXT: vpcomgtb %xmm2, %xmm3, %xmm8
2387 ; XOP-FALLBACK-NEXT: vpcomgtb %xmm1, %xmm0, %xmm5
2388 ; XOP-FALLBACK-NEXT: vpminsb %xmm2, %xmm3, %xmm6
2389 ; XOP-FALLBACK-NEXT: vpminsb %xmm1, %xmm0, %xmm7
2390 ; XOP-FALLBACK-NEXT: vpmaxsb %xmm2, %xmm3, %xmm2
2391 ; XOP-FALLBACK-NEXT: vpsubb %xmm6, %xmm2, %xmm2
2392 ; XOP-FALLBACK-NEXT: vpmaxsb %xmm1, %xmm0, %xmm1
2393 ; XOP-FALLBACK-NEXT: vpsubb %xmm7, %xmm1, %xmm1
2394 ; XOP-FALLBACK-NEXT: vpcmpeqd %xmm6, %xmm6, %xmm6
2395 ; XOP-FALLBACK-NEXT: vpshlb %xmm6, %xmm2, %xmm2
2396 ; XOP-FALLBACK-NEXT: vpshlb %xmm6, %xmm1, %xmm1
2397 ; XOP-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2398 ; XOP-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
2399 ; XOP-FALLBACK-NEXT: vpor %xmm7, %xmm5, %xmm5
2400 ; XOP-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2401 ; XOP-FALLBACK-NEXT: vpmullw %xmm4, %xmm6, %xmm4
2402 ; XOP-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2403 ; XOP-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2404 ; XOP-FALLBACK-NEXT: vpmullw %xmm5, %xmm1, %xmm1
2405 ; XOP-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm5 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30]
2406 ; XOP-FALLBACK-NEXT: vpperm %xmm5, %xmm4, %xmm1, %xmm1
2407 ; XOP-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2408 ; XOP-FALLBACK-NEXT: vpor %xmm7, %xmm8, %xmm6
2409 ; XOP-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2410 ; XOP-FALLBACK-NEXT: vpmullw %xmm7, %xmm4, %xmm4
2411 ; XOP-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2412 ; XOP-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
2413 ; XOP-FALLBACK-NEXT: vpmullw %xmm6, %xmm2, %xmm2
2414 ; XOP-FALLBACK-NEXT: vpperm %xmm5, %xmm4, %xmm2, %xmm2
2415 ; XOP-FALLBACK-NEXT: vpaddb %xmm3, %xmm2, %xmm2
2416 ; XOP-FALLBACK-NEXT: vpaddb %xmm0, %xmm1, %xmm0
2417 ; XOP-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
2418 ; XOP-FALLBACK-NEXT: retq
2419 ;
2420 ; XOPAVX1-LABEL: vec256_i8_signed_reg_reg:
2421 ; XOPAVX1: # %bb.0:
2422 ; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
2423 ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
2424 ; XOPAVX1-NEXT: vpcomgtb %xmm2, %xmm3, %xmm8
2425 ; XOPAVX1-NEXT: vpcomgtb %xmm1, %xmm0, %xmm5
2426 ; XOPAVX1-NEXT: vpminsb %xmm2, %xmm3, %xmm6
2427 ; XOPAVX1-NEXT: vpminsb %xmm1, %xmm0, %xmm7
2428 ; XOPAVX1-NEXT: vpmaxsb %xmm2, %xmm3, %xmm2
2429 ; XOPAVX1-NEXT: vpsubb %xmm6, %xmm2, %xmm2
2430 ; XOPAVX1-NEXT: vpmaxsb %xmm1, %xmm0, %xmm1
2431 ; XOPAVX1-NEXT: vpsubb %xmm7, %xmm1, %xmm1
2432 ; XOPAVX1-NEXT: vpcmpeqd %xmm6, %xmm6, %xmm6
2433 ; XOPAVX1-NEXT: vpshlb %xmm6, %xmm2, %xmm2
2434 ; XOPAVX1-NEXT: vpshlb %xmm6, %xmm1, %xmm1
2435 ; XOPAVX1-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2436 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
2437 ; XOPAVX1-NEXT: vpor %xmm7, %xmm5, %xmm5
2438 ; XOPAVX1-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2439 ; XOPAVX1-NEXT: vpmullw %xmm4, %xmm6, %xmm4
2440 ; XOPAVX1-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2441 ; XOPAVX1-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2442 ; XOPAVX1-NEXT: vpmullw %xmm5, %xmm1, %xmm1
2443 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm5 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30]
2444 ; XOPAVX1-NEXT: vpperm %xmm5, %xmm4, %xmm1, %xmm1
2445 ; XOPAVX1-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2446 ; XOPAVX1-NEXT: vpor %xmm7, %xmm8, %xmm6
2447 ; XOPAVX1-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2448 ; XOPAVX1-NEXT: vpmullw %xmm7, %xmm4, %xmm4
2449 ; XOPAVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2450 ; XOPAVX1-NEXT: vpmovzxbw {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
2451 ; XOPAVX1-NEXT: vpmullw %xmm6, %xmm2, %xmm2
2452 ; XOPAVX1-NEXT: vpperm %xmm5, %xmm4, %xmm2, %xmm2
2453 ; XOPAVX1-NEXT: vpaddb %xmm3, %xmm2, %xmm2
2454 ; XOPAVX1-NEXT: vpaddb %xmm0, %xmm1, %xmm0
2455 ; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
2456 ; XOPAVX1-NEXT: retq
2457 ;
2458 ; AVX512F-LABEL: vec256_i8_signed_reg_reg:
2459 ; AVX512F: # %bb.0:
2460 ; AVX512F-NEXT: vpcmpgtb %ymm1, %ymm0, %ymm2
2461 ; AVX512F-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
2462 ; AVX512F-NEXT: vpminsb %ymm1, %ymm0, %ymm3
2463 ; AVX512F-NEXT: vpmaxsb %ymm1, %ymm0, %ymm1
2464 ; AVX512F-NEXT: vpsubb %ymm3, %ymm1, %ymm1
2465 ; AVX512F-NEXT: vpsrlw $1, %ymm1, %ymm1
2466 ; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
2467 ; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2468 ; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2469 ; AVX512F-NEXT: vpmullw %ymm4, %ymm3, %ymm3
2470 ; AVX512F-NEXT: vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2471 ; AVX512F-NEXT: vpand %ymm4, %ymm3, %ymm3
2472 ; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2473 ; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2474 ; AVX512F-NEXT: vpmullw %ymm2, %ymm1, %ymm1
2475 ; AVX512F-NEXT: vpand %ymm4, %ymm1, %ymm1
2476 ; AVX512F-NEXT: vpackuswb %ymm3, %ymm1, %ymm1
2477 ; AVX512F-NEXT: vpaddb %ymm0, %ymm1, %ymm0
2478 ; AVX512F-NEXT: retq
2479 ;
2480 ; AVX512VL-FALLBACK-LABEL: vec256_i8_signed_reg_reg:
2481 ; AVX512VL-FALLBACK: # %bb.0:
2482 ; AVX512VL-FALLBACK-NEXT: vpminsb %ymm1, %ymm0, %ymm2
2483 ; AVX512VL-FALLBACK-NEXT: vpmaxsb %ymm1, %ymm0, %ymm3
2484 ; AVX512VL-FALLBACK-NEXT: vpsubb %ymm2, %ymm3, %ymm2
2485 ; AVX512VL-FALLBACK-NEXT: vpsrlw $1, %ymm2, %ymm2
2486 ; AVX512VL-FALLBACK-NEXT: vpcmpgtb %ymm1, %ymm0, %ymm1
2487 ; AVX512VL-FALLBACK-NEXT: vpternlogq $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
2488 ; AVX512VL-FALLBACK-NEXT: vpsubb %ymm1, %ymm2, %ymm1
2489 ; AVX512VL-FALLBACK-NEXT: vpaddb %ymm0, %ymm1, %ymm0
2490 ; AVX512VL-FALLBACK-NEXT: retq
2491 ;
2492 ; AVX512BW-FALLBACK-LABEL: vec256_i8_signed_reg_reg:
2493 ; AVX512BW-FALLBACK: # %bb.0:
2494 ; AVX512BW-FALLBACK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
2495 ; AVX512BW-FALLBACK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
2496 ; AVX512BW-FALLBACK-NEXT: vpcmpgtb %zmm1, %zmm0, %k1
2497 ; AVX512BW-FALLBACK-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
2498 ; AVX512BW-FALLBACK-NEXT: vmovdqa {{.*#+}} ymm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
2499 ; AVX512BW-FALLBACK-NEXT: vmovdqu8 %zmm2, %zmm3 {%k1}
2500 ; AVX512BW-FALLBACK-NEXT: vpminsb %ymm1, %ymm0, %ymm2
2501 ; AVX512BW-FALLBACK-NEXT: vpmaxsb %ymm1, %ymm0, %ymm1
2502 ; AVX512BW-FALLBACK-NEXT: vpsubb %ymm2, %ymm1, %ymm1
2503 ; AVX512BW-FALLBACK-NEXT: vpsrlw $1, %ymm1, %ymm1
2504 ; AVX512BW-FALLBACK-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
2505 ; AVX512BW-FALLBACK-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
2506 ; AVX512BW-FALLBACK-NEXT: vpmovzxbw {{.*#+}} zmm2 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero,ymm3[16],zero,ymm3[17],zero,ymm3[18],zero,ymm3[19],zero,ymm3[20],zero,ymm3[21],zero,ymm3[22],zero,ymm3[23],zero,ymm3[24],zero,ymm3[25],zero,ymm3[26],zero,ymm3[27],zero,ymm3[28],zero,ymm3[29],zero,ymm3[30],zero,ymm3[31],zero
2507 ; AVX512BW-FALLBACK-NEXT: vpmullw %zmm2, %zmm1, %zmm1
2508 ; AVX512BW-FALLBACK-NEXT: vpmovwb %zmm1, %ymm1
2509 ; AVX512BW-FALLBACK-NEXT: vpaddb %ymm0, %ymm1, %ymm0
2510 ; AVX512BW-FALLBACK-NEXT: retq
2511 ;
2512 ; AVX512VLBW-LABEL: vec256_i8_signed_reg_reg:
2513 ; AVX512VLBW: # %bb.0:
2514 ; AVX512VLBW-NEXT: vpcmpgtb %ymm1, %ymm0, %k1
2515 ; AVX512VLBW-NEXT: vpminsb %ymm1, %ymm0, %ymm2
2516 ; AVX512VLBW-NEXT: vpmaxsb %ymm1, %ymm0, %ymm1
2517 ; AVX512VLBW-NEXT: vpsubb %ymm2, %ymm1, %ymm1
2518 ; AVX512VLBW-NEXT: vpsrlw $1, %ymm1, %ymm1
2519 ; AVX512VLBW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
2520 ; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
2521 ; AVX512VLBW-NEXT: vpsubb %ymm1, %ymm2, %ymm1 {%k1}
2522 ; AVX512VLBW-NEXT: vpaddb %ymm0, %ymm1, %ymm0
2523 ; AVX512VLBW-NEXT: retq
2524 %t3 = icmp sgt <32 x i8> %a1, %a2 ; signed
2525 %t4 = select <32 x i1> %t3, <32 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <32 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
2526 %t5 = select <32 x i1> %t3, <32 x i8> %a2, <32 x i8> %a1
2527 %t6 = select <32 x i1> %t3, <32 x i8> %a1, <32 x i8> %a2
2528 %t7 = sub <32 x i8> %t6, %t5
2529 %t8 = lshr <32 x i8> %t7, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
2530 %t9 = mul nsw <32 x i8> %t8, %t4 ; signed
2531 %a10 = add nsw <32 x i8> %t9, %a1 ; signed
2532 ret <32 x i8> %a10
2533 }
2535 define <32 x i8> @vec256_i8_unsigned_reg_reg(<32 x i8> %a1, <32 x i8> %a2) nounwind {
2536 ; AVX1-FALLBACK-LABEL: vec256_i8_unsigned_reg_reg:
2537 ; AVX1-FALLBACK: # %bb.0:
2538 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm1, %xmm3
2539 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm2
2540 ; AVX1-FALLBACK-NEXT: vpminub %xmm3, %xmm2, %xmm4
2541 ; AVX1-FALLBACK-NEXT: vpcmpeqb %xmm4, %xmm2, %xmm5
2542 ; AVX1-FALLBACK-NEXT: vpcmpeqd %xmm6, %xmm6, %xmm6
2543 ; AVX1-FALLBACK-NEXT: vpxor %xmm6, %xmm5, %xmm8
2544 ; AVX1-FALLBACK-NEXT: vpminub %xmm1, %xmm0, %xmm7
2545 ; AVX1-FALLBACK-NEXT: vpcmpeqb %xmm7, %xmm0, %xmm5
2546 ; AVX1-FALLBACK-NEXT: vpxor %xmm6, %xmm5, %xmm5
2547 ; AVX1-FALLBACK-NEXT: vpmaxub %xmm3, %xmm2, %xmm3
2548 ; AVX1-FALLBACK-NEXT: vpmaxub %xmm1, %xmm0, %xmm1
2549 ; AVX1-FALLBACK-NEXT: vpsubb %xmm7, %xmm1, %xmm1
2550 ; AVX1-FALLBACK-NEXT: vpsubb %xmm4, %xmm3, %xmm3
2551 ; AVX1-FALLBACK-NEXT: vpsrlw $1, %xmm3, %xmm3
2552 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
2553 ; AVX1-FALLBACK-NEXT: vpand %xmm4, %xmm3, %xmm3
2554 ; AVX1-FALLBACK-NEXT: vpsrlw $1, %xmm1, %xmm1
2555 ; AVX1-FALLBACK-NEXT: vpand %xmm4, %xmm1, %xmm1
2556 ; AVX1-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2557 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm6 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
2558 ; AVX1-FALLBACK-NEXT: vpor %xmm6, %xmm5, %xmm5
2559 ; AVX1-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2560 ; AVX1-FALLBACK-NEXT: vpmullw %xmm7, %xmm4, %xmm4
2561 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255]
2562 ; AVX1-FALLBACK-NEXT: vpand %xmm7, %xmm4, %xmm4
2563 ; AVX1-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2564 ; AVX1-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2565 ; AVX1-FALLBACK-NEXT: vpmullw %xmm5, %xmm1, %xmm1
2566 ; AVX1-FALLBACK-NEXT: vpand %xmm7, %xmm1, %xmm1
2567 ; AVX1-FALLBACK-NEXT: vpackuswb %xmm4, %xmm1, %xmm1
2568 ; AVX1-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2569 ; AVX1-FALLBACK-NEXT: vpor %xmm6, %xmm8, %xmm5
2570 ; AVX1-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2571 ; AVX1-FALLBACK-NEXT: vpmullw %xmm6, %xmm4, %xmm4
2572 ; AVX1-FALLBACK-NEXT: vpand %xmm7, %xmm4, %xmm4
2573 ; AVX1-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
2574 ; AVX1-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2575 ; AVX1-FALLBACK-NEXT: vpmullw %xmm5, %xmm3, %xmm3
2576 ; AVX1-FALLBACK-NEXT: vpand %xmm7, %xmm3, %xmm3
2577 ; AVX1-FALLBACK-NEXT: vpackuswb %xmm4, %xmm3, %xmm3
2578 ; AVX1-FALLBACK-NEXT: vpaddb %xmm2, %xmm3, %xmm2
2579 ; AVX1-FALLBACK-NEXT: vpaddb %xmm0, %xmm1, %xmm0
2580 ; AVX1-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
2581 ; AVX1-FALLBACK-NEXT: retq
2582 ;
2583 ; AVX2-LABEL: vec256_i8_unsigned_reg_reg:
2584 ; AVX2: # %bb.0:
2585 ; AVX2-NEXT: vpminub %ymm1, %ymm0, %ymm2
2586 ; AVX2-NEXT: vpcmpeqb %ymm2, %ymm0, %ymm3
2587 ; AVX2-NEXT: vpcmpeqd %ymm4, %ymm4, %ymm4
2588 ; AVX2-NEXT: vpxor %ymm4, %ymm3, %ymm3
2589 ; AVX2-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
2590 ; AVX2-NEXT: vpmaxub %ymm1, %ymm0, %ymm1
2591 ; AVX2-NEXT: vpsubb %ymm2, %ymm1, %ymm1
2592 ; AVX2-NEXT: vpsrlw $1, %ymm1, %ymm1
2593 ; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
2594 ; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2595 ; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2596 ; AVX2-NEXT: vpmullw %ymm4, %ymm2, %ymm2
2597 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2598 ; AVX2-NEXT: vpand %ymm4, %ymm2, %ymm2
2599 ; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2600 ; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2601 ; AVX2-NEXT: vpmullw %ymm3, %ymm1, %ymm1
2602 ; AVX2-NEXT: vpand %ymm4, %ymm1, %ymm1
2603 ; AVX2-NEXT: vpackuswb %ymm2, %ymm1, %ymm1
2604 ; AVX2-NEXT: vpaddb %ymm0, %ymm1, %ymm0
2605 ; AVX2-NEXT: retq
2606 ;
2607 ; XOP-FALLBACK-LABEL: vec256_i8_unsigned_reg_reg:
2608 ; XOP-FALLBACK: # %bb.0:
2609 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm1, %xmm2
2610 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm3
2611 ; XOP-FALLBACK-NEXT: vpcomgtub %xmm2, %xmm3, %xmm8
2612 ; XOP-FALLBACK-NEXT: vpcomgtub %xmm1, %xmm0, %xmm5
2613 ; XOP-FALLBACK-NEXT: vpminub %xmm2, %xmm3, %xmm6
2614 ; XOP-FALLBACK-NEXT: vpminub %xmm1, %xmm0, %xmm7
2615 ; XOP-FALLBACK-NEXT: vpmaxub %xmm2, %xmm3, %xmm2
2616 ; XOP-FALLBACK-NEXT: vpsubb %xmm6, %xmm2, %xmm2
2617 ; XOP-FALLBACK-NEXT: vpmaxub %xmm1, %xmm0, %xmm1
2618 ; XOP-FALLBACK-NEXT: vpsubb %xmm7, %xmm1, %xmm1
2619 ; XOP-FALLBACK-NEXT: vpcmpeqd %xmm6, %xmm6, %xmm6
2620 ; XOP-FALLBACK-NEXT: vpshlb %xmm6, %xmm2, %xmm2
2621 ; XOP-FALLBACK-NEXT: vpshlb %xmm6, %xmm1, %xmm1
2622 ; XOP-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2623 ; XOP-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
2624 ; XOP-FALLBACK-NEXT: vpor %xmm7, %xmm5, %xmm5
2625 ; XOP-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2626 ; XOP-FALLBACK-NEXT: vpmullw %xmm4, %xmm6, %xmm4
2627 ; XOP-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2628 ; XOP-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2629 ; XOP-FALLBACK-NEXT: vpmullw %xmm5, %xmm1, %xmm1
2630 ; XOP-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm5 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30]
2631 ; XOP-FALLBACK-NEXT: vpperm %xmm5, %xmm4, %xmm1, %xmm1
2632 ; XOP-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2633 ; XOP-FALLBACK-NEXT: vpor %xmm7, %xmm8, %xmm6
2634 ; XOP-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2635 ; XOP-FALLBACK-NEXT: vpmullw %xmm7, %xmm4, %xmm4
2636 ; XOP-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2637 ; XOP-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
2638 ; XOP-FALLBACK-NEXT: vpmullw %xmm6, %xmm2, %xmm2
2639 ; XOP-FALLBACK-NEXT: vpperm %xmm5, %xmm4, %xmm2, %xmm2
2640 ; XOP-FALLBACK-NEXT: vpaddb %xmm3, %xmm2, %xmm2
2641 ; XOP-FALLBACK-NEXT: vpaddb %xmm0, %xmm1, %xmm0
2642 ; XOP-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
2643 ; XOP-FALLBACK-NEXT: retq
2644 ;
2645 ; XOPAVX1-LABEL: vec256_i8_unsigned_reg_reg:
2646 ; XOPAVX1: # %bb.0:
2647 ; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
2648 ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
2649 ; XOPAVX1-NEXT: vpcomgtub %xmm2, %xmm3, %xmm8
2650 ; XOPAVX1-NEXT: vpcomgtub %xmm1, %xmm0, %xmm5
2651 ; XOPAVX1-NEXT: vpminub %xmm2, %xmm3, %xmm6
2652 ; XOPAVX1-NEXT: vpminub %xmm1, %xmm0, %xmm7
2653 ; XOPAVX1-NEXT: vpmaxub %xmm2, %xmm3, %xmm2
2654 ; XOPAVX1-NEXT: vpsubb %xmm6, %xmm2, %xmm2
2655 ; XOPAVX1-NEXT: vpmaxub %xmm1, %xmm0, %xmm1
2656 ; XOPAVX1-NEXT: vpsubb %xmm7, %xmm1, %xmm1
2657 ; XOPAVX1-NEXT: vpcmpeqd %xmm6, %xmm6, %xmm6
2658 ; XOPAVX1-NEXT: vpshlb %xmm6, %xmm2, %xmm2
2659 ; XOPAVX1-NEXT: vpshlb %xmm6, %xmm1, %xmm1
2660 ; XOPAVX1-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2661 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
2662 ; XOPAVX1-NEXT: vpor %xmm7, %xmm5, %xmm5
2663 ; XOPAVX1-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2664 ; XOPAVX1-NEXT: vpmullw %xmm4, %xmm6, %xmm4
2665 ; XOPAVX1-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2666 ; XOPAVX1-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2667 ; XOPAVX1-NEXT: vpmullw %xmm5, %xmm1, %xmm1
2668 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm5 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30]
2669 ; XOPAVX1-NEXT: vpperm %xmm5, %xmm4, %xmm1, %xmm1
2670 ; XOPAVX1-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2671 ; XOPAVX1-NEXT: vpor %xmm7, %xmm8, %xmm6
2672 ; XOPAVX1-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2673 ; XOPAVX1-NEXT: vpmullw %xmm7, %xmm4, %xmm4
2674 ; XOPAVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2675 ; XOPAVX1-NEXT: vpmovzxbw {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
2676 ; XOPAVX1-NEXT: vpmullw %xmm6, %xmm2, %xmm2
2677 ; XOPAVX1-NEXT: vpperm %xmm5, %xmm4, %xmm2, %xmm2
2678 ; XOPAVX1-NEXT: vpaddb %xmm3, %xmm2, %xmm2
2679 ; XOPAVX1-NEXT: vpaddb %xmm0, %xmm1, %xmm0
2680 ; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
2681 ; XOPAVX1-NEXT: retq
2682 ;
2683 ; AVX512F-LABEL: vec256_i8_unsigned_reg_reg:
2684 ; AVX512F: # %bb.0:
2685 ; AVX512F-NEXT: vpminub %ymm1, %ymm0, %ymm2
2686 ; AVX512F-NEXT: vpcmpeqb %ymm2, %ymm0, %ymm3
2687 ; AVX512F-NEXT: vpternlogq $15, %zmm3, %zmm3, %zmm3
2688 ; AVX512F-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
2689 ; AVX512F-NEXT: vpmaxub %ymm1, %ymm0, %ymm1
2690 ; AVX512F-NEXT: vpsubb %ymm2, %ymm1, %ymm1
2691 ; AVX512F-NEXT: vpsrlw $1, %ymm1, %ymm1
2692 ; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
2693 ; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2694 ; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2695 ; AVX512F-NEXT: vpmullw %ymm4, %ymm2, %ymm2
2696 ; AVX512F-NEXT: vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2697 ; AVX512F-NEXT: vpand %ymm4, %ymm2, %ymm2
2698 ; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2699 ; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2700 ; AVX512F-NEXT: vpmullw %ymm3, %ymm1, %ymm1
2701 ; AVX512F-NEXT: vpand %ymm4, %ymm1, %ymm1
2702 ; AVX512F-NEXT: vpackuswb %ymm2, %ymm1, %ymm1
2703 ; AVX512F-NEXT: vpaddb %ymm0, %ymm1, %ymm0
2704 ; AVX512F-NEXT: retq
2705 ;
2706 ; AVX512VL-FALLBACK-LABEL: vec256_i8_unsigned_reg_reg:
2707 ; AVX512VL-FALLBACK: # %bb.0:
2708 ; AVX512VL-FALLBACK-NEXT: vpminub %ymm1, %ymm0, %ymm2
2709 ; AVX512VL-FALLBACK-NEXT: vpmaxub %ymm1, %ymm0, %ymm1
2710 ; AVX512VL-FALLBACK-NEXT: vpsubb %ymm2, %ymm1, %ymm1
2711 ; AVX512VL-FALLBACK-NEXT: vpsrlw $1, %ymm1, %ymm1
2712 ; AVX512VL-FALLBACK-NEXT: vpcmpeqb %ymm2, %ymm0, %ymm2
2713 ; AVX512VL-FALLBACK-NEXT: vpternlogq $15, %ymm2, %ymm2, %ymm2
2714 ; AVX512VL-FALLBACK-NEXT: vpternlogq $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm1
2715 ; AVX512VL-FALLBACK-NEXT: vpsubb %ymm2, %ymm1, %ymm1
2716 ; AVX512VL-FALLBACK-NEXT: vpaddb %ymm0, %ymm1, %ymm0
2717 ; AVX512VL-FALLBACK-NEXT: retq
2718 ;
2719 ; AVX512BW-FALLBACK-LABEL: vec256_i8_unsigned_reg_reg:
2720 ; AVX512BW-FALLBACK: # %bb.0:
2721 ; AVX512BW-FALLBACK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
2722 ; AVX512BW-FALLBACK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
2723 ; AVX512BW-FALLBACK-NEXT: vpcmpnleub %zmm1, %zmm0, %k1
2724 ; AVX512BW-FALLBACK-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
2725 ; AVX512BW-FALLBACK-NEXT: vmovdqa {{.*#+}} ymm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
2726 ; AVX512BW-FALLBACK-NEXT: vmovdqu8 %zmm2, %zmm3 {%k1}
2727 ; AVX512BW-FALLBACK-NEXT: vpminub %ymm1, %ymm0, %ymm2
2728 ; AVX512BW-FALLBACK-NEXT: vpmaxub %ymm1, %ymm0, %ymm1
2729 ; AVX512BW-FALLBACK-NEXT: vpsubb %ymm2, %ymm1, %ymm1
2730 ; AVX512BW-FALLBACK-NEXT: vpsrlw $1, %ymm1, %ymm1
2731 ; AVX512BW-FALLBACK-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
2732 ; AVX512BW-FALLBACK-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
2733 ; AVX512BW-FALLBACK-NEXT: vpmovzxbw {{.*#+}} zmm2 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero,ymm3[16],zero,ymm3[17],zero,ymm3[18],zero,ymm3[19],zero,ymm3[20],zero,ymm3[21],zero,ymm3[22],zero,ymm3[23],zero,ymm3[24],zero,ymm3[25],zero,ymm3[26],zero,ymm3[27],zero,ymm3[28],zero,ymm3[29],zero,ymm3[30],zero,ymm3[31],zero
2734 ; AVX512BW-FALLBACK-NEXT: vpmullw %zmm2, %zmm1, %zmm1
2735 ; AVX512BW-FALLBACK-NEXT: vpmovwb %zmm1, %ymm1
2736 ; AVX512BW-FALLBACK-NEXT: vpaddb %ymm0, %ymm1, %ymm0
2737 ; AVX512BW-FALLBACK-NEXT: retq
2738 ;
2739 ; AVX512VLBW-LABEL: vec256_i8_unsigned_reg_reg:
2740 ; AVX512VLBW: # %bb.0:
2741 ; AVX512VLBW-NEXT: vpcmpnleub %ymm1, %ymm0, %k1
2742 ; AVX512VLBW-NEXT: vpminub %ymm1, %ymm0, %ymm2
2743 ; AVX512VLBW-NEXT: vpmaxub %ymm1, %ymm0, %ymm1
2744 ; AVX512VLBW-NEXT: vpsubb %ymm2, %ymm1, %ymm1
2745 ; AVX512VLBW-NEXT: vpsrlw $1, %ymm1, %ymm1
2746 ; AVX512VLBW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
2747 ; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
2748 ; AVX512VLBW-NEXT: vpsubb %ymm1, %ymm2, %ymm1 {%k1}
2749 ; AVX512VLBW-NEXT: vpaddb %ymm0, %ymm1, %ymm0
2750 ; AVX512VLBW-NEXT: retq
2751 %t3 = icmp ugt <32 x i8> %a1, %a2
2752 %t4 = select <32 x i1> %t3, <32 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <32 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
2753 %t5 = select <32 x i1> %t3, <32 x i8> %a2, <32 x i8> %a1
2754 %t6 = select <32 x i1> %t3, <32 x i8> %a1, <32 x i8> %a2
2755 %t7 = sub <32 x i8> %t6, %t5
2756 %t8 = lshr <32 x i8> %t7, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
2757 %t9 = mul <32 x i8> %t8, %t4
2758 %a10 = add <32 x i8> %t9, %a1
2759 ret <32 x i8> %a10
2760 }
2762 ; Values are loaded. Only check signed case.
2764 define <32 x i8> @vec256_i8_signed_mem_reg(ptr %a1_addr, <32 x i8> %a2) nounwind {
2765 ; AVX1-FALLBACK-LABEL: vec256_i8_signed_mem_reg:
2766 ; AVX1-FALLBACK: # %bb.0:
2767 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm3
2768 ; AVX1-FALLBACK-NEXT: vmovdqa (%rdi), %xmm1
2769 ; AVX1-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm2
2770 ; AVX1-FALLBACK-NEXT: vpcmpgtb %xmm3, %xmm2, %xmm8
2771 ; AVX1-FALLBACK-NEXT: vpcmpgtb %xmm0, %xmm1, %xmm5
2772 ; AVX1-FALLBACK-NEXT: vpminsb %xmm3, %xmm2, %xmm6
2773 ; AVX1-FALLBACK-NEXT: vpminsb %xmm0, %xmm1, %xmm7
2774 ; AVX1-FALLBACK-NEXT: vpmaxsb %xmm3, %xmm2, %xmm3
2775 ; AVX1-FALLBACK-NEXT: vpsubb %xmm6, %xmm3, %xmm3
2776 ; AVX1-FALLBACK-NEXT: vpmaxsb %xmm0, %xmm1, %xmm0
2777 ; AVX1-FALLBACK-NEXT: vpsubb %xmm7, %xmm0, %xmm0
2778 ; AVX1-FALLBACK-NEXT: vpsrlw $1, %xmm3, %xmm3
2779 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
2780 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm3, %xmm3
2781 ; AVX1-FALLBACK-NEXT: vpsrlw $1, %xmm0, %xmm0
2782 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm0, %xmm0
2783 ; AVX1-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2784 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
2785 ; AVX1-FALLBACK-NEXT: vpor %xmm7, %xmm5, %xmm5
2786 ; AVX1-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2787 ; AVX1-FALLBACK-NEXT: vpmullw %xmm4, %xmm6, %xmm4
2788 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]
2789 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm4, %xmm4
2790 ; AVX1-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2791 ; AVX1-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2792 ; AVX1-FALLBACK-NEXT: vpmullw %xmm5, %xmm0, %xmm0
2793 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm0, %xmm0
2794 ; AVX1-FALLBACK-NEXT: vpackuswb %xmm4, %xmm0, %xmm0
2795 ; AVX1-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2796 ; AVX1-FALLBACK-NEXT: vpor %xmm7, %xmm8, %xmm5
2797 ; AVX1-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2798 ; AVX1-FALLBACK-NEXT: vpmullw %xmm7, %xmm4, %xmm4
2799 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm4, %xmm4
2800 ; AVX1-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
2801 ; AVX1-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2802 ; AVX1-FALLBACK-NEXT: vpmullw %xmm5, %xmm3, %xmm3
2803 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm3, %xmm3
2804 ; AVX1-FALLBACK-NEXT: vpackuswb %xmm4, %xmm3, %xmm3
2805 ; AVX1-FALLBACK-NEXT: vpaddb %xmm2, %xmm3, %xmm2
2806 ; AVX1-FALLBACK-NEXT: vpaddb %xmm1, %xmm0, %xmm0
2807 ; AVX1-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
2808 ; AVX1-FALLBACK-NEXT: retq
2809 ;
2810 ; AVX2-LABEL: vec256_i8_signed_mem_reg:
2811 ; AVX2: # %bb.0:
2812 ; AVX2-NEXT: vmovdqa (%rdi), %ymm1
2813 ; AVX2-NEXT: vpcmpgtb %ymm0, %ymm1, %ymm2
2814 ; AVX2-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
2815 ; AVX2-NEXT: vpminsb %ymm0, %ymm1, %ymm3
2816 ; AVX2-NEXT: vpmaxsb %ymm0, %ymm1, %ymm0
2817 ; AVX2-NEXT: vpsubb %ymm3, %ymm0, %ymm0
2818 ; AVX2-NEXT: vpsrlw $1, %ymm0, %ymm0
2819 ; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
2820 ; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2821 ; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2822 ; AVX2-NEXT: vpmullw %ymm4, %ymm3, %ymm3
2823 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2824 ; AVX2-NEXT: vpand %ymm4, %ymm3, %ymm3
2825 ; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2826 ; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2827 ; AVX2-NEXT: vpmullw %ymm2, %ymm0, %ymm0
2828 ; AVX2-NEXT: vpand %ymm4, %ymm0, %ymm0
2829 ; AVX2-NEXT: vpackuswb %ymm3, %ymm0, %ymm0
2830 ; AVX2-NEXT: vpaddb %ymm1, %ymm0, %ymm0
2831 ; AVX2-NEXT: retq
2832 ;
2833 ; XOP-FALLBACK-LABEL: vec256_i8_signed_mem_reg:
2834 ; XOP-FALLBACK: # %bb.0:
2835 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm2
2836 ; XOP-FALLBACK-NEXT: vmovdqa (%rdi), %xmm1
2837 ; XOP-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm3
2838 ; XOP-FALLBACK-NEXT: vpcomgtb %xmm2, %xmm3, %xmm8
2839 ; XOP-FALLBACK-NEXT: vpcomgtb %xmm0, %xmm1, %xmm5
2840 ; XOP-FALLBACK-NEXT: vpminsb %xmm2, %xmm3, %xmm6
2841 ; XOP-FALLBACK-NEXT: vpminsb %xmm0, %xmm1, %xmm7
2842 ; XOP-FALLBACK-NEXT: vpmaxsb %xmm2, %xmm3, %xmm2
2843 ; XOP-FALLBACK-NEXT: vpsubb %xmm6, %xmm2, %xmm2
2844 ; XOP-FALLBACK-NEXT: vpmaxsb %xmm0, %xmm1, %xmm0
2845 ; XOP-FALLBACK-NEXT: vpsubb %xmm7, %xmm0, %xmm0
2846 ; XOP-FALLBACK-NEXT: vpcmpeqd %xmm6, %xmm6, %xmm6
2847 ; XOP-FALLBACK-NEXT: vpshlb %xmm6, %xmm2, %xmm2
2848 ; XOP-FALLBACK-NEXT: vpshlb %xmm6, %xmm0, %xmm0
2849 ; XOP-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2850 ; XOP-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
2851 ; XOP-FALLBACK-NEXT: vpor %xmm7, %xmm5, %xmm5
2852 ; XOP-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2853 ; XOP-FALLBACK-NEXT: vpmullw %xmm4, %xmm6, %xmm4
2854 ; XOP-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2855 ; XOP-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2856 ; XOP-FALLBACK-NEXT: vpmullw %xmm5, %xmm0, %xmm0
2857 ; XOP-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm5 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30]
2858 ; XOP-FALLBACK-NEXT: vpperm %xmm5, %xmm4, %xmm0, %xmm0
2859 ; XOP-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2860 ; XOP-FALLBACK-NEXT: vpor %xmm7, %xmm8, %xmm6
2861 ; XOP-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2862 ; XOP-FALLBACK-NEXT: vpmullw %xmm7, %xmm4, %xmm4
2863 ; XOP-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2864 ; XOP-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
2865 ; XOP-FALLBACK-NEXT: vpmullw %xmm6, %xmm2, %xmm2
2866 ; XOP-FALLBACK-NEXT: vpperm %xmm5, %xmm4, %xmm2, %xmm2
2867 ; XOP-FALLBACK-NEXT: vpaddb %xmm3, %xmm2, %xmm2
2868 ; XOP-FALLBACK-NEXT: vpaddb %xmm1, %xmm0, %xmm0
2869 ; XOP-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
2870 ; XOP-FALLBACK-NEXT: retq
2871 ;
2872 ; XOPAVX1-LABEL: vec256_i8_signed_mem_reg:
2873 ; XOPAVX1: # %bb.0:
2874 ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
2875 ; XOPAVX1-NEXT: vmovdqa (%rdi), %xmm1
2876 ; XOPAVX1-NEXT: vmovdqa 16(%rdi), %xmm3
2877 ; XOPAVX1-NEXT: vpcomgtb %xmm2, %xmm3, %xmm8
2878 ; XOPAVX1-NEXT: vpcomgtb %xmm0, %xmm1, %xmm5
2879 ; XOPAVX1-NEXT: vpminsb %xmm2, %xmm3, %xmm6
2880 ; XOPAVX1-NEXT: vpminsb %xmm0, %xmm1, %xmm7
2881 ; XOPAVX1-NEXT: vpmaxsb %xmm2, %xmm3, %xmm2
2882 ; XOPAVX1-NEXT: vpsubb %xmm6, %xmm2, %xmm2
2883 ; XOPAVX1-NEXT: vpmaxsb %xmm0, %xmm1, %xmm0
2884 ; XOPAVX1-NEXT: vpsubb %xmm7, %xmm0, %xmm0
2885 ; XOPAVX1-NEXT: vpcmpeqd %xmm6, %xmm6, %xmm6
2886 ; XOPAVX1-NEXT: vpshlb %xmm6, %xmm2, %xmm2
2887 ; XOPAVX1-NEXT: vpshlb %xmm6, %xmm0, %xmm0
2888 ; XOPAVX1-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2889 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
2890 ; XOPAVX1-NEXT: vpor %xmm7, %xmm5, %xmm5
2891 ; XOPAVX1-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2892 ; XOPAVX1-NEXT: vpmullw %xmm4, %xmm6, %xmm4
2893 ; XOPAVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2894 ; XOPAVX1-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
2895 ; XOPAVX1-NEXT: vpmullw %xmm5, %xmm0, %xmm0
2896 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm5 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30]
2897 ; XOPAVX1-NEXT: vpperm %xmm5, %xmm4, %xmm0, %xmm0
2898 ; XOPAVX1-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2899 ; XOPAVX1-NEXT: vpor %xmm7, %xmm8, %xmm6
2900 ; XOPAVX1-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
2901 ; XOPAVX1-NEXT: vpmullw %xmm7, %xmm4, %xmm4
2902 ; XOPAVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
2903 ; XOPAVX1-NEXT: vpmovzxbw {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
2904 ; XOPAVX1-NEXT: vpmullw %xmm6, %xmm2, %xmm2
2905 ; XOPAVX1-NEXT: vpperm %xmm5, %xmm4, %xmm2, %xmm2
2906 ; XOPAVX1-NEXT: vpaddb %xmm3, %xmm2, %xmm2
2907 ; XOPAVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
2908 ; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
2909 ; XOPAVX1-NEXT: retq
2910 ;
2911 ; AVX512F-LABEL: vec256_i8_signed_mem_reg:
2912 ; AVX512F: # %bb.0:
2913 ; AVX512F-NEXT: vmovdqa (%rdi), %ymm1
2914 ; AVX512F-NEXT: vpcmpgtb %ymm0, %ymm1, %ymm2
2915 ; AVX512F-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
2916 ; AVX512F-NEXT: vpminsb %ymm0, %ymm1, %ymm3
2917 ; AVX512F-NEXT: vpmaxsb %ymm0, %ymm1, %ymm0
2918 ; AVX512F-NEXT: vpsubb %ymm3, %ymm0, %ymm0
2919 ; AVX512F-NEXT: vpsrlw $1, %ymm0, %ymm0
2920 ; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
2921 ; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2922 ; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
2923 ; AVX512F-NEXT: vpmullw %ymm4, %ymm3, %ymm3
2924 ; AVX512F-NEXT: vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
2925 ; AVX512F-NEXT: vpand %ymm4, %ymm3, %ymm3
2926 ; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2927 ; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
2928 ; AVX512F-NEXT: vpmullw %ymm2, %ymm0, %ymm0
2929 ; AVX512F-NEXT: vpand %ymm4, %ymm0, %ymm0
2930 ; AVX512F-NEXT: vpackuswb %ymm3, %ymm0, %ymm0
2931 ; AVX512F-NEXT: vpaddb %ymm1, %ymm0, %ymm0
2932 ; AVX512F-NEXT: retq
2933 ;
2934 ; AVX512VL-FALLBACK-LABEL: vec256_i8_signed_mem_reg:
2935 ; AVX512VL-FALLBACK: # %bb.0:
2936 ; AVX512VL-FALLBACK-NEXT: vmovdqa (%rdi), %ymm1
2937 ; AVX512VL-FALLBACK-NEXT: vpminsb %ymm0, %ymm1, %ymm2
2938 ; AVX512VL-FALLBACK-NEXT: vpmaxsb %ymm0, %ymm1, %ymm3
2939 ; AVX512VL-FALLBACK-NEXT: vpsubb %ymm2, %ymm3, %ymm2
2940 ; AVX512VL-FALLBACK-NEXT: vpsrlw $1, %ymm2, %ymm2
2941 ; AVX512VL-FALLBACK-NEXT: vpcmpgtb %ymm0, %ymm1, %ymm0
2942 ; AVX512VL-FALLBACK-NEXT: vpternlogq $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
2943 ; AVX512VL-FALLBACK-NEXT: vpsubb %ymm0, %ymm2, %ymm0
2944 ; AVX512VL-FALLBACK-NEXT: vpaddb %ymm1, %ymm0, %ymm0
2945 ; AVX512VL-FALLBACK-NEXT: retq
2946 ;
2947 ; AVX512BW-FALLBACK-LABEL: vec256_i8_signed_mem_reg:
2948 ; AVX512BW-FALLBACK: # %bb.0:
2949 ; AVX512BW-FALLBACK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
2950 ; AVX512BW-FALLBACK-NEXT: vmovdqa (%rdi), %ymm1
2951 ; AVX512BW-FALLBACK-NEXT: vpcmpgtb %zmm0, %zmm1, %k1
2952 ; AVX512BW-FALLBACK-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
2953 ; AVX512BW-FALLBACK-NEXT: vmovdqa {{.*#+}} ymm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
2954 ; AVX512BW-FALLBACK-NEXT: vmovdqu8 %zmm2, %zmm3 {%k1}
2955 ; AVX512BW-FALLBACK-NEXT: vpminsb %ymm0, %ymm1, %ymm2
2956 ; AVX512BW-FALLBACK-NEXT: vpmaxsb %ymm0, %ymm1, %ymm0
2957 ; AVX512BW-FALLBACK-NEXT: vpsubb %ymm2, %ymm0, %ymm0
2958 ; AVX512BW-FALLBACK-NEXT: vpsrlw $1, %ymm0, %ymm0
2959 ; AVX512BW-FALLBACK-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
2960 ; AVX512BW-FALLBACK-NEXT: vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
2961 ; AVX512BW-FALLBACK-NEXT: vpmovzxbw {{.*#+}} zmm2 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero,ymm3[16],zero,ymm3[17],zero,ymm3[18],zero,ymm3[19],zero,ymm3[20],zero,ymm3[21],zero,ymm3[22],zero,ymm3[23],zero,ymm3[24],zero,ymm3[25],zero,ymm3[26],zero,ymm3[27],zero,ymm3[28],zero,ymm3[29],zero,ymm3[30],zero,ymm3[31],zero
2962 ; AVX512BW-FALLBACK-NEXT: vpmullw %zmm2, %zmm0, %zmm0
2963 ; AVX512BW-FALLBACK-NEXT: vpmovwb %zmm0, %ymm0
2964 ; AVX512BW-FALLBACK-NEXT: vpaddb %ymm1, %ymm0, %ymm0
2965 ; AVX512BW-FALLBACK-NEXT: retq
2966 ;
2967 ; AVX512VLBW-LABEL: vec256_i8_signed_mem_reg:
2968 ; AVX512VLBW: # %bb.0:
2969 ; AVX512VLBW-NEXT: vmovdqa (%rdi), %ymm1
2970 ; AVX512VLBW-NEXT: vpcmpgtb %ymm0, %ymm1, %k1
2971 ; AVX512VLBW-NEXT: vpminsb %ymm0, %ymm1, %ymm2
2972 ; AVX512VLBW-NEXT: vpmaxsb %ymm0, %ymm1, %ymm0
2973 ; AVX512VLBW-NEXT: vpsubb %ymm2, %ymm0, %ymm0
2974 ; AVX512VLBW-NEXT: vpsrlw $1, %ymm0, %ymm0
2975 ; AVX512VLBW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
2976 ; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
2977 ; AVX512VLBW-NEXT: vpsubb %ymm0, %ymm2, %ymm0 {%k1}
2978 ; AVX512VLBW-NEXT: vpaddb %ymm1, %ymm0, %ymm0
2979 ; AVX512VLBW-NEXT: retq
2980 %a1 = load <32 x i8>, ptr %a1_addr
2981 %t3 = icmp sgt <32 x i8> %a1, %a2 ; signed
2982 %t4 = select <32 x i1> %t3, <32 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <32 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
2983 %t5 = select <32 x i1> %t3, <32 x i8> %a2, <32 x i8> %a1
2984 %t6 = select <32 x i1> %t3, <32 x i8> %a1, <32 x i8> %a2
2985 %t7 = sub <32 x i8> %t6, %t5
2986 %t8 = lshr <32 x i8> %t7, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
2987 %t9 = mul nsw <32 x i8> %t8, %t4 ; signed
2988 %a10 = add nsw <32 x i8> %t9, %a1 ; signed
2989 ret <32 x i8> %a10
2990 }
2992 define <32 x i8> @vec256_i8_signed_reg_mem(<32 x i8> %a1, ptr %a2_addr) nounwind {
2993 ; AVX1-FALLBACK-LABEL: vec256_i8_signed_reg_mem:
2994 ; AVX1-FALLBACK: # %bb.0:
2995 ; AVX1-FALLBACK-NEXT: vmovdqa (%rdi), %xmm2
2996 ; AVX1-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm3
2997 ; AVX1-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm1
2998 ; AVX1-FALLBACK-NEXT: vpcmpgtb %xmm3, %xmm1, %xmm8
2999 ; AVX1-FALLBACK-NEXT: vpcmpgtb %xmm2, %xmm0, %xmm5
3000 ; AVX1-FALLBACK-NEXT: vpminsb %xmm3, %xmm1, %xmm6
3001 ; AVX1-FALLBACK-NEXT: vpminsb %xmm2, %xmm0, %xmm7
3002 ; AVX1-FALLBACK-NEXT: vpmaxsb %xmm3, %xmm1, %xmm3
3003 ; AVX1-FALLBACK-NEXT: vpsubb %xmm6, %xmm3, %xmm3
3004 ; AVX1-FALLBACK-NEXT: vpmaxsb %xmm2, %xmm0, %xmm2
3005 ; AVX1-FALLBACK-NEXT: vpsubb %xmm7, %xmm2, %xmm2
3006 ; AVX1-FALLBACK-NEXT: vpsrlw $1, %xmm3, %xmm3
3007 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
3008 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm3, %xmm3
3009 ; AVX1-FALLBACK-NEXT: vpsrlw $1, %xmm2, %xmm2
3010 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm2, %xmm2
3011 ; AVX1-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3012 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
3013 ; AVX1-FALLBACK-NEXT: vpor %xmm7, %xmm5, %xmm5
3014 ; AVX1-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3015 ; AVX1-FALLBACK-NEXT: vpmullw %xmm4, %xmm6, %xmm4
3016 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]
3017 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm4, %xmm4
3018 ; AVX1-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
3019 ; AVX1-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
3020 ; AVX1-FALLBACK-NEXT: vpmullw %xmm5, %xmm2, %xmm2
3021 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm2, %xmm2
3022 ; AVX1-FALLBACK-NEXT: vpackuswb %xmm4, %xmm2, %xmm2
3023 ; AVX1-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3024 ; AVX1-FALLBACK-NEXT: vpor %xmm7, %xmm8, %xmm5
3025 ; AVX1-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3026 ; AVX1-FALLBACK-NEXT: vpmullw %xmm7, %xmm4, %xmm4
3027 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm4, %xmm4
3028 ; AVX1-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
3029 ; AVX1-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
3030 ; AVX1-FALLBACK-NEXT: vpmullw %xmm5, %xmm3, %xmm3
3031 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm3, %xmm3
3032 ; AVX1-FALLBACK-NEXT: vpackuswb %xmm4, %xmm3, %xmm3
3033 ; AVX1-FALLBACK-NEXT: vpaddb %xmm1, %xmm3, %xmm1
3034 ; AVX1-FALLBACK-NEXT: vpaddb %xmm0, %xmm2, %xmm0
3035 ; AVX1-FALLBACK-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
3036 ; AVX1-FALLBACK-NEXT: retq
3037 ;
3038 ; AVX2-LABEL: vec256_i8_signed_reg_mem:
3039 ; AVX2: # %bb.0:
3040 ; AVX2-NEXT: vmovdqa (%rdi), %ymm1
3041 ; AVX2-NEXT: vpcmpgtb %ymm1, %ymm0, %ymm2
3042 ; AVX2-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
3043 ; AVX2-NEXT: vpminsb %ymm1, %ymm0, %ymm3
3044 ; AVX2-NEXT: vpmaxsb %ymm1, %ymm0, %ymm1
3045 ; AVX2-NEXT: vpsubb %ymm3, %ymm1, %ymm1
3046 ; AVX2-NEXT: vpsrlw $1, %ymm1, %ymm1
3047 ; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
3048 ; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
3049 ; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
3050 ; AVX2-NEXT: vpmullw %ymm4, %ymm3, %ymm3
3051 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
3052 ; AVX2-NEXT: vpand %ymm4, %ymm3, %ymm3
3053 ; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
3054 ; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
3055 ; AVX2-NEXT: vpmullw %ymm2, %ymm1, %ymm1
3056 ; AVX2-NEXT: vpand %ymm4, %ymm1, %ymm1
3057 ; AVX2-NEXT: vpackuswb %ymm3, %ymm1, %ymm1
3058 ; AVX2-NEXT: vpaddb %ymm0, %ymm1, %ymm0
3059 ; AVX2-NEXT: retq
3060 ;
3061 ; XOP-FALLBACK-LABEL: vec256_i8_signed_reg_mem:
3062 ; XOP-FALLBACK: # %bb.0:
3063 ; XOP-FALLBACK-NEXT: vmovdqa (%rdi), %xmm1
3064 ; XOP-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm2
3065 ; XOP-FALLBACK-NEXT: vextractf128 $1, %ymm0, %xmm3
3066 ; XOP-FALLBACK-NEXT: vpcomgtb %xmm2, %xmm3, %xmm8
3067 ; XOP-FALLBACK-NEXT: vpcomgtb %xmm1, %xmm0, %xmm5
3068 ; XOP-FALLBACK-NEXT: vpminsb %xmm2, %xmm3, %xmm6
3069 ; XOP-FALLBACK-NEXT: vpminsb %xmm1, %xmm0, %xmm7
3070 ; XOP-FALLBACK-NEXT: vpmaxsb %xmm2, %xmm3, %xmm2
3071 ; XOP-FALLBACK-NEXT: vpsubb %xmm6, %xmm2, %xmm2
3072 ; XOP-FALLBACK-NEXT: vpmaxsb %xmm1, %xmm0, %xmm1
3073 ; XOP-FALLBACK-NEXT: vpsubb %xmm7, %xmm1, %xmm1
3074 ; XOP-FALLBACK-NEXT: vpcmpeqd %xmm6, %xmm6, %xmm6
3075 ; XOP-FALLBACK-NEXT: vpshlb %xmm6, %xmm2, %xmm2
3076 ; XOP-FALLBACK-NEXT: vpshlb %xmm6, %xmm1, %xmm1
3077 ; XOP-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3078 ; XOP-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
3079 ; XOP-FALLBACK-NEXT: vpor %xmm7, %xmm5, %xmm5
3080 ; XOP-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3081 ; XOP-FALLBACK-NEXT: vpmullw %xmm4, %xmm6, %xmm4
3082 ; XOP-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
3083 ; XOP-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
3084 ; XOP-FALLBACK-NEXT: vpmullw %xmm5, %xmm1, %xmm1
3085 ; XOP-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm5 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30]
3086 ; XOP-FALLBACK-NEXT: vpperm %xmm5, %xmm4, %xmm1, %xmm1
3087 ; XOP-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3088 ; XOP-FALLBACK-NEXT: vpor %xmm7, %xmm8, %xmm6
3089 ; XOP-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3090 ; XOP-FALLBACK-NEXT: vpmullw %xmm7, %xmm4, %xmm4
3091 ; XOP-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
3092 ; XOP-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
3093 ; XOP-FALLBACK-NEXT: vpmullw %xmm6, %xmm2, %xmm2
3094 ; XOP-FALLBACK-NEXT: vpperm %xmm5, %xmm4, %xmm2, %xmm2
3095 ; XOP-FALLBACK-NEXT: vpaddb %xmm3, %xmm2, %xmm2
3096 ; XOP-FALLBACK-NEXT: vpaddb %xmm0, %xmm1, %xmm0
3097 ; XOP-FALLBACK-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
3098 ; XOP-FALLBACK-NEXT: retq
3099 ;
3100 ; XOPAVX1-LABEL: vec256_i8_signed_reg_mem:
3101 ; XOPAVX1: # %bb.0:
3102 ; XOPAVX1-NEXT: vmovdqa (%rdi), %xmm1
3103 ; XOPAVX1-NEXT: vmovdqa 16(%rdi), %xmm2
3104 ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
3105 ; XOPAVX1-NEXT: vpcomgtb %xmm2, %xmm3, %xmm8
3106 ; XOPAVX1-NEXT: vpcomgtb %xmm1, %xmm0, %xmm5
3107 ; XOPAVX1-NEXT: vpminsb %xmm2, %xmm3, %xmm6
3108 ; XOPAVX1-NEXT: vpminsb %xmm1, %xmm0, %xmm7
3109 ; XOPAVX1-NEXT: vpmaxsb %xmm2, %xmm3, %xmm2
3110 ; XOPAVX1-NEXT: vpsubb %xmm6, %xmm2, %xmm2
3111 ; XOPAVX1-NEXT: vpmaxsb %xmm1, %xmm0, %xmm1
3112 ; XOPAVX1-NEXT: vpsubb %xmm7, %xmm1, %xmm1
3113 ; XOPAVX1-NEXT: vpcmpeqd %xmm6, %xmm6, %xmm6
3114 ; XOPAVX1-NEXT: vpshlb %xmm6, %xmm2, %xmm2
3115 ; XOPAVX1-NEXT: vpshlb %xmm6, %xmm1, %xmm1
3116 ; XOPAVX1-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3117 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
3118 ; XOPAVX1-NEXT: vpor %xmm7, %xmm5, %xmm5
3119 ; XOPAVX1-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3120 ; XOPAVX1-NEXT: vpmullw %xmm4, %xmm6, %xmm4
3121 ; XOPAVX1-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
3122 ; XOPAVX1-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
3123 ; XOPAVX1-NEXT: vpmullw %xmm5, %xmm1, %xmm1
3124 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm5 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30]
3125 ; XOPAVX1-NEXT: vpperm %xmm5, %xmm4, %xmm1, %xmm1
3126 ; XOPAVX1-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3127 ; XOPAVX1-NEXT: vpor %xmm7, %xmm8, %xmm6
3128 ; XOPAVX1-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3129 ; XOPAVX1-NEXT: vpmullw %xmm7, %xmm4, %xmm4
3130 ; XOPAVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
3131 ; XOPAVX1-NEXT: vpmovzxbw {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
3132 ; XOPAVX1-NEXT: vpmullw %xmm6, %xmm2, %xmm2
3133 ; XOPAVX1-NEXT: vpperm %xmm5, %xmm4, %xmm2, %xmm2
3134 ; XOPAVX1-NEXT: vpaddb %xmm3, %xmm2, %xmm2
3135 ; XOPAVX1-NEXT: vpaddb %xmm0, %xmm1, %xmm0
3136 ; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
3137 ; XOPAVX1-NEXT: retq
3138 ;
3139 ; AVX512F-LABEL: vec256_i8_signed_reg_mem:
3140 ; AVX512F: # %bb.0:
3141 ; AVX512F-NEXT: vmovdqa (%rdi), %ymm1
3142 ; AVX512F-NEXT: vpcmpgtb %ymm1, %ymm0, %ymm2
3143 ; AVX512F-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
3144 ; AVX512F-NEXT: vpminsb %ymm1, %ymm0, %ymm3
3145 ; AVX512F-NEXT: vpmaxsb %ymm1, %ymm0, %ymm1
3146 ; AVX512F-NEXT: vpsubb %ymm3, %ymm1, %ymm1
3147 ; AVX512F-NEXT: vpsrlw $1, %ymm1, %ymm1
3148 ; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
3149 ; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
3150 ; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
3151 ; AVX512F-NEXT: vpmullw %ymm4, %ymm3, %ymm3
3152 ; AVX512F-NEXT: vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
3153 ; AVX512F-NEXT: vpand %ymm4, %ymm3, %ymm3
3154 ; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
3155 ; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
3156 ; AVX512F-NEXT: vpmullw %ymm2, %ymm1, %ymm1
3157 ; AVX512F-NEXT: vpand %ymm4, %ymm1, %ymm1
3158 ; AVX512F-NEXT: vpackuswb %ymm3, %ymm1, %ymm1
3159 ; AVX512F-NEXT: vpaddb %ymm0, %ymm1, %ymm0
3160 ; AVX512F-NEXT: retq
3161 ;
3162 ; AVX512VL-FALLBACK-LABEL: vec256_i8_signed_reg_mem:
3163 ; AVX512VL-FALLBACK: # %bb.0:
3164 ; AVX512VL-FALLBACK-NEXT: vmovdqa (%rdi), %ymm1
3165 ; AVX512VL-FALLBACK-NEXT: vpminsb %ymm1, %ymm0, %ymm2
3166 ; AVX512VL-FALLBACK-NEXT: vpmaxsb %ymm1, %ymm0, %ymm3
3167 ; AVX512VL-FALLBACK-NEXT: vpsubb %ymm2, %ymm3, %ymm2
3168 ; AVX512VL-FALLBACK-NEXT: vpsrlw $1, %ymm2, %ymm2
3169 ; AVX512VL-FALLBACK-NEXT: vpcmpgtb %ymm1, %ymm0, %ymm1
3170 ; AVX512VL-FALLBACK-NEXT: vpternlogq $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
3171 ; AVX512VL-FALLBACK-NEXT: vpsubb %ymm1, %ymm2, %ymm1
3172 ; AVX512VL-FALLBACK-NEXT: vpaddb %ymm0, %ymm1, %ymm0
3173 ; AVX512VL-FALLBACK-NEXT: retq
3174 ;
3175 ; AVX512BW-FALLBACK-LABEL: vec256_i8_signed_reg_mem:
3176 ; AVX512BW-FALLBACK: # %bb.0:
3177 ; AVX512BW-FALLBACK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
3178 ; AVX512BW-FALLBACK-NEXT: vmovdqa (%rdi), %ymm1
3179 ; AVX512BW-FALLBACK-NEXT: vpcmpgtb %zmm1, %zmm0, %k1
3180 ; AVX512BW-FALLBACK-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
3181 ; AVX512BW-FALLBACK-NEXT: vmovdqa {{.*#+}} ymm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
3182 ; AVX512BW-FALLBACK-NEXT: vmovdqu8 %zmm2, %zmm3 {%k1}
3183 ; AVX512BW-FALLBACK-NEXT: vpminsb %ymm1, %ymm0, %ymm2
3184 ; AVX512BW-FALLBACK-NEXT: vpmaxsb %ymm1, %ymm0, %ymm1
3185 ; AVX512BW-FALLBACK-NEXT: vpsubb %ymm2, %ymm1, %ymm1
3186 ; AVX512BW-FALLBACK-NEXT: vpsrlw $1, %ymm1, %ymm1
3187 ; AVX512BW-FALLBACK-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
3188 ; AVX512BW-FALLBACK-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
3189 ; AVX512BW-FALLBACK-NEXT: vpmovzxbw {{.*#+}} zmm2 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero,ymm3[16],zero,ymm3[17],zero,ymm3[18],zero,ymm3[19],zero,ymm3[20],zero,ymm3[21],zero,ymm3[22],zero,ymm3[23],zero,ymm3[24],zero,ymm3[25],zero,ymm3[26],zero,ymm3[27],zero,ymm3[28],zero,ymm3[29],zero,ymm3[30],zero,ymm3[31],zero
3190 ; AVX512BW-FALLBACK-NEXT: vpmullw %zmm2, %zmm1, %zmm1
3191 ; AVX512BW-FALLBACK-NEXT: vpmovwb %zmm1, %ymm1
3192 ; AVX512BW-FALLBACK-NEXT: vpaddb %ymm0, %ymm1, %ymm0
3193 ; AVX512BW-FALLBACK-NEXT: retq
3194 ;
3195 ; AVX512VLBW-LABEL: vec256_i8_signed_reg_mem:
3196 ; AVX512VLBW: # %bb.0:
3197 ; AVX512VLBW-NEXT: vmovdqa (%rdi), %ymm1
3198 ; AVX512VLBW-NEXT: vpcmpgtb %ymm1, %ymm0, %k1
3199 ; AVX512VLBW-NEXT: vpminsb %ymm1, %ymm0, %ymm2
3200 ; AVX512VLBW-NEXT: vpmaxsb %ymm1, %ymm0, %ymm1
3201 ; AVX512VLBW-NEXT: vpsubb %ymm2, %ymm1, %ymm1
3202 ; AVX512VLBW-NEXT: vpsrlw $1, %ymm1, %ymm1
3203 ; AVX512VLBW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
3204 ; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
3205 ; AVX512VLBW-NEXT: vpsubb %ymm1, %ymm2, %ymm1 {%k1}
3206 ; AVX512VLBW-NEXT: vpaddb %ymm0, %ymm1, %ymm0
3207 ; AVX512VLBW-NEXT: retq
3208 %a2 = load <32 x i8>, ptr %a2_addr
3209 %t3 = icmp sgt <32 x i8> %a1, %a2 ; signed
3210 %t4 = select <32 x i1> %t3, <32 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <32 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
3211 %t5 = select <32 x i1> %t3, <32 x i8> %a2, <32 x i8> %a1
3212 %t6 = select <32 x i1> %t3, <32 x i8> %a1, <32 x i8> %a2
3213 %t7 = sub <32 x i8> %t6, %t5
3214 %t8 = lshr <32 x i8> %t7, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
3215 %t9 = mul nsw <32 x i8> %t8, %t4 ; signed
3216 %a10 = add nsw <32 x i8> %t9, %a1 ; signed
3217 ret <32 x i8> %a10
3218 }
3220 define <32 x i8> @vec256_i8_signed_mem_mem(ptr %a1_addr, ptr %a2_addr) nounwind {
3221 ; AVX1-FALLBACK-LABEL: vec256_i8_signed_mem_mem:
3222 ; AVX1-FALLBACK: # %bb.0:
3223 ; AVX1-FALLBACK-NEXT: vmovdqa (%rsi), %xmm2
3224 ; AVX1-FALLBACK-NEXT: vmovdqa 16(%rsi), %xmm3
3225 ; AVX1-FALLBACK-NEXT: vmovdqa (%rdi), %xmm0
3226 ; AVX1-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm1
3227 ; AVX1-FALLBACK-NEXT: vpcmpgtb %xmm3, %xmm1, %xmm8
3228 ; AVX1-FALLBACK-NEXT: vpcmpgtb %xmm2, %xmm0, %xmm5
3229 ; AVX1-FALLBACK-NEXT: vpminsb %xmm3, %xmm1, %xmm6
3230 ; AVX1-FALLBACK-NEXT: vpminsb %xmm2, %xmm0, %xmm7
3231 ; AVX1-FALLBACK-NEXT: vpmaxsb %xmm3, %xmm1, %xmm3
3232 ; AVX1-FALLBACK-NEXT: vpsubb %xmm6, %xmm3, %xmm3
3233 ; AVX1-FALLBACK-NEXT: vpmaxsb %xmm2, %xmm0, %xmm2
3234 ; AVX1-FALLBACK-NEXT: vpsubb %xmm7, %xmm2, %xmm2
3235 ; AVX1-FALLBACK-NEXT: vpsrlw $1, %xmm3, %xmm3
3236 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
3237 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm3, %xmm3
3238 ; AVX1-FALLBACK-NEXT: vpsrlw $1, %xmm2, %xmm2
3239 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm2, %xmm2
3240 ; AVX1-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3241 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
3242 ; AVX1-FALLBACK-NEXT: vpor %xmm7, %xmm5, %xmm5
3243 ; AVX1-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3244 ; AVX1-FALLBACK-NEXT: vpmullw %xmm4, %xmm6, %xmm4
3245 ; AVX1-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]
3246 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm4, %xmm4
3247 ; AVX1-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
3248 ; AVX1-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
3249 ; AVX1-FALLBACK-NEXT: vpmullw %xmm5, %xmm2, %xmm2
3250 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm2, %xmm2
3251 ; AVX1-FALLBACK-NEXT: vpackuswb %xmm4, %xmm2, %xmm2
3252 ; AVX1-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3253 ; AVX1-FALLBACK-NEXT: vpor %xmm7, %xmm8, %xmm5
3254 ; AVX1-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3255 ; AVX1-FALLBACK-NEXT: vpmullw %xmm7, %xmm4, %xmm4
3256 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm4, %xmm4
3257 ; AVX1-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
3258 ; AVX1-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
3259 ; AVX1-FALLBACK-NEXT: vpmullw %xmm5, %xmm3, %xmm3
3260 ; AVX1-FALLBACK-NEXT: vpand %xmm6, %xmm3, %xmm3
3261 ; AVX1-FALLBACK-NEXT: vpackuswb %xmm4, %xmm3, %xmm3
3262 ; AVX1-FALLBACK-NEXT: vpaddb %xmm1, %xmm3, %xmm1
3263 ; AVX1-FALLBACK-NEXT: vpaddb %xmm0, %xmm2, %xmm0
3264 ; AVX1-FALLBACK-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
3265 ; AVX1-FALLBACK-NEXT: retq
3266 ;
3267 ; AVX2-LABEL: vec256_i8_signed_mem_mem:
3268 ; AVX2: # %bb.0:
3269 ; AVX2-NEXT: vmovdqa (%rdi), %ymm0
3270 ; AVX2-NEXT: vmovdqa (%rsi), %ymm1
3271 ; AVX2-NEXT: vpcmpgtb %ymm1, %ymm0, %ymm2
3272 ; AVX2-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
3273 ; AVX2-NEXT: vpminsb %ymm1, %ymm0, %ymm3
3274 ; AVX2-NEXT: vpmaxsb %ymm1, %ymm0, %ymm1
3275 ; AVX2-NEXT: vpsubb %ymm3, %ymm1, %ymm1
3276 ; AVX2-NEXT: vpsrlw $1, %ymm1, %ymm1
3277 ; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
3278 ; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
3279 ; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
3280 ; AVX2-NEXT: vpmullw %ymm4, %ymm3, %ymm3
3281 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
3282 ; AVX2-NEXT: vpand %ymm4, %ymm3, %ymm3
3283 ; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
3284 ; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
3285 ; AVX2-NEXT: vpmullw %ymm2, %ymm1, %ymm1
3286 ; AVX2-NEXT: vpand %ymm4, %ymm1, %ymm1
3287 ; AVX2-NEXT: vpackuswb %ymm3, %ymm1, %ymm1
3288 ; AVX2-NEXT: vpaddb %ymm0, %ymm1, %ymm0
3289 ; AVX2-NEXT: retq
3290 ;
3291 ; XOP-FALLBACK-LABEL: vec256_i8_signed_mem_mem:
3292 ; XOP-FALLBACK: # %bb.0:
3293 ; XOP-FALLBACK-NEXT: vmovdqa (%rsi), %xmm0
3294 ; XOP-FALLBACK-NEXT: vmovdqa 16(%rsi), %xmm1
3295 ; XOP-FALLBACK-NEXT: vmovdqa (%rdi), %xmm2
3296 ; XOP-FALLBACK-NEXT: vmovdqa 16(%rdi), %xmm3
3297 ; XOP-FALLBACK-NEXT: vpcomgtb %xmm1, %xmm3, %xmm8
3298 ; XOP-FALLBACK-NEXT: vpcomgtb %xmm0, %xmm2, %xmm5
3299 ; XOP-FALLBACK-NEXT: vpminsb %xmm1, %xmm3, %xmm6
3300 ; XOP-FALLBACK-NEXT: vpminsb %xmm0, %xmm2, %xmm7
3301 ; XOP-FALLBACK-NEXT: vpmaxsb %xmm1, %xmm3, %xmm1
3302 ; XOP-FALLBACK-NEXT: vpsubb %xmm6, %xmm1, %xmm1
3303 ; XOP-FALLBACK-NEXT: vpmaxsb %xmm0, %xmm2, %xmm0
3304 ; XOP-FALLBACK-NEXT: vpsubb %xmm7, %xmm0, %xmm0
3305 ; XOP-FALLBACK-NEXT: vpcmpeqd %xmm6, %xmm6, %xmm6
3306 ; XOP-FALLBACK-NEXT: vpshlb %xmm6, %xmm1, %xmm1
3307 ; XOP-FALLBACK-NEXT: vpshlb %xmm6, %xmm0, %xmm0
3308 ; XOP-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3309 ; XOP-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
3310 ; XOP-FALLBACK-NEXT: vpor %xmm7, %xmm5, %xmm5
3311 ; XOP-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3312 ; XOP-FALLBACK-NEXT: vpmullw %xmm4, %xmm6, %xmm4
3313 ; XOP-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
3314 ; XOP-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
3315 ; XOP-FALLBACK-NEXT: vpmullw %xmm5, %xmm0, %xmm0
3316 ; XOP-FALLBACK-NEXT: vmovdqa {{.*#+}} xmm5 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30]
3317 ; XOP-FALLBACK-NEXT: vpperm %xmm5, %xmm4, %xmm0, %xmm0
3318 ; XOP-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3319 ; XOP-FALLBACK-NEXT: vpor %xmm7, %xmm8, %xmm6
3320 ; XOP-FALLBACK-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3321 ; XOP-FALLBACK-NEXT: vpmullw %xmm7, %xmm4, %xmm4
3322 ; XOP-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
3323 ; XOP-FALLBACK-NEXT: vpmovzxbw {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
3324 ; XOP-FALLBACK-NEXT: vpmullw %xmm6, %xmm1, %xmm1
3325 ; XOP-FALLBACK-NEXT: vpperm %xmm5, %xmm4, %xmm1, %xmm1
3326 ; XOP-FALLBACK-NEXT: vpaddb %xmm3, %xmm1, %xmm1
3327 ; XOP-FALLBACK-NEXT: vpaddb %xmm2, %xmm0, %xmm0
3328 ; XOP-FALLBACK-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
3329 ; XOP-FALLBACK-NEXT: retq
3330 ;
3331 ; XOPAVX1-LABEL: vec256_i8_signed_mem_mem:
3332 ; XOPAVX1: # %bb.0:
3333 ; XOPAVX1-NEXT: vmovdqa (%rsi), %xmm0
3334 ; XOPAVX1-NEXT: vmovdqa 16(%rsi), %xmm1
3335 ; XOPAVX1-NEXT: vmovdqa (%rdi), %xmm2
3336 ; XOPAVX1-NEXT: vmovdqa 16(%rdi), %xmm3
3337 ; XOPAVX1-NEXT: vpcomgtb %xmm1, %xmm3, %xmm8
3338 ; XOPAVX1-NEXT: vpcomgtb %xmm0, %xmm2, %xmm5
3339 ; XOPAVX1-NEXT: vpminsb %xmm1, %xmm3, %xmm6
3340 ; XOPAVX1-NEXT: vpminsb %xmm0, %xmm2, %xmm7
3341 ; XOPAVX1-NEXT: vpmaxsb %xmm1, %xmm3, %xmm1
3342 ; XOPAVX1-NEXT: vpsubb %xmm6, %xmm1, %xmm1
3343 ; XOPAVX1-NEXT: vpmaxsb %xmm0, %xmm2, %xmm0
3344 ; XOPAVX1-NEXT: vpsubb %xmm7, %xmm0, %xmm0
3345 ; XOPAVX1-NEXT: vpcmpeqd %xmm6, %xmm6, %xmm6
3346 ; XOPAVX1-NEXT: vpshlb %xmm6, %xmm1, %xmm1
3347 ; XOPAVX1-NEXT: vpshlb %xmm6, %xmm0, %xmm0
3348 ; XOPAVX1-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3349 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
3350 ; XOPAVX1-NEXT: vpor %xmm7, %xmm5, %xmm5
3351 ; XOPAVX1-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3352 ; XOPAVX1-NEXT: vpmullw %xmm4, %xmm6, %xmm4
3353 ; XOPAVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
3354 ; XOPAVX1-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
3355 ; XOPAVX1-NEXT: vpmullw %xmm5, %xmm0, %xmm0
3356 ; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm5 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30]
3357 ; XOPAVX1-NEXT: vpperm %xmm5, %xmm4, %xmm0, %xmm0
3358 ; XOPAVX1-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3359 ; XOPAVX1-NEXT: vpor %xmm7, %xmm8, %xmm6
3360 ; XOPAVX1-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
3361 ; XOPAVX1-NEXT: vpmullw %xmm7, %xmm4, %xmm4
3362 ; XOPAVX1-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
3363 ; XOPAVX1-NEXT: vpmovzxbw {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
3364 ; XOPAVX1-NEXT: vpmullw %xmm6, %xmm1, %xmm1
3365 ; XOPAVX1-NEXT: vpperm %xmm5, %xmm4, %xmm1, %xmm1
3366 ; XOPAVX1-NEXT: vpaddb %xmm3, %xmm1, %xmm1
3367 ; XOPAVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0
3368 ; XOPAVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
3369 ; XOPAVX1-NEXT: retq
3370 ;
3371 ; AVX512F-LABEL: vec256_i8_signed_mem_mem:
3372 ; AVX512F: # %bb.0:
3373 ; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
3374 ; AVX512F-NEXT: vmovdqa (%rsi), %ymm1
3375 ; AVX512F-NEXT: vpcmpgtb %ymm1, %ymm0, %ymm2
3376 ; AVX512F-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
3377 ; AVX512F-NEXT: vpminsb %ymm1, %ymm0, %ymm3
3378 ; AVX512F-NEXT: vpmaxsb %ymm1, %ymm0, %ymm1
3379 ; AVX512F-NEXT: vpsubb %ymm3, %ymm1, %ymm1
3380 ; AVX512F-NEXT: vpsrlw $1, %ymm1, %ymm1
3381 ; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
3382 ; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
3383 ; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
3384 ; AVX512F-NEXT: vpmullw %ymm4, %ymm3, %ymm3
3385 ; AVX512F-NEXT: vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
3386 ; AVX512F-NEXT: vpand %ymm4, %ymm3, %ymm3
3387 ; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
3388 ; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
3389 ; AVX512F-NEXT: vpmullw %ymm2, %ymm1, %ymm1
3390 ; AVX512F-NEXT: vpand %ymm4, %ymm1, %ymm1
3391 ; AVX512F-NEXT: vpackuswb %ymm3, %ymm1, %ymm1
3392 ; AVX512F-NEXT: vpaddb %ymm0, %ymm1, %ymm0
3393 ; AVX512F-NEXT: retq
3394 ;
3395 ; AVX512VL-FALLBACK-LABEL: vec256_i8_signed_mem_mem:
3396 ; AVX512VL-FALLBACK: # %bb.0:
3397 ; AVX512VL-FALLBACK-NEXT: vmovdqa (%rdi), %ymm0
3398 ; AVX512VL-FALLBACK-NEXT: vmovdqa (%rsi), %ymm1
3399 ; AVX512VL-FALLBACK-NEXT: vpminsb %ymm1, %ymm0, %ymm2
3400 ; AVX512VL-FALLBACK-NEXT: vpmaxsb %ymm1, %ymm0, %ymm3
3401 ; AVX512VL-FALLBACK-NEXT: vpsubb %ymm2, %ymm3, %ymm2
3402 ; AVX512VL-FALLBACK-NEXT: vpsrlw $1, %ymm2, %ymm2
3403 ; AVX512VL-FALLBACK-NEXT: vpcmpgtb %ymm1, %ymm0, %ymm1
3404 ; AVX512VL-FALLBACK-NEXT: vpternlogq $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
3405 ; AVX512VL-FALLBACK-NEXT: vpsubb %ymm1, %ymm2, %ymm1
3406 ; AVX512VL-FALLBACK-NEXT: vpaddb %ymm0, %ymm1, %ymm0
3407 ; AVX512VL-FALLBACK-NEXT: retq
3408 ;
3409 ; AVX512BW-FALLBACK-LABEL: vec256_i8_signed_mem_mem:
3410 ; AVX512BW-FALLBACK: # %bb.0:
3411 ; AVX512BW-FALLBACK-NEXT: vmovdqa (%rdi), %ymm0
3412 ; AVX512BW-FALLBACK-NEXT: vmovdqa (%rsi), %ymm1
3413 ; AVX512BW-FALLBACK-NEXT: vpcmpgtb %zmm1, %zmm0, %k1
3414 ; AVX512BW-FALLBACK-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
3415 ; AVX512BW-FALLBACK-NEXT: vmovdqa {{.*#+}} ymm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
3416 ; AVX512BW-FALLBACK-NEXT: vmovdqu8 %zmm2, %zmm3 {%k1}
3417 ; AVX512BW-FALLBACK-NEXT: vpminsb %ymm1, %ymm0, %ymm2
3418 ; AVX512BW-FALLBACK-NEXT: vpmaxsb %ymm1, %ymm0, %ymm1
3419 ; AVX512BW-FALLBACK-NEXT: vpsubb %ymm2, %ymm1, %ymm1
3420 ; AVX512BW-FALLBACK-NEXT: vpsrlw $1, %ymm1, %ymm1
3421 ; AVX512BW-FALLBACK-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
3422 ; AVX512BW-FALLBACK-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
3423 ; AVX512BW-FALLBACK-NEXT: vpmovzxbw {{.*#+}} zmm2 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero,ymm3[16],zero,ymm3[17],zero,ymm3[18],zero,ymm3[19],zero,ymm3[20],zero,ymm3[21],zero,ymm3[22],zero,ymm3[23],zero,ymm3[24],zero,ymm3[25],zero,ymm3[26],zero,ymm3[27],zero,ymm3[28],zero,ymm3[29],zero,ymm3[30],zero,ymm3[31],zero
3424 ; AVX512BW-FALLBACK-NEXT: vpmullw %zmm2, %zmm1, %zmm1
3425 ; AVX512BW-FALLBACK-NEXT: vpmovwb %zmm1, %ymm1
3426 ; AVX512BW-FALLBACK-NEXT: vpaddb %ymm0, %ymm1, %ymm0
3427 ; AVX512BW-FALLBACK-NEXT: retq
3428 ;
3429 ; AVX512VLBW-LABEL: vec256_i8_signed_mem_mem:
3430 ; AVX512VLBW: # %bb.0:
3431 ; AVX512VLBW-NEXT: vmovdqa (%rdi), %ymm0
3432 ; AVX512VLBW-NEXT: vmovdqa (%rsi), %ymm1
3433 ; AVX512VLBW-NEXT: vpcmpgtb %ymm1, %ymm0, %k1
3434 ; AVX512VLBW-NEXT: vpminsb %ymm1, %ymm0, %ymm2
3435 ; AVX512VLBW-NEXT: vpmaxsb %ymm1, %ymm0, %ymm1
3436 ; AVX512VLBW-NEXT: vpsubb %ymm2, %ymm1, %ymm1
3437 ; AVX512VLBW-NEXT: vpsrlw $1, %ymm1, %ymm1
3438 ; AVX512VLBW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
3439 ; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
3440 ; AVX512VLBW-NEXT: vpsubb %ymm1, %ymm2, %ymm1 {%k1}
3441 ; AVX512VLBW-NEXT: vpaddb %ymm0, %ymm1, %ymm0
3442 ; AVX512VLBW-NEXT: retq
3443 %a1 = load <32 x i8>, ptr %a1_addr
3444 %a2 = load <32 x i8>, ptr %a2_addr
3445 %t3 = icmp sgt <32 x i8> %a1, %a2 ; signed
3446 %t4 = select <32 x i1> %t3, <32 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <32 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
3447 %t5 = select <32 x i1> %t3, <32 x i8> %a2, <32 x i8> %a1
3448 %t6 = select <32 x i1> %t3, <32 x i8> %a1, <32 x i8> %a2
3449 %t7 = sub <32 x i8> %t6, %t5
3450 %t8 = lshr <32 x i8> %t7, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
3451 %t9 = mul nsw <32 x i8> %t8, %t4 ; signed
3452 %a10 = add nsw <32 x i8> %t9, %a1 ; signed
3453 ret <32 x i8> %a10
3454 }