| blob | 36dbb46f0b0a451439dee89927f9b7183b8858e4 |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=SSE2
3 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
4 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX256,AVX2
5 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX256,AVX512,AVX512F
6 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=AVX,AVX256,AVX512,AVX512BW
8 define i32 @_Z10test_shortPsS_i_128(i16* nocapture readonly, i16* nocapture readonly, i32) local_unnamed_addr #0 {
9 ; SSE2-LABEL: _Z10test_shortPsS_i_128:
10 ; SSE2: # %bb.0: # %entry
11 ; SSE2-NEXT: movl %edx, %eax
12 ; SSE2-NEXT: pxor %xmm0, %xmm0
13 ; SSE2-NEXT: xorl %ecx, %ecx
14 ; SSE2-NEXT: .p2align 4, 0x90
15 ; SSE2-NEXT: .LBB0_1: # %vector.body
16 ; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
17 ; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero
18 ; SSE2-NEXT: movq {{.*#+}} xmm2 = mem[0],zero
19 ; SSE2-NEXT: movdqa %xmm2, %xmm3
20 ; SSE2-NEXT: pmulhw %xmm1, %xmm3
21 ; SSE2-NEXT: pmullw %xmm1, %xmm2
22 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
23 ; SSE2-NEXT: paddd %xmm2, %xmm0
24 ; SSE2-NEXT: addq $8, %rcx
25 ; SSE2-NEXT: cmpq %rcx, %rax
26 ; SSE2-NEXT: jne .LBB0_1
27 ; SSE2-NEXT: # %bb.2: # %middle.block
28 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
29 ; SSE2-NEXT: paddd %xmm0, %xmm1
30 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
31 ; SSE2-NEXT: paddd %xmm1, %xmm0
32 ; SSE2-NEXT: movd %xmm0, %eax
33 ; SSE2-NEXT: retq
34 ;
35 ; AVX-LABEL: _Z10test_shortPsS_i_128:
36 ; AVX: # %bb.0: # %entry
37 ; AVX-NEXT: movl %edx, %eax
38 ; AVX-NEXT: vpxor %xmm0, %xmm0, %xmm0
39 ; AVX-NEXT: xorl %ecx, %ecx
40 ; AVX-NEXT: .p2align 4, 0x90
41 ; AVX-NEXT: .LBB0_1: # %vector.body
42 ; AVX-NEXT: # =>This Inner Loop Header: Depth=1
43 ; AVX-NEXT: vpmovsxwd (%rdi,%rcx,2), %xmm1
44 ; AVX-NEXT: vpmovsxwd (%rsi,%rcx,2), %xmm2
45 ; AVX-NEXT: vpmulld %xmm1, %xmm2, %xmm1
46 ; AVX-NEXT: vpaddd %xmm0, %xmm1, %xmm0
47 ; AVX-NEXT: addq $8, %rcx
48 ; AVX-NEXT: cmpq %rcx, %rax
49 ; AVX-NEXT: jne .LBB0_1
50 ; AVX-NEXT: # %bb.2: # %middle.block
51 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
52 ; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
53 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
54 ; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
55 ; AVX-NEXT: vmovd %xmm0, %eax
56 ; AVX-NEXT: retq
57 entry:
58 %3 = zext i32 %2 to i64
59 br label %vector.body
61 vector.body:
62 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ]
63 %vec.phi = phi <4 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ]
64 %4 = getelementptr inbounds i16, i16* %0, i64 %index
65 %5 = bitcast i16* %4 to <4 x i16>*
66 %wide.load = load <4 x i16>, <4 x i16>* %5, align 2
67 %6 = sext <4 x i16> %wide.load to <4 x i32>
68 %7 = getelementptr inbounds i16, i16* %1, i64 %index
69 %8 = bitcast i16* %7 to <4 x i16>*
70 %wide.load14 = load <4 x i16>, <4 x i16>* %8, align 2
71 %9 = sext <4 x i16> %wide.load14 to <4 x i32>
72 %10 = mul nsw <4 x i32> %9, %6
73 %11 = add nsw <4 x i32> %10, %vec.phi
74 %index.next = add i64 %index, 8
75 %12 = icmp eq i64 %index.next, %3
76 br i1 %12, label %middle.block, label %vector.body
78 middle.block:
79 %rdx.shuf15 = shufflevector <4 x i32> %11, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
80 %bin.rdx16 = add <4 x i32> %11, %rdx.shuf15
81 %rdx.shuf17 = shufflevector <4 x i32> %bin.rdx16, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
82 %bin.rdx18 = add <4 x i32> %bin.rdx16, %rdx.shuf17
83 %13 = extractelement <4 x i32> %bin.rdx18, i32 0
84 ret i32 %13
85 }
87 define i32 @_Z10test_shortPsS_i_256(i16* nocapture readonly, i16* nocapture readonly, i32) local_unnamed_addr #0 {
88 ; SSE2-LABEL: _Z10test_shortPsS_i_256:
89 ; SSE2: # %bb.0: # %entry
90 ; SSE2-NEXT: movl %edx, %eax
91 ; SSE2-NEXT: pxor %xmm0, %xmm0
92 ; SSE2-NEXT: xorl %ecx, %ecx
93 ; SSE2-NEXT: pxor %xmm1, %xmm1
94 ; SSE2-NEXT: .p2align 4, 0x90
95 ; SSE2-NEXT: .LBB1_1: # %vector.body
96 ; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
97 ; SSE2-NEXT: movdqu (%rdi,%rcx,2), %xmm2
98 ; SSE2-NEXT: movdqu (%rsi,%rcx,2), %xmm3
99 ; SSE2-NEXT: pmaddwd %xmm2, %xmm3
100 ; SSE2-NEXT: paddd %xmm3, %xmm1
101 ; SSE2-NEXT: addq $8, %rcx
102 ; SSE2-NEXT: cmpq %rcx, %rax
103 ; SSE2-NEXT: jne .LBB1_1
104 ; SSE2-NEXT: # %bb.2: # %middle.block
105 ; SSE2-NEXT: paddd %xmm0, %xmm1
106 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
107 ; SSE2-NEXT: paddd %xmm1, %xmm0
108 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
109 ; SSE2-NEXT: paddd %xmm0, %xmm1
110 ; SSE2-NEXT: movd %xmm1, %eax
111 ; SSE2-NEXT: retq
112 ;
113 ; AVX1-LABEL: _Z10test_shortPsS_i_256:
114 ; AVX1: # %bb.0: # %entry
115 ; AVX1-NEXT: movl %edx, %eax
116 ; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0
117 ; AVX1-NEXT: xorl %ecx, %ecx
118 ; AVX1-NEXT: .p2align 4, 0x90
119 ; AVX1-NEXT: .LBB1_1: # %vector.body
120 ; AVX1-NEXT: # =>This Inner Loop Header: Depth=1
121 ; AVX1-NEXT: vmovdqu (%rsi,%rcx,2), %xmm1
122 ; AVX1-NEXT: vpmaddwd (%rdi,%rcx,2), %xmm1, %xmm1
123 ; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm1
124 ; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
125 ; AVX1-NEXT: addq $8, %rcx
126 ; AVX1-NEXT: cmpq %rcx, %rax
127 ; AVX1-NEXT: jne .LBB1_1
128 ; AVX1-NEXT: # %bb.2: # %middle.block
129 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
130 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
131 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
132 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
133 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
134 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
135 ; AVX1-NEXT: vmovd %xmm0, %eax
136 ; AVX1-NEXT: vzeroupper
137 ; AVX1-NEXT: retq
138 ;
139 ; AVX256-LABEL: _Z10test_shortPsS_i_256:
140 ; AVX256: # %bb.0: # %entry
141 ; AVX256-NEXT: movl %edx, %eax
142 ; AVX256-NEXT: vpxor %xmm0, %xmm0, %xmm0
143 ; AVX256-NEXT: xorl %ecx, %ecx
144 ; AVX256-NEXT: .p2align 4, 0x90
145 ; AVX256-NEXT: .LBB1_1: # %vector.body
146 ; AVX256-NEXT: # =>This Inner Loop Header: Depth=1
147 ; AVX256-NEXT: vmovdqu (%rsi,%rcx,2), %xmm1
148 ; AVX256-NEXT: vpmaddwd (%rdi,%rcx,2), %xmm1, %xmm1
149 ; AVX256-NEXT: vpaddd %ymm0, %ymm1, %ymm0
150 ; AVX256-NEXT: addq $8, %rcx
151 ; AVX256-NEXT: cmpq %rcx, %rax
152 ; AVX256-NEXT: jne .LBB1_1
153 ; AVX256-NEXT: # %bb.2: # %middle.block
154 ; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1
155 ; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0
156 ; AVX256-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
157 ; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0
158 ; AVX256-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
159 ; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0
160 ; AVX256-NEXT: vmovd %xmm0, %eax
161 ; AVX256-NEXT: vzeroupper
162 ; AVX256-NEXT: retq
163 entry:
164 %3 = zext i32 %2 to i64
165 br label %vector.body
167 vector.body:
168 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ]
169 %vec.phi = phi <8 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ]
170 %4 = getelementptr inbounds i16, i16* %0, i64 %index
171 %5 = bitcast i16* %4 to <8 x i16>*
172 %wide.load = load <8 x i16>, <8 x i16>* %5, align 2
173 %6 = sext <8 x i16> %wide.load to <8 x i32>
174 %7 = getelementptr inbounds i16, i16* %1, i64 %index
175 %8 = bitcast i16* %7 to <8 x i16>*
176 %wide.load14 = load <8 x i16>, <8 x i16>* %8, align 2
177 %9 = sext <8 x i16> %wide.load14 to <8 x i32>
178 %10 = mul nsw <8 x i32> %9, %6
179 %11 = add nsw <8 x i32> %10, %vec.phi
180 %index.next = add i64 %index, 8
181 %12 = icmp eq i64 %index.next, %3
182 br i1 %12, label %middle.block, label %vector.body
184 middle.block:
185 %rdx.shuf = shufflevector <8 x i32> %11, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
186 %bin.rdx = add <8 x i32> %11, %rdx.shuf
187 %rdx.shuf15 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
188 %bin.rdx16 = add <8 x i32> %bin.rdx, %rdx.shuf15
189 %rdx.shuf17 = shufflevector <8 x i32> %bin.rdx16, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
190 %bin.rdx18 = add <8 x i32> %bin.rdx16, %rdx.shuf17
191 %13 = extractelement <8 x i32> %bin.rdx18, i32 0
192 ret i32 %13
193 }
195 define i32 @_Z10test_shortPsS_i_512(i16* nocapture readonly, i16* nocapture readonly, i32) local_unnamed_addr #0 {
196 ; SSE2-LABEL: _Z10test_shortPsS_i_512:
197 ; SSE2: # %bb.0: # %entry
198 ; SSE2-NEXT: movl %edx, %eax
199 ; SSE2-NEXT: pxor %xmm0, %xmm0
200 ; SSE2-NEXT: xorl %ecx, %ecx
201 ; SSE2-NEXT: pxor %xmm2, %xmm2
202 ; SSE2-NEXT: pxor %xmm1, %xmm1
203 ; SSE2-NEXT: .p2align 4, 0x90
204 ; SSE2-NEXT: .LBB2_1: # %vector.body
205 ; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
206 ; SSE2-NEXT: movdqu (%rdi,%rcx,2), %xmm3
207 ; SSE2-NEXT: movdqu 16(%rdi,%rcx,2), %xmm4
208 ; SSE2-NEXT: movdqu (%rsi,%rcx,2), %xmm5
209 ; SSE2-NEXT: pmaddwd %xmm3, %xmm5
210 ; SSE2-NEXT: paddd %xmm5, %xmm2
211 ; SSE2-NEXT: movdqu 16(%rsi,%rcx,2), %xmm3
212 ; SSE2-NEXT: pmaddwd %xmm4, %xmm3
213 ; SSE2-NEXT: paddd %xmm3, %xmm1
214 ; SSE2-NEXT: addq $16, %rcx
215 ; SSE2-NEXT: cmpq %rcx, %rax
216 ; SSE2-NEXT: jne .LBB2_1
217 ; SSE2-NEXT: # %bb.2: # %middle.block
218 ; SSE2-NEXT: paddd %xmm0, %xmm2
219 ; SSE2-NEXT: paddd %xmm0, %xmm1
220 ; SSE2-NEXT: paddd %xmm2, %xmm1
221 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
222 ; SSE2-NEXT: paddd %xmm1, %xmm0
223 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
224 ; SSE2-NEXT: paddd %xmm0, %xmm1
225 ; SSE2-NEXT: movd %xmm1, %eax
226 ; SSE2-NEXT: retq
227 ;
228 ; AVX1-LABEL: _Z10test_shortPsS_i_512:
229 ; AVX1: # %bb.0: # %entry
230 ; AVX1-NEXT: movl %edx, %eax
231 ; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0
232 ; AVX1-NEXT: xorl %ecx, %ecx
233 ; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
234 ; AVX1-NEXT: .p2align 4, 0x90
235 ; AVX1-NEXT: .LBB2_1: # %vector.body
236 ; AVX1-NEXT: # =>This Inner Loop Header: Depth=1
237 ; AVX1-NEXT: vmovdqu (%rsi,%rcx,2), %xmm2
238 ; AVX1-NEXT: vmovdqu 16(%rsi,%rcx,2), %xmm3
239 ; AVX1-NEXT: vpmaddwd 16(%rdi,%rcx,2), %xmm3, %xmm3
240 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4
241 ; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
242 ; AVX1-NEXT: vpmaddwd (%rdi,%rcx,2), %xmm2, %xmm2
243 ; AVX1-NEXT: vpaddd %xmm1, %xmm2, %xmm1
244 ; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1
245 ; AVX1-NEXT: addq $16, %rcx
246 ; AVX1-NEXT: cmpq %rcx, %rax
247 ; AVX1-NEXT: jne .LBB2_1
248 ; AVX1-NEXT: # %bb.2: # %middle.block
249 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
250 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
251 ; AVX1-NEXT: vpaddd %xmm3, %xmm2, %xmm2
252 ; AVX1-NEXT: vpaddd %xmm2, %xmm0, %xmm0
253 ; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm0
254 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
255 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
256 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
257 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
258 ; AVX1-NEXT: vmovd %xmm0, %eax
259 ; AVX1-NEXT: vzeroupper
260 ; AVX1-NEXT: retq
261 ;
262 ; AVX2-LABEL: _Z10test_shortPsS_i_512:
263 ; AVX2: # %bb.0: # %entry
264 ; AVX2-NEXT: movl %edx, %eax
265 ; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
266 ; AVX2-NEXT: xorl %ecx, %ecx
267 ; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
268 ; AVX2-NEXT: .p2align 4, 0x90
269 ; AVX2-NEXT: .LBB2_1: # %vector.body
270 ; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
271 ; AVX2-NEXT: vmovdqu (%rsi,%rcx,2), %ymm2
272 ; AVX2-NEXT: vpmaddwd (%rdi,%rcx,2), %ymm2, %ymm2
273 ; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
274 ; AVX2-NEXT: addq $16, %rcx
275 ; AVX2-NEXT: cmpq %rcx, %rax
276 ; AVX2-NEXT: jne .LBB2_1
277 ; AVX2-NEXT: # %bb.2: # %middle.block
278 ; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
279 ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
280 ; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
281 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
282 ; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
283 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
284 ; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
285 ; AVX2-NEXT: vmovd %xmm0, %eax
286 ; AVX2-NEXT: vzeroupper
287 ; AVX2-NEXT: retq
288 ;
289 ; AVX512-LABEL: _Z10test_shortPsS_i_512:
290 ; AVX512: # %bb.0: # %entry
291 ; AVX512-NEXT: movl %edx, %eax
292 ; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
293 ; AVX512-NEXT: xorl %ecx, %ecx
294 ; AVX512-NEXT: .p2align 4, 0x90
295 ; AVX512-NEXT: .LBB2_1: # %vector.body
296 ; AVX512-NEXT: # =>This Inner Loop Header: Depth=1
297 ; AVX512-NEXT: vmovdqu (%rsi,%rcx,2), %ymm1
298 ; AVX512-NEXT: vpmaddwd (%rdi,%rcx,2), %ymm1, %ymm1
299 ; AVX512-NEXT: vpaddd %zmm0, %zmm1, %zmm0
300 ; AVX512-NEXT: addq $16, %rcx
301 ; AVX512-NEXT: cmpq %rcx, %rax
302 ; AVX512-NEXT: jne .LBB2_1
303 ; AVX512-NEXT: # %bb.2: # %middle.block
304 ; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
305 ; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
306 ; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
307 ; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
308 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
309 ; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
310 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
311 ; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
312 ; AVX512-NEXT: vmovd %xmm0, %eax
313 ; AVX512-NEXT: vzeroupper
314 ; AVX512-NEXT: retq
315 entry:
316 %3 = zext i32 %2 to i64
317 br label %vector.body
319 vector.body:
320 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ]
321 %vec.phi = phi <16 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ]
322 %4 = getelementptr inbounds i16, i16* %0, i64 %index
323 %5 = bitcast i16* %4 to <16 x i16>*
324 %wide.load = load <16 x i16>, <16 x i16>* %5, align 2
325 %6 = sext <16 x i16> %wide.load to <16 x i32>
326 %7 = getelementptr inbounds i16, i16* %1, i64 %index
327 %8 = bitcast i16* %7 to <16 x i16>*
328 %wide.load14 = load <16 x i16>, <16 x i16>* %8, align 2
329 %9 = sext <16 x i16> %wide.load14 to <16 x i32>
330 %10 = mul nsw <16 x i32> %9, %6
331 %11 = add nsw <16 x i32> %10, %vec.phi
332 %index.next = add i64 %index, 16
333 %12 = icmp eq i64 %index.next, %3
334 br i1 %12, label %middle.block, label %vector.body
336 middle.block:
337 %rdx.shuf1 = shufflevector <16 x i32> %11, <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
338 %bin.rdx1 = add <16 x i32> %11, %rdx.shuf1
339 %rdx.shuf = shufflevector <16 x i32> %bin.rdx1, <16 x i32> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
340 %bin.rdx = add <16 x i32> %bin.rdx1, %rdx.shuf
341 %rdx.shuf15 = shufflevector <16 x i32> %bin.rdx, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
342 %bin.rdx16 = add <16 x i32> %bin.rdx, %rdx.shuf15
343 %rdx.shuf17 = shufflevector <16 x i32> %bin.rdx16, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
344 %bin.rdx18 = add <16 x i32> %bin.rdx16, %rdx.shuf17
345 %13 = extractelement <16 x i32> %bin.rdx18, i32 0
346 ret i32 %13
347 }
349 define i32 @_Z10test_shortPsS_i_1024(i16* nocapture readonly, i16* nocapture readonly, i32) local_unnamed_addr #0 {
350 ; SSE2-LABEL: _Z10test_shortPsS_i_1024:
351 ; SSE2: # %bb.0: # %entry
352 ; SSE2-NEXT: movl %edx, %eax
353 ; SSE2-NEXT: pxor %xmm8, %xmm8
354 ; SSE2-NEXT: xorl %ecx, %ecx
355 ; SSE2-NEXT: pxor %xmm2, %xmm2
356 ; SSE2-NEXT: pxor %xmm4, %xmm4
357 ; SSE2-NEXT: pxor %xmm1, %xmm1
358 ; SSE2-NEXT: pxor %xmm3, %xmm3
359 ; SSE2-NEXT: .p2align 4, 0x90
360 ; SSE2-NEXT: .LBB3_1: # %vector.body
361 ; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
362 ; SSE2-NEXT: movdqu (%rdi,%rcx,2), %xmm5
363 ; SSE2-NEXT: movdqu 16(%rdi,%rcx,2), %xmm6
364 ; SSE2-NEXT: movdqu 32(%rdi,%rcx,2), %xmm7
365 ; SSE2-NEXT: movdqu 48(%rdi,%rcx,2), %xmm9
366 ; SSE2-NEXT: movdqu (%rsi,%rcx,2), %xmm0
367 ; SSE2-NEXT: pmaddwd %xmm5, %xmm0
368 ; SSE2-NEXT: paddd %xmm0, %xmm2
369 ; SSE2-NEXT: movdqu 16(%rsi,%rcx,2), %xmm0
370 ; SSE2-NEXT: pmaddwd %xmm6, %xmm0
371 ; SSE2-NEXT: paddd %xmm0, %xmm4
372 ; SSE2-NEXT: movdqu 32(%rsi,%rcx,2), %xmm0
373 ; SSE2-NEXT: pmaddwd %xmm7, %xmm0
374 ; SSE2-NEXT: paddd %xmm0, %xmm1
375 ; SSE2-NEXT: movdqu 48(%rsi,%rcx,2), %xmm0
376 ; SSE2-NEXT: pmaddwd %xmm9, %xmm0
377 ; SSE2-NEXT: paddd %xmm0, %xmm3
378 ; SSE2-NEXT: addq $16, %rcx
379 ; SSE2-NEXT: cmpq %rcx, %rax
380 ; SSE2-NEXT: jne .LBB3_1
381 ; SSE2-NEXT: # %bb.2: # %middle.block
382 ; SSE2-NEXT: paddd %xmm8, %xmm4
383 ; SSE2-NEXT: paddd %xmm8, %xmm3
384 ; SSE2-NEXT: paddd %xmm4, %xmm3
385 ; SSE2-NEXT: paddd %xmm8, %xmm2
386 ; SSE2-NEXT: paddd %xmm8, %xmm1
387 ; SSE2-NEXT: paddd %xmm3, %xmm1
388 ; SSE2-NEXT: paddd %xmm2, %xmm1
389 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
390 ; SSE2-NEXT: paddd %xmm1, %xmm0
391 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
392 ; SSE2-NEXT: paddd %xmm0, %xmm1
393 ; SSE2-NEXT: movd %xmm1, %eax
394 ; SSE2-NEXT: retq
395 ;
396 ; AVX1-LABEL: _Z10test_shortPsS_i_1024:
397 ; AVX1: # %bb.0: # %entry
398 ; AVX1-NEXT: movl %edx, %eax
399 ; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0
400 ; AVX1-NEXT: xorl %ecx, %ecx
401 ; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
402 ; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
403 ; AVX1-NEXT: .p2align 4, 0x90
404 ; AVX1-NEXT: .LBB3_1: # %vector.body
405 ; AVX1-NEXT: # =>This Inner Loop Header: Depth=1
406 ; AVX1-NEXT: vmovdqu (%rsi,%rcx,2), %xmm3
407 ; AVX1-NEXT: vmovdqu 16(%rsi,%rcx,2), %xmm4
408 ; AVX1-NEXT: vmovdqu 32(%rsi,%rcx,2), %xmm5
409 ; AVX1-NEXT: vmovdqu 48(%rsi,%rcx,2), %xmm6
410 ; AVX1-NEXT: vpmaddwd 48(%rdi,%rcx,2), %xmm6, %xmm6
411 ; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm7
412 ; AVX1-NEXT: vpaddd %xmm7, %xmm6, %xmm6
413 ; AVX1-NEXT: vpmaddwd 32(%rdi,%rcx,2), %xmm5, %xmm5
414 ; AVX1-NEXT: vpaddd %xmm2, %xmm5, %xmm2
415 ; AVX1-NEXT: vinsertf128 $1, %xmm6, %ymm2, %ymm2
416 ; AVX1-NEXT: vpmaddwd 16(%rdi,%rcx,2), %xmm4, %xmm4
417 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
418 ; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4
419 ; AVX1-NEXT: vpmaddwd (%rdi,%rcx,2), %xmm3, %xmm3
420 ; AVX1-NEXT: vpaddd %xmm1, %xmm3, %xmm1
421 ; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm1, %ymm1
422 ; AVX1-NEXT: addq $16, %rcx
423 ; AVX1-NEXT: cmpq %rcx, %rax
424 ; AVX1-NEXT: jne .LBB3_1
425 ; AVX1-NEXT: # %bb.2: # %middle.block
426 ; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm3
427 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4
428 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5
429 ; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
430 ; AVX1-NEXT: vpaddd %xmm5, %xmm2, %xmm2
431 ; AVX1-NEXT: vpaddd %xmm2, %xmm5, %xmm2
432 ; AVX1-NEXT: vpaddd %xmm2, %xmm4, %xmm2
433 ; AVX1-NEXT: vpaddd %xmm3, %xmm0, %xmm0
434 ; AVX1-NEXT: vpaddd %xmm2, %xmm0, %xmm0
435 ; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm0
436 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
437 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
438 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
439 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
440 ; AVX1-NEXT: vmovd %xmm0, %eax
441 ; AVX1-NEXT: vzeroupper
442 ; AVX1-NEXT: retq
443 ;
444 ; AVX2-LABEL: _Z10test_shortPsS_i_1024:
445 ; AVX2: # %bb.0: # %entry
446 ; AVX2-NEXT: movl %edx, %eax
447 ; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
448 ; AVX2-NEXT: xorl %ecx, %ecx
449 ; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
450 ; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
451 ; AVX2-NEXT: .p2align 4, 0x90
452 ; AVX2-NEXT: .LBB3_1: # %vector.body
453 ; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
454 ; AVX2-NEXT: vmovdqu (%rsi,%rcx,2), %ymm3
455 ; AVX2-NEXT: vmovdqu 32(%rsi,%rcx,2), %ymm4
456 ; AVX2-NEXT: vpmaddwd 32(%rdi,%rcx,2), %ymm4, %ymm4
457 ; AVX2-NEXT: vpaddd %ymm2, %ymm4, %ymm2
458 ; AVX2-NEXT: vpmaddwd (%rdi,%rcx,2), %ymm3, %ymm3
459 ; AVX2-NEXT: vpaddd %ymm1, %ymm3, %ymm1
460 ; AVX2-NEXT: addq $16, %rcx
461 ; AVX2-NEXT: cmpq %rcx, %rax
462 ; AVX2-NEXT: jne .LBB3_1
463 ; AVX2-NEXT: # %bb.2: # %middle.block
464 ; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm1
465 ; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
466 ; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
467 ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
468 ; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
469 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
470 ; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
471 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
472 ; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
473 ; AVX2-NEXT: vmovd %xmm0, %eax
474 ; AVX2-NEXT: vzeroupper
475 ; AVX2-NEXT: retq
476 ;
477 ; AVX512F-LABEL: _Z10test_shortPsS_i_1024:
478 ; AVX512F: # %bb.0: # %entry
479 ; AVX512F-NEXT: movl %edx, %eax
480 ; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
481 ; AVX512F-NEXT: xorl %ecx, %ecx
482 ; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
483 ; AVX512F-NEXT: .p2align 4, 0x90
484 ; AVX512F-NEXT: .LBB3_1: # %vector.body
485 ; AVX512F-NEXT: # =>This Inner Loop Header: Depth=1
486 ; AVX512F-NEXT: vmovdqu (%rsi,%rcx,2), %ymm2
487 ; AVX512F-NEXT: vmovdqu 32(%rsi,%rcx,2), %ymm3
488 ; AVX512F-NEXT: vpmaddwd 32(%rdi,%rcx,2), %ymm3, %ymm3
489 ; AVX512F-NEXT: vpmaddwd (%rdi,%rcx,2), %ymm2, %ymm2
490 ; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2
491 ; AVX512F-NEXT: vpaddd %zmm1, %zmm2, %zmm1
492 ; AVX512F-NEXT: addq $16, %rcx
493 ; AVX512F-NEXT: cmpq %rcx, %rax
494 ; AVX512F-NEXT: jne .LBB3_1
495 ; AVX512F-NEXT: # %bb.2: # %middle.block
496 ; AVX512F-NEXT: vpaddd %zmm0, %zmm1, %zmm0
497 ; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
498 ; AVX512F-NEXT: vpaddd %zmm1, %zmm0, %zmm0
499 ; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
500 ; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
501 ; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
502 ; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
503 ; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
504 ; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
505 ; AVX512F-NEXT: vmovd %xmm0, %eax
506 ; AVX512F-NEXT: vzeroupper
507 ; AVX512F-NEXT: retq
508 ;
509 ; AVX512BW-LABEL: _Z10test_shortPsS_i_1024:
510 ; AVX512BW: # %bb.0: # %entry
511 ; AVX512BW-NEXT: movl %edx, %eax
512 ; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
513 ; AVX512BW-NEXT: xorl %ecx, %ecx
514 ; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
515 ; AVX512BW-NEXT: .p2align 4, 0x90
516 ; AVX512BW-NEXT: .LBB3_1: # %vector.body
517 ; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
518 ; AVX512BW-NEXT: vmovdqu64 (%rsi,%rcx,2), %zmm2
519 ; AVX512BW-NEXT: vpmaddwd (%rdi,%rcx,2), %zmm2, %zmm2
520 ; AVX512BW-NEXT: vpaddd %zmm1, %zmm2, %zmm1
521 ; AVX512BW-NEXT: addq $16, %rcx
522 ; AVX512BW-NEXT: cmpq %rcx, %rax
523 ; AVX512BW-NEXT: jne .LBB3_1
524 ; AVX512BW-NEXT: # %bb.2: # %middle.block
525 ; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
526 ; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
527 ; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
528 ; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
529 ; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
530 ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
531 ; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
532 ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
533 ; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
534 ; AVX512BW-NEXT: vmovd %xmm0, %eax
535 ; AVX512BW-NEXT: vzeroupper
536 ; AVX512BW-NEXT: retq
537 entry:
538 %3 = zext i32 %2 to i64
539 br label %vector.body
541 vector.body:
542 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ]
543 %vec.phi = phi <32 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ]
544 %4 = getelementptr inbounds i16, i16* %0, i64 %index
545 %5 = bitcast i16* %4 to <32 x i16>*
546 %wide.load = load <32 x i16>, <32 x i16>* %5, align 2
547 %6 = sext <32 x i16> %wide.load to <32 x i32>
548 %7 = getelementptr inbounds i16, i16* %1, i64 %index
549 %8 = bitcast i16* %7 to <32 x i16>*
550 %wide.load14 = load <32 x i16>, <32 x i16>* %8, align 2
551 %9 = sext <32 x i16> %wide.load14 to <32 x i32>
552 %10 = mul nsw <32 x i32> %9, %6
553 %11 = add nsw <32 x i32> %10, %vec.phi
554 %index.next = add i64 %index, 16
555 %12 = icmp eq i64 %index.next, %3
556 br i1 %12, label %middle.block, label %vector.body
558 middle.block:
559 %rdx.shuf2 = shufflevector <32 x i32> %11, <32 x i32> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
560 %bin.rdx2 = add <32 x i32> %11, %rdx.shuf2
561 %rdx.shuf1 = shufflevector <32 x i32> %bin.rdx2, <32 x i32> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
562 %bin.rdx1 = add <32 x i32> %bin.rdx2, %rdx.shuf1
563 %rdx.shuf = shufflevector <32 x i32> %bin.rdx1, <32 x i32> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
564 %bin.rdx = add <32 x i32> %bin.rdx1, %rdx.shuf
565 %rdx.shuf15 = shufflevector <32 x i32> %bin.rdx, <32 x i32> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
566 %bin.rdx16 = add <32 x i32> %bin.rdx, %rdx.shuf15
567 %rdx.shuf17 = shufflevector <32 x i32> %bin.rdx16, <32 x i32> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
568 %bin.rdx18 = add <32 x i32> %bin.rdx16, %rdx.shuf17
569 %13 = extractelement <32 x i32> %bin.rdx18, i32 0
570 ret i32 %13
571 }
573 define i32 @_Z9test_charPcS_i_128(i8* nocapture readonly, i8* nocapture readonly, i32) local_unnamed_addr #0 {
574 ; SSE2-LABEL: _Z9test_charPcS_i_128:
575 ; SSE2: # %bb.0: # %entry
576 ; SSE2-NEXT: movl %edx, %eax
577 ; SSE2-NEXT: pxor %xmm0, %xmm0
578 ; SSE2-NEXT: xorl %ecx, %ecx
579 ; SSE2-NEXT: .p2align 4, 0x90
580 ; SSE2-NEXT: .LBB4_1: # %vector.body
581 ; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
582 ; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
583 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
584 ; SSE2-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
585 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
586 ; SSE2-NEXT: psraw $8, %xmm1
587 ; SSE2-NEXT: psraw $8, %xmm2
588 ; SSE2-NEXT: pmullw %xmm1, %xmm2
589 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
590 ; SSE2-NEXT: psrad $16, %xmm1
591 ; SSE2-NEXT: paddd %xmm1, %xmm0
592 ; SSE2-NEXT: addq $16, %rcx
593 ; SSE2-NEXT: cmpq %rcx, %rax
594 ; SSE2-NEXT: jne .LBB4_1
595 ; SSE2-NEXT: # %bb.2: # %middle.block
596 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
597 ; SSE2-NEXT: paddd %xmm0, %xmm1
598 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
599 ; SSE2-NEXT: paddd %xmm1, %xmm0
600 ; SSE2-NEXT: movd %xmm0, %eax
601 ; SSE2-NEXT: retq
602 ;
603 ; AVX-LABEL: _Z9test_charPcS_i_128:
604 ; AVX: # %bb.0: # %entry
605 ; AVX-NEXT: movl %edx, %eax
606 ; AVX-NEXT: vpxor %xmm0, %xmm0, %xmm0
607 ; AVX-NEXT: xorl %ecx, %ecx
608 ; AVX-NEXT: .p2align 4, 0x90
609 ; AVX-NEXT: .LBB4_1: # %vector.body
610 ; AVX-NEXT: # =>This Inner Loop Header: Depth=1
611 ; AVX-NEXT: vpmovsxbd (%rdi,%rcx), %xmm1
612 ; AVX-NEXT: vpmovsxbd (%rsi,%rcx), %xmm2
613 ; AVX-NEXT: vpmulld %xmm1, %xmm2, %xmm1
614 ; AVX-NEXT: vpaddd %xmm0, %xmm1, %xmm0
615 ; AVX-NEXT: addq $16, %rcx
616 ; AVX-NEXT: cmpq %rcx, %rax
617 ; AVX-NEXT: jne .LBB4_1
618 ; AVX-NEXT: # %bb.2: # %middle.block
619 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
620 ; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
621 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
622 ; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
623 ; AVX-NEXT: vmovd %xmm0, %eax
624 ; AVX-NEXT: retq
625 entry:
626 %3 = zext i32 %2 to i64
627 br label %vector.body
629 vector.body:
630 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ]
631 %vec.phi = phi <4 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ]
632 %4 = getelementptr inbounds i8, i8* %0, i64 %index
633 %5 = bitcast i8* %4 to <4 x i8>*
634 %wide.load = load <4 x i8>, <4 x i8>* %5, align 1
635 %6 = sext <4 x i8> %wide.load to <4 x i32>
636 %7 = getelementptr inbounds i8, i8* %1, i64 %index
637 %8 = bitcast i8* %7 to <4 x i8>*
638 %wide.load14 = load <4 x i8>, <4 x i8>* %8, align 1
639 %9 = sext <4 x i8> %wide.load14 to <4 x i32>
640 %10 = mul nsw <4 x i32> %9, %6
641 %11 = add nsw <4 x i32> %10, %vec.phi
642 %index.next = add i64 %index, 16
643 %12 = icmp eq i64 %index.next, %3
644 br i1 %12, label %middle.block, label %vector.body
646 middle.block:
647 %rdx.shuf17 = shufflevector <4 x i32> %11, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
648 %bin.rdx18 = add <4 x i32> %11, %rdx.shuf17
649 %rdx.shuf19 = shufflevector <4 x i32> %bin.rdx18, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
650 %bin.rdx20 = add <4 x i32> %bin.rdx18, %rdx.shuf19
651 %13 = extractelement <4 x i32> %bin.rdx20, i32 0
652 ret i32 %13
653 }
655 define i32 @_Z9test_charPcS_i_256(i8* nocapture readonly, i8* nocapture readonly, i32) local_unnamed_addr #0 {
656 ; SSE2-LABEL: _Z9test_charPcS_i_256:
657 ; SSE2: # %bb.0: # %entry
658 ; SSE2-NEXT: movl %edx, %eax
659 ; SSE2-NEXT: pxor %xmm0, %xmm0
660 ; SSE2-NEXT: xorl %ecx, %ecx
661 ; SSE2-NEXT: pxor %xmm1, %xmm1
662 ; SSE2-NEXT: .p2align 4, 0x90
663 ; SSE2-NEXT: .LBB5_1: # %vector.body
664 ; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
665 ; SSE2-NEXT: movq {{.*#+}} xmm2 = mem[0],zero
666 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
667 ; SSE2-NEXT: movq {{.*#+}} xmm3 = mem[0],zero
668 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
669 ; SSE2-NEXT: psraw $8, %xmm2
670 ; SSE2-NEXT: psraw $8, %xmm3
671 ; SSE2-NEXT: pmaddwd %xmm2, %xmm3
672 ; SSE2-NEXT: paddd %xmm3, %xmm1
673 ; SSE2-NEXT: addq $16, %rcx
674 ; SSE2-NEXT: cmpq %rcx, %rax
675 ; SSE2-NEXT: jne .LBB5_1
676 ; SSE2-NEXT: # %bb.2: # %middle.block
677 ; SSE2-NEXT: paddd %xmm0, %xmm1
678 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
679 ; SSE2-NEXT: paddd %xmm1, %xmm0
680 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
681 ; SSE2-NEXT: paddd %xmm0, %xmm1
682 ; SSE2-NEXT: movd %xmm1, %eax
683 ; SSE2-NEXT: retq
684 ;
685 ; AVX1-LABEL: _Z9test_charPcS_i_256:
686 ; AVX1: # %bb.0: # %entry
687 ; AVX1-NEXT: movl %edx, %eax
688 ; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0
689 ; AVX1-NEXT: xorl %ecx, %ecx
690 ; AVX1-NEXT: .p2align 4, 0x90
691 ; AVX1-NEXT: .LBB5_1: # %vector.body
692 ; AVX1-NEXT: # =>This Inner Loop Header: Depth=1
693 ; AVX1-NEXT: vpmovsxbw (%rdi,%rcx), %xmm1
694 ; AVX1-NEXT: vpmovsxbw (%rsi,%rcx), %xmm2
695 ; AVX1-NEXT: vpmaddwd %xmm1, %xmm2, %xmm1
696 ; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm1
697 ; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
698 ; AVX1-NEXT: addq $16, %rcx
699 ; AVX1-NEXT: cmpq %rcx, %rax
700 ; AVX1-NEXT: jne .LBB5_1
701 ; AVX1-NEXT: # %bb.2: # %middle.block
702 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
703 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
704 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
705 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
706 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
707 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
708 ; AVX1-NEXT: vmovd %xmm0, %eax
709 ; AVX1-NEXT: vzeroupper
710 ; AVX1-NEXT: retq
711 ;
712 ; AVX256-LABEL: _Z9test_charPcS_i_256:
713 ; AVX256: # %bb.0: # %entry
714 ; AVX256-NEXT: movl %edx, %eax
715 ; AVX256-NEXT: vpxor %xmm0, %xmm0, %xmm0
716 ; AVX256-NEXT: xorl %ecx, %ecx
717 ; AVX256-NEXT: .p2align 4, 0x90
718 ; AVX256-NEXT: .LBB5_1: # %vector.body
719 ; AVX256-NEXT: # =>This Inner Loop Header: Depth=1
720 ; AVX256-NEXT: vpmovsxbw (%rdi,%rcx), %xmm1
721 ; AVX256-NEXT: vpmovsxbw (%rsi,%rcx), %xmm2
722 ; AVX256-NEXT: vpmaddwd %xmm1, %xmm2, %xmm1
723 ; AVX256-NEXT: vpaddd %ymm0, %ymm1, %ymm0
724 ; AVX256-NEXT: addq $16, %rcx
725 ; AVX256-NEXT: cmpq %rcx, %rax
726 ; AVX256-NEXT: jne .LBB5_1
727 ; AVX256-NEXT: # %bb.2: # %middle.block
728 ; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1
729 ; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0
730 ; AVX256-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
731 ; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0
732 ; AVX256-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
733 ; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0
734 ; AVX256-NEXT: vmovd %xmm0, %eax
735 ; AVX256-NEXT: vzeroupper
736 ; AVX256-NEXT: retq
737 entry:
738 %3 = zext i32 %2 to i64
739 br label %vector.body
741 vector.body:
742 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ]
743 %vec.phi = phi <8 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ]
744 %4 = getelementptr inbounds i8, i8* %0, i64 %index
745 %5 = bitcast i8* %4 to <8 x i8>*
746 %wide.load = load <8 x i8>, <8 x i8>* %5, align 1
747 %6 = sext <8 x i8> %wide.load to <8 x i32>
748 %7 = getelementptr inbounds i8, i8* %1, i64 %index
749 %8 = bitcast i8* %7 to <8 x i8>*
750 %wide.load14 = load <8 x i8>, <8 x i8>* %8, align 1
751 %9 = sext <8 x i8> %wide.load14 to <8 x i32>
752 %10 = mul nsw <8 x i32> %9, %6
753 %11 = add nsw <8 x i32> %10, %vec.phi
754 %index.next = add i64 %index, 16
755 %12 = icmp eq i64 %index.next, %3
756 br i1 %12, label %middle.block, label %vector.body
758 middle.block:
759 %rdx.shuf15 = shufflevector <8 x i32> %11, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
760 %bin.rdx16 = add <8 x i32> %11, %rdx.shuf15
761 %rdx.shuf17 = shufflevector <8 x i32> %bin.rdx16, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
762 %bin.rdx18 = add <8 x i32> %bin.rdx16, %rdx.shuf17
763 %rdx.shuf19 = shufflevector <8 x i32> %bin.rdx18, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
764 %bin.rdx20 = add <8 x i32> %bin.rdx18, %rdx.shuf19
765 %13 = extractelement <8 x i32> %bin.rdx20, i32 0
766 ret i32 %13
767 }
769 define i32 @_Z9test_charPcS_i_512(i8* nocapture readonly, i8* nocapture readonly, i32) local_unnamed_addr #0 {
770 ; SSE2-LABEL: _Z9test_charPcS_i_512:
771 ; SSE2: # %bb.0: # %entry
772 ; SSE2-NEXT: movl %edx, %eax
773 ; SSE2-NEXT: pxor %xmm0, %xmm0
774 ; SSE2-NEXT: xorl %ecx, %ecx
775 ; SSE2-NEXT: pxor %xmm2, %xmm2
776 ; SSE2-NEXT: pxor %xmm1, %xmm1
777 ; SSE2-NEXT: .p2align 4, 0x90
778 ; SSE2-NEXT: .LBB6_1: # %vector.body
779 ; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
780 ; SSE2-NEXT: movdqu (%rdi,%rcx), %xmm3
781 ; SSE2-NEXT: movdqu (%rsi,%rcx), %xmm4
782 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15]
783 ; SSE2-NEXT: psraw $8, %xmm5
784 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm4[8],xmm6[9],xmm4[9],xmm6[10],xmm4[10],xmm6[11],xmm4[11],xmm6[12],xmm4[12],xmm6[13],xmm4[13],xmm6[14],xmm4[14],xmm6[15],xmm4[15]
785 ; SSE2-NEXT: psraw $8, %xmm6
786 ; SSE2-NEXT: pmaddwd %xmm5, %xmm6
787 ; SSE2-NEXT: paddd %xmm6, %xmm1
788 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
789 ; SSE2-NEXT: psraw $8, %xmm3
790 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
791 ; SSE2-NEXT: psraw $8, %xmm4
792 ; SSE2-NEXT: pmaddwd %xmm3, %xmm4
793 ; SSE2-NEXT: paddd %xmm4, %xmm2
794 ; SSE2-NEXT: addq $16, %rcx
795 ; SSE2-NEXT: cmpq %rcx, %rax
796 ; SSE2-NEXT: jne .LBB6_1
797 ; SSE2-NEXT: # %bb.2: # %middle.block
798 ; SSE2-NEXT: paddd %xmm0, %xmm2
799 ; SSE2-NEXT: paddd %xmm0, %xmm1
800 ; SSE2-NEXT: paddd %xmm2, %xmm1
801 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
802 ; SSE2-NEXT: paddd %xmm1, %xmm0
803 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
804 ; SSE2-NEXT: paddd %xmm0, %xmm1
805 ; SSE2-NEXT: movd %xmm1, %eax
806 ; SSE2-NEXT: retq
807 ;
808 ; AVX1-LABEL: _Z9test_charPcS_i_512:
809 ; AVX1: # %bb.0: # %entry
810 ; AVX1-NEXT: movl %edx, %eax
811 ; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0
812 ; AVX1-NEXT: xorl %ecx, %ecx
813 ; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
814 ; AVX1-NEXT: .p2align 4, 0x90
815 ; AVX1-NEXT: .LBB6_1: # %vector.body
816 ; AVX1-NEXT: # =>This Inner Loop Header: Depth=1
817 ; AVX1-NEXT: vpmovsxbw (%rdi,%rcx), %xmm2
818 ; AVX1-NEXT: vpmovsxbw 8(%rdi,%rcx), %xmm3
819 ; AVX1-NEXT: vpmovsxbw (%rsi,%rcx), %xmm4
820 ; AVX1-NEXT: vpmaddwd %xmm2, %xmm4, %xmm2
821 ; AVX1-NEXT: vpmovsxbw 8(%rsi,%rcx), %xmm4
822 ; AVX1-NEXT: vpmaddwd %xmm3, %xmm4, %xmm3
823 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4
824 ; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
825 ; AVX1-NEXT: vpaddd %xmm1, %xmm2, %xmm1
826 ; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1
827 ; AVX1-NEXT: addq $16, %rcx
828 ; AVX1-NEXT: cmpq %rcx, %rax
829 ; AVX1-NEXT: jne .LBB6_1
830 ; AVX1-NEXT: # %bb.2: # %middle.block
831 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
832 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
833 ; AVX1-NEXT: vpaddd %xmm3, %xmm2, %xmm2
834 ; AVX1-NEXT: vpaddd %xmm2, %xmm0, %xmm0
835 ; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm0
836 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
837 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
838 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
839 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
840 ; AVX1-NEXT: vmovd %xmm0, %eax
841 ; AVX1-NEXT: vzeroupper
842 ; AVX1-NEXT: retq
843 ;
844 ; AVX2-LABEL: _Z9test_charPcS_i_512:
845 ; AVX2: # %bb.0: # %entry
846 ; AVX2-NEXT: movl %edx, %eax
847 ; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
848 ; AVX2-NEXT: xorl %ecx, %ecx
849 ; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
850 ; AVX2-NEXT: .p2align 4, 0x90
851 ; AVX2-NEXT: .LBB6_1: # %vector.body
852 ; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
853 ; AVX2-NEXT: vpmovsxbw (%rdi,%rcx), %ymm2
854 ; AVX2-NEXT: vpmovsxbw (%rsi,%rcx), %ymm3
855 ; AVX2-NEXT: vpmaddwd %ymm2, %ymm3, %ymm2
856 ; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
857 ; AVX2-NEXT: addq $16, %rcx
858 ; AVX2-NEXT: cmpq %rcx, %rax
859 ; AVX2-NEXT: jne .LBB6_1
860 ; AVX2-NEXT: # %bb.2: # %middle.block
861 ; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
862 ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
863 ; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
864 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
865 ; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
866 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
867 ; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
868 ; AVX2-NEXT: vmovd %xmm0, %eax
869 ; AVX2-NEXT: vzeroupper
870 ; AVX2-NEXT: retq
871 ;
872 ; AVX512-LABEL: _Z9test_charPcS_i_512:
873 ; AVX512: # %bb.0: # %entry
874 ; AVX512-NEXT: movl %edx, %eax
875 ; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
876 ; AVX512-NEXT: xorl %ecx, %ecx
877 ; AVX512-NEXT: .p2align 4, 0x90
878 ; AVX512-NEXT: .LBB6_1: # %vector.body
879 ; AVX512-NEXT: # =>This Inner Loop Header: Depth=1
880 ; AVX512-NEXT: vpmovsxbw (%rdi,%rcx), %ymm1
881 ; AVX512-NEXT: vpmovsxbw (%rsi,%rcx), %ymm2
882 ; AVX512-NEXT: vpmaddwd %ymm1, %ymm2, %ymm1
883 ; AVX512-NEXT: vpaddd %zmm0, %zmm1, %zmm0
884 ; AVX512-NEXT: addq $16, %rcx
885 ; AVX512-NEXT: cmpq %rcx, %rax
886 ; AVX512-NEXT: jne .LBB6_1
887 ; AVX512-NEXT: # %bb.2: # %middle.block
888 ; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
889 ; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
890 ; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
891 ; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
892 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
893 ; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
894 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
895 ; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
896 ; AVX512-NEXT: vmovd %xmm0, %eax
897 ; AVX512-NEXT: vzeroupper
898 ; AVX512-NEXT: retq
899 entry:
900 %3 = zext i32 %2 to i64
901 br label %vector.body
903 vector.body:
904 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ]
905 %vec.phi = phi <16 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ]
906 %4 = getelementptr inbounds i8, i8* %0, i64 %index
907 %5 = bitcast i8* %4 to <16 x i8>*
908 %wide.load = load <16 x i8>, <16 x i8>* %5, align 1
909 %6 = sext <16 x i8> %wide.load to <16 x i32>
910 %7 = getelementptr inbounds i8, i8* %1, i64 %index
911 %8 = bitcast i8* %7 to <16 x i8>*
912 %wide.load14 = load <16 x i8>, <16 x i8>* %8, align 1
913 %9 = sext <16 x i8> %wide.load14 to <16 x i32>
914 %10 = mul nsw <16 x i32> %9, %6
915 %11 = add nsw <16 x i32> %10, %vec.phi
916 %index.next = add i64 %index, 16
917 %12 = icmp eq i64 %index.next, %3
918 br i1 %12, label %middle.block, label %vector.body
920 middle.block:
921 %rdx.shuf = shufflevector <16 x i32> %11, <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
922 %bin.rdx = add <16 x i32> %11, %rdx.shuf
923 %rdx.shuf15 = shufflevector <16 x i32> %bin.rdx, <16 x i32> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
924 %bin.rdx16 = add <16 x i32> %bin.rdx, %rdx.shuf15
925 %rdx.shuf17 = shufflevector <16 x i32> %bin.rdx16, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
926 %bin.rdx18 = add <16 x i32> %bin.rdx16, %rdx.shuf17
927 %rdx.shuf19 = shufflevector <16 x i32> %bin.rdx18, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
928 %bin.rdx20 = add <16 x i32> %bin.rdx18, %rdx.shuf19
929 %13 = extractelement <16 x i32> %bin.rdx20, i32 0
930 ret i32 %13
931 }
933 define i32 @_Z9test_charPcS_i_1024(i8* nocapture readonly, i8* nocapture readonly, i32) local_unnamed_addr #0 {
934 ; SSE2-LABEL: _Z9test_charPcS_i_1024:
935 ; SSE2: # %bb.0: # %entry
936 ; SSE2-NEXT: movl %edx, %eax
937 ; SSE2-NEXT: pxor %xmm8, %xmm8
938 ; SSE2-NEXT: xorl %ecx, %ecx
939 ; SSE2-NEXT: pxor %xmm2, %xmm2
940 ; SSE2-NEXT: pxor %xmm4, %xmm4
941 ; SSE2-NEXT: pxor %xmm1, %xmm1
942 ; SSE2-NEXT: pxor %xmm3, %xmm3
943 ; SSE2-NEXT: .p2align 4, 0x90
944 ; SSE2-NEXT: .LBB7_1: # %vector.body
945 ; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
946 ; SSE2-NEXT: movdqu (%rdi,%rcx), %xmm10
947 ; SSE2-NEXT: movdqu 16(%rdi,%rcx), %xmm7
948 ; SSE2-NEXT: movdqu (%rsi,%rcx), %xmm9
949 ; SSE2-NEXT: movdqu 16(%rsi,%rcx), %xmm0
950 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm7[8],xmm5[9],xmm7[9],xmm5[10],xmm7[10],xmm5[11],xmm7[11],xmm5[12],xmm7[12],xmm5[13],xmm7[13],xmm5[14],xmm7[14],xmm5[15],xmm7[15]
951 ; SSE2-NEXT: psraw $8, %xmm5
952 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
953 ; SSE2-NEXT: psraw $8, %xmm6
954 ; SSE2-NEXT: pmaddwd %xmm5, %xmm6
955 ; SSE2-NEXT: paddd %xmm6, %xmm3
956 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3],xmm5[4],xmm7[4],xmm5[5],xmm7[5],xmm5[6],xmm7[6],xmm5[7],xmm7[7]
957 ; SSE2-NEXT: psraw $8, %xmm5
958 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
959 ; SSE2-NEXT: psraw $8, %xmm0
960 ; SSE2-NEXT: pmaddwd %xmm5, %xmm0
961 ; SSE2-NEXT: paddd %xmm0, %xmm1
962 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm10[8],xmm0[9],xmm10[9],xmm0[10],xmm10[10],xmm0[11],xmm10[11],xmm0[12],xmm10[12],xmm0[13],xmm10[13],xmm0[14],xmm10[14],xmm0[15],xmm10[15]
963 ; SSE2-NEXT: psraw $8, %xmm0
964 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm9[8],xmm5[9],xmm9[9],xmm5[10],xmm9[10],xmm5[11],xmm9[11],xmm5[12],xmm9[12],xmm5[13],xmm9[13],xmm5[14],xmm9[14],xmm5[15],xmm9[15]
965 ; SSE2-NEXT: psraw $8, %xmm5
966 ; SSE2-NEXT: pmaddwd %xmm0, %xmm5
967 ; SSE2-NEXT: paddd %xmm5, %xmm4
968 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3],xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
969 ; SSE2-NEXT: psraw $8, %xmm0
970 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm9[0],xmm5[1],xmm9[1],xmm5[2],xmm9[2],xmm5[3],xmm9[3],xmm5[4],xmm9[4],xmm5[5],xmm9[5],xmm5[6],xmm9[6],xmm5[7],xmm9[7]
971 ; SSE2-NEXT: psraw $8, %xmm5
972 ; SSE2-NEXT: pmaddwd %xmm0, %xmm5
973 ; SSE2-NEXT: paddd %xmm5, %xmm2
974 ; SSE2-NEXT: addq $32, %rcx
975 ; SSE2-NEXT: cmpq %rcx, %rax
976 ; SSE2-NEXT: jne .LBB7_1
977 ; SSE2-NEXT: # %bb.2: # %middle.block
978 ; SSE2-NEXT: paddd %xmm8, %xmm4
979 ; SSE2-NEXT: paddd %xmm8, %xmm3
980 ; SSE2-NEXT: paddd %xmm4, %xmm3
981 ; SSE2-NEXT: paddd %xmm8, %xmm2
982 ; SSE2-NEXT: paddd %xmm8, %xmm1
983 ; SSE2-NEXT: paddd %xmm3, %xmm1
984 ; SSE2-NEXT: paddd %xmm2, %xmm1
985 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
986 ; SSE2-NEXT: paddd %xmm1, %xmm0
987 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
988 ; SSE2-NEXT: paddd %xmm0, %xmm1
989 ; SSE2-NEXT: movd %xmm1, %eax
990 ; SSE2-NEXT: retq
991 ;
992 ; AVX1-LABEL: _Z9test_charPcS_i_1024:
993 ; AVX1: # %bb.0: # %entry
994 ; AVX1-NEXT: movl %edx, %eax
995 ; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0
996 ; AVX1-NEXT: xorl %ecx, %ecx
997 ; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
998 ; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
999 ; AVX1-NEXT: .p2align 4, 0x90
1000 ; AVX1-NEXT: .LBB7_1: # %vector.body
1001 ; AVX1-NEXT: # =>This Inner Loop Header: Depth=1
1002 ; AVX1-NEXT: vpmovsxbw (%rdi,%rcx), %xmm3
1003 ; AVX1-NEXT: vpmovsxbw 8(%rdi,%rcx), %xmm4
1004 ; AVX1-NEXT: vpmovsxbw 16(%rdi,%rcx), %xmm5
1005 ; AVX1-NEXT: vpmovsxbw 24(%rdi,%rcx), %xmm6
1006 ; AVX1-NEXT: vpmovsxbw (%rsi,%rcx), %xmm7
1007 ; AVX1-NEXT: vpmaddwd %xmm3, %xmm7, %xmm3
1008 ; AVX1-NEXT: vpmovsxbw 8(%rsi,%rcx), %xmm7
1009 ; AVX1-NEXT: vpmaddwd %xmm4, %xmm7, %xmm4
1010 ; AVX1-NEXT: vpmovsxbw 16(%rsi,%rcx), %xmm7
1011 ; AVX1-NEXT: vpmaddwd %xmm5, %xmm7, %xmm5
1012 ; AVX1-NEXT: vpmovsxbw 24(%rsi,%rcx), %xmm7
1013 ; AVX1-NEXT: vpmaddwd %xmm6, %xmm7, %xmm6
1014 ; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm7
1015 ; AVX1-NEXT: vpaddd %xmm7, %xmm6, %xmm6
1016 ; AVX1-NEXT: vpaddd %xmm2, %xmm5, %xmm2
1017 ; AVX1-NEXT: vinsertf128 $1, %xmm6, %ymm2, %ymm2
1018 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
1019 ; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4
1020 ; AVX1-NEXT: vpaddd %xmm1, %xmm3, %xmm1
1021 ; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm1, %ymm1
1022 ; AVX1-NEXT: addq $32, %rcx
1023 ; AVX1-NEXT: cmpq %rcx, %rax
1024 ; AVX1-NEXT: jne .LBB7_1
1025 ; AVX1-NEXT: # %bb.2: # %middle.block
1026 ; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm3
1027 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4
1028 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5
1029 ; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
1030 ; AVX1-NEXT: vpaddd %xmm5, %xmm2, %xmm2
1031 ; AVX1-NEXT: vpaddd %xmm2, %xmm5, %xmm2
1032 ; AVX1-NEXT: vpaddd %xmm2, %xmm4, %xmm2
1033 ; AVX1-NEXT: vpaddd %xmm3, %xmm0, %xmm0
1034 ; AVX1-NEXT: vpaddd %xmm2, %xmm0, %xmm0
1035 ; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm0
1036 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1037 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1038 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1039 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1040 ; AVX1-NEXT: vmovd %xmm0, %eax
1041 ; AVX1-NEXT: vzeroupper
1042 ; AVX1-NEXT: retq
1043 ;
1044 ; AVX2-LABEL: _Z9test_charPcS_i_1024:
1045 ; AVX2: # %bb.0: # %entry
1046 ; AVX2-NEXT: movl %edx, %eax
1047 ; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
1048 ; AVX2-NEXT: xorl %ecx, %ecx
1049 ; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
1050 ; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
1051 ; AVX2-NEXT: .p2align 4, 0x90
1052 ; AVX2-NEXT: .LBB7_1: # %vector.body
1053 ; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
1054 ; AVX2-NEXT: vpmovsxbw (%rdi,%rcx), %ymm3
1055 ; AVX2-NEXT: vpmovsxbw 16(%rdi,%rcx), %ymm4
1056 ; AVX2-NEXT: vpmovsxbw (%rsi,%rcx), %ymm5
1057 ; AVX2-NEXT: vpmaddwd %ymm3, %ymm5, %ymm3
1058 ; AVX2-NEXT: vpaddd %ymm1, %ymm3, %ymm1
1059 ; AVX2-NEXT: vpmovsxbw 16(%rsi,%rcx), %ymm3
1060 ; AVX2-NEXT: vpmaddwd %ymm4, %ymm3, %ymm3
1061 ; AVX2-NEXT: vpaddd %ymm2, %ymm3, %ymm2
1062 ; AVX2-NEXT: addq $32, %rcx
1063 ; AVX2-NEXT: cmpq %rcx, %rax
1064 ; AVX2-NEXT: jne .LBB7_1
1065 ; AVX2-NEXT: # %bb.2: # %middle.block
1066 ; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm1
1067 ; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
1068 ; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
1069 ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
1070 ; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1071 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1072 ; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1073 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1074 ; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1075 ; AVX2-NEXT: vmovd %xmm0, %eax
1076 ; AVX2-NEXT: vzeroupper
1077 ; AVX2-NEXT: retq
1078 ;
1079 ; AVX512F-LABEL: _Z9test_charPcS_i_1024:
1080 ; AVX512F: # %bb.0: # %entry
1081 ; AVX512F-NEXT: movl %edx, %eax
1082 ; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
1083 ; AVX512F-NEXT: xorl %ecx, %ecx
1084 ; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
1085 ; AVX512F-NEXT: .p2align 4, 0x90
1086 ; AVX512F-NEXT: .LBB7_1: # %vector.body
1087 ; AVX512F-NEXT: # =>This Inner Loop Header: Depth=1
1088 ; AVX512F-NEXT: vpmovsxbw (%rdi,%rcx), %ymm2
1089 ; AVX512F-NEXT: vpmovsxbw 16(%rdi,%rcx), %ymm3
1090 ; AVX512F-NEXT: vpmovsxbw (%rsi,%rcx), %ymm4
1091 ; AVX512F-NEXT: vpmaddwd %ymm2, %ymm4, %ymm2
1092 ; AVX512F-NEXT: vpmovsxbw 16(%rsi,%rcx), %ymm4
1093 ; AVX512F-NEXT: vpmaddwd %ymm3, %ymm4, %ymm3
1094 ; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2
1095 ; AVX512F-NEXT: vpaddd %zmm1, %zmm2, %zmm1
1096 ; AVX512F-NEXT: addq $32, %rcx
1097 ; AVX512F-NEXT: cmpq %rcx, %rax
1098 ; AVX512F-NEXT: jne .LBB7_1
1099 ; AVX512F-NEXT: # %bb.2: # %middle.block
1100 ; AVX512F-NEXT: vpaddd %zmm0, %zmm1, %zmm0
1101 ; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
1102 ; AVX512F-NEXT: vpaddd %zmm1, %zmm0, %zmm0
1103 ; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
1104 ; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1105 ; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1106 ; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1107 ; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1108 ; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1109 ; AVX512F-NEXT: vmovd %xmm0, %eax
1110 ; AVX512F-NEXT: vzeroupper
1111 ; AVX512F-NEXT: retq
1112 ;
1113 ; AVX512BW-LABEL: _Z9test_charPcS_i_1024:
1114 ; AVX512BW: # %bb.0: # %entry
1115 ; AVX512BW-NEXT: movl %edx, %eax
1116 ; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
1117 ; AVX512BW-NEXT: xorl %ecx, %ecx
1118 ; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
1119 ; AVX512BW-NEXT: .p2align 4, 0x90
1120 ; AVX512BW-NEXT: .LBB7_1: # %vector.body
1121 ; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
1122 ; AVX512BW-NEXT: vpmovsxbw (%rdi,%rcx), %zmm2
1123 ; AVX512BW-NEXT: vpmovsxbw (%rsi,%rcx), %zmm3
1124 ; AVX512BW-NEXT: vpmaddwd %zmm2, %zmm3, %zmm2
1125 ; AVX512BW-NEXT: vpaddd %zmm1, %zmm2, %zmm1
1126 ; AVX512BW-NEXT: addq $32, %rcx
1127 ; AVX512BW-NEXT: cmpq %rcx, %rax
1128 ; AVX512BW-NEXT: jne .LBB7_1
1129 ; AVX512BW-NEXT: # %bb.2: # %middle.block
1130 ; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
1131 ; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
1132 ; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
1133 ; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
1134 ; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1135 ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1136 ; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1137 ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1138 ; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1139 ; AVX512BW-NEXT: vmovd %xmm0, %eax
1140 ; AVX512BW-NEXT: vzeroupper
1141 ; AVX512BW-NEXT: retq
1142 entry:
1143 %3 = zext i32 %2 to i64
1144 br label %vector.body
1146 vector.body:
1147 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ]
1148 %vec.phi = phi <32 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ]
1149 %4 = getelementptr inbounds i8, i8* %0, i64 %index
1150 %5 = bitcast i8* %4 to <32 x i8>*
1151 %wide.load = load <32 x i8>, <32 x i8>* %5, align 1
1152 %6 = sext <32 x i8> %wide.load to <32 x i32>
1153 %7 = getelementptr inbounds i8, i8* %1, i64 %index
1154 %8 = bitcast i8* %7 to <32 x i8>*
1155 %wide.load14 = load <32 x i8>, <32 x i8>* %8, align 1
1156 %9 = sext <32 x i8> %wide.load14 to <32 x i32>
1157 %10 = mul nsw <32 x i32> %9, %6
1158 %11 = add nsw <32 x i32> %10, %vec.phi
1159 %index.next = add i64 %index, 32
1160 %12 = icmp eq i64 %index.next, %3
1161 br i1 %12, label %middle.block, label %vector.body
1163 middle.block:
1164 %rdx.shuf1 = shufflevector <32 x i32> %11, <32 x i32> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1165 %bin.rdx1 = add <32 x i32> %11, %rdx.shuf1
1166 %rdx.shuf = shufflevector <32 x i32> %bin.rdx1, <32 x i32> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1167 %bin.rdx = add <32 x i32> %bin.rdx1, %rdx.shuf
1168 %rdx.shuf15 = shufflevector <32 x i32> %bin.rdx, <32 x i32> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1169 %bin.rdx32 = add <32 x i32> %bin.rdx, %rdx.shuf15
1170 %rdx.shuf17 = shufflevector <32 x i32> %bin.rdx32, <32 x i32> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1171 %bin.rdx18 = add <32 x i32> %bin.rdx32, %rdx.shuf17
1172 %rdx.shuf19 = shufflevector <32 x i32> %bin.rdx18, <32 x i32> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1173 %bin.rdx20 = add <32 x i32> %bin.rdx18, %rdx.shuf19
1174 %13 = extractelement <32 x i32> %bin.rdx20, i32 0
1175 ret i32 %13
1176 }
1178 define i32 @test_unsigned_short_128(i16* nocapture readonly, i16* nocapture readonly, i32) local_unnamed_addr #0 {
1179 ; SSE2-LABEL: test_unsigned_short_128:
1180 ; SSE2: # %bb.0: # %entry
1181 ; SSE2-NEXT: movl %edx, %eax
1182 ; SSE2-NEXT: pxor %xmm0, %xmm0
1183 ; SSE2-NEXT: xorl %ecx, %ecx
1184 ; SSE2-NEXT: .p2align 4, 0x90
1185 ; SSE2-NEXT: .LBB8_1: # %vector.body
1186 ; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
1187 ; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero
1188 ; SSE2-NEXT: movq {{.*#+}} xmm2 = mem[0],zero
1189 ; SSE2-NEXT: movdqa %xmm2, %xmm3
1190 ; SSE2-NEXT: pmulhuw %xmm1, %xmm3
1191 ; SSE2-NEXT: pmullw %xmm1, %xmm2
1192 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
1193 ; SSE2-NEXT: paddd %xmm2, %xmm0
1194 ; SSE2-NEXT: addq $16, %rcx
1195 ; SSE2-NEXT: cmpq %rcx, %rax
1196 ; SSE2-NEXT: jne .LBB8_1
1197 ; SSE2-NEXT: # %bb.2: # %middle.block
1198 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1199 ; SSE2-NEXT: paddd %xmm0, %xmm1
1200 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
1201 ; SSE2-NEXT: paddd %xmm1, %xmm0
1202 ; SSE2-NEXT: movd %xmm0, %eax
1203 ; SSE2-NEXT: retq
1204 ;
1205 ; AVX-LABEL: test_unsigned_short_128:
1206 ; AVX: # %bb.0: # %entry
1207 ; AVX-NEXT: movl %edx, %eax
1208 ; AVX-NEXT: vpxor %xmm0, %xmm0, %xmm0
1209 ; AVX-NEXT: xorl %ecx, %ecx
1210 ; AVX-NEXT: .p2align 4, 0x90
1211 ; AVX-NEXT: .LBB8_1: # %vector.body
1212 ; AVX-NEXT: # =>This Inner Loop Header: Depth=1
1213 ; AVX-NEXT: vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1214 ; AVX-NEXT: vpmovzxwd {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1215 ; AVX-NEXT: vpmulld %xmm1, %xmm2, %xmm1
1216 ; AVX-NEXT: vpaddd %xmm0, %xmm1, %xmm0
1217 ; AVX-NEXT: addq $16, %rcx
1218 ; AVX-NEXT: cmpq %rcx, %rax
1219 ; AVX-NEXT: jne .LBB8_1
1220 ; AVX-NEXT: # %bb.2: # %middle.block
1221 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1222 ; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1223 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1224 ; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1225 ; AVX-NEXT: vmovd %xmm0, %eax
1226 ; AVX-NEXT: retq
1227 entry:
1228 %3 = zext i32 %2 to i64
1229 br label %vector.body
1231 vector.body:
1232 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ]
1233 %vec.phi = phi <4 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ]
1234 %4 = getelementptr inbounds i16, i16* %0, i64 %index
1235 %5 = bitcast i16* %4 to <4 x i16>*
1236 %wide.load = load <4 x i16>, <4 x i16>* %5, align 2
1237 %6 = zext <4 x i16> %wide.load to <4 x i32>
1238 %7 = getelementptr inbounds i16, i16* %1, i64 %index
1239 %8 = bitcast i16* %7 to <4 x i16>*
1240 %wide.load14 = load <4 x i16>, <4 x i16>* %8, align 2
1241 %9 = zext <4 x i16> %wide.load14 to <4 x i32>
1242 %10 = mul nsw <4 x i32> %9, %6
1243 %11 = add nsw <4 x i32> %10, %vec.phi
1244 %index.next = add i64 %index, 16
1245 %12 = icmp eq i64 %index.next, %3
1246 br i1 %12, label %middle.block, label %vector.body
1248 middle.block:
1249 %rdx.shuf15 = shufflevector <4 x i32> %11, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
1250 %bin.rdx16 = add <4 x i32> %11, %rdx.shuf15
1251 %rdx.shuf17 = shufflevector <4 x i32> %bin.rdx16, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
1252 %bin.rdx18 = add <4 x i32> %bin.rdx16, %rdx.shuf17
1253 %13 = extractelement <4 x i32> %bin.rdx18, i32 0
1254 ret i32 %13
1255 }
1257 define i32 @test_unsigned_short_256(i16* nocapture readonly, i16* nocapture readonly, i32) local_unnamed_addr #0 {
1258 ; SSE2-LABEL: test_unsigned_short_256:
1259 ; SSE2: # %bb.0: # %entry
1260 ; SSE2-NEXT: movl %edx, %eax
1261 ; SSE2-NEXT: pxor %xmm0, %xmm0
1262 ; SSE2-NEXT: xorl %ecx, %ecx
1263 ; SSE2-NEXT: pxor %xmm1, %xmm1
1264 ; SSE2-NEXT: .p2align 4, 0x90
1265 ; SSE2-NEXT: .LBB9_1: # %vector.body
1266 ; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
1267 ; SSE2-NEXT: movdqu (%rdi,%rcx,2), %xmm2
1268 ; SSE2-NEXT: movdqu (%rsi,%rcx,2), %xmm3
1269 ; SSE2-NEXT: movdqa %xmm3, %xmm4
1270 ; SSE2-NEXT: pmulhuw %xmm2, %xmm4
1271 ; SSE2-NEXT: pmullw %xmm2, %xmm3
1272 ; SSE2-NEXT: movdqa %xmm3, %xmm2
1273 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
1274 ; SSE2-NEXT: paddd %xmm2, %xmm0
1275 ; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
1276 ; SSE2-NEXT: paddd %xmm3, %xmm1
1277 ; SSE2-NEXT: addq $16, %rcx
1278 ; SSE2-NEXT: cmpq %rcx, %rax
1279 ; SSE2-NEXT: jne .LBB9_1
1280 ; SSE2-NEXT: # %bb.2: # %middle.block
1281 ; SSE2-NEXT: paddd %xmm1, %xmm0
1282 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1283 ; SSE2-NEXT: paddd %xmm0, %xmm1
1284 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
1285 ; SSE2-NEXT: paddd %xmm1, %xmm0
1286 ; SSE2-NEXT: movd %xmm0, %eax
1287 ; SSE2-NEXT: retq
1288 ;
1289 ; AVX1-LABEL: test_unsigned_short_256:
1290 ; AVX1: # %bb.0: # %entry
1291 ; AVX1-NEXT: movl %edx, %eax
1292 ; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0
1293 ; AVX1-NEXT: xorl %ecx, %ecx
1294 ; AVX1-NEXT: .p2align 4, 0x90
1295 ; AVX1-NEXT: .LBB9_1: # %vector.body
1296 ; AVX1-NEXT: # =>This Inner Loop Header: Depth=1
1297 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1298 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1299 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1300 ; AVX1-NEXT: vpmulld %xmm1, %xmm3, %xmm1
1301 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1302 ; AVX1-NEXT: vpmulld %xmm2, %xmm3, %xmm2
1303 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
1304 ; AVX1-NEXT: vpaddd %xmm3, %xmm1, %xmm1
1305 ; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm0
1306 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
1307 ; AVX1-NEXT: addq $16, %rcx
1308 ; AVX1-NEXT: cmpq %rcx, %rax
1309 ; AVX1-NEXT: jne .LBB9_1
1310 ; AVX1-NEXT: # %bb.2: # %middle.block
1311 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
1312 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1313 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1314 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1315 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1316 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1317 ; AVX1-NEXT: vmovd %xmm0, %eax
1318 ; AVX1-NEXT: vzeroupper
1319 ; AVX1-NEXT: retq
1320 ;
1321 ; AVX256-LABEL: test_unsigned_short_256:
1322 ; AVX256: # %bb.0: # %entry
1323 ; AVX256-NEXT: movl %edx, %eax
1324 ; AVX256-NEXT: vpxor %xmm0, %xmm0, %xmm0
1325 ; AVX256-NEXT: xorl %ecx, %ecx
1326 ; AVX256-NEXT: .p2align 4, 0x90
1327 ; AVX256-NEXT: .LBB9_1: # %vector.body
1328 ; AVX256-NEXT: # =>This Inner Loop Header: Depth=1
1329 ; AVX256-NEXT: vpmovzxwd {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
1330 ; AVX256-NEXT: vpmovzxwd {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
1331 ; AVX256-NEXT: vpmulld %ymm1, %ymm2, %ymm1
1332 ; AVX256-NEXT: vpaddd %ymm0, %ymm1, %ymm0
1333 ; AVX256-NEXT: addq $16, %rcx
1334 ; AVX256-NEXT: cmpq %rcx, %rax
1335 ; AVX256-NEXT: jne .LBB9_1
1336 ; AVX256-NEXT: # %bb.2: # %middle.block
1337 ; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1
1338 ; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1339 ; AVX256-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1340 ; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1341 ; AVX256-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1342 ; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1343 ; AVX256-NEXT: vmovd %xmm0, %eax
1344 ; AVX256-NEXT: vzeroupper
1345 ; AVX256-NEXT: retq
1346 entry:
1347 %3 = zext i32 %2 to i64
1348 br label %vector.body
1350 vector.body:
1351 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ]
1352 %vec.phi = phi <8 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ]
1353 %4 = getelementptr inbounds i16, i16* %0, i64 %index
1354 %5 = bitcast i16* %4 to <8 x i16>*
1355 %wide.load = load <8 x i16>, <8 x i16>* %5, align 2
1356 %6 = zext <8 x i16> %wide.load to <8 x i32>
1357 %7 = getelementptr inbounds i16, i16* %1, i64 %index
1358 %8 = bitcast i16* %7 to <8 x i16>*
1359 %wide.load14 = load <8 x i16>, <8 x i16>* %8, align 2
1360 %9 = zext <8 x i16> %wide.load14 to <8 x i32>
1361 %10 = mul nsw <8 x i32> %9, %6
1362 %11 = add nsw <8 x i32> %10, %vec.phi
1363 %index.next = add i64 %index, 16
1364 %12 = icmp eq i64 %index.next, %3
1365 br i1 %12, label %middle.block, label %vector.body
1367 middle.block:
1368 %rdx.shuf = shufflevector <8 x i32> %11, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
1369 %bin.rdx = add <8 x i32> %11, %rdx.shuf
1370 %rdx.shuf15 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1371 %bin.rdx16 = add <8 x i32> %bin.rdx, %rdx.shuf15
1372 %rdx.shuf17 = shufflevector <8 x i32> %bin.rdx16, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1373 %bin.rdx18 = add <8 x i32> %bin.rdx16, %rdx.shuf17
1374 %13 = extractelement <8 x i32> %bin.rdx18, i32 0
1375 ret i32 %13
1376 }
1378 define i32 @test_unsigned_short_512(i16* nocapture readonly, i16* nocapture readonly, i32) local_unnamed_addr #0 {
1379 ; SSE2-LABEL: test_unsigned_short_512:
1380 ; SSE2: # %bb.0: # %entry
1381 ; SSE2-NEXT: movl %edx, %eax
1382 ; SSE2-NEXT: pxor %xmm0, %xmm0
1383 ; SSE2-NEXT: xorl %ecx, %ecx
1384 ; SSE2-NEXT: pxor %xmm1, %xmm1
1385 ; SSE2-NEXT: pxor %xmm3, %xmm3
1386 ; SSE2-NEXT: pxor %xmm2, %xmm2
1387 ; SSE2-NEXT: .p2align 4, 0x90
1388 ; SSE2-NEXT: .LBB10_1: # %vector.body
1389 ; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
1390 ; SSE2-NEXT: movdqu (%rdi,%rcx,2), %xmm4
1391 ; SSE2-NEXT: movdqu 16(%rdi,%rcx,2), %xmm8
1392 ; SSE2-NEXT: movdqu (%rsi,%rcx,2), %xmm6
1393 ; SSE2-NEXT: movdqu 16(%rsi,%rcx,2), %xmm7
1394 ; SSE2-NEXT: movdqa %xmm6, %xmm5
1395 ; SSE2-NEXT: pmulhuw %xmm4, %xmm5
1396 ; SSE2-NEXT: pmullw %xmm4, %xmm6
1397 ; SSE2-NEXT: movdqa %xmm6, %xmm4
1398 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
1399 ; SSE2-NEXT: paddd %xmm4, %xmm0
1400 ; SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
1401 ; SSE2-NEXT: paddd %xmm6, %xmm1
1402 ; SSE2-NEXT: movdqa %xmm7, %xmm4
1403 ; SSE2-NEXT: pmulhuw %xmm8, %xmm4
1404 ; SSE2-NEXT: pmullw %xmm8, %xmm7
1405 ; SSE2-NEXT: movdqa %xmm7, %xmm5
1406 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
1407 ; SSE2-NEXT: paddd %xmm5, %xmm3
1408 ; SSE2-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm4[4],xmm7[5],xmm4[5],xmm7[6],xmm4[6],xmm7[7],xmm4[7]
1409 ; SSE2-NEXT: paddd %xmm7, %xmm2
1410 ; SSE2-NEXT: addq $16, %rcx
1411 ; SSE2-NEXT: cmpq %rcx, %rax
1412 ; SSE2-NEXT: jne .LBB10_1
1413 ; SSE2-NEXT: # %bb.2: # %middle.block
1414 ; SSE2-NEXT: paddd %xmm3, %xmm0
1415 ; SSE2-NEXT: paddd %xmm2, %xmm1
1416 ; SSE2-NEXT: paddd %xmm0, %xmm1
1417 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
1418 ; SSE2-NEXT: paddd %xmm1, %xmm0
1419 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1420 ; SSE2-NEXT: paddd %xmm0, %xmm1
1421 ; SSE2-NEXT: movd %xmm1, %eax
1422 ; SSE2-NEXT: retq
1423 ;
1424 ; AVX1-LABEL: test_unsigned_short_512:
1425 ; AVX1: # %bb.0: # %entry
1426 ; AVX1-NEXT: movl %edx, %eax
1427 ; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0
1428 ; AVX1-NEXT: xorl %ecx, %ecx
1429 ; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
1430 ; AVX1-NEXT: .p2align 4, 0x90
1431 ; AVX1-NEXT: .LBB10_1: # %vector.body
1432 ; AVX1-NEXT: # =>This Inner Loop Header: Depth=1
1433 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1434 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1435 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1436 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm5 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1437 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm6 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1438 ; AVX1-NEXT: vpmulld %xmm2, %xmm6, %xmm2
1439 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm6 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1440 ; AVX1-NEXT: vpmulld %xmm3, %xmm6, %xmm3
1441 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm6 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1442 ; AVX1-NEXT: vpmulld %xmm4, %xmm6, %xmm4
1443 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm6 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1444 ; AVX1-NEXT: vpmulld %xmm5, %xmm6, %xmm5
1445 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm6
1446 ; AVX1-NEXT: vpaddd %xmm6, %xmm2, %xmm2
1447 ; AVX1-NEXT: vpaddd %xmm1, %xmm3, %xmm1
1448 ; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1
1449 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
1450 ; AVX1-NEXT: vpaddd %xmm2, %xmm4, %xmm2
1451 ; AVX1-NEXT: vpaddd %xmm0, %xmm5, %xmm0
1452 ; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
1453 ; AVX1-NEXT: addq $16, %rcx
1454 ; AVX1-NEXT: cmpq %rcx, %rax
1455 ; AVX1-NEXT: jne .LBB10_1
1456 ; AVX1-NEXT: # %bb.2: # %middle.block
1457 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
1458 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
1459 ; AVX1-NEXT: vpaddd %xmm3, %xmm2, %xmm2
1460 ; AVX1-NEXT: vpaddd %xmm2, %xmm1, %xmm1
1461 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1462 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1463 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1464 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1465 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1466 ; AVX1-NEXT: vmovd %xmm0, %eax
1467 ; AVX1-NEXT: vzeroupper
1468 ; AVX1-NEXT: retq
1469 ;
1470 ; AVX2-LABEL: test_unsigned_short_512:
1471 ; AVX2: # %bb.0: # %entry
1472 ; AVX2-NEXT: movl %edx, %eax
1473 ; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
1474 ; AVX2-NEXT: xorl %ecx, %ecx
1475 ; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
1476 ; AVX2-NEXT: .p2align 4, 0x90
1477 ; AVX2-NEXT: .LBB10_1: # %vector.body
1478 ; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
1479 ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
1480 ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
1481 ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
1482 ; AVX2-NEXT: vpmulld %ymm2, %ymm4, %ymm2
1483 ; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
1484 ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
1485 ; AVX2-NEXT: vpmulld %ymm3, %ymm2, %ymm2
1486 ; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
1487 ; AVX2-NEXT: addq $16, %rcx
1488 ; AVX2-NEXT: cmpq %rcx, %rax
1489 ; AVX2-NEXT: jne .LBB10_1
1490 ; AVX2-NEXT: # %bb.2: # %middle.block
1491 ; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0
1492 ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
1493 ; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1494 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1495 ; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1496 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1497 ; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1498 ; AVX2-NEXT: vmovd %xmm0, %eax
1499 ; AVX2-NEXT: vzeroupper
1500 ; AVX2-NEXT: retq
1501 ;
1502 ; AVX512-LABEL: test_unsigned_short_512:
1503 ; AVX512: # %bb.0: # %entry
1504 ; AVX512-NEXT: movl %edx, %eax
1505 ; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
1506 ; AVX512-NEXT: xorl %ecx, %ecx
1507 ; AVX512-NEXT: .p2align 4, 0x90
1508 ; AVX512-NEXT: .LBB10_1: # %vector.body
1509 ; AVX512-NEXT: # =>This Inner Loop Header: Depth=1
1510 ; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
1511 ; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
1512 ; AVX512-NEXT: vpmulld %zmm1, %zmm2, %zmm1
1513 ; AVX512-NEXT: vpaddd %zmm0, %zmm1, %zmm0
1514 ; AVX512-NEXT: addq $16, %rcx
1515 ; AVX512-NEXT: cmpq %rcx, %rax
1516 ; AVX512-NEXT: jne .LBB10_1
1517 ; AVX512-NEXT: # %bb.2: # %middle.block
1518 ; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
1519 ; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
1520 ; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
1521 ; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1522 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1523 ; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1524 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1525 ; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1526 ; AVX512-NEXT: vmovd %xmm0, %eax
1527 ; AVX512-NEXT: vzeroupper
1528 ; AVX512-NEXT: retq
1529 entry:
1530 %3 = zext i32 %2 to i64
1531 br label %vector.body
1533 vector.body:
1534 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ]
1535 %vec.phi = phi <16 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ]
1536 %4 = getelementptr inbounds i16, i16* %0, i64 %index
1537 %5 = bitcast i16* %4 to <16 x i16>*
1538 %wide.load = load <16 x i16>, <16 x i16>* %5, align 2
1539 %6 = zext <16 x i16> %wide.load to <16 x i32>
1540 %7 = getelementptr inbounds i16, i16* %1, i64 %index
1541 %8 = bitcast i16* %7 to <16 x i16>*
1542 %wide.load14 = load <16 x i16>, <16 x i16>* %8, align 2
1543 %9 = zext <16 x i16> %wide.load14 to <16 x i32>
1544 %10 = mul nsw <16 x i32> %9, %6
1545 %11 = add nsw <16 x i32> %10, %vec.phi
1546 %index.next = add i64 %index, 16
1547 %12 = icmp eq i64 %index.next, %3
1548 br i1 %12, label %middle.block, label %vector.body
1550 middle.block:
1551 %rdx.shuf1 = shufflevector <16 x i32> %11, <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1552 %bin.rdx1 = add <16 x i32> %11, %rdx.shuf1
1553 %rdx.shuf = shufflevector <16 x i32> %bin.rdx1, <16 x i32> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1554 %bin.rdx = add <16 x i32> %bin.rdx1, %rdx.shuf
1555 %rdx.shuf15 = shufflevector <16 x i32> %bin.rdx, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1556 %bin.rdx16 = add <16 x i32> %bin.rdx, %rdx.shuf15
1557 %rdx.shuf17 = shufflevector <16 x i32> %bin.rdx16, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1558 %bin.rdx18 = add <16 x i32> %bin.rdx16, %rdx.shuf17
1559 %13 = extractelement <16 x i32> %bin.rdx18, i32 0
1560 ret i32 %13
1561 }
1563 define i32 @test_unsigned_short_1024(i16* nocapture readonly, i16* nocapture readonly, i32) local_unnamed_addr #0 {
1564 ; SSE2-LABEL: test_unsigned_short_1024:
1565 ; SSE2: # %bb.0: # %entry
1566 ; SSE2-NEXT: movl %edx, %eax
1567 ; SSE2-NEXT: pxor %xmm8, %xmm8
1568 ; SSE2-NEXT: xorl %ecx, %ecx
1569 ; SSE2-NEXT: pxor %xmm3, %xmm3
1570 ; SSE2-NEXT: pxor %xmm9, %xmm9
1571 ; SSE2-NEXT: pxor %xmm10, %xmm10
1572 ; SSE2-NEXT: pxor %xmm4, %xmm4
1573 ; SSE2-NEXT: pxor %xmm6, %xmm6
1574 ; SSE2-NEXT: pxor %xmm5, %xmm5
1575 ; SSE2-NEXT: pxor %xmm7, %xmm7
1576 ; SSE2-NEXT: .p2align 4, 0x90
1577 ; SSE2-NEXT: .LBB11_1: # %vector.body
1578 ; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
1579 ; SSE2-NEXT: movdqu 48(%rdi,%rcx,2), %xmm0
1580 ; SSE2-NEXT: movdqu 48(%rsi,%rcx,2), %xmm1
1581 ; SSE2-NEXT: movdqa %xmm1, %xmm2
1582 ; SSE2-NEXT: pmulhuw %xmm0, %xmm2
1583 ; SSE2-NEXT: pmullw %xmm0, %xmm1
1584 ; SSE2-NEXT: movdqa %xmm1, %xmm0
1585 ; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
1586 ; SSE2-NEXT: paddd %xmm0, %xmm7
1587 ; SSE2-NEXT: movdqu 32(%rdi,%rcx,2), %xmm0
1588 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
1589 ; SSE2-NEXT: movdqu 32(%rsi,%rcx,2), %xmm2
1590 ; SSE2-NEXT: paddd %xmm1, %xmm5
1591 ; SSE2-NEXT: movdqa %xmm2, %xmm1
1592 ; SSE2-NEXT: pmulhuw %xmm0, %xmm1
1593 ; SSE2-NEXT: pmullw %xmm0, %xmm2
1594 ; SSE2-NEXT: movdqa %xmm2, %xmm0
1595 ; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
1596 ; SSE2-NEXT: paddd %xmm0, %xmm6
1597 ; SSE2-NEXT: movdqu (%rdi,%rcx,2), %xmm0
1598 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
1599 ; SSE2-NEXT: movdqu (%rsi,%rcx,2), %xmm1
1600 ; SSE2-NEXT: paddd %xmm2, %xmm4
1601 ; SSE2-NEXT: movdqa %xmm1, %xmm2
1602 ; SSE2-NEXT: pmulhuw %xmm0, %xmm2
1603 ; SSE2-NEXT: pmullw %xmm0, %xmm1
1604 ; SSE2-NEXT: movdqa %xmm1, %xmm0
1605 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
1606 ; SSE2-NEXT: paddd %xmm0, %xmm8
1607 ; SSE2-NEXT: movdqu 16(%rdi,%rcx,2), %xmm0
1608 ; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1609 ; SSE2-NEXT: movdqu 16(%rsi,%rcx,2), %xmm2
1610 ; SSE2-NEXT: paddd %xmm1, %xmm3
1611 ; SSE2-NEXT: movdqa %xmm2, %xmm1
1612 ; SSE2-NEXT: pmulhuw %xmm0, %xmm1
1613 ; SSE2-NEXT: pmullw %xmm0, %xmm2
1614 ; SSE2-NEXT: movdqa %xmm2, %xmm0
1615 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1616 ; SSE2-NEXT: paddd %xmm0, %xmm9
1617 ; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
1618 ; SSE2-NEXT: paddd %xmm2, %xmm10
1619 ; SSE2-NEXT: addq $16, %rcx
1620 ; SSE2-NEXT: cmpq %rcx, %rax
1621 ; SSE2-NEXT: jne .LBB11_1
1622 ; SSE2-NEXT: # %bb.2: # %middle.block
1623 ; SSE2-NEXT: paddd %xmm6, %xmm3
1624 ; SSE2-NEXT: paddd %xmm7, %xmm10
1625 ; SSE2-NEXT: paddd %xmm3, %xmm10
1626 ; SSE2-NEXT: paddd %xmm4, %xmm8
1627 ; SSE2-NEXT: paddd %xmm5, %xmm9
1628 ; SSE2-NEXT: paddd %xmm10, %xmm9
1629 ; SSE2-NEXT: paddd %xmm8, %xmm9
1630 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm9[2,3,0,1]
1631 ; SSE2-NEXT: paddd %xmm9, %xmm0
1632 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1633 ; SSE2-NEXT: paddd %xmm0, %xmm1
1634 ; SSE2-NEXT: movd %xmm1, %eax
1635 ; SSE2-NEXT: retq
1636 ;
1637 ; AVX1-LABEL: test_unsigned_short_1024:
1638 ; AVX1: # %bb.0: # %entry
1639 ; AVX1-NEXT: movl %edx, %eax
1640 ; AVX1-NEXT: vpxor %xmm8, %xmm8, %xmm8
1641 ; AVX1-NEXT: xorl %ecx, %ecx
1642 ; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
1643 ; AVX1-NEXT: vpxor %xmm9, %xmm9, %xmm9
1644 ; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
1645 ; AVX1-NEXT: .p2align 4, 0x90
1646 ; AVX1-NEXT: .LBB11_1: # %vector.body
1647 ; AVX1-NEXT: # =>This Inner Loop Header: Depth=1
1648 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1649 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm5 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1650 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm6 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1651 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm7 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1652 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1653 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm12 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1654 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm10 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1655 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm11 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1656 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1657 ; AVX1-NEXT: vpmulld %xmm4, %xmm1, %xmm1
1658 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1659 ; AVX1-NEXT: vpmulld %xmm5, %xmm4, %xmm4
1660 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm5 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1661 ; AVX1-NEXT: vpmulld %xmm6, %xmm5, %xmm5
1662 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm6 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1663 ; AVX1-NEXT: vpmulld %xmm7, %xmm6, %xmm6
1664 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm7 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1665 ; AVX1-NEXT: vpmulld %xmm0, %xmm7, %xmm13
1666 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm7 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1667 ; AVX1-NEXT: vpmulld %xmm12, %xmm7, %xmm7
1668 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1669 ; AVX1-NEXT: vpmulld %xmm10, %xmm0, %xmm10
1670 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
1671 ; AVX1-NEXT: vpmulld %xmm11, %xmm0, %xmm11
1672 ; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm0
1673 ; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm0
1674 ; AVX1-NEXT: vpaddd %xmm2, %xmm4, %xmm1
1675 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm2
1676 ; AVX1-NEXT: vextractf128 $1, %ymm8, %xmm0
1677 ; AVX1-NEXT: vpaddd %xmm0, %xmm5, %xmm0
1678 ; AVX1-NEXT: vpaddd %xmm8, %xmm6, %xmm1
1679 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm8
1680 ; AVX1-NEXT: vextractf128 $1, %ymm9, %xmm0
1681 ; AVX1-NEXT: vpaddd %xmm0, %xmm13, %xmm0
1682 ; AVX1-NEXT: vpaddd %xmm9, %xmm7, %xmm1
1683 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm9
1684 ; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm0
1685 ; AVX1-NEXT: vpaddd %xmm0, %xmm10, %xmm0
1686 ; AVX1-NEXT: vpaddd %xmm3, %xmm11, %xmm1
1687 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm3
1688 ; AVX1-NEXT: addq $16, %rcx
1689 ; AVX1-NEXT: cmpq %rcx, %rax
1690 ; AVX1-NEXT: jne .LBB11_1
1691 ; AVX1-NEXT: # %bb.2: # %middle.block
1692 ; AVX1-NEXT: vpaddd %xmm3, %xmm2, %xmm0
1693 ; AVX1-NEXT: vextractf128 $1, %ymm8, %xmm1
1694 ; AVX1-NEXT: vextractf128 $1, %ymm9, %xmm4
1695 ; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
1696 ; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm3
1697 ; AVX1-NEXT: vpaddd %xmm3, %xmm2, %xmm2
1698 ; AVX1-NEXT: vpaddd %xmm2, %xmm4, %xmm2
1699 ; AVX1-NEXT: vpaddd %xmm2, %xmm1, %xmm1
1700 ; AVX1-NEXT: vpaddd %xmm0, %xmm9, %xmm0
1701 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1702 ; AVX1-NEXT: vpaddd %xmm0, %xmm8, %xmm0
1703 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1704 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1705 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1706 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1707 ; AVX1-NEXT: vmovd %xmm0, %eax
1708 ; AVX1-NEXT: vzeroupper
1709 ; AVX1-NEXT: retq
1710 ;
1711 ; AVX2-LABEL: test_unsigned_short_1024:
1712 ; AVX2: # %bb.0: # %entry
1713 ; AVX2-NEXT: movl %edx, %eax
1714 ; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
1715 ; AVX2-NEXT: xorl %ecx, %ecx
1716 ; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
1717 ; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
1718 ; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
1719 ; AVX2-NEXT: .p2align 4, 0x90
1720 ; AVX2-NEXT: .LBB11_1: # %vector.body
1721 ; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
1722 ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
1723 ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm5 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
1724 ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm6 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
1725 ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm7 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
1726 ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm8 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
1727 ; AVX2-NEXT: vpmulld %ymm4, %ymm8, %ymm4
1728 ; AVX2-NEXT: vpaddd %ymm2, %ymm4, %ymm2
1729 ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
1730 ; AVX2-NEXT: vpmulld %ymm5, %ymm4, %ymm4
1731 ; AVX2-NEXT: vpaddd %ymm1, %ymm4, %ymm1
1732 ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
1733 ; AVX2-NEXT: vpmulld %ymm6, %ymm4, %ymm4
1734 ; AVX2-NEXT: vpaddd %ymm0, %ymm4, %ymm0
1735 ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
1736 ; AVX2-NEXT: vpmulld %ymm7, %ymm4, %ymm4
1737 ; AVX2-NEXT: vpaddd %ymm3, %ymm4, %ymm3
1738 ; AVX2-NEXT: addq $16, %rcx
1739 ; AVX2-NEXT: cmpq %rcx, %rax
1740 ; AVX2-NEXT: jne .LBB11_1
1741 ; AVX2-NEXT: # %bb.2: # %middle.block
1742 ; AVX2-NEXT: vpaddd %ymm2, %ymm0, %ymm0
1743 ; AVX2-NEXT: vpaddd %ymm3, %ymm1, %ymm1
1744 ; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0
1745 ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
1746 ; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1747 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1748 ; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1749 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1750 ; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1751 ; AVX2-NEXT: vmovd %xmm0, %eax
1752 ; AVX2-NEXT: vzeroupper
1753 ; AVX2-NEXT: retq
1754 ;
1755 ; AVX512-LABEL: test_unsigned_short_1024:
1756 ; AVX512: # %bb.0: # %entry
1757 ; AVX512-NEXT: movl %edx, %eax
1758 ; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
1759 ; AVX512-NEXT: xorl %ecx, %ecx
1760 ; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
1761 ; AVX512-NEXT: .p2align 4, 0x90
1762 ; AVX512-NEXT: .LBB11_1: # %vector.body
1763 ; AVX512-NEXT: # =>This Inner Loop Header: Depth=1
1764 ; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
1765 ; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
1766 ; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
1767 ; AVX512-NEXT: vpmulld %zmm2, %zmm4, %zmm2
1768 ; AVX512-NEXT: vpaddd %zmm1, %zmm2, %zmm1
1769 ; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
1770 ; AVX512-NEXT: vpmulld %zmm3, %zmm2, %zmm2
1771 ; AVX512-NEXT: vpaddd %zmm0, %zmm2, %zmm0
1772 ; AVX512-NEXT: addq $16, %rcx
1773 ; AVX512-NEXT: cmpq %rcx, %rax
1774 ; AVX512-NEXT: jne .LBB11_1
1775 ; AVX512-NEXT: # %bb.2: # %middle.block
1776 ; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
1777 ; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
1778 ; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
1779 ; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
1780 ; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1781 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1782 ; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1783 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1784 ; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
1785 ; AVX512-NEXT: vmovd %xmm0, %eax
1786 ; AVX512-NEXT: vzeroupper
1787 ; AVX512-NEXT: retq
1788 entry:
1789 %3 = zext i32 %2 to i64
1790 br label %vector.body
1792 vector.body:
1793 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ]
1794 %vec.phi = phi <32 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ]
1795 %4 = getelementptr inbounds i16, i16* %0, i64 %index
1796 %5 = bitcast i16* %4 to <32 x i16>*
1797 %wide.load = load <32 x i16>, <32 x i16>* %5, align 2
1798 %6 = zext <32 x i16> %wide.load to <32 x i32>
1799 %7 = getelementptr inbounds i16, i16* %1, i64 %index
1800 %8 = bitcast i16* %7 to <32 x i16>*
1801 %wide.load14 = load <32 x i16>, <32 x i16>* %8, align 2
1802 %9 = zext <32 x i16> %wide.load14 to <32 x i32>
1803 %10 = mul nsw <32 x i32> %9, %6
1804 %11 = add nsw <32 x i32> %10, %vec.phi
1805 %index.next = add i64 %index, 16
1806 %12 = icmp eq i64 %index.next, %3
1807 br i1 %12, label %middle.block, label %vector.body
1809 middle.block:
1810 %rdx.shuf2 = shufflevector <32 x i32> %11, <32 x i32> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1811 %bin.rdx2 = add <32 x i32> %11, %rdx.shuf2
1812 %rdx.shuf1 = shufflevector <32 x i32> %bin.rdx2, <32 x i32> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1813 %bin.rdx1 = add <32 x i32> %bin.rdx2, %rdx.shuf1
1814 %rdx.shuf = shufflevector <32 x i32> %bin.rdx1, <32 x i32> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1815 %bin.rdx = add <32 x i32> %bin.rdx1, %rdx.shuf
1816 %rdx.shuf15 = shufflevector <32 x i32> %bin.rdx, <32 x i32> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1817 %bin.rdx16 = add <32 x i32> %bin.rdx, %rdx.shuf15
1818 %rdx.shuf17 = shufflevector <32 x i32> %bin.rdx16, <32 x i32> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1819 %bin.rdx18 = add <32 x i32> %bin.rdx16, %rdx.shuf17
1820 %13 = extractelement <32 x i32> %bin.rdx18, i32 0
1821 ret i32 %13
1822 }
1824 define <4 x i32> @pmaddwd_8(<8 x i16> %A, <8 x i16> %B) {
1825 ; SSE2-LABEL: pmaddwd_8:
1826 ; SSE2: # %bb.0:
1827 ; SSE2-NEXT: pmaddwd %xmm1, %xmm0
1828 ; SSE2-NEXT: retq
1829 ;
1830 ; AVX-LABEL: pmaddwd_8:
1831 ; AVX: # %bb.0:
1832 ; AVX-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
1833 ; AVX-NEXT: retq
1834 %a = sext <8 x i16> %A to <8 x i32>
1835 %b = sext <8 x i16> %B to <8 x i32>
1836 %m = mul nsw <8 x i32> %a, %b
1837 %odd = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
1838 %even = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
1839 %ret = add <4 x i32> %odd, %even
1840 ret <4 x i32> %ret
1841 }
1843 define <4 x i32> @pmaddwd_8_swapped(<8 x i16> %A, <8 x i16> %B) {
1844 ; SSE2-LABEL: pmaddwd_8_swapped:
1845 ; SSE2: # %bb.0:
1846 ; SSE2-NEXT: pmaddwd %xmm1, %xmm0
1847 ; SSE2-NEXT: retq
1848 ;
1849 ; AVX-LABEL: pmaddwd_8_swapped:
1850 ; AVX: # %bb.0:
1851 ; AVX-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
1852 ; AVX-NEXT: retq
1853 %a = sext <8 x i16> %A to <8 x i32>
1854 %b = sext <8 x i16> %B to <8 x i32>
1855 %m = mul nsw <8 x i32> %a, %b
1856 %odd = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
1857 %even = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
1858 %ret = add <4 x i32> %even, %odd
1859 ret <4 x i32> %ret
1860 }
1862 define <4 x i32> @larger_mul(<16 x i16> %A, <16 x i16> %B) {
1863 ; SSE2-LABEL: larger_mul:
1864 ; SSE2: # %bb.0:
1865 ; SSE2-NEXT: movdqa %xmm0, %xmm1
1866 ; SSE2-NEXT: pmulhw %xmm2, %xmm1
1867 ; SSE2-NEXT: pmullw %xmm2, %xmm0
1868 ; SSE2-NEXT: movdqa %xmm0, %xmm2
1869 ; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
1870 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1871 ; SSE2-NEXT: movdqa %xmm0, %xmm1
1872 ; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,2],xmm2[0,2]
1873 ; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,3],xmm2[1,3]
1874 ; SSE2-NEXT: paddd %xmm1, %xmm0
1875 ; SSE2-NEXT: retq
1876 ;
1877 ; AVX1-LABEL: larger_mul:
1878 ; AVX1: # %bb.0:
1879 ; AVX1-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
1880 ; AVX1-NEXT: vzeroupper
1881 ; AVX1-NEXT: retq
1882 ;
1883 ; AVX2-LABEL: larger_mul:
1884 ; AVX2: # %bb.0:
1885 ; AVX2-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
1886 ; AVX2-NEXT: vzeroupper
1887 ; AVX2-NEXT: retq
1888 ;
1889 ; AVX512-LABEL: larger_mul:
1890 ; AVX512: # %bb.0:
1891 ; AVX512-NEXT: vpmovsxwd %ymm0, %zmm0
1892 ; AVX512-NEXT: vpmovsxwd %ymm1, %zmm1
1893 ; AVX512-NEXT: vpmulld %zmm1, %zmm0, %zmm0
1894 ; AVX512-NEXT: vpextrd $2, %xmm0, %eax
1895 ; AVX512-NEXT: vpinsrd $1, %eax, %xmm0, %xmm1
1896 ; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm2
1897 ; AVX512-NEXT: vmovd %xmm2, %eax
1898 ; AVX512-NEXT: vpinsrd $2, %eax, %xmm1, %xmm1
1899 ; AVX512-NEXT: vpextrd $2, %xmm2, %eax
1900 ; AVX512-NEXT: vpinsrd $3, %eax, %xmm1, %xmm1
1901 ; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
1902 ; AVX512-NEXT: vpextrd $1, %xmm2, %eax
1903 ; AVX512-NEXT: vpinsrd $2, %eax, %xmm0, %xmm0
1904 ; AVX512-NEXT: vpextrd $3, %xmm2, %eax
1905 ; AVX512-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0
1906 ; AVX512-NEXT: vpaddd %xmm0, %xmm1, %xmm0
1907 ; AVX512-NEXT: vzeroupper
1908 ; AVX512-NEXT: retq
1909 %a = sext <16 x i16> %A to <16 x i32>
1910 %b = sext <16 x i16> %B to <16 x i32>
1911 %m = mul nsw <16 x i32> %a, %b
1912 %odd = shufflevector <16 x i32> %m, <16 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
1913 %even = shufflevector <16 x i32> %m, <16 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
1914 %ret = add <4 x i32> %odd, %even
1915 ret <4 x i32> %ret
1916 }
1918 define <8 x i32> @pmaddwd_16(<16 x i16> %A, <16 x i16> %B) {
1919 ; SSE2-LABEL: pmaddwd_16:
1920 ; SSE2: # %bb.0:
1921 ; SSE2-NEXT: pmaddwd %xmm2, %xmm0
1922 ; SSE2-NEXT: pmaddwd %xmm3, %xmm1
1923 ; SSE2-NEXT: retq
1924 ;
1925 ; AVX1-LABEL: pmaddwd_16:
1926 ; AVX1: # %bb.0:
1927 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
1928 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
1929 ; AVX1-NEXT: vpmaddwd %xmm3, %xmm2, %xmm2
1930 ; AVX1-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
1931 ; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
1932 ; AVX1-NEXT: retq
1933 ;
1934 ; AVX256-LABEL: pmaddwd_16:
1935 ; AVX256: # %bb.0:
1936 ; AVX256-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
1937 ; AVX256-NEXT: retq
1938 %a = sext <16 x i16> %A to <16 x i32>
1939 %b = sext <16 x i16> %B to <16 x i32>
1940 %m = mul nsw <16 x i32> %a, %b
1941 %odd = shufflevector <16 x i32> %m, <16 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
1942 %even = shufflevector <16 x i32> %m, <16 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
1943 %ret = add <8 x i32> %odd, %even
1944 ret <8 x i32> %ret
1945 }
1947 define <16 x i32> @pmaddwd_32(<32 x i16> %A, <32 x i16> %B) {
1948 ; SSE2-LABEL: pmaddwd_32:
1949 ; SSE2: # %bb.0:
1950 ; SSE2-NEXT: pmaddwd %xmm4, %xmm0
1951 ; SSE2-NEXT: pmaddwd %xmm5, %xmm1
1952 ; SSE2-NEXT: pmaddwd %xmm6, %xmm2
1953 ; SSE2-NEXT: pmaddwd %xmm7, %xmm3
1954 ; SSE2-NEXT: retq
1955 ;
1956 ; AVX1-LABEL: pmaddwd_32:
1957 ; AVX1: # %bb.0:
1958 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4
1959 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5
1960 ; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm6
1961 ; AVX1-NEXT: vpmaddwd %xmm6, %xmm4, %xmm4
1962 ; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm6
1963 ; AVX1-NEXT: vpmaddwd %xmm6, %xmm5, %xmm5
1964 ; AVX1-NEXT: vpmaddwd %xmm2, %xmm0, %xmm0
1965 ; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm0
1966 ; AVX1-NEXT: vpmaddwd %xmm3, %xmm1, %xmm1
1967 ; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm1, %ymm1
1968 ; AVX1-NEXT: retq
1969 ;
1970 ; AVX2-LABEL: pmaddwd_32:
1971 ; AVX2: # %bb.0:
1972 ; AVX2-NEXT: vpmaddwd %ymm2, %ymm0, %ymm0
1973 ; AVX2-NEXT: vpmaddwd %ymm3, %ymm1, %ymm1
1974 ; AVX2-NEXT: retq
1975 ;
1976 ; AVX512F-LABEL: pmaddwd_32:
1977 ; AVX512F: # %bb.0:
1978 ; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm2
1979 ; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm3
1980 ; AVX512F-NEXT: vpmaddwd %ymm2, %ymm3, %ymm2
1981 ; AVX512F-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
1982 ; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
1983 ; AVX512F-NEXT: retq
1984 ;
1985 ; AVX512BW-LABEL: pmaddwd_32:
1986 ; AVX512BW: # %bb.0:
1987 ; AVX512BW-NEXT: vpmaddwd %zmm1, %zmm0, %zmm0
1988 ; AVX512BW-NEXT: retq
1989 %a = sext <32 x i16> %A to <32 x i32>
1990 %b = sext <32 x i16> %B to <32 x i32>
1991 %m = mul nsw <32 x i32> %a, %b
1992 %odd = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
1993 %even = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
1994 %ret = add <16 x i32> %odd, %even
1995 ret <16 x i32> %ret
1996 }
1998 define <4 x i32> @pmaddwd_const(<8 x i16> %A) {
1999 ; SSE2-LABEL: pmaddwd_const:
2000 ; SSE2: # %bb.0:
2001 ; SSE2-NEXT: pmaddwd {{.*}}(%rip), %xmm0
2002 ; SSE2-NEXT: retq
2003 ;
2004 ; AVX-LABEL: pmaddwd_const:
2005 ; AVX: # %bb.0:
2006 ; AVX-NEXT: vpmaddwd {{.*}}(%rip), %xmm0, %xmm0
2007 ; AVX-NEXT: retq
2008 %a = sext <8 x i16> %A to <8 x i32>
2009 %m = mul nsw <8 x i32> %a, <i32 32767, i32 -32768, i32 0, i32 0, i32 1, i32 7, i32 42, i32 32>
2010 %odd = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
2011 %even = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
2012 %ret = add <4 x i32> %odd, %even
2013 ret <4 x i32> %ret
2014 }
2016 ; Do not select unsigned i16 multiplication
2017 define <4 x i32> @pmaddwd_negative1(<8 x i16> %A, <8 x i16> %B) {
2018 ; SSE2-LABEL: pmaddwd_negative1:
2019 ; SSE2: # %bb.0:
2020 ; SSE2-NEXT: movdqa %xmm0, %xmm2
2021 ; SSE2-NEXT: pmulhuw %xmm1, %xmm2
2022 ; SSE2-NEXT: pmullw %xmm1, %xmm0
2023 ; SSE2-NEXT: movdqa %xmm0, %xmm1
2024 ; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
2025 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
2026 ; SSE2-NEXT: movdqa %xmm0, %xmm2
2027 ; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm1[0,2]
2028 ; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
2029 ; SSE2-NEXT: paddd %xmm2, %xmm0
2030 ; SSE2-NEXT: retq
2031 ;
2032 ; AVX1-LABEL: pmaddwd_negative1:
2033 ; AVX1: # %bb.0:
2034 ; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
2035 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
2036 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
2037 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
2038 ; AVX1-NEXT: vpmulld %xmm2, %xmm3, %xmm2
2039 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
2040 ; AVX1-NEXT: vpmulld %xmm1, %xmm0, %xmm0
2041 ; AVX1-NEXT: vphaddd %xmm2, %xmm0, %xmm0
2042 ; AVX1-NEXT: retq
2043 ;
2044 ; AVX256-LABEL: pmaddwd_negative1:
2045 ; AVX256: # %bb.0:
2046 ; AVX256-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
2047 ; AVX256-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
2048 ; AVX256-NEXT: vpmulld %ymm1, %ymm0, %ymm0
2049 ; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1
2050 ; AVX256-NEXT: vphaddd %xmm1, %xmm0, %xmm0
2051 ; AVX256-NEXT: vzeroupper
2052 ; AVX256-NEXT: retq
2053 %a = zext <8 x i16> %A to <8 x i32>
2054 %b = zext <8 x i16> %B to <8 x i32>
2055 %m = mul nuw <8 x i32> %a, %b
2056 %odd = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
2057 %even = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
2058 %ret = add <4 x i32> %odd, %even
2059 ret <4 x i32> %ret
2060 }
2062 ; Do not select if constant is too large
2063 define <4 x i32> @pmaddwd_negative2(<8 x i16> %A) {
2064 ; SSE2-LABEL: pmaddwd_negative2:
2065 ; SSE2: # %bb.0:
2066 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
2067 ; SSE2-NEXT: psrad $16, %xmm1
2068 ; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
2069 ; SSE2-NEXT: psrad $16, %xmm0
2070 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
2071 ; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1,7,42,32]
2072 ; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
2073 ; SSE2-NEXT: pmuludq %xmm2, %xmm4
2074 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
2075 ; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [32768,4294934528,0,0]
2076 ; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
2077 ; SSE2-NEXT: pmuludq %xmm2, %xmm6
2078 ; SSE2-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,2],xmm4[0,2]
2079 ; SSE2-NEXT: pmuludq %xmm3, %xmm0
2080 ; SSE2-NEXT: pmuludq %xmm5, %xmm1
2081 ; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,2],xmm0[0,2]
2082 ; SSE2-NEXT: paddd %xmm6, %xmm1
2083 ; SSE2-NEXT: movdqa %xmm1, %xmm0
2084 ; SSE2-NEXT: retq
2085 ;
2086 ; AVX1-LABEL: pmaddwd_negative2:
2087 ; AVX1: # %bb.0:
2088 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
2089 ; AVX1-NEXT: vpmovsxwd %xmm1, %xmm1
2090 ; AVX1-NEXT: vpmovsxwd %xmm0, %xmm0
2091 ; AVX1-NEXT: vpmulld {{.*}}(%rip), %xmm0, %xmm0
2092 ; AVX1-NEXT: vpmulld {{.*}}(%rip), %xmm1, %xmm1
2093 ; AVX1-NEXT: vphaddd %xmm1, %xmm0, %xmm0
2094 ; AVX1-NEXT: retq
2095 ;
2096 ; AVX256-LABEL: pmaddwd_negative2:
2097 ; AVX256: # %bb.0:
2098 ; AVX256-NEXT: vpmovsxwd %xmm0, %ymm0
2099 ; AVX256-NEXT: vpmulld {{.*}}(%rip), %ymm0, %ymm0
2100 ; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1
2101 ; AVX256-NEXT: vphaddd %xmm1, %xmm0, %xmm0
2102 ; AVX256-NEXT: vzeroupper
2103 ; AVX256-NEXT: retq
2104 %a = sext <8 x i16> %A to <8 x i32>
2105 %m = mul nsw <8 x i32> %a, <i32 32768, i32 -32768, i32 0, i32 0, i32 1, i32 7, i32 42, i32 32>
2106 %odd = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
2107 %even = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
2108 %ret = add <4 x i32> %odd, %even
2109 ret <4 x i32> %ret
2110 }
2112 define <4 x i32> @jumbled_indices4(<8 x i16> %A, <8 x i16> %B) {
2113 ; SSE2-LABEL: jumbled_indices4:
2114 ; SSE2: # %bb.0:
2115 ; SSE2-NEXT: pmaddwd %xmm1, %xmm0
2116 ; SSE2-NEXT: retq
2117 ;
2118 ; AVX-LABEL: jumbled_indices4:
2119 ; AVX: # %bb.0:
2120 ; AVX-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
2121 ; AVX-NEXT: retq
2122 %exta = sext <8 x i16> %A to <8 x i32>
2123 %extb = sext <8 x i16> %B to <8 x i32>
2124 %m = mul <8 x i32> %exta, %extb
2125 %sa = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 3, i32 1, i32 5, i32 6>
2126 %sb = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 2, i32 0, i32 4, i32 7>
2127 %a = add <4 x i32> %sa, %sb
2128 ret <4 x i32> %a
2129 }
2131 define <8 x i32> @jumbled_indices8(<16 x i16> %A, <16 x i16> %B) {
2132 ; SSE2-LABEL: jumbled_indices8:
2133 ; SSE2: # %bb.0:
2134 ; SSE2-NEXT: pmaddwd %xmm2, %xmm0
2135 ; SSE2-NEXT: pmaddwd %xmm3, %xmm1
2136 ; SSE2-NEXT: retq
2137 ;
2138 ; AVX1-LABEL: jumbled_indices8:
2139 ; AVX1: # %bb.0:
2140 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
2141 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
2142 ; AVX1-NEXT: vpmaddwd %xmm3, %xmm2, %xmm2
2143 ; AVX1-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
2144 ; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
2145 ; AVX1-NEXT: retq
2146 ;
2147 ; AVX256-LABEL: jumbled_indices8:
2148 ; AVX256: # %bb.0:
2149 ; AVX256-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
2150 ; AVX256-NEXT: retq
2151 %exta = sext <16 x i16> %A to <16 x i32>
2152 %extb = sext <16 x i16> %B to <16 x i32>
2153 %m = mul <16 x i32> %exta, %extb
2154 %sa = shufflevector <16 x i32> %m, <16 x i32> undef, <8 x i32> <i32 0, i32 2, i32 7, i32 4, i32 11, i32 8, i32 15, i32 12>
2155 %sb = shufflevector <16 x i32> %m, <16 x i32> undef, <8 x i32> <i32 1, i32 3, i32 6, i32 5, i32 10, i32 9, i32 14, i32 13>
2156 %a = add <8 x i32> %sa, %sb
2157 ret <8 x i32> %a
2158 }
2160 define <16 x i32> @jumbled_indices16(<32 x i16> %A, <32 x i16> %B) {
2161 ; SSE2-LABEL: jumbled_indices16:
2162 ; SSE2: # %bb.0:
2163 ; SSE2-NEXT: pmaddwd %xmm4, %xmm0
2164 ; SSE2-NEXT: pmaddwd %xmm5, %xmm1
2165 ; SSE2-NEXT: pmaddwd %xmm6, %xmm2
2166 ; SSE2-NEXT: pmaddwd %xmm7, %xmm3
2167 ; SSE2-NEXT: retq
2168 ;
2169 ; AVX1-LABEL: jumbled_indices16:
2170 ; AVX1: # %bb.0:
2171 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4
2172 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5
2173 ; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm6
2174 ; AVX1-NEXT: vpmaddwd %xmm6, %xmm4, %xmm4
2175 ; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm6
2176 ; AVX1-NEXT: vpmaddwd %xmm6, %xmm5, %xmm5
2177 ; AVX1-NEXT: vpmaddwd %xmm2, %xmm0, %xmm0
2178 ; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm0
2179 ; AVX1-NEXT: vpmaddwd %xmm3, %xmm1, %xmm1
2180 ; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm1, %ymm1
2181 ; AVX1-NEXT: retq
2182 ;
2183 ; AVX2-LABEL: jumbled_indices16:
2184 ; AVX2: # %bb.0:
2185 ; AVX2-NEXT: vpmaddwd %ymm2, %ymm0, %ymm0
2186 ; AVX2-NEXT: vpmaddwd %ymm3, %ymm1, %ymm1
2187 ; AVX2-NEXT: retq
2188 ;
2189 ; AVX512F-LABEL: jumbled_indices16:
2190 ; AVX512F: # %bb.0:
2191 ; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm2
2192 ; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm3
2193 ; AVX512F-NEXT: vpmaddwd %ymm2, %ymm3, %ymm2
2194 ; AVX512F-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
2195 ; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
2196 ; AVX512F-NEXT: retq
2197 ;
2198 ; AVX512BW-LABEL: jumbled_indices16:
2199 ; AVX512BW: # %bb.0:
2200 ; AVX512BW-NEXT: vpmaddwd %zmm1, %zmm0, %zmm0
2201 ; AVX512BW-NEXT: retq
2202 %exta = sext <32 x i16> %A to <32 x i32>
2203 %extb = sext <32 x i16> %B to <32 x i32>
2204 %m = mul <32 x i32> %exta, %extb
2205 %sa = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 2, i32 0, i32 5, i32 6, i32 11, i32 9, i32 15, i32 12, i32 17, i32 18, i32 20, i32 23, i32 27, i32 24, i32 31, i32 29>
2206 %sb = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 3, i32 1, i32 4, i32 7, i32 10, i32 8, i32 14, i32 13, i32 16, i32 19, i32 21, i32 22, i32 26, i32 25, i32 30, i32 28>
2207 %a = add <16 x i32> %sa, %sb
2208 ret <16 x i32> %a
2209 }
2211 define <32 x i32> @jumbled_indices32(<64 x i16> %A, <64 x i16> %B) {
2212 ; SSE2-LABEL: jumbled_indices32:
2213 ; SSE2: # %bb.0:
2214 ; SSE2-NEXT: movq %rdi, %rax
2215 ; SSE2-NEXT: pmaddwd {{[0-9]+}}(%rsp), %xmm0
2216 ; SSE2-NEXT: pmaddwd {{[0-9]+}}(%rsp), %xmm1
2217 ; SSE2-NEXT: pmaddwd {{[0-9]+}}(%rsp), %xmm2
2218 ; SSE2-NEXT: pmaddwd {{[0-9]+}}(%rsp), %xmm3
2219 ; SSE2-NEXT: pmaddwd {{[0-9]+}}(%rsp), %xmm4
2220 ; SSE2-NEXT: pmaddwd {{[0-9]+}}(%rsp), %xmm5
2221 ; SSE2-NEXT: pmaddwd {{[0-9]+}}(%rsp), %xmm6
2222 ; SSE2-NEXT: pmaddwd {{[0-9]+}}(%rsp), %xmm7
2223 ; SSE2-NEXT: movdqa %xmm7, 112(%rdi)
2224 ; SSE2-NEXT: movdqa %xmm6, 96(%rdi)
2225 ; SSE2-NEXT: movdqa %xmm5, 80(%rdi)
2226 ; SSE2-NEXT: movdqa %xmm4, 64(%rdi)
2227 ; SSE2-NEXT: movdqa %xmm3, 48(%rdi)
2228 ; SSE2-NEXT: movdqa %xmm2, 32(%rdi)
2229 ; SSE2-NEXT: movdqa %xmm1, 16(%rdi)
2230 ; SSE2-NEXT: movdqa %xmm0, (%rdi)
2231 ; SSE2-NEXT: retq
2232 ;
2233 ; AVX1-LABEL: jumbled_indices32:
2234 ; AVX1: # %bb.0:
2235 ; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm8
2236 ; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm9
2237 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm10
2238 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm11
2239 ; AVX1-NEXT: vextractf128 $1, %ymm7, %xmm12
2240 ; AVX1-NEXT: vpmaddwd %xmm12, %xmm8, %xmm8
2241 ; AVX1-NEXT: vextractf128 $1, %ymm6, %xmm12
2242 ; AVX1-NEXT: vpmaddwd %xmm12, %xmm9, %xmm9
2243 ; AVX1-NEXT: vextractf128 $1, %ymm5, %xmm12
2244 ; AVX1-NEXT: vpmaddwd %xmm12, %xmm10, %xmm10
2245 ; AVX1-NEXT: vextractf128 $1, %ymm4, %xmm12
2246 ; AVX1-NEXT: vpmaddwd %xmm12, %xmm11, %xmm11
2247 ; AVX1-NEXT: vpmaddwd %xmm4, %xmm0, %xmm0
2248 ; AVX1-NEXT: vinsertf128 $1, %xmm11, %ymm0, %ymm0
2249 ; AVX1-NEXT: vpmaddwd %xmm5, %xmm1, %xmm1
2250 ; AVX1-NEXT: vinsertf128 $1, %xmm10, %ymm1, %ymm1
2251 ; AVX1-NEXT: vpmaddwd %xmm6, %xmm2, %xmm2
2252 ; AVX1-NEXT: vinsertf128 $1, %xmm9, %ymm2, %ymm2
2253 ; AVX1-NEXT: vpmaddwd %xmm7, %xmm3, %xmm3
2254 ; AVX1-NEXT: vinsertf128 $1, %xmm8, %ymm3, %ymm3
2255 ; AVX1-NEXT: retq
2256 ;
2257 ; AVX2-LABEL: jumbled_indices32:
2258 ; AVX2: # %bb.0:
2259 ; AVX2-NEXT: vpmaddwd %ymm4, %ymm0, %ymm0
2260 ; AVX2-NEXT: vpmaddwd %ymm5, %ymm1, %ymm1
2261 ; AVX2-NEXT: vpmaddwd %ymm6, %ymm2, %ymm2
2262 ; AVX2-NEXT: vpmaddwd %ymm7, %ymm3, %ymm3
2263 ; AVX2-NEXT: retq
2264 ;
2265 ; AVX512F-LABEL: jumbled_indices32:
2266 ; AVX512F: # %bb.0:
2267 ; AVX512F-NEXT: vpmaddwd %ymm5, %ymm1, %ymm1
2268 ; AVX512F-NEXT: vpmaddwd %ymm4, %ymm0, %ymm0
2269 ; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
2270 ; AVX512F-NEXT: vpmaddwd %ymm7, %ymm3, %ymm1
2271 ; AVX512F-NEXT: vpmaddwd %ymm6, %ymm2, %ymm2
2272 ; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
2273 ; AVX512F-NEXT: retq
2274 ;
2275 ; AVX512BW-LABEL: jumbled_indices32:
2276 ; AVX512BW: # %bb.0:
2277 ; AVX512BW-NEXT: vpmaddwd %zmm2, %zmm0, %zmm0
2278 ; AVX512BW-NEXT: vpmaddwd %zmm3, %zmm1, %zmm1
2279 ; AVX512BW-NEXT: retq
2280 %exta = sext <64 x i16> %A to <64 x i32>
2281 %extb = sext <64 x i16> %B to <64 x i32>
2282 %m = mul <64 x i32> %exta, %extb
2283 %sa = shufflevector <64 x i32> %m, <64 x i32> undef, <32 x i32> <i32 1, i32 2, i32 6, i32 5, i32 10, i32 8, i32 14, i32 12, i32 19, i32 17, i32 22, i32 20, i32 25, i32 27, i32 30, i32 28, i32 32, i32 34, i32 37, i32 38, i32 41, i32 43, i32 45, i32 47, i32 50, i32 48, i32 52, i32 54, i32 59, i32 56, i32 61, i32 63>
2284 %sb = shufflevector <64 x i32> %m, <64 x i32> undef, <32 x i32> <i32 0, i32 3, i32 7, i32 4, i32 11, i32 9, i32 15, i32 13, i32 18, i32 16, i32 23, i32 21, i32 24, i32 26, i32 31, i32 29, i32 33, i32 35, i32 36, i32 39, i32 40, i32 42, i32 44, i32 46, i32 51, i32 49, i32 53, i32 55, i32 58, i32 57, i32 60, i32 62>
2285 %a = add <32 x i32> %sa, %sb
2286 ret <32 x i32> %a
2287 }
2289 ; NOTE: We're testing with loads because ABI lowering creates a concat_vectors that extract_vector_elt creation can see through.
2290 ; This would require the combine to recreate the concat_vectors.
2291 define <4 x i32> @pmaddwd_128(<8 x i16>* %Aptr, <8 x i16>* %Bptr) {
2292 ; SSE2-LABEL: pmaddwd_128:
2293 ; SSE2: # %bb.0:
2294 ; SSE2-NEXT: movdqa (%rdi), %xmm0
2295 ; SSE2-NEXT: pmaddwd (%rsi), %xmm0
2296 ; SSE2-NEXT: retq
2297 ;
2298 ; AVX-LABEL: pmaddwd_128:
2299 ; AVX: # %bb.0:
2300 ; AVX-NEXT: vmovdqa (%rdi), %xmm0
2301 ; AVX-NEXT: vpmaddwd (%rsi), %xmm0, %xmm0
2302 ; AVX-NEXT: retq
2303 %A = load <8 x i16>, <8 x i16>* %Aptr
2304 %B = load <8 x i16>, <8 x i16>* %Bptr
2305 %A_even = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
2306 %A_odd = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
2307 %B_even = shufflevector <8 x i16> %B, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
2308 %B_odd = shufflevector <8 x i16> %B, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
2309 %A_even_ext = sext <4 x i16> %A_even to <4 x i32>
2310 %B_even_ext = sext <4 x i16> %B_even to <4 x i32>
2311 %A_odd_ext = sext <4 x i16> %A_odd to <4 x i32>
2312 %B_odd_ext = sext <4 x i16> %B_odd to <4 x i32>
2313 %even_mul = mul <4 x i32> %A_even_ext, %B_even_ext
2314 %odd_mul = mul <4 x i32> %A_odd_ext, %B_odd_ext
2315 %add = add <4 x i32> %even_mul, %odd_mul
2316 ret <4 x i32> %add
2317 }
2319 define <8 x i32> @pmaddwd_256(<16 x i16>* %Aptr, <16 x i16>* %Bptr) {
2320 ; SSE2-LABEL: pmaddwd_256:
2321 ; SSE2: # %bb.0:
2322 ; SSE2-NEXT: movdqa (%rdi), %xmm0
2323 ; SSE2-NEXT: movdqa 16(%rdi), %xmm1
2324 ; SSE2-NEXT: pmaddwd (%rsi), %xmm0
2325 ; SSE2-NEXT: pmaddwd 16(%rsi), %xmm1
2326 ; SSE2-NEXT: retq
2327 ;
2328 ; AVX1-LABEL: pmaddwd_256:
2329 ; AVX1: # %bb.0:
2330 ; AVX1-NEXT: vmovdqa (%rdi), %xmm0
2331 ; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1
2332 ; AVX1-NEXT: vpmaddwd 16(%rsi), %xmm1, %xmm1
2333 ; AVX1-NEXT: vpmaddwd (%rsi), %xmm0, %xmm0
2334 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
2335 ; AVX1-NEXT: retq
2336 ;
2337 ; AVX256-LABEL: pmaddwd_256:
2338 ; AVX256: # %bb.0:
2339 ; AVX256-NEXT: vmovdqa (%rdi), %ymm0
2340 ; AVX256-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0
2341 ; AVX256-NEXT: retq
2342 %A = load <16 x i16>, <16 x i16>* %Aptr
2343 %B = load <16 x i16>, <16 x i16>* %Bptr
2344 %A_even = shufflevector <16 x i16> %A, <16 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
2345 %A_odd = shufflevector <16 x i16> %A, <16 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
2346 %B_even = shufflevector <16 x i16> %B, <16 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
2347 %B_odd = shufflevector <16 x i16> %B, <16 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
2348 %A_even_ext = sext <8 x i16> %A_even to <8 x i32>
2349 %B_even_ext = sext <8 x i16> %B_even to <8 x i32>
2350 %A_odd_ext = sext <8 x i16> %A_odd to <8 x i32>
2351 %B_odd_ext = sext <8 x i16> %B_odd to <8 x i32>
2352 %even_mul = mul <8 x i32> %A_even_ext, %B_even_ext
2353 %odd_mul = mul <8 x i32> %A_odd_ext, %B_odd_ext
2354 %add = add <8 x i32> %even_mul, %odd_mul
2355 ret <8 x i32> %add
2356 }
2358 define <16 x i32> @pmaddwd_512(<32 x i16>* %Aptr, <32 x i16>* %Bptr) {
2359 ; SSE2-LABEL: pmaddwd_512:
2360 ; SSE2: # %bb.0:
2361 ; SSE2-NEXT: movdqa (%rdi), %xmm0
2362 ; SSE2-NEXT: movdqa 16(%rdi), %xmm1
2363 ; SSE2-NEXT: movdqa 32(%rdi), %xmm2
2364 ; SSE2-NEXT: movdqa 48(%rdi), %xmm3
2365 ; SSE2-NEXT: pmaddwd (%rsi), %xmm0
2366 ; SSE2-NEXT: pmaddwd 16(%rsi), %xmm1
2367 ; SSE2-NEXT: pmaddwd 32(%rsi), %xmm2
2368 ; SSE2-NEXT: pmaddwd 48(%rsi), %xmm3
2369 ; SSE2-NEXT: retq
2370 ;
2371 ; AVX1-LABEL: pmaddwd_512:
2372 ; AVX1: # %bb.0:
2373 ; AVX1-NEXT: vmovdqa (%rdi), %xmm0
2374 ; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1
2375 ; AVX1-NEXT: vmovdqa 32(%rdi), %xmm2
2376 ; AVX1-NEXT: vmovdqa 48(%rdi), %xmm3
2377 ; AVX1-NEXT: vpmaddwd 16(%rsi), %xmm1, %xmm1
2378 ; AVX1-NEXT: vpmaddwd (%rsi), %xmm0, %xmm0
2379 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
2380 ; AVX1-NEXT: vpmaddwd 48(%rsi), %xmm3, %xmm1
2381 ; AVX1-NEXT: vpmaddwd 32(%rsi), %xmm2, %xmm2
2382 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1
2383 ; AVX1-NEXT: retq
2384 ;
2385 ; AVX2-LABEL: pmaddwd_512:
2386 ; AVX2: # %bb.0:
2387 ; AVX2-NEXT: vmovdqa (%rdi), %ymm0
2388 ; AVX2-NEXT: vmovdqa 32(%rdi), %ymm1
2389 ; AVX2-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0
2390 ; AVX2-NEXT: vpmaddwd 32(%rsi), %ymm1, %ymm1
2391 ; AVX2-NEXT: retq
2392 ;
2393 ; AVX512F-LABEL: pmaddwd_512:
2394 ; AVX512F: # %bb.0:
2395 ; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
2396 ; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1
2397 ; AVX512F-NEXT: vpmaddwd 32(%rsi), %ymm1, %ymm1
2398 ; AVX512F-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0
2399 ; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
2400 ; AVX512F-NEXT: retq
2401 ;
2402 ; AVX512BW-LABEL: pmaddwd_512:
2403 ; AVX512BW: # %bb.0:
2404 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
2405 ; AVX512BW-NEXT: vpmaddwd (%rsi), %zmm0, %zmm0
2406 ; AVX512BW-NEXT: retq
2407 %A = load <32 x i16>, <32 x i16>* %Aptr
2408 %B = load <32 x i16>, <32 x i16>* %Bptr
2409 %A_even = shufflevector <32 x i16> %A, <32 x i16> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
2410 %A_odd = shufflevector <32 x i16> %A, <32 x i16> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
2411 %B_even = shufflevector <32 x i16> %B, <32 x i16> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
2412 %B_odd = shufflevector <32 x i16> %B, <32 x i16> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
2413 %A_even_ext = sext <16 x i16> %A_even to <16 x i32>
2414 %B_even_ext = sext <16 x i16> %B_even to <16 x i32>
2415 %A_odd_ext = sext <16 x i16> %A_odd to <16 x i32>
2416 %B_odd_ext = sext <16 x i16> %B_odd to <16 x i32>
2417 %even_mul = mul <16 x i32> %A_even_ext, %B_even_ext
2418 %odd_mul = mul <16 x i32> %A_odd_ext, %B_odd_ext
2419 %add = add <16 x i32> %even_mul, %odd_mul
2420 ret <16 x i32> %add
2421 }
2423 define <32 x i32> @pmaddwd_1024(<64 x i16>* %Aptr, <64 x i16>* %Bptr) {
2424 ; SSE2-LABEL: pmaddwd_1024:
2425 ; SSE2: # %bb.0:
2426 ; SSE2-NEXT: movq %rdi, %rax
2427 ; SSE2-NEXT: movdqa (%rsi), %xmm0
2428 ; SSE2-NEXT: movdqa 16(%rsi), %xmm1
2429 ; SSE2-NEXT: movdqa 32(%rsi), %xmm2
2430 ; SSE2-NEXT: movdqa 48(%rsi), %xmm3
2431 ; SSE2-NEXT: pmaddwd (%rdx), %xmm0
2432 ; SSE2-NEXT: pmaddwd 16(%rdx), %xmm1
2433 ; SSE2-NEXT: pmaddwd 32(%rdx), %xmm2
2434 ; SSE2-NEXT: pmaddwd 48(%rdx), %xmm3
2435 ; SSE2-NEXT: movdqa 64(%rsi), %xmm4
2436 ; SSE2-NEXT: pmaddwd 64(%rdx), %xmm4
2437 ; SSE2-NEXT: movdqa 80(%rsi), %xmm5
2438 ; SSE2-NEXT: pmaddwd 80(%rdx), %xmm5
2439 ; SSE2-NEXT: movdqa 96(%rsi), %xmm6
2440 ; SSE2-NEXT: pmaddwd 96(%rdx), %xmm6
2441 ; SSE2-NEXT: movdqa 112(%rsi), %xmm7
2442 ; SSE2-NEXT: pmaddwd 112(%rdx), %xmm7
2443 ; SSE2-NEXT: movdqa %xmm7, 112(%rdi)
2444 ; SSE2-NEXT: movdqa %xmm6, 96(%rdi)
2445 ; SSE2-NEXT: movdqa %xmm5, 80(%rdi)
2446 ; SSE2-NEXT: movdqa %xmm4, 64(%rdi)
2447 ; SSE2-NEXT: movdqa %xmm3, 48(%rdi)
2448 ; SSE2-NEXT: movdqa %xmm2, 32(%rdi)
2449 ; SSE2-NEXT: movdqa %xmm1, 16(%rdi)
2450 ; SSE2-NEXT: movdqa %xmm0, (%rdi)
2451 ; SSE2-NEXT: retq
2452 ;
2453 ; AVX1-LABEL: pmaddwd_1024:
2454 ; AVX1: # %bb.0:
2455 ; AVX1-NEXT: vmovdqa (%rdi), %xmm0
2456 ; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1
2457 ; AVX1-NEXT: vmovdqa 32(%rdi), %xmm2
2458 ; AVX1-NEXT: vmovdqa 48(%rdi), %xmm3
2459 ; AVX1-NEXT: vpmaddwd 16(%rsi), %xmm1, %xmm1
2460 ; AVX1-NEXT: vpmaddwd (%rsi), %xmm0, %xmm0
2461 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
2462 ; AVX1-NEXT: vpmaddwd 48(%rsi), %xmm3, %xmm1
2463 ; AVX1-NEXT: vpmaddwd 32(%rsi), %xmm2, %xmm2
2464 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1
2465 ; AVX1-NEXT: vmovdqa 80(%rdi), %xmm2
2466 ; AVX1-NEXT: vpmaddwd 80(%rsi), %xmm2, %xmm2
2467 ; AVX1-NEXT: vmovdqa 64(%rdi), %xmm3
2468 ; AVX1-NEXT: vpmaddwd 64(%rsi), %xmm3, %xmm3
2469 ; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2
2470 ; AVX1-NEXT: vmovdqa 112(%rdi), %xmm3
2471 ; AVX1-NEXT: vpmaddwd 112(%rsi), %xmm3, %xmm3
2472 ; AVX1-NEXT: vmovdqa 96(%rdi), %xmm4
2473 ; AVX1-NEXT: vpmaddwd 96(%rsi), %xmm4, %xmm4
2474 ; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm4, %ymm3
2475 ; AVX1-NEXT: retq
2476 ;
2477 ; AVX2-LABEL: pmaddwd_1024:
2478 ; AVX2: # %bb.0:
2479 ; AVX2-NEXT: vmovdqa (%rdi), %ymm0
2480 ; AVX2-NEXT: vmovdqa 32(%rdi), %ymm1
2481 ; AVX2-NEXT: vmovdqa 64(%rdi), %ymm2
2482 ; AVX2-NEXT: vmovdqa 96(%rdi), %ymm3
2483 ; AVX2-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0
2484 ; AVX2-NEXT: vpmaddwd 32(%rsi), %ymm1, %ymm1
2485 ; AVX2-NEXT: vpmaddwd 64(%rsi), %ymm2, %ymm2
2486 ; AVX2-NEXT: vpmaddwd 96(%rsi), %ymm3, %ymm3
2487 ; AVX2-NEXT: retq
2488 ;
2489 ; AVX512F-LABEL: pmaddwd_1024:
2490 ; AVX512F: # %bb.0:
2491 ; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
2492 ; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1
2493 ; AVX512F-NEXT: vmovdqa 64(%rdi), %ymm2
2494 ; AVX512F-NEXT: vmovdqa 96(%rdi), %ymm3
2495 ; AVX512F-NEXT: vpmaddwd 32(%rsi), %ymm1, %ymm1
2496 ; AVX512F-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0
2497 ; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
2498 ; AVX512F-NEXT: vpmaddwd 96(%rsi), %ymm3, %ymm1
2499 ; AVX512F-NEXT: vpmaddwd 64(%rsi), %ymm2, %ymm2
2500 ; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
2501 ; AVX512F-NEXT: retq
2502 ;
2503 ; AVX512BW-LABEL: pmaddwd_1024:
2504 ; AVX512BW: # %bb.0:
2505 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
2506 ; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm1
2507 ; AVX512BW-NEXT: vpmaddwd (%rsi), %zmm0, %zmm0
2508 ; AVX512BW-NEXT: vpmaddwd 64(%rsi), %zmm1, %zmm1
2509 ; AVX512BW-NEXT: retq
2510 %A = load <64 x i16>, <64 x i16>* %Aptr
2511 %B = load <64 x i16>, <64 x i16>* %Bptr
2512 %A_even = shufflevector <64 x i16> %A, <64 x i16> undef, <32 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30, i32 32, i32 34, i32 36, i32 38, i32 40, i32 42, i32 44, i32 46, i32 48, i32 50, i32 52, i32 54, i32 56, i32 58, i32 60, i32 62>
2513 %A_odd = shufflevector <64 x i16> %A, <64 x i16> undef, <32 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 33, i32 35, i32 37, i32 39, i32 41, i32 43, i32 45, i32 47, i32 49, i32 51, i32 53, i32 55, i32 57, i32 59, i32 61, i32 63>
2514 %B_even = shufflevector <64 x i16> %B, <64 x i16> undef, <32 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30, i32 32, i32 34, i32 36, i32 38, i32 40, i32 42, i32 44, i32 46, i32 48, i32 50, i32 52, i32 54, i32 56, i32 58, i32 60, i32 62>
2515 %B_odd = shufflevector <64 x i16> %B, <64 x i16> undef, <32 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 33, i32 35, i32 37, i32 39, i32 41, i32 43, i32 45, i32 47, i32 49, i32 51, i32 53, i32 55, i32 57, i32 59, i32 61, i32 63>
2516 %A_even_ext = sext <32 x i16> %A_even to <32 x i32>
2517 %B_even_ext = sext <32 x i16> %B_even to <32 x i32>
2518 %A_odd_ext = sext <32 x i16> %A_odd to <32 x i32>
2519 %B_odd_ext = sext <32 x i16> %B_odd to <32 x i32>
2520 %even_mul = mul <32 x i32> %A_even_ext, %B_even_ext
2521 %odd_mul = mul <32 x i32> %A_odd_ext, %B_odd_ext
2522 %add = add <32 x i32> %even_mul, %odd_mul
2523 ret <32 x i32> %add
2524 }
2526 define <4 x i32> @pmaddwd_commuted_mul(<8 x i16>* %Aptr, <8 x i16>* %Bptr) {
2527 ; SSE2-LABEL: pmaddwd_commuted_mul:
2528 ; SSE2: # %bb.0:
2529 ; SSE2-NEXT: movdqa (%rdi), %xmm0
2530 ; SSE2-NEXT: pmaddwd (%rsi), %xmm0
2531 ; SSE2-NEXT: retq
2532 ;
2533 ; AVX-LABEL: pmaddwd_commuted_mul:
2534 ; AVX: # %bb.0:
2535 ; AVX-NEXT: vmovdqa (%rdi), %xmm0
2536 ; AVX-NEXT: vpmaddwd (%rsi), %xmm0, %xmm0
2537 ; AVX-NEXT: retq
2538 %A = load <8 x i16>, <8 x i16>* %Aptr
2539 %B = load <8 x i16>, <8 x i16>* %Bptr
2540 %A_even = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
2541 %A_odd = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
2542 %B_even = shufflevector <8 x i16> %B, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
2543 %B_odd = shufflevector <8 x i16> %B, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
2544 %A_even_ext = sext <4 x i16> %A_even to <4 x i32>
2545 %B_even_ext = sext <4 x i16> %B_even to <4 x i32>
2546 %A_odd_ext = sext <4 x i16> %A_odd to <4 x i32>
2547 %B_odd_ext = sext <4 x i16> %B_odd to <4 x i32>
2548 %even_mul = mul <4 x i32> %A_even_ext, %B_even_ext
2549 %odd_mul = mul <4 x i32> %B_odd_ext, %A_odd_ext ; Different order than previous mul
2550 %add = add <4 x i32> %even_mul, %odd_mul
2551 ret <4 x i32> %add
2552 }
2554 define <4 x i32> @pmaddwd_swapped_indices(<8 x i16>* %Aptr, <8 x i16>* %Bptr) {
2555 ; SSE2-LABEL: pmaddwd_swapped_indices:
2556 ; SSE2: # %bb.0:
2557 ; SSE2-NEXT: movdqa (%rdi), %xmm0
2558 ; SSE2-NEXT: pmaddwd (%rsi), %xmm0
2559 ; SSE2-NEXT: retq
2560 ;
2561 ; AVX-LABEL: pmaddwd_swapped_indices:
2562 ; AVX: # %bb.0:
2563 ; AVX-NEXT: vmovdqa (%rdi), %xmm0
2564 ; AVX-NEXT: vpmaddwd (%rsi), %xmm0, %xmm0
2565 ; AVX-NEXT: retq
2566 %A = load <8 x i16>, <8 x i16>* %Aptr
2567 %B = load <8 x i16>, <8 x i16>* %Bptr
2568 %A_even = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 1, i32 2, i32 5, i32 6> ; indices aren't all even
2569 %A_odd = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 0, i32 3, i32 4, i32 7> ; indices aren't all odd
2570 %B_even = shufflevector <8 x i16> %B, <8 x i16> undef, <4 x i32> <i32 1, i32 2, i32 5, i32 6> ; same indices as A
2571 %B_odd = shufflevector <8 x i16> %B, <8 x i16> undef, <4 x i32> <i32 0, i32 3, i32 4, i32 7> ; same indices as A
2572 %A_even_ext = sext <4 x i16> %A_even to <4 x i32>
2573 %B_even_ext = sext <4 x i16> %B_even to <4 x i32>
2574 %A_odd_ext = sext <4 x i16> %A_odd to <4 x i32>
2575 %B_odd_ext = sext <4 x i16> %B_odd to <4 x i32>
2576 %even_mul = mul <4 x i32> %A_even_ext, %B_even_ext
2577 %odd_mul = mul <4 x i32> %A_odd_ext, %B_odd_ext
2578 %add = add <4 x i32> %even_mul, %odd_mul
2579 ret <4 x i32> %add
2580 }
2582 ; Negative test were indices aren't paired properly
2583 define <4 x i32> @pmaddwd_bad_indices(<8 x i16>* %Aptr, <8 x i16>* %Bptr) {
2584 ; SSE2-LABEL: pmaddwd_bad_indices:
2585 ; SSE2: # %bb.0:
2586 ; SSE2-NEXT: movdqa (%rdi), %xmm0
2587 ; SSE2-NEXT: movdqa (%rsi), %xmm1
2588 ; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm0[2,1,2,3,4,5,6,7]
2589 ; SSE2-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,6,7]
2590 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
2591 ; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,0,3,2,4,5,6,7]
2592 ; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7]
2593 ; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7]
2594 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2595 ; SSE2-NEXT: pshuflw {{.*#+}} xmm3 = xmm1[0,2,2,3,4,5,6,7]
2596 ; SSE2-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
2597 ; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
2598 ; SSE2-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]
2599 ; SSE2-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,7]
2600 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
2601 ; SSE2-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,0,3,2,4,5,6,7]
2602 ; SSE2-NEXT: movdqa %xmm2, %xmm4
2603 ; SSE2-NEXT: pmulhw %xmm3, %xmm4
2604 ; SSE2-NEXT: pmullw %xmm3, %xmm2
2605 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
2606 ; SSE2-NEXT: movdqa %xmm0, %xmm3
2607 ; SSE2-NEXT: pmulhw %xmm1, %xmm3
2608 ; SSE2-NEXT: pmullw %xmm1, %xmm0
2609 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
2610 ; SSE2-NEXT: paddd %xmm2, %xmm0
2611 ; SSE2-NEXT: retq
2612 ;
2613 ; AVX-LABEL: pmaddwd_bad_indices:
2614 ; AVX: # %bb.0:
2615 ; AVX-NEXT: vmovdqa (%rdi), %xmm0
2616 ; AVX-NEXT: vmovdqa (%rsi), %xmm1
2617 ; AVX-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[2,3,4,5,10,11,12,13,12,13,10,11,12,13,14,15]
2618 ; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,6,7,8,9,14,15,8,9,14,15,12,13,14,15]
2619 ; AVX-NEXT: vpshufb {{.*#+}} xmm3 = xmm1[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
2620 ; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15]
2621 ; AVX-NEXT: vpmovsxwd %xmm2, %xmm2
2622 ; AVX-NEXT: vpmovsxwd %xmm3, %xmm3
2623 ; AVX-NEXT: vpmulld %xmm3, %xmm2, %xmm2
2624 ; AVX-NEXT: vpmovsxwd %xmm0, %xmm0
2625 ; AVX-NEXT: vpmovsxwd %xmm1, %xmm1
2626 ; AVX-NEXT: vpmulld %xmm1, %xmm0, %xmm0
2627 ; AVX-NEXT: vpaddd %xmm0, %xmm2, %xmm0
2628 ; AVX-NEXT: retq
2629 %A = load <8 x i16>, <8 x i16>* %Aptr
2630 %B = load <8 x i16>, <8 x i16>* %Bptr
2631 %A_even = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
2632 %A_odd = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
2633 %B_even = shufflevector <8 x i16> %B, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6> ; different indices than A
2634 %B_odd = shufflevector <8 x i16> %B, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7> ; different indices than A
2635 %A_even_ext = sext <4 x i16> %A_even to <4 x i32>
2636 %B_even_ext = sext <4 x i16> %B_even to <4 x i32>
2637 %A_odd_ext = sext <4 x i16> %A_odd to <4 x i32>
2638 %B_odd_ext = sext <4 x i16> %B_odd to <4 x i32>
2639 %even_mul = mul <4 x i32> %A_even_ext, %B_even_ext
2640 %odd_mul = mul <4 x i32> %A_odd_ext, %B_odd_ext
2641 %add = add <4 x i32> %even_mul, %odd_mul
2642 ret <4 x i32> %add
2643 }
2645 ; This test contains two multiplies joined by an add. The result of that add is then reduced to a single element.
2646 ; SelectionDAGBuilder should tag the joining add as a vector reduction. We need to recognize that both sides can use pmaddwd
2647 define i32 @madd_double_reduction(<8 x i16>* %arg, <8 x i16>* %arg1, <8 x i16>* %arg2, <8 x i16>* %arg3) {
2648 ; SSE2-LABEL: madd_double_reduction:
2649 ; SSE2: # %bb.0:
2650 ; SSE2-NEXT: movdqu (%rdi), %xmm0
2651 ; SSE2-NEXT: movdqu (%rsi), %xmm1
2652 ; SSE2-NEXT: pmaddwd %xmm0, %xmm1
2653 ; SSE2-NEXT: movdqu (%rdx), %xmm0
2654 ; SSE2-NEXT: movdqu (%rcx), %xmm2
2655 ; SSE2-NEXT: pmaddwd %xmm0, %xmm2
2656 ; SSE2-NEXT: paddd %xmm1, %xmm2
2657 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
2658 ; SSE2-NEXT: paddd %xmm2, %xmm0
2659 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
2660 ; SSE2-NEXT: paddd %xmm0, %xmm1
2661 ; SSE2-NEXT: movd %xmm1, %eax
2662 ; SSE2-NEXT: retq
2663 ;
2664 ; AVX-LABEL: madd_double_reduction:
2665 ; AVX: # %bb.0:
2666 ; AVX-NEXT: vmovdqu (%rdi), %xmm0
2667 ; AVX-NEXT: vmovdqu (%rdx), %xmm1
2668 ; AVX-NEXT: vpmaddwd (%rcx), %xmm1, %xmm1
2669 ; AVX-NEXT: vpmaddwd (%rsi), %xmm0, %xmm0
2670 ; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
2671 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
2672 ; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
2673 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
2674 ; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
2675 ; AVX-NEXT: vmovd %xmm0, %eax
2676 ; AVX-NEXT: retq
2677 %tmp = load <8 x i16>, <8 x i16>* %arg, align 1
2678 %tmp6 = load <8 x i16>, <8 x i16>* %arg1, align 1
2679 %tmp7 = sext <8 x i16> %tmp to <8 x i32>
2680 %tmp17 = sext <8 x i16> %tmp6 to <8 x i32>
2681 %tmp19 = mul nsw <8 x i32> %tmp7, %tmp17
2682 %tmp20 = load <8 x i16>, <8 x i16>* %arg2, align 1
2683 %tmp21 = load <8 x i16>, <8 x i16>* %arg3, align 1
2684 %tmp22 = sext <8 x i16> %tmp20 to <8 x i32>
2685 %tmp23 = sext <8 x i16> %tmp21 to <8 x i32>
2686 %tmp25 = mul nsw <8 x i32> %tmp22, %tmp23
2687 %tmp26 = add nuw nsw <8 x i32> %tmp25, %tmp19
2688 %tmp29 = shufflevector <8 x i32> %tmp26, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
2689 %tmp30 = add <8 x i32> %tmp26, %tmp29
2690 %tmp31 = shufflevector <8 x i32> %tmp30, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
2691 %tmp32 = add <8 x i32> %tmp30, %tmp31
2692 %tmp33 = shufflevector <8 x i32> %tmp32, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
2693 %tmp34 = add <8 x i32> %tmp32, %tmp33
2694 %tmp35 = extractelement <8 x i32> %tmp34, i64 0
2695 ret i32 %tmp35
2696 }
2698 define i32 @madd_quad_reduction(<8 x i16>* %arg, <8 x i16>* %arg1, <8 x i16>* %arg2, <8 x i16>* %arg3, <8 x i16>* %arg4, <8 x i16>* %arg5, <8 x i16>* %arg6, <8 x i16>* %arg7) {
2699 ; SSE2-LABEL: madd_quad_reduction:
2700 ; SSE2: # %bb.0:
2701 ; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r10
2702 ; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax
2703 ; SSE2-NEXT: movdqu (%rdi), %xmm0
2704 ; SSE2-NEXT: movdqu (%rsi), %xmm1
2705 ; SSE2-NEXT: pmaddwd %xmm0, %xmm1
2706 ; SSE2-NEXT: movdqu (%rdx), %xmm0
2707 ; SSE2-NEXT: movdqu (%rcx), %xmm2
2708 ; SSE2-NEXT: pmaddwd %xmm0, %xmm2
2709 ; SSE2-NEXT: movdqu (%r8), %xmm0
2710 ; SSE2-NEXT: movdqu (%r9), %xmm3
2711 ; SSE2-NEXT: pmaddwd %xmm0, %xmm3
2712 ; SSE2-NEXT: paddd %xmm2, %xmm3
2713 ; SSE2-NEXT: movdqu (%rax), %xmm0
2714 ; SSE2-NEXT: movdqu (%r10), %xmm2
2715 ; SSE2-NEXT: pmaddwd %xmm0, %xmm2
2716 ; SSE2-NEXT: paddd %xmm3, %xmm2
2717 ; SSE2-NEXT: paddd %xmm1, %xmm2
2718 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
2719 ; SSE2-NEXT: paddd %xmm2, %xmm0
2720 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
2721 ; SSE2-NEXT: paddd %xmm0, %xmm1
2722 ; SSE2-NEXT: movd %xmm1, %eax
2723 ; SSE2-NEXT: retq
2724 ;
2725 ; AVX-LABEL: madd_quad_reduction:
2726 ; AVX: # %bb.0:
2727 ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %r10
2728 ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax
2729 ; AVX-NEXT: vmovdqu (%rdi), %xmm0
2730 ; AVX-NEXT: vmovdqu (%rdx), %xmm1
2731 ; AVX-NEXT: vpmaddwd (%rcx), %xmm1, %xmm1
2732 ; AVX-NEXT: vpmaddwd (%rsi), %xmm0, %xmm0
2733 ; AVX-NEXT: vmovdqu (%r8), %xmm2
2734 ; AVX-NEXT: vpmaddwd (%r9), %xmm2, %xmm2
2735 ; AVX-NEXT: vpaddd %xmm2, %xmm1, %xmm1
2736 ; AVX-NEXT: vmovdqu (%rax), %xmm2
2737 ; AVX-NEXT: vpmaddwd (%r10), %xmm2, %xmm2
2738 ; AVX-NEXT: vpaddd %xmm2, %xmm1, %xmm1
2739 ; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
2740 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
2741 ; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
2742 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
2743 ; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
2744 ; AVX-NEXT: vmovd %xmm0, %eax
2745 ; AVX-NEXT: retq
2746 %tmp = load <8 x i16>, <8 x i16>* %arg, align 1
2747 %tmp6 = load <8 x i16>, <8 x i16>* %arg1, align 1
2748 %tmp7 = sext <8 x i16> %tmp to <8 x i32>
2749 %tmp17 = sext <8 x i16> %tmp6 to <8 x i32>
2750 %tmp19 = mul nsw <8 x i32> %tmp7, %tmp17
2751 %tmp20 = load <8 x i16>, <8 x i16>* %arg2, align 1
2752 %tmp21 = load <8 x i16>, <8 x i16>* %arg3, align 1
2753 %tmp22 = sext <8 x i16> %tmp20 to <8 x i32>
2754 %tmp23 = sext <8 x i16> %tmp21 to <8 x i32>
2755 %tmp25 = mul nsw <8 x i32> %tmp22, %tmp23
2756 %tmp26 = add nuw nsw <8 x i32> %tmp25, %tmp19
2758 %tmp40 = load <8 x i16>, <8 x i16>* %arg4, align 1
2759 %tmp41 = load <8 x i16>, <8 x i16>* %arg5, align 1
2760 %tmp42 = sext <8 x i16> %tmp40 to <8 x i32>
2761 %tmp43 = sext <8 x i16> %tmp41 to <8 x i32>
2762 %tmp45 = mul nsw <8 x i32> %tmp42, %tmp43
2763 %tmp56 = add nuw nsw <8 x i32> %tmp26, %tmp45
2765 %tmp50 = load <8 x i16>, <8 x i16>* %arg6, align 1
2766 %tmp51 = load <8 x i16>, <8 x i16>* %arg7, align 1
2767 %tmp52 = sext <8 x i16> %tmp50 to <8 x i32>
2768 %tmp53 = sext <8 x i16> %tmp51 to <8 x i32>
2769 %tmp55 = mul nsw <8 x i32> %tmp52, %tmp53
2770 %tmp57 = add nuw nsw <8 x i32> %tmp55, %tmp56
2772 %tmp29 = shufflevector <8 x i32> %tmp57, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
2773 %tmp30 = add <8 x i32> %tmp57, %tmp29
2774 %tmp31 = shufflevector <8 x i32> %tmp30, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
2775 %tmp32 = add <8 x i32> %tmp30, %tmp31
2776 %tmp33 = shufflevector <8 x i32> %tmp32, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
2777 %tmp34 = add <8 x i32> %tmp32, %tmp33
2778 %tmp35 = extractelement <8 x i32> %tmp34, i64 0
2779 ret i32 %tmp35
2780 }
2782 define i64 @sum_and_sum_of_squares(i8* %a, i32 %n) {
2783 ; SSE2-LABEL: sum_and_sum_of_squares:
2784 ; SSE2: # %bb.0: # %entry
2785 ; SSE2-NEXT: movl %esi, %eax
2786 ; SSE2-NEXT: pxor %xmm0, %xmm0
2787 ; SSE2-NEXT: pxor %xmm1, %xmm1
2788 ; SSE2-NEXT: pxor %xmm2, %xmm2
2789 ; SSE2-NEXT: pxor %xmm3, %xmm3
2790 ; SSE2-NEXT: .p2align 4, 0x90
2791 ; SSE2-NEXT: .LBB33_1: # %vector.body
2792 ; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
2793 ; SSE2-NEXT: movq {{.*#+}} xmm4 = mem[0],zero
2794 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
2795 ; SSE2-NEXT: movdqa %xmm4, %xmm5
2796 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
2797 ; SSE2-NEXT: paddd %xmm5, %xmm2
2798 ; SSE2-NEXT: movdqa %xmm4, %xmm5
2799 ; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
2800 ; SSE2-NEXT: paddd %xmm5, %xmm3
2801 ; SSE2-NEXT: pmaddwd %xmm4, %xmm4
2802 ; SSE2-NEXT: paddd %xmm4, %xmm1
2803 ; SSE2-NEXT: addq $8, %rdi
2804 ; SSE2-NEXT: addq $-8, %rax
2805 ; SSE2-NEXT: jne .LBB33_1
2806 ; SSE2-NEXT: # %bb.2: # %middle.block
2807 ; SSE2-NEXT: paddd %xmm0, %xmm1
2808 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
2809 ; SSE2-NEXT: paddd %xmm1, %xmm0
2810 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
2811 ; SSE2-NEXT: paddd %xmm0, %xmm1
2812 ; SSE2-NEXT: movd %xmm1, %eax
2813 ; SSE2-NEXT: retq
2814 ;
2815 ; AVX1-LABEL: sum_and_sum_of_squares:
2816 ; AVX1: # %bb.0: # %entry
2817 ; AVX1-NEXT: movl %esi, %eax
2818 ; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0
2819 ; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
2820 ; AVX1-NEXT: .p2align 4, 0x90
2821 ; AVX1-NEXT: .LBB33_1: # %vector.body
2822 ; AVX1-NEXT: # =>This Inner Loop Header: Depth=1
2823 ; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
2824 ; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
2825 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4
2826 ; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm4
2827 ; AVX1-NEXT: vpaddd %xmm1, %xmm2, %xmm1
2828 ; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm1, %ymm1
2829 ; AVX1-NEXT: vpmaddwd %xmm2, %xmm2, %xmm2
2830 ; AVX1-NEXT: vpmaddwd %xmm3, %xmm3, %xmm3
2831 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
2832 ; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
2833 ; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm0
2834 ; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0
2835 ; AVX1-NEXT: addq $8, %rdi
2836 ; AVX1-NEXT: addq $-8, %rax
2837 ; AVX1-NEXT: jne .LBB33_1
2838 ; AVX1-NEXT: # %bb.2: # %middle.block
2839 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
2840 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
2841 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
2842 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
2843 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
2844 ; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
2845 ; AVX1-NEXT: vmovd %xmm0, %eax
2846 ; AVX1-NEXT: vzeroupper
2847 ; AVX1-NEXT: retq
2848 ;
2849 ; AVX256-LABEL: sum_and_sum_of_squares:
2850 ; AVX256: # %bb.0: # %entry
2851 ; AVX256-NEXT: movl %esi, %eax
2852 ; AVX256-NEXT: vpxor %xmm0, %xmm0, %xmm0
2853 ; AVX256-NEXT: vpxor %xmm1, %xmm1, %xmm1
2854 ; AVX256-NEXT: .p2align 4, 0x90
2855 ; AVX256-NEXT: .LBB33_1: # %vector.body
2856 ; AVX256-NEXT: # =>This Inner Loop Header: Depth=1
2857 ; AVX256-NEXT: vpmovzxbd {{.*#+}} ymm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
2858 ; AVX256-NEXT: vpaddd %ymm1, %ymm2, %ymm1
2859 ; AVX256-NEXT: vpmaddwd %ymm2, %ymm2, %ymm2
2860 ; AVX256-NEXT: vpaddd %ymm0, %ymm2, %ymm0
2861 ; AVX256-NEXT: addq $8, %rdi
2862 ; AVX256-NEXT: addq $-8, %rax
2863 ; AVX256-NEXT: jne .LBB33_1
2864 ; AVX256-NEXT: # %bb.2: # %middle.block
2865 ; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1
2866 ; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0
2867 ; AVX256-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
2868 ; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0
2869 ; AVX256-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
2870 ; AVX256-NEXT: vpaddd %xmm1, %xmm0, %xmm0
2871 ; AVX256-NEXT: vmovd %xmm0, %eax
2872 ; AVX256-NEXT: vzeroupper
2873 ; AVX256-NEXT: retq
2874 entry:
2875 %0 = zext i32 %n to i64
2876 br label %vector.body
2878 vector.body:
2879 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ]
2880 %vec.phi = phi <8 x i32> [ %6, %vector.body ], [ zeroinitializer, %entry ]
2881 %sum.phi = phi <8 x i32> [ %4, %vector.body ], [ zeroinitializer, %entry ]
2882 %1 = getelementptr inbounds i8, i8* %a, i64 %index
2883 %2 = bitcast i8* %1 to <8 x i8>*
2884 %wide.load = load <8 x i8>, <8 x i8>* %2, align 1
2885 %3 = zext <8 x i8> %wide.load to <8 x i32>
2886 %4 = add nsw <8 x i32> %3, %sum.phi
2887 %5 = mul nsw <8 x i32> %3, %3
2888 %6 = add nsw <8 x i32> %5, %vec.phi
2889 %index.next = add i64 %index, 8
2890 %7 = icmp eq i64 %index.next, %0
2891 br i1 %7, label %middle.block, label %vector.body
2893 middle.block:
2894 %rdx.shuf35 = shufflevector <8 x i32> %4, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
2895 %bin.rdx36 = add <8 x i32> %4, %rdx.shuf35
2896 %rdx.shuf37 = shufflevector <8 x i32> %bin.rdx36, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
2897 %bin.rdx38 = add <8 x i32> %bin.rdx36, %rdx.shuf37
2898 %rdx.shuf39 = shufflevector <8 x i32> %bin.rdx38, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
2899 %bin.rdx40 = add <8 x i32> %bin.rdx38, %rdx.shuf39
2900 %8 = extractelement <8 x i32> %bin.rdx40, i32 0
2901 %rdx.shuf = shufflevector <8 x i32> %6, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
2902 %bin.rdx = add <8 x i32> %6, %rdx.shuf
2903 %rdx.shuf31 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
2904 %bin.rdx32 = add <8 x i32> %bin.rdx, %rdx.shuf31
2905 %rdx.shuf33 = shufflevector <8 x i32> %bin.rdx32, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
2906 %bin.rdx34 = add <8 x i32> %bin.rdx32, %rdx.shuf33
2907 %9 = extractelement <8 x i32> %bin.rdx34, i32 0
2908 %tmp = zext i32 %8 to i64
2909 %tmp28 = shl nuw i64 %tmp, 32
2910 %tmp29 = zext i32 %9 to i64
2911 ret i64 %tmp29
2912 }