| blob | a6d31bcf1cbbab4027bd3315eb18eb2acbe6b76d |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=+sse2 | FileCheck %s --check-prefix=SSE --check-prefix=SSE2
3 ; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=+sse4.2 | FileCheck %s --check-prefix=SSE --check-prefix=SSE42
4 ; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=+avx | FileCheck %s --check-prefix=AVX --check-prefix=AVX1
5 ; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2,AVX2-SLOW
6 ; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2,AVX2-FAST,AVX2-FAST-ALL
7 ; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2,AVX2-FAST,AVX2-FAST-PERLANE
8 ; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=+xop | FileCheck %s --check-prefix=XOP
10 define void @v3i64(<2 x i64> %a, <2 x i64> %b, <3 x i64>* %p) nounwind {
11 ; SSE2-LABEL: v3i64:
12 ; SSE2: # %bb.0:
13 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
14 ; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
15 ; SSE2-NEXT: movq %xmm2, 16(%rdi)
16 ; SSE2-NEXT: movdqa %xmm0, (%rdi)
17 ; SSE2-NEXT: retq
18 ;
19 ; SSE42-LABEL: v3i64:
20 ; SSE42: # %bb.0:
21 ; SSE42-NEXT: pextrq $1, %xmm0, 16(%rdi)
22 ; SSE42-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
23 ; SSE42-NEXT: movdqa %xmm0, (%rdi)
24 ; SSE42-NEXT: retq
25 ;
26 ; AVX-LABEL: v3i64:
27 ; AVX: # %bb.0:
28 ; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm0[0],xmm1[0]
29 ; AVX-NEXT: vpextrq $1, %xmm0, 16(%rdi)
30 ; AVX-NEXT: vmovdqa %xmm1, (%rdi)
31 ; AVX-NEXT: retq
32 ;
33 ; XOP-LABEL: v3i64:
34 ; XOP: # %bb.0:
35 ; XOP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm0[0],xmm1[0]
36 ; XOP-NEXT: vpextrq $1, %xmm0, 16(%rdi)
37 ; XOP-NEXT: vmovdqa %xmm1, (%rdi)
38 ; XOP-NEXT: retq
39 %r = shufflevector <2 x i64> %a, <2 x i64> %b, <3 x i32> <i32 0, i32 2, i32 1>
40 store <3 x i64> %r, <3 x i64>* %p
41 ret void
42 }
43 define void @v3f64(<2 x double> %a, <2 x double> %b, <3 x double>* %p) nounwind {
44 ; SSE-LABEL: v3f64:
45 ; SSE: # %bb.0:
46 ; SSE-NEXT: movhps %xmm0, 16(%rdi)
47 ; SSE-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
48 ; SSE-NEXT: movaps %xmm0, (%rdi)
49 ; SSE-NEXT: retq
50 ;
51 ; AVX-LABEL: v3f64:
52 ; AVX: # %bb.0:
53 ; AVX-NEXT: vmovlhps {{.*#+}} xmm1 = xmm0[0],xmm1[0]
54 ; AVX-NEXT: vmovhps %xmm0, 16(%rdi)
55 ; AVX-NEXT: vmovaps %xmm1, (%rdi)
56 ; AVX-NEXT: retq
57 ;
58 ; XOP-LABEL: v3f64:
59 ; XOP: # %bb.0:
60 ; XOP-NEXT: vmovlhps {{.*#+}} xmm1 = xmm0[0],xmm1[0]
61 ; XOP-NEXT: vmovhps %xmm0, 16(%rdi)
62 ; XOP-NEXT: vmovaps %xmm1, (%rdi)
63 ; XOP-NEXT: retq
64 %r = shufflevector <2 x double> %a, <2 x double> %b, <3 x i32> <i32 0, i32 2, i32 1>
65 store <3 x double> %r, <3 x double>* %p
66 ret void
67 }
69 define void @v3i32(<2 x i32> %a, <2 x i32> %b, <3 x i32>* %p) nounwind {
70 ; SSE2-LABEL: v3i32:
71 ; SSE2: # %bb.0:
72 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
73 ; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
74 ; SSE2-NEXT: movd %xmm2, 8(%rdi)
75 ; SSE2-NEXT: movq %xmm0, (%rdi)
76 ; SSE2-NEXT: retq
77 ;
78 ; SSE42-LABEL: v3i32:
79 ; SSE42: # %bb.0:
80 ; SSE42-NEXT: extractps $1, %xmm0, 8(%rdi)
81 ; SSE42-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
82 ; SSE42-NEXT: movlps %xmm0, (%rdi)
83 ; SSE42-NEXT: retq
84 ;
85 ; AVX-LABEL: v3i32:
86 ; AVX: # %bb.0:
87 ; AVX-NEXT: vunpcklps {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
88 ; AVX-NEXT: vextractps $1, %xmm0, 8(%rdi)
89 ; AVX-NEXT: vmovlps %xmm1, (%rdi)
90 ; AVX-NEXT: retq
91 ;
92 ; XOP-LABEL: v3i32:
93 ; XOP: # %bb.0:
94 ; XOP-NEXT: vunpcklps {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
95 ; XOP-NEXT: vextractps $1, %xmm0, 8(%rdi)
96 ; XOP-NEXT: vmovlps %xmm1, (%rdi)
97 ; XOP-NEXT: retq
98 %r = shufflevector <2 x i32> %a, <2 x i32> %b, <3 x i32> <i32 0, i32 2, i32 1>
99 store <3 x i32> %r, <3 x i32>* %p
100 ret void
101 }
103 define void @v5i16(<4 x i16> %a, <4 x i16> %b, <5 x i16>* %p) nounwind {
104 ; SSE2-LABEL: v5i16:
105 ; SSE2: # %bb.0:
106 ; SSE2-NEXT: psrlq $16, %xmm1
107 ; SSE2-NEXT: pextrw $3, %xmm0, %eax
108 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
109 ; SSE2-NEXT: movw %ax, 8(%rdi)
110 ; SSE2-NEXT: movq %xmm0, (%rdi)
111 ; SSE2-NEXT: retq
112 ;
113 ; SSE42-LABEL: v5i16:
114 ; SSE42: # %bb.0:
115 ; SSE42-NEXT: psrlq $16, %xmm1
116 ; SSE42-NEXT: pextrw $3, %xmm0, 8(%rdi)
117 ; SSE42-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
118 ; SSE42-NEXT: movq %xmm0, (%rdi)
119 ; SSE42-NEXT: retq
120 ;
121 ; AVX-LABEL: v5i16:
122 ; AVX: # %bb.0:
123 ; AVX-NEXT: vpsrlq $16, %xmm1, %xmm1
124 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
125 ; AVX-NEXT: vpextrw $3, %xmm0, 8(%rdi)
126 ; AVX-NEXT: vmovq %xmm1, (%rdi)
127 ; AVX-NEXT: retq
128 ;
129 ; XOP-LABEL: v5i16:
130 ; XOP: # %bb.0:
131 ; XOP-NEXT: vpsrlq $16, %xmm1, %xmm1
132 ; XOP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
133 ; XOP-NEXT: vpextrw $3, %xmm0, 8(%rdi)
134 ; XOP-NEXT: vmovq %xmm1, (%rdi)
135 ; XOP-NEXT: retq
136 %r = shufflevector <4 x i16> %a, <4 x i16> %b, <5 x i32> <i32 0, i32 5, i32 1, i32 6, i32 3>
137 store <5 x i16> %r, <5 x i16>* %p
138 ret void
139 }
141 define void @v5i32(<4 x i32> %a, <4 x i32> %b, <5 x i32>* %p) nounwind {
142 ; SSE2-LABEL: v5i32:
143 ; SSE2: # %bb.0:
144 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,2,2,3]
145 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
146 ; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
147 ; SSE2-NEXT: movd %xmm2, 16(%rdi)
148 ; SSE2-NEXT: movdqa %xmm0, (%rdi)
149 ; SSE2-NEXT: retq
150 ;
151 ; SSE42-LABEL: v5i32:
152 ; SSE42: # %bb.0:
153 ; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,2]
154 ; SSE42-NEXT: pmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero
155 ; SSE42-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
156 ; SSE42-NEXT: pextrd $3, %xmm0, 16(%rdi)
157 ; SSE42-NEXT: movdqa %xmm2, (%rdi)
158 ; SSE42-NEXT: retq
159 ;
160 ; AVX1-LABEL: v5i32:
161 ; AVX1: # %bb.0:
162 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,2]
163 ; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero
164 ; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
165 ; AVX1-NEXT: vpextrd $3, %xmm0, 16(%rdi)
166 ; AVX1-NEXT: vmovdqa %xmm1, (%rdi)
167 ; AVX1-NEXT: retq
168 ;
169 ; AVX2-LABEL: v5i32:
170 ; AVX2: # %bb.0:
171 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,2]
172 ; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero
173 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
174 ; AVX2-NEXT: vpextrd $3, %xmm0, 16(%rdi)
175 ; AVX2-NEXT: vmovdqa %xmm1, (%rdi)
176 ; AVX2-NEXT: retq
177 ;
178 ; XOP-LABEL: v5i32:
179 ; XOP: # %bb.0:
180 ; XOP-NEXT: vpperm {{.*#+}} xmm1 = xmm0[0,1,2,3],xmm1[4,5,6,7],xmm0[4,5,6,7],xmm1[8,9,10,11]
181 ; XOP-NEXT: vpextrd $3, %xmm0, 16(%rdi)
182 ; XOP-NEXT: vmovdqa %xmm1, (%rdi)
183 ; XOP-NEXT: retq
184 %r = shufflevector <4 x i32> %a, <4 x i32> %b, <5 x i32> <i32 0, i32 5, i32 1, i32 6, i32 3>
185 store <5 x i32> %r, <5 x i32>* %p
186 ret void
187 }
189 define void @v5f32(<4 x float> %a, <4 x float> %b, <5 x float>* %p) nounwind {
190 ; SSE2-LABEL: v5f32:
191 ; SSE2: # %bb.0:
192 ; SSE2-NEXT: movaps %xmm0, %xmm2
193 ; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[1,2]
194 ; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2,1,3]
195 ; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
196 ; SSE2-NEXT: movss %xmm0, 16(%rdi)
197 ; SSE2-NEXT: movaps %xmm2, (%rdi)
198 ; SSE2-NEXT: retq
199 ;
200 ; SSE42-LABEL: v5f32:
201 ; SSE42: # %bb.0:
202 ; SSE42-NEXT: extractps $3, %xmm0, 16(%rdi)
203 ; SSE42-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[1,2]
204 ; SSE42-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2,1,3]
205 ; SSE42-NEXT: movaps %xmm0, (%rdi)
206 ; SSE42-NEXT: retq
207 ;
208 ; AVX-LABEL: v5f32:
209 ; AVX: # %bb.0:
210 ; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm0[0,1],xmm1[1,2]
211 ; AVX-NEXT: vpermilps {{.*#+}} xmm1 = xmm1[0,2,1,3]
212 ; AVX-NEXT: vextractps $3, %xmm0, 16(%rdi)
213 ; AVX-NEXT: vmovaps %xmm1, (%rdi)
214 ; AVX-NEXT: retq
215 ;
216 ; XOP-LABEL: v5f32:
217 ; XOP: # %bb.0:
218 ; XOP-NEXT: vshufps {{.*#+}} xmm1 = xmm0[0,1],xmm1[1,2]
219 ; XOP-NEXT: vpermilps {{.*#+}} xmm1 = xmm1[0,2,1,3]
220 ; XOP-NEXT: vextractps $3, %xmm0, 16(%rdi)
221 ; XOP-NEXT: vmovaps %xmm1, (%rdi)
222 ; XOP-NEXT: retq
223 %r = shufflevector <4 x float> %a, <4 x float> %b, <5 x i32> <i32 0, i32 5, i32 1, i32 6, i32 3>
224 store <5 x float> %r, <5 x float>* %p
225 ret void
226 }
228 define void @v7i8(<4 x i8> %a, <4 x i8> %b, <7 x i8>* %p) nounwind {
229 ; SSE2-LABEL: v7i8:
230 ; SSE2: # %bb.0:
231 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
232 ; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,1,3,4,5,6,7]
233 ; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [0,255,0,255,0,255,255,255,255,255,255,255,255,255,255,255]
234 ; SSE2-NEXT: movdqa %xmm1, -{{[0-9]+}}(%rsp)
235 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
236 ; SSE2-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,2,3,0,4,5,6,7]
237 ; SSE2-NEXT: pand %xmm2, %xmm1
238 ; SSE2-NEXT: pandn %xmm0, %xmm2
239 ; SSE2-NEXT: por %xmm1, %xmm2
240 ; SSE2-NEXT: movb -{{[0-9]+}}(%rsp), %al
241 ; SSE2-NEXT: movb %al, 6(%rdi)
242 ; SSE2-NEXT: movd %xmm2, (%rdi)
243 ; SSE2-NEXT: pextrw $2, %xmm2, %eax
244 ; SSE2-NEXT: movw %ax, 4(%rdi)
245 ; SSE2-NEXT: retq
246 ;
247 ; SSE42-LABEL: v7i8:
248 ; SSE42: # %bb.0:
249 ; SSE42-NEXT: pextrb $0, %xmm1, 6(%rdi)
250 ; SSE42-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
251 ; SSE42-NEXT: pshufb {{.*#+}} xmm1 = xmm1[1,4,7,4,3,6,0,u,u,u,u,u,u,u,u,u]
252 ; SSE42-NEXT: pextrw $2, %xmm1, 4(%rdi)
253 ; SSE42-NEXT: movd %xmm1, (%rdi)
254 ; SSE42-NEXT: retq
255 ;
256 ; AVX-LABEL: v7i8:
257 ; AVX: # %bb.0:
258 ; AVX-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
259 ; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[1,4,7,4,3,6,0,u,u,u,u,u,u,u,u,u]
260 ; AVX-NEXT: vpextrb $0, %xmm1, 6(%rdi)
261 ; AVX-NEXT: vpextrw $2, %xmm0, 4(%rdi)
262 ; AVX-NEXT: vmovd %xmm0, (%rdi)
263 ; AVX-NEXT: retq
264 ;
265 ; XOP-LABEL: v7i8:
266 ; XOP: # %bb.0:
267 ; XOP-NEXT: vpperm {{.*#+}} xmm0 = xmm0[0],xmm1[2],xmm0[3],xmm1[2],xmm0[1],xmm1[3,0,u,u,u,u,u,u,u,u,u]
268 ; XOP-NEXT: vpextrb $0, %xmm1, 6(%rdi)
269 ; XOP-NEXT: vpextrw $2, %xmm0, 4(%rdi)
270 ; XOP-NEXT: vmovd %xmm0, (%rdi)
271 ; XOP-NEXT: retq
272 %r = shufflevector <4 x i8> %a, <4 x i8> %b, <7 x i32> <i32 0, i32 6, i32 3, i32 6, i32 1, i32 7, i32 4>
273 store <7 x i8> %r, <7 x i8>* %p
274 ret void
275 }
277 define void @v7i16(<4 x i16> %a, <4 x i16> %b, <7 x i16>* %p) nounwind {
278 ; SSE2-LABEL: v7i16:
279 ; SSE2: # %bb.0:
280 ; SSE2-NEXT: movd %xmm1, %eax
281 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
282 ; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm1[0,1,0,3,4,5,6,7]
283 ; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7]
284 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
285 ; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,2,3,2,4,5,6,7]
286 ; SSE2-NEXT: pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,5,6,4,7]
287 ; SSE2-NEXT: movw %ax, 12(%rdi)
288 ; SSE2-NEXT: movq %xmm0, (%rdi)
289 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
290 ; SSE2-NEXT: movd %xmm0, 8(%rdi)
291 ; SSE2-NEXT: retq
292 ;
293 ; SSE42-LABEL: v7i16:
294 ; SSE42: # %bb.0:
295 ; SSE42-NEXT: pextrw $0, %xmm1, 12(%rdi)
296 ; SSE42-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
297 ; SSE42-NEXT: pshufb {{.*#+}} xmm1 = xmm1[2,3,8,9,14,15,8,9,6,7,12,13,0,1,14,15]
298 ; SSE42-NEXT: pextrd $2, %xmm1, 8(%rdi)
299 ; SSE42-NEXT: movq %xmm1, (%rdi)
300 ; SSE42-NEXT: retq
301 ;
302 ; AVX-LABEL: v7i16:
303 ; AVX: # %bb.0:
304 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
305 ; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,3,8,9,14,15,8,9,6,7,12,13,0,1,14,15]
306 ; AVX-NEXT: vpextrw $0, %xmm1, 12(%rdi)
307 ; AVX-NEXT: vpextrd $2, %xmm0, 8(%rdi)
308 ; AVX-NEXT: vmovq %xmm0, (%rdi)
309 ; AVX-NEXT: retq
310 ;
311 ; XOP-LABEL: v7i16:
312 ; XOP: # %bb.0:
313 ; XOP-NEXT: vpperm {{.*#+}} xmm0 = xmm0[0,1],xmm1[4,5],xmm0[6,7],xmm1[4,5],xmm0[2,3],xmm1[6,7,0,1],xmm0[6,7]
314 ; XOP-NEXT: vpextrw $0, %xmm1, 12(%rdi)
315 ; XOP-NEXT: vpextrd $2, %xmm0, 8(%rdi)
316 ; XOP-NEXT: vmovq %xmm0, (%rdi)
317 ; XOP-NEXT: retq
318 %r = shufflevector <4 x i16> %a, <4 x i16> %b, <7 x i32> <i32 0, i32 6, i32 3, i32 6, i32 1, i32 7, i32 4>
319 store <7 x i16> %r, <7 x i16>* %p
320 ret void
321 }
324 define void @v7i32(<4 x i32> %a, <4 x i32> %b, <7 x i32>* %p) nounwind {
325 ; SSE2-LABEL: v7i32:
326 ; SSE2: # %bb.0:
327 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,2,2,2]
328 ; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,1,0,3]
329 ; SSE2-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
330 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
331 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
332 ; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
333 ; SSE2-NEXT: movd %xmm1, 24(%rdi)
334 ; SSE2-NEXT: movq %xmm0, 16(%rdi)
335 ; SSE2-NEXT: movdqa %xmm3, (%rdi)
336 ; SSE2-NEXT: retq
337 ;
338 ; SSE42-LABEL: v7i32:
339 ; SSE42: # %bb.0:
340 ; SSE42-NEXT: movdqa %xmm0, %xmm2
341 ; SSE42-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5],xmm2[6,7]
342 ; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,3,2]
343 ; SSE42-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
344 ; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
345 ; SSE42-NEXT: movd %xmm1, 24(%rdi)
346 ; SSE42-NEXT: movq %xmm0, 16(%rdi)
347 ; SSE42-NEXT: movdqa %xmm2, (%rdi)
348 ; SSE42-NEXT: retq
349 ;
350 ; AVX-LABEL: v7i32:
351 ; AVX: # %bb.0:
352 ; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm0[0,1],xmm1[2],xmm0[3]
353 ; AVX-NEXT: vpermilps {{.*#+}} xmm2 = xmm2[0,2,3,2]
354 ; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
355 ; AVX-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[1,3,2,3]
356 ; AVX-NEXT: vmovss %xmm1, 24(%rdi)
357 ; AVX-NEXT: vmovlps %xmm0, 16(%rdi)
358 ; AVX-NEXT: vmovaps %xmm2, (%rdi)
359 ; AVX-NEXT: retq
360 ;
361 ; XOP-LABEL: v7i32:
362 ; XOP: # %bb.0:
363 ; XOP-NEXT: vblendps {{.*#+}} xmm2 = xmm0[0,1],xmm1[2],xmm0[3]
364 ; XOP-NEXT: vpermilps {{.*#+}} xmm2 = xmm2[0,2,3,2]
365 ; XOP-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
366 ; XOP-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[1,3,2,3]
367 ; XOP-NEXT: vmovss %xmm1, 24(%rdi)
368 ; XOP-NEXT: vmovlps %xmm0, 16(%rdi)
369 ; XOP-NEXT: vmovaps %xmm2, (%rdi)
370 ; XOP-NEXT: retq
371 %r = shufflevector <4 x i32> %a, <4 x i32> %b, <7 x i32> <i32 0, i32 6, i32 3, i32 6, i32 1, i32 7, i32 4>
372 store <7 x i32> %r, <7 x i32>* %p
373 ret void
374 }
376 define void @v12i8(<8 x i8> %a, <8 x i8> %b, <12 x i8>* %p) nounwind {
377 ; SSE2-LABEL: v12i8:
378 ; SSE2: # %bb.0:
379 ; SSE2-NEXT: pxor %xmm2, %xmm2
380 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
381 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
382 ; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,3,1,3,4,5,6,7]
383 ; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]
384 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]
385 ; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,1,4,5,6,7]
386 ; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,4]
387 ; SSE2-NEXT: packuswb %xmm2, %xmm0
388 ; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [255,255,0,255,255,0,255,255,0,255,255,0,255,255,255,255]
389 ; SSE2-NEXT: pand %xmm2, %xmm0
390 ; SSE2-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,0,1,1,4,5,6,7]
391 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,1,3]
392 ; SSE2-NEXT: pandn %xmm1, %xmm2
393 ; SSE2-NEXT: por %xmm0, %xmm2
394 ; SSE2-NEXT: movq %xmm2, (%rdi)
395 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
396 ; SSE2-NEXT: movd %xmm0, 8(%rdi)
397 ; SSE2-NEXT: retq
398 ;
399 ; SSE42-LABEL: v12i8:
400 ; SSE42: # %bb.0:
401 ; SSE42-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
402 ; SSE42-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0,8,1,2,10,3,4,12,5,6,14,7,u,u,u,u]
403 ; SSE42-NEXT: pextrd $2, %xmm0, 8(%rdi)
404 ; SSE42-NEXT: movq %xmm0, (%rdi)
405 ; SSE42-NEXT: retq
406 ;
407 ; AVX-LABEL: v12i8:
408 ; AVX: # %bb.0:
409 ; AVX-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
410 ; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,8,1,2,10,3,4,12,5,6,14,7,u,u,u,u]
411 ; AVX-NEXT: vpextrd $2, %xmm0, 8(%rdi)
412 ; AVX-NEXT: vmovq %xmm0, (%rdi)
413 ; AVX-NEXT: retq
414 ;
415 ; XOP-LABEL: v12i8:
416 ; XOP: # %bb.0:
417 ; XOP-NEXT: vpperm {{.*#+}} xmm0 = xmm0[0,4],xmm1[0],xmm0[1,5],xmm1[1],xmm0[2,6],xmm1[2],xmm0[3,7],xmm1[3],xmm0[u,u,u,u]
418 ; XOP-NEXT: vpextrd $2, %xmm0, 8(%rdi)
419 ; XOP-NEXT: vmovq %xmm0, (%rdi)
420 ; XOP-NEXT: retq
421 %r = shufflevector <8 x i8> %a, <8 x i8> %b, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
422 store <12 x i8> %r, <12 x i8>* %p
423 ret void
424 }
426 define void @v12i16(<8 x i16> %a, <8 x i16> %b, <12 x i16>* %p) nounwind {
427 ; SSE2-LABEL: v12i16:
428 ; SSE2: # %bb.0:
429 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,0,0,0]
430 ; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [65535,65535,0,65535,65535,0,65535,65535]
431 ; SSE2-NEXT: pshufhw {{.*#+}} xmm4 = xmm0[0,1,2,3,6,5,4,7]
432 ; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,3,2,1]
433 ; SSE2-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,2,2,1,4,5,6,7]
434 ; SSE2-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,5,6,4]
435 ; SSE2-NEXT: pand %xmm3, %xmm4
436 ; SSE2-NEXT: pandn %xmm2, %xmm3
437 ; SSE2-NEXT: por %xmm4, %xmm3
438 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
439 ; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [0,65535,65535,0,65535,65535,65535,65535]
440 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
441 ; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,1,3,4,5,6,7]
442 ; SSE2-NEXT: pand %xmm2, %xmm0
443 ; SSE2-NEXT: pandn %xmm1, %xmm2
444 ; SSE2-NEXT: por %xmm0, %xmm2
445 ; SSE2-NEXT: movq %xmm2, 16(%rdi)
446 ; SSE2-NEXT: movdqa %xmm3, (%rdi)
447 ; SSE2-NEXT: retq
448 ;
449 ; SSE42-LABEL: v12i16:
450 ; SSE42: # %bb.0:
451 ; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
452 ; SSE42-NEXT: pshufd {{.*#+}} xmm3 = xmm0[3,1,2,3]
453 ; SSE42-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,3,1,3,4,5,6,7]
454 ; SSE42-NEXT: pblendw {{.*#+}} xmm3 = xmm2[0],xmm3[1,2],xmm2[3],xmm3[4,5,6,7]
455 ; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
456 ; SSE42-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,u,u,2,3,10,11,u,u,4,5,12,13]
457 ; SSE42-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
458 ; SSE42-NEXT: movdqa %xmm0, (%rdi)
459 ; SSE42-NEXT: movq %xmm3, 16(%rdi)
460 ; SSE42-NEXT: retq
461 ;
462 ; AVX1-LABEL: v12i16:
463 ; AVX1: # %bb.0:
464 ; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
465 ; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[3,1,2,3]
466 ; AVX1-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,3,1,3,4,5,6,7]
467 ; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2],xmm2[3],xmm3[4,5,6,7]
468 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
469 ; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,u,u,2,3,10,11,u,u,4,5,12,13]
470 ; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
471 ; AVX1-NEXT: vmovdqa %xmm0, (%rdi)
472 ; AVX1-NEXT: vmovq %xmm2, 16(%rdi)
473 ; AVX1-NEXT: retq
474 ;
475 ; AVX2-SLOW-LABEL: v12i16:
476 ; AVX2-SLOW: # %bb.0:
477 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
478 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[3,1,2,3]
479 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,3,1,3,4,5,6,7]
480 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2],xmm2[3],xmm3[4,5,6,7]
481 ; AVX2-SLOW-NEXT: vpbroadcastd %xmm1, %xmm1
482 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,u,u,2,3,10,11,u,u,4,5,12,13]
483 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
484 ; AVX2-SLOW-NEXT: vmovdqa %xmm0, (%rdi)
485 ; AVX2-SLOW-NEXT: vmovq %xmm2, 16(%rdi)
486 ; AVX2-SLOW-NEXT: retq
487 ;
488 ; AVX2-FAST-LABEL: v12i16:
489 ; AVX2-FAST: # %bb.0:
490 ; AVX2-FAST-NEXT: vpbroadcastd %xmm1, %xmm2
491 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[0,1,8,9,u,u,2,3,10,11,u,u,4,5,12,13]
492 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2],xmm3[3,4],xmm2[5],xmm3[6,7]
493 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
494 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,6,7,14,15,u,u,u,u,u,u,u,u,u,u]
495 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5,6,7]
496 ; AVX2-FAST-NEXT: vmovq %xmm0, 16(%rdi)
497 ; AVX2-FAST-NEXT: vmovdqa %xmm2, (%rdi)
498 ; AVX2-FAST-NEXT: retq
499 ;
500 ; XOP-LABEL: v12i16:
501 ; XOP: # %bb.0:
502 ; XOP-NEXT: vpperm {{.*#+}} xmm2 = xmm0[0,1,8,9],xmm1[0,1],xmm0[2,3,10,11],xmm1[2,3],xmm0[4,5,12,13]
503 ; XOP-NEXT: vpperm {{.*#+}} xmm0 = xmm1[4,5],xmm0[6,7,14,15],xmm1[6,7],xmm0[u,u,u,u,u,u,u,u]
504 ; XOP-NEXT: vmovq %xmm0, 16(%rdi)
505 ; XOP-NEXT: vmovdqa %xmm2, (%rdi)
506 ; XOP-NEXT: retq
507 %r = shufflevector <8 x i16> %a, <8 x i16> %b, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
508 store <12 x i16> %r, <12 x i16>* %p
509 ret void
510 }
512 define void @v12i32(<8 x i32> %a, <8 x i32> %b, <12 x i32>* %p) nounwind {
513 ; SSE2-LABEL: v12i32:
514 ; SSE2: # %bb.0:
515 ; SSE2-NEXT: movaps %xmm2, %xmm3
516 ; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm0[1,3]
517 ; SSE2-NEXT: movaps %xmm0, %xmm4
518 ; SSE2-NEXT: unpcklps {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
519 ; SSE2-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm3[0,2]
520 ; SSE2-NEXT: movaps %xmm0, %xmm3
521 ; SSE2-NEXT: unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm1[1]
522 ; SSE2-NEXT: movaps %xmm2, %xmm5
523 ; SSE2-NEXT: shufps {{.*#+}} xmm5 = xmm5[1,0],xmm1[1,0]
524 ; SSE2-NEXT: shufps {{.*#+}} xmm5 = xmm5[2,0],xmm3[0,2]
525 ; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,3],xmm2[3,3]
526 ; SSE2-NEXT: unpckhps {{.*#+}} xmm2 = xmm2[2],xmm0[2],xmm2[3],xmm0[3]
527 ; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,3],xmm1[0,2]
528 ; SSE2-NEXT: movaps %xmm2, 32(%rdi)
529 ; SSE2-NEXT: movaps %xmm5, 16(%rdi)
530 ; SSE2-NEXT: movaps %xmm4, (%rdi)
531 ; SSE2-NEXT: retq
532 ;
533 ; SSE42-LABEL: v12i32:
534 ; SSE42: # %bb.0:
535 ; SSE42-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,0,1,1]
536 ; SSE42-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,1,0,1]
537 ; SSE42-NEXT: pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,3],xmm4[4,5,6,7]
538 ; SSE42-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,1,0,1]
539 ; SSE42-NEXT: pblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5],xmm4[6,7]
540 ; SSE42-NEXT: pshufd {{.*#+}} xmm4 = xmm1[1,1,2,2]
541 ; SSE42-NEXT: pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm2[2,3],xmm4[4,5,6,7]
542 ; SSE42-NEXT: pblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm0[4,5],xmm4[6,7]
543 ; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
544 ; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
545 ; SSE42-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm0[2,3],xmm2[4,5,6,7]
546 ; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,2,3,3]
547 ; SSE42-NEXT: pblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5],xmm2[6,7]
548 ; SSE42-NEXT: movdqa %xmm0, 32(%rdi)
549 ; SSE42-NEXT: movdqa %xmm4, 16(%rdi)
550 ; SSE42-NEXT: movdqa %xmm3, (%rdi)
551 ; SSE42-NEXT: retq
552 ;
553 ; AVX1-LABEL: v12i32:
554 ; AVX1: # %bb.0:
555 ; AVX1-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,0,1]
556 ; AVX1-NEXT: vmovsldup {{.*#+}} ymm2 = ymm2[0,0,2,2,4,4,6,6]
557 ; AVX1-NEXT: vpermilps {{.*#+}} ymm3 = ymm0[0,u,u,1,5,u,u,6]
558 ; AVX1-NEXT: vblendps {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3,4,5],ymm2[6],ymm3[7]
559 ; AVX1-NEXT: vpermilps {{.*#+}} xmm3 = xmm1[0,1,0,1]
560 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm3
561 ; AVX1-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]
562 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
563 ; AVX1-NEXT: vshufps {{.*#+}} xmm0 = xmm0[3,3],xmm3[3,3]
564 ; AVX1-NEXT: vpermilps {{.*#+}} xmm1 = xmm1[2,3,2,3]
565 ; AVX1-NEXT: vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3]
566 ; AVX1-NEXT: vmovaps %xmm0, 32(%rdi)
567 ; AVX1-NEXT: vmovaps %ymm2, (%rdi)
568 ; AVX1-NEXT: vzeroupper
569 ; AVX1-NEXT: retq
570 ;
571 ; AVX2-SLOW-LABEL: v12i32:
572 ; AVX2-SLOW: # %bb.0:
573 ; AVX2-SLOW-NEXT: vmovaps {{.*#+}} ymm2 = <0,4,u,1,5,u,2,6>
574 ; AVX2-SLOW-NEXT: vpermps %ymm0, %ymm2, %ymm2
575 ; AVX2-SLOW-NEXT: vbroadcastsd %xmm1, %ymm3
576 ; AVX2-SLOW-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]
577 ; AVX2-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm3
578 ; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[3,3],xmm3[3,3]
579 ; AVX2-SLOW-NEXT: vpermilps {{.*#+}} xmm1 = xmm1[2,3,2,3]
580 ; AVX2-SLOW-NEXT: vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3]
581 ; AVX2-SLOW-NEXT: vmovaps %xmm0, 32(%rdi)
582 ; AVX2-SLOW-NEXT: vmovaps %ymm2, (%rdi)
583 ; AVX2-SLOW-NEXT: vzeroupper
584 ; AVX2-SLOW-NEXT: retq
585 ;
586 ; AVX2-FAST-ALL-LABEL: v12i32:
587 ; AVX2-FAST-ALL: # %bb.0:
588 ; AVX2-FAST-ALL-NEXT: vmovaps {{.*#+}} ymm2 = <0,4,u,1,5,u,2,6>
589 ; AVX2-FAST-ALL-NEXT: vpermps %ymm0, %ymm2, %ymm2
590 ; AVX2-FAST-ALL-NEXT: vbroadcastsd %xmm1, %ymm3
591 ; AVX2-FAST-ALL-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]
592 ; AVX2-FAST-ALL-NEXT: vmovaps {{.*#+}} ymm3 = <u,3,7,u,u,u,u,u>
593 ; AVX2-FAST-ALL-NEXT: vpermps %ymm0, %ymm3, %ymm0
594 ; AVX2-FAST-ALL-NEXT: vpermilps {{.*#+}} xmm1 = xmm1[2,3,2,3]
595 ; AVX2-FAST-ALL-NEXT: vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3]
596 ; AVX2-FAST-ALL-NEXT: vmovaps %xmm0, 32(%rdi)
597 ; AVX2-FAST-ALL-NEXT: vmovaps %ymm2, (%rdi)
598 ; AVX2-FAST-ALL-NEXT: vzeroupper
599 ; AVX2-FAST-ALL-NEXT: retq
600 ;
601 ; AVX2-FAST-PERLANE-LABEL: v12i32:
602 ; AVX2-FAST-PERLANE: # %bb.0:
603 ; AVX2-FAST-PERLANE-NEXT: vmovaps {{.*#+}} ymm2 = <0,4,u,1,5,u,2,6>
604 ; AVX2-FAST-PERLANE-NEXT: vpermps %ymm0, %ymm2, %ymm2
605 ; AVX2-FAST-PERLANE-NEXT: vbroadcastsd %xmm1, %ymm3
606 ; AVX2-FAST-PERLANE-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]
607 ; AVX2-FAST-PERLANE-NEXT: vextractf128 $1, %ymm0, %xmm3
608 ; AVX2-FAST-PERLANE-NEXT: vshufps {{.*#+}} xmm0 = xmm0[3,3],xmm3[3,3]
609 ; AVX2-FAST-PERLANE-NEXT: vpermilps {{.*#+}} xmm1 = xmm1[2,3,2,3]
610 ; AVX2-FAST-PERLANE-NEXT: vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3]
611 ; AVX2-FAST-PERLANE-NEXT: vmovaps %xmm0, 32(%rdi)
612 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm2, (%rdi)
613 ; AVX2-FAST-PERLANE-NEXT: vzeroupper
614 ; AVX2-FAST-PERLANE-NEXT: retq
615 ;
616 ; XOP-LABEL: v12i32:
617 ; XOP: # %bb.0:
618 ; XOP-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,0,1]
619 ; XOP-NEXT: vpermil2ps {{.*#+}} ymm2 = ymm0[0],ymm2[0],ymm0[u,1,5,u],ymm2[6],ymm0[6]
620 ; XOP-NEXT: vpermilps {{.*#+}} xmm3 = xmm1[0,1,0,1]
621 ; XOP-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm3
622 ; XOP-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]
623 ; XOP-NEXT: vextractf128 $1, %ymm0, %xmm3
624 ; XOP-NEXT: vshufps {{.*#+}} xmm0 = xmm0[3,3],xmm3[3,3]
625 ; XOP-NEXT: vpermilps {{.*#+}} xmm1 = xmm1[2,3,2,3]
626 ; XOP-NEXT: vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3]
627 ; XOP-NEXT: vmovaps %xmm0, 32(%rdi)
628 ; XOP-NEXT: vmovaps %ymm2, (%rdi)
629 ; XOP-NEXT: vzeroupper
630 ; XOP-NEXT: retq
631 %r = shufflevector <8 x i32> %a, <8 x i32> %b, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
632 store <12 x i32> %r, <12 x i32>* %p
633 ret void
634 }
636 define void @pr29025(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <12 x i8> *%p) nounwind {
637 ; SSE2-LABEL: pr29025:
638 ; SSE2: # %bb.0:
639 ; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
640 ; SSE2-NEXT: pxor %xmm1, %xmm1
641 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
642 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
643 ; SSE2-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,3,1,3,4,5,6,7]
644 ; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]
645 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]
646 ; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,1,4,5,6,7]
647 ; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,4]
648 ; SSE2-NEXT: packuswb %xmm1, %xmm0
649 ; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [255,255,0,255,255,0,255,255,0,255,255,0,255,255,255,255]
650 ; SSE2-NEXT: pand %xmm1, %xmm0
651 ; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,1,1,4,5,6,7]
652 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,1,3]
653 ; SSE2-NEXT: pandn %xmm2, %xmm1
654 ; SSE2-NEXT: por %xmm0, %xmm1
655 ; SSE2-NEXT: movq %xmm1, (%rdi)
656 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
657 ; SSE2-NEXT: movd %xmm0, 8(%rdi)
658 ; SSE2-NEXT: retq
659 ;
660 ; SSE42-LABEL: pr29025:
661 ; SSE42: # %bb.0:
662 ; SSE42-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
663 ; SSE42-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
664 ; SSE42-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0,8,1,2,10,3,4,12,5,6,14,7,u,u,u,u]
665 ; SSE42-NEXT: pextrd $2, %xmm0, 8(%rdi)
666 ; SSE42-NEXT: movq %xmm0, (%rdi)
667 ; SSE42-NEXT: retq
668 ;
669 ; AVX-LABEL: pr29025:
670 ; AVX: # %bb.0:
671 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
672 ; AVX-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
673 ; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,8,1,2,10,3,4,12,5,6,14,7,u,u,u,u]
674 ; AVX-NEXT: vpextrd $2, %xmm0, 8(%rdi)
675 ; AVX-NEXT: vmovq %xmm0, (%rdi)
676 ; AVX-NEXT: retq
677 ;
678 ; XOP-LABEL: pr29025:
679 ; XOP: # %bb.0:
680 ; XOP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
681 ; XOP-NEXT: vpperm {{.*#+}} xmm0 = xmm0[0,4],xmm2[0],xmm0[1,5],xmm2[1],xmm0[2,6],xmm2[2],xmm0[3,7],xmm2[3],xmm0[u,u,u,u]
682 ; XOP-NEXT: vpextrd $2, %xmm0, 8(%rdi)
683 ; XOP-NEXT: vmovq %xmm0, (%rdi)
684 ; XOP-NEXT: retq
685 %s1 = shufflevector <4 x i8> %a, <4 x i8> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
686 %s2 = shufflevector <4 x i8> %c, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
687 %r = shufflevector <8 x i8> %s1, <8 x i8> %s2, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
688 store <12 x i8> %r, <12 x i8>* %p, align 1
689 ret void
690 }
692 define void @interleave_24i8_out(<24 x i8>* %p, <8 x i8>* %q1, <8 x i8>* %q2, <8 x i8>* %q3) nounwind {
693 ; SSE2-LABEL: interleave_24i8_out:
694 ; SSE2: # %bb.0:
695 ; SSE2-NEXT: movdqu (%rdi), %xmm0
696 ; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero
697 ; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [255,255,0,255,255,0,255,255,255,255,255,255,255,255,255,255]
698 ; SSE2-NEXT: movdqa %xmm0, %xmm2
699 ; SSE2-NEXT: pand %xmm4, %xmm2
700 ; SSE2-NEXT: pandn %xmm1, %xmm4
701 ; SSE2-NEXT: por %xmm2, %xmm4
702 ; SSE2-NEXT: pxor %xmm2, %xmm2
703 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
704 ; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [65535,0,65535,65535,0,65535,65535,0]
705 ; SSE2-NEXT: pand %xmm5, %xmm4
706 ; SSE2-NEXT: movdqa %xmm0, %xmm3
707 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
708 ; SSE2-NEXT: pandn %xmm3, %xmm5
709 ; SSE2-NEXT: por %xmm4, %xmm5
710 ; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm5[0,2,1,3]
711 ; SSE2-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,7,6,5]
712 ; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,3,2,1]
713 ; SSE2-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,3,2,1,4,5,6,7]
714 ; SSE2-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,6,5,4,7]
715 ; SSE2-NEXT: packuswb %xmm4, %xmm4
716 ; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [0,255,255,0,255,255,0,255,255,255,255,255,255,255,255,255]
717 ; SSE2-NEXT: movdqa %xmm0, %xmm6
718 ; SSE2-NEXT: pand %xmm5, %xmm6
719 ; SSE2-NEXT: pandn %xmm1, %xmm5
720 ; SSE2-NEXT: por %xmm6, %xmm5
721 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
722 ; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [65535,65535,0,65535,65535,0,65535,65535]
723 ; SSE2-NEXT: pand %xmm6, %xmm5
724 ; SSE2-NEXT: pandn %xmm3, %xmm6
725 ; SSE2-NEXT: por %xmm5, %xmm6
726 ; SSE2-NEXT: pshuflw {{.*#+}} xmm5 = xmm6[2,1,0,3,4,5,6,7]
727 ; SSE2-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,5,4,7]
728 ; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,3,2,1]
729 ; SSE2-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[1,2,3,0,4,5,6,7]
730 ; SSE2-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,6,7,4]
731 ; SSE2-NEXT: packuswb %xmm5, %xmm5
732 ; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [255,0,255,255,0,255,255,0,255,255,255,255,255,255,255,255]
733 ; SSE2-NEXT: pand %xmm6, %xmm0
734 ; SSE2-NEXT: pandn %xmm1, %xmm6
735 ; SSE2-NEXT: por %xmm0, %xmm6
736 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
737 ; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [0,65535,65535,0,65535,65535,0,65535]
738 ; SSE2-NEXT: pand %xmm0, %xmm6
739 ; SSE2-NEXT: pandn %xmm3, %xmm0
740 ; SSE2-NEXT: por %xmm6, %xmm0
741 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0]
742 ; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,5]
743 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0]
744 ; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7]
745 ; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]
746 ; SSE2-NEXT: packuswb %xmm0, %xmm0
747 ; SSE2-NEXT: movq %xmm4, (%rsi)
748 ; SSE2-NEXT: movq %xmm5, (%rdx)
749 ; SSE2-NEXT: movq %xmm0, (%rcx)
750 ; SSE2-NEXT: retq
751 ;
752 ; SSE42-LABEL: interleave_24i8_out:
753 ; SSE42: # %bb.0:
754 ; SSE42-NEXT: movdqu (%rdi), %xmm0
755 ; SSE42-NEXT: movq {{.*#+}} xmm1 = mem[0],zero
756 ; SSE42-NEXT: movdqa %xmm1, %xmm2
757 ; SSE42-NEXT: pshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,xmm2[2,5,u,u,u,u,u,u,u,u]
758 ; SSE42-NEXT: movdqa %xmm0, %xmm3
759 ; SSE42-NEXT: pshufb {{.*#+}} xmm3 = xmm3[0,3,6,9,12,15],zero,zero,xmm3[u,u,u,u,u,u,u,u]
760 ; SSE42-NEXT: por %xmm2, %xmm3
761 ; SSE42-NEXT: movdqa %xmm1, %xmm2
762 ; SSE42-NEXT: pshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,xmm2[0,3,6,u,u,u,u,u,u,u,u]
763 ; SSE42-NEXT: movdqa %xmm0, %xmm4
764 ; SSE42-NEXT: pshufb {{.*#+}} xmm4 = xmm4[1,4,7,10,13],zero,zero,zero,xmm4[u,u,u,u,u,u,u,u]
765 ; SSE42-NEXT: por %xmm2, %xmm4
766 ; SSE42-NEXT: pshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm1[1,4,7,u,u,u,u,u,u,u,u]
767 ; SSE42-NEXT: pshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,14],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
768 ; SSE42-NEXT: por %xmm1, %xmm0
769 ; SSE42-NEXT: movq %xmm3, (%rsi)
770 ; SSE42-NEXT: movq %xmm4, (%rdx)
771 ; SSE42-NEXT: movq %xmm0, (%rcx)
772 ; SSE42-NEXT: retq
773 ;
774 ; AVX-LABEL: interleave_24i8_out:
775 ; AVX: # %bb.0:
776 ; AVX-NEXT: vmovdqu (%rdi), %xmm0
777 ; AVX-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
778 ; AVX-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,xmm1[2,5,u,u,u,u,u,u,u,u]
779 ; AVX-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[0,3,6,9,12,15],zero,zero,xmm0[u,u,u,u,u,u,u,u]
780 ; AVX-NEXT: vpor %xmm2, %xmm3, %xmm2
781 ; AVX-NEXT: vpshufb {{.*#+}} xmm3 = zero,zero,zero,zero,zero,xmm1[0,3,6,u,u,u,u,u,u,u,u]
782 ; AVX-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[1,4,7,10,13],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
783 ; AVX-NEXT: vpor %xmm3, %xmm4, %xmm3
784 ; AVX-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm1[1,4,7,u,u,u,u,u,u,u,u]
785 ; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,14],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
786 ; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0
787 ; AVX-NEXT: vmovq %xmm2, (%rsi)
788 ; AVX-NEXT: vmovq %xmm3, (%rdx)
789 ; AVX-NEXT: vmovq %xmm0, (%rcx)
790 ; AVX-NEXT: retq
791 ;
792 ; XOP-LABEL: interleave_24i8_out:
793 ; XOP: # %bb.0:
794 ; XOP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
795 ; XOP-NEXT: vmovdqu (%rdi), %xmm1
796 ; XOP-NEXT: vpperm {{.*#+}} xmm2 = xmm1[0,3,6,9,12,15],xmm0[2,5],xmm1[u,u,u,u,u,u,u,u]
797 ; XOP-NEXT: vpperm {{.*#+}} xmm3 = xmm1[1,4,7,10,13],xmm0[0,3,6],xmm1[u,u,u,u,u,u,u,u]
798 ; XOP-NEXT: vpperm {{.*#+}} xmm0 = xmm1[2,5,8,11,14],xmm0[1,4,7],xmm1[u,u,u,u,u,u,u,u]
799 ; XOP-NEXT: vmovq %xmm2, (%rsi)
800 ; XOP-NEXT: vmovq %xmm3, (%rdx)
801 ; XOP-NEXT: vmovq %xmm0, (%rcx)
802 ; XOP-NEXT: retq
803 %wide.vec = load <24 x i8>, <24 x i8>* %p, align 4
804 %s1 = shufflevector <24 x i8> %wide.vec, <24 x i8> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
805 %s2 = shufflevector <24 x i8> %wide.vec, <24 x i8> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
806 %s3 = shufflevector <24 x i8> %wide.vec, <24 x i8> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
807 store <8 x i8> %s1, <8 x i8>* %q1, align 4
808 store <8 x i8> %s2, <8 x i8>* %q2, align 4
809 store <8 x i8> %s3, <8 x i8>* %q3, align 4
810 ret void
811 }
813 define void @interleave_24i8_in(<24 x i8>* %p, <8 x i8>* %q1, <8 x i8>* %q2, <8 x i8>* %q3) nounwind {
814 ; SSE2-LABEL: interleave_24i8_in:
815 ; SSE2: # %bb.0:
816 ; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero
817 ; SSE2-NEXT: movq {{.*#+}} xmm2 = mem[0],zero
818 ; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
819 ; SSE2-NEXT: pxor %xmm3, %xmm3
820 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
821 ; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[1,1,2,2]
822 ; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [65535,65535,0,65535,65535,0,65535,65535]
823 ; SSE2-NEXT: pand %xmm5, %xmm4
824 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
825 ; SSE2-NEXT: pshuflw {{.*#+}} xmm3 = xmm2[3,3,3,3,4,5,6,7]
826 ; SSE2-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,4,4]
827 ; SSE2-NEXT: pandn %xmm3, %xmm5
828 ; SSE2-NEXT: por %xmm4, %xmm5
829 ; SSE2-NEXT: movdqa %xmm1, %xmm3
830 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
831 ; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,2,1]
832 ; SSE2-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,2,2,4,5,6,7]
833 ; SSE2-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,4,5]
834 ; SSE2-NEXT: packuswb %xmm5, %xmm3
835 ; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [255,255,0,255,255,0,255,255,0,255,255,0,255,255,0,255]
836 ; SSE2-NEXT: pand %xmm4, %xmm3
837 ; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,1,0,1]
838 ; SSE2-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[0,0,0,0,4,5,6,7]
839 ; SSE2-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,6,6]
840 ; SSE2-NEXT: pandn %xmm5, %xmm4
841 ; SSE2-NEXT: por %xmm3, %xmm4
842 ; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
843 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,1,2,3]
844 ; SSE2-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,1,0,4,5,6,7]
845 ; SSE2-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,7]
846 ; SSE2-NEXT: packuswb %xmm1, %xmm1
847 ; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [255,0,255,255,0,255,255,0,255,255,255,255,255,255,255,255]
848 ; SSE2-NEXT: pand %xmm2, %xmm1
849 ; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,3,3,4,5,6,7]
850 ; SSE2-NEXT: pandn %xmm0, %xmm2
851 ; SSE2-NEXT: por %xmm1, %xmm2
852 ; SSE2-NEXT: movq %xmm2, 16(%rdi)
853 ; SSE2-NEXT: movdqu %xmm4, (%rdi)
854 ; SSE2-NEXT: retq
855 ;
856 ; SSE42-LABEL: interleave_24i8_in:
857 ; SSE42: # %bb.0:
858 ; SSE42-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
859 ; SSE42-NEXT: movq {{.*#+}} xmm1 = mem[0],zero
860 ; SSE42-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
861 ; SSE42-NEXT: movq {{.*#+}} xmm1 = mem[0],zero
862 ; SSE42-NEXT: movdqa %xmm0, %xmm2
863 ; SSE42-NEXT: pshufb {{.*#+}} xmm2 = xmm2[0,8],zero,xmm2[1,9],zero,xmm2[2,10],zero,xmm2[3,11],zero,xmm2[4,12],zero,xmm2[5]
864 ; SSE42-NEXT: movdqa %xmm1, %xmm3
865 ; SSE42-NEXT: pshufb {{.*#+}} xmm3 = zero,zero,xmm3[0],zero,zero,xmm3[1],zero,zero,xmm3[2],zero,zero,xmm3[3],zero,zero,xmm3[4],zero
866 ; SSE42-NEXT: por %xmm2, %xmm3
867 ; SSE42-NEXT: pshufb {{.*#+}} xmm0 = xmm0[13],zero,xmm0[6,14],zero,xmm0[7,15],zero,xmm0[u,u,u,u,u,u,u,u]
868 ; SSE42-NEXT: pshufb {{.*#+}} xmm1 = zero,xmm1[5],zero,zero,xmm1[6],zero,zero,xmm1[7,u,u,u,u,u,u,u,u]
869 ; SSE42-NEXT: por %xmm0, %xmm1
870 ; SSE42-NEXT: movq %xmm1, 16(%rdi)
871 ; SSE42-NEXT: movdqu %xmm3, (%rdi)
872 ; SSE42-NEXT: retq
873 ;
874 ; AVX-LABEL: interleave_24i8_in:
875 ; AVX: # %bb.0:
876 ; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
877 ; AVX-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
878 ; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
879 ; AVX-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
880 ; AVX-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[0,8],zero,xmm0[1,9],zero,xmm0[2,10],zero,xmm0[3,11],zero,xmm0[4,12],zero,xmm0[5]
881 ; AVX-NEXT: vpshufb {{.*#+}} xmm3 = zero,zero,xmm1[0],zero,zero,xmm1[1],zero,zero,xmm1[2],zero,zero,xmm1[3],zero,zero,xmm1[4],zero
882 ; AVX-NEXT: vpor %xmm3, %xmm2, %xmm2
883 ; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[13],zero,xmm0[6,14],zero,xmm0[7,15],zero,xmm0[u,u,u,u,u,u,u,u]
884 ; AVX-NEXT: vpshufb {{.*#+}} xmm1 = zero,xmm1[5],zero,zero,xmm1[6],zero,zero,xmm1[7,u,u,u,u,u,u,u,u]
885 ; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0
886 ; AVX-NEXT: vmovq %xmm0, 16(%rdi)
887 ; AVX-NEXT: vmovdqu %xmm2, (%rdi)
888 ; AVX-NEXT: retq
889 ;
890 ; XOP-LABEL: interleave_24i8_in:
891 ; XOP: # %bb.0:
892 ; XOP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
893 ; XOP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
894 ; XOP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
895 ; XOP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
896 ; XOP-NEXT: vpperm {{.*#+}} xmm2 = xmm0[0,8],xmm1[0],xmm0[1,9],xmm1[1],xmm0[2,10],xmm1[2],xmm0[3,11],xmm1[3],xmm0[4,12],xmm1[4],xmm0[5]
897 ; XOP-NEXT: vpperm {{.*#+}} xmm0 = xmm0[13],xmm1[5],xmm0[6,14],xmm1[6],xmm0[7,15],xmm1[7],xmm0[u,u,u,u,u,u,u,u]
898 ; XOP-NEXT: vmovq %xmm0, 16(%rdi)
899 ; XOP-NEXT: vmovdqu %xmm2, (%rdi)
900 ; XOP-NEXT: retq
901 %s1 = load <8 x i8>, <8 x i8>* %q1, align 4
902 %s2 = load <8 x i8>, <8 x i8>* %q2, align 4
903 %s3 = load <8 x i8>, <8 x i8>* %q3, align 4
904 %t1 = shufflevector <8 x i8> %s1, <8 x i8> %s2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
905 %t2 = shufflevector <8 x i8> %s3, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
906 %interleaved = shufflevector <16 x i8> %t1, <16 x i8> %t2, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
907 store <24 x i8> %interleaved, <24 x i8>* %p, align 4
908 ret void
909 }
912 define void @interleave_24i16_out(<24 x i16>* %p, <8 x i16>* %q1, <8 x i16>* %q2, <8 x i16>* %q3) nounwind {
913 ; SSE2-LABEL: interleave_24i16_out:
914 ; SSE2: # %bb.0:
915 ; SSE2-NEXT: movdqu (%rdi), %xmm3
916 ; SSE2-NEXT: movdqu 16(%rdi), %xmm2
917 ; SSE2-NEXT: movdqu 32(%rdi), %xmm8
918 ; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [65535,0,65535,65535,0,65535,65535,0]
919 ; SSE2-NEXT: movdqa %xmm3, %xmm4
920 ; SSE2-NEXT: pand %xmm1, %xmm4
921 ; SSE2-NEXT: pandn %xmm2, %xmm1
922 ; SSE2-NEXT: por %xmm4, %xmm1
923 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]
924 ; SSE2-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7]
925 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]
926 ; SSE2-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,1,4,5,6,7]
927 ; SSE2-NEXT: pshufhw {{.*#+}} xmm4 = xmm2[0,1,2,3,4,7,6,7]
928 ; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm8[0,1,2,1]
929 ; SSE2-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,6,5]
930 ; SSE2-NEXT: shufps {{.*#+}} xmm5 = xmm5[3,0],xmm4[2,0]
931 ; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm5[2,0]
932 ; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [65535,65535,0,65535,65535,0,65535,65535]
933 ; SSE2-NEXT: movdqa %xmm4, %xmm5
934 ; SSE2-NEXT: pandn %xmm2, %xmm5
935 ; SSE2-NEXT: movdqa %xmm3, %xmm6
936 ; SSE2-NEXT: pand %xmm4, %xmm6
937 ; SSE2-NEXT: por %xmm5, %xmm6
938 ; SSE2-NEXT: pshuflw {{.*#+}} xmm5 = xmm6[2,1,2,3,4,5,6,7]
939 ; SSE2-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,4,7]
940 ; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,3,2,3]
941 ; SSE2-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[1,2,3,0,4,5,6,7]
942 ; SSE2-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,5,5]
943 ; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [65535,65535,65535,65535,65535,0,0,0]
944 ; SSE2-NEXT: pand %xmm6, %xmm5
945 ; SSE2-NEXT: pshuflw {{.*#+}} xmm7 = xmm8[0,3,2,3,4,5,6,7]
946 ; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,1,0,3]
947 ; SSE2-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,4,5,6]
948 ; SSE2-NEXT: movdqa %xmm6, %xmm0
949 ; SSE2-NEXT: pandn %xmm7, %xmm0
950 ; SSE2-NEXT: por %xmm5, %xmm0
951 ; SSE2-NEXT: pand %xmm4, %xmm2
952 ; SSE2-NEXT: pandn %xmm3, %xmm4
953 ; SSE2-NEXT: por %xmm2, %xmm4
954 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[3,1,2,0]
955 ; SSE2-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,6,7]
956 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,1,0,3]
957 ; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,0,3,4,5,6,7]
958 ; SSE2-NEXT: pand %xmm6, %xmm2
959 ; SSE2-NEXT: pshufhw {{.*#+}} xmm3 = xmm8[0,1,2,3,4,7,6,7]
960 ; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,2,0]
961 ; SSE2-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,4,5]
962 ; SSE2-NEXT: pandn %xmm3, %xmm6
963 ; SSE2-NEXT: por %xmm2, %xmm6
964 ; SSE2-NEXT: movups %xmm1, (%rsi)
965 ; SSE2-NEXT: movdqu %xmm0, (%rdx)
966 ; SSE2-NEXT: movdqu %xmm6, (%rcx)
967 ; SSE2-NEXT: retq
968 ;
969 ; SSE42-LABEL: interleave_24i16_out:
970 ; SSE42: # %bb.0:
971 ; SSE42-NEXT: movdqu (%rdi), %xmm0
972 ; SSE42-NEXT: movdqu 16(%rdi), %xmm1
973 ; SSE42-NEXT: movdqu 32(%rdi), %xmm2
974 ; SSE42-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,1,2,1]
975 ; SSE42-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,6,5]
976 ; SSE42-NEXT: movdqa %xmm0, %xmm4
977 ; SSE42-NEXT: pblendw {{.*#+}} xmm4 = xmm4[0],xmm1[1],xmm4[2,3],xmm1[4],xmm4[5,6],xmm1[7]
978 ; SSE42-NEXT: pshufb {{.*#+}} xmm4 = xmm4[0,1,6,7,12,13,2,3,8,9,14,15,u,u,u,u]
979 ; SSE42-NEXT: pblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5],xmm3[6,7]
980 ; SSE42-NEXT: movdqa %xmm2, %xmm3
981 ; SSE42-NEXT: pshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u,u,u,u,0,1,6,7,12,13]
982 ; SSE42-NEXT: movdqa %xmm0, %xmm5
983 ; SSE42-NEXT: pblendw {{.*#+}} xmm5 = xmm5[0,1],xmm1[2],xmm5[3,4],xmm1[5],xmm5[6,7]
984 ; SSE42-NEXT: pshufb {{.*#+}} xmm5 = xmm5[2,3,8,9,14,15,4,5,10,11,u,u,u,u,u,u]
985 ; SSE42-NEXT: pblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4],xmm3[5,6,7]
986 ; SSE42-NEXT: pshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,u,u,u,u,2,3,8,9,14,15]
987 ; SSE42-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]
988 ; SSE42-NEXT: pshufb {{.*#+}} xmm1 = xmm1[4,5,10,11,0,1,6,7,12,13,u,u,u,u,u,u]
989 ; SSE42-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm2[5,6,7]
990 ; SSE42-NEXT: movdqu %xmm4, (%rsi)
991 ; SSE42-NEXT: movdqu %xmm5, (%rdx)
992 ; SSE42-NEXT: movdqu %xmm1, (%rcx)
993 ; SSE42-NEXT: retq
994 ;
995 ; AVX1-LABEL: interleave_24i16_out:
996 ; AVX1: # %bb.0:
997 ; AVX1-NEXT: vmovdqu (%rdi), %xmm0
998 ; AVX1-NEXT: vmovdqu 16(%rdi), %xmm1
999 ; AVX1-NEXT: vmovdqu 32(%rdi), %xmm2
1000 ; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,1,2,1]
1001 ; AVX1-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,6,5]
1002 ; AVX1-NEXT: vpblendw {{.*#+}} xmm4 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]
1003 ; AVX1-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[0,1,6,7,12,13,2,3,8,9,14,15,u,u,u,u]
1004 ; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3,4,5],xmm3[6,7]
1005 ; AVX1-NEXT: vpshufb {{.*#+}} xmm4 = xmm2[u,u,u,u,u,u,u,u,u,u,0,1,6,7,12,13]
1006 ; AVX1-NEXT: vpblendw {{.*#+}} xmm5 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
1007 ; AVX1-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[2,3,8,9,14,15,4,5,10,11,u,u,u,u,u,u]
1008 ; AVX1-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3,4],xmm4[5,6,7]
1009 ; AVX1-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,u,u,u,u,2,3,8,9,14,15]
1010 ; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]
1011 ; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,u,u,u,u,u,u]
1012 ; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm2[5,6,7]
1013 ; AVX1-NEXT: vmovdqu %xmm3, (%rsi)
1014 ; AVX1-NEXT: vmovdqu %xmm4, (%rdx)
1015 ; AVX1-NEXT: vmovdqu %xmm0, (%rcx)
1016 ; AVX1-NEXT: retq
1017 ;
1018 ; AVX2-LABEL: interleave_24i16_out:
1019 ; AVX2: # %bb.0:
1020 ; AVX2-NEXT: vmovdqu (%rdi), %xmm0
1021 ; AVX2-NEXT: vmovdqu 16(%rdi), %xmm1
1022 ; AVX2-NEXT: vmovdqu 32(%rdi), %xmm2
1023 ; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0,1],xmm2[2],xmm0[3,4],xmm2[5],xmm0[6,7]
1024 ; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm1[1],xmm3[2,3],xmm1[4],xmm3[5,6],xmm1[7]
1025 ; AVX2-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11]
1026 ; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1,2],xmm2[3],xmm0[4,5],xmm2[6],xmm0[7]
1027 ; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm1[2],xmm4[3,4],xmm1[5],xmm4[6,7]
1028 ; AVX2-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13]
1029 ; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3],xmm2[4],xmm0[5,6],xmm2[7]
1030 ; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]
1031 ; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15]
1032 ; AVX2-NEXT: vmovdqu %xmm3, (%rsi)
1033 ; AVX2-NEXT: vmovdqu %xmm4, (%rdx)
1034 ; AVX2-NEXT: vmovdqu %xmm0, (%rcx)
1035 ; AVX2-NEXT: retq
1036 ;
1037 ; XOP-LABEL: interleave_24i16_out:
1038 ; XOP: # %bb.0:
1039 ; XOP-NEXT: vmovdqu (%rdi), %xmm0
1040 ; XOP-NEXT: vmovdqu 16(%rdi), %xmm1
1041 ; XOP-NEXT: vmovdqu 32(%rdi), %xmm2
1042 ; XOP-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]
1043 ; XOP-NEXT: vpperm {{.*#+}} xmm3 = xmm3[0,1,6,7,12,13,2,3,8,9,14,15],xmm2[4,5,10,11]
1044 ; XOP-NEXT: vpblendw {{.*#+}} xmm4 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
1045 ; XOP-NEXT: vpperm {{.*#+}} xmm4 = xmm4[2,3,8,9,14,15,4,5,10,11],xmm2[0,1,6,7,12,13]
1046 ; XOP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]
1047 ; XOP-NEXT: vpperm {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13],xmm2[2,3,8,9,14,15]
1048 ; XOP-NEXT: vmovdqu %xmm3, (%rsi)
1049 ; XOP-NEXT: vmovdqu %xmm4, (%rdx)
1050 ; XOP-NEXT: vmovdqu %xmm0, (%rcx)
1051 ; XOP-NEXT: retq
1052 %wide.vec = load <24 x i16>, <24 x i16>* %p, align 4
1053 %s1 = shufflevector <24 x i16> %wide.vec, <24 x i16> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
1054 %s2 = shufflevector <24 x i16> %wide.vec, <24 x i16> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
1055 %s3 = shufflevector <24 x i16> %wide.vec, <24 x i16> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
1056 store <8 x i16> %s1, <8 x i16>* %q1, align 4
1057 store <8 x i16> %s2, <8 x i16>* %q2, align 4
1058 store <8 x i16> %s3, <8 x i16>* %q3, align 4
1059 ret void
1060 }
1062 define void @interleave_24i16_out_reverse(<24 x i16>* %p, <8 x i16>* %q1, <8 x i16>* %q2, <8 x i16>* %q3) nounwind {
1063 ; SSE2-LABEL: interleave_24i16_out_reverse:
1064 ; SSE2: # %bb.0:
1065 ; SSE2-NEXT: movdqu (%rdi), %xmm8
1066 ; SSE2-NEXT: movdqu 16(%rdi), %xmm1
1067 ; SSE2-NEXT: movdqu 32(%rdi), %xmm3
1068 ; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [65535,0,65535,65535,0,65535,65535,0]
1069 ; SSE2-NEXT: movdqa %xmm1, %xmm4
1070 ; SSE2-NEXT: pand %xmm2, %xmm4
1071 ; SSE2-NEXT: pandn %xmm3, %xmm2
1072 ; SSE2-NEXT: por %xmm4, %xmm2
1073 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3]
1074 ; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,0,3,4,5,6,7]
1075 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,1]
1076 ; SSE2-NEXT: pshufhw {{.*#+}} xmm4 = xmm2[0,1,2,3,5,6,6,7]
1077 ; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm8[0,1,2,1]
1078 ; SSE2-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,5,6]
1079 ; SSE2-NEXT: shufps {{.*#+}} xmm5 = xmm5[3,0],xmm4[2,0]
1080 ; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,0,1,2,4,5,6,7]
1081 ; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm5[2,0]
1082 ; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [65535,65535,0,65535,65535,0,65535,65535]
1083 ; SSE2-NEXT: movdqa %xmm4, %xmm5
1084 ; SSE2-NEXT: pandn %xmm1, %xmm5
1085 ; SSE2-NEXT: movdqa %xmm3, %xmm6
1086 ; SSE2-NEXT: pand %xmm4, %xmm6
1087 ; SSE2-NEXT: por %xmm5, %xmm6
1088 ; SSE2-NEXT: pshuflw {{.*#+}} xmm5 = xmm6[0,3,2,3,4,5,6,7]
1089 ; SSE2-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,5,6,7]
1090 ; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,2,2,1]
1091 ; SSE2-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[2,1,0,3,4,5,6,7]
1092 ; SSE2-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,6,6,6]
1093 ; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [65535,65535,65535,65535,65535,0,0,0]
1094 ; SSE2-NEXT: pand %xmm6, %xmm5
1095 ; SSE2-NEXT: pshufhw {{.*#+}} xmm7 = xmm8[0,1,2,3,4,7,6,7]
1096 ; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,1,2,0]
1097 ; SSE2-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5,4,7]
1098 ; SSE2-NEXT: movdqa %xmm6, %xmm0
1099 ; SSE2-NEXT: pandn %xmm7, %xmm0
1100 ; SSE2-NEXT: por %xmm5, %xmm0
1101 ; SSE2-NEXT: pand %xmm4, %xmm1
1102 ; SSE2-NEXT: pandn %xmm3, %xmm4
1103 ; SSE2-NEXT: por %xmm1, %xmm4
1104 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm4[3,1,2,0]
1105 ; SSE2-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]
1106 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
1107 ; SSE2-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,0,1,2,4,5,6,7]
1108 ; SSE2-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,7,7,7]
1109 ; SSE2-NEXT: pand %xmm6, %xmm1
1110 ; SSE2-NEXT: pshuflw {{.*#+}} xmm3 = xmm8[0,3,2,3,4,5,6,7]
1111 ; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,3]
1112 ; SSE2-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,5,4]
1113 ; SSE2-NEXT: pandn %xmm3, %xmm6
1114 ; SSE2-NEXT: por %xmm1, %xmm6
1115 ; SSE2-NEXT: movups %xmm2, (%rsi)
1116 ; SSE2-NEXT: movdqu %xmm0, (%rdx)
1117 ; SSE2-NEXT: movdqu %xmm6, (%rcx)
1118 ; SSE2-NEXT: retq
1119 ;
1120 ; SSE42-LABEL: interleave_24i16_out_reverse:
1121 ; SSE42: # %bb.0:
1122 ; SSE42-NEXT: movdqu (%rdi), %xmm0
1123 ; SSE42-NEXT: movdqu 16(%rdi), %xmm1
1124 ; SSE42-NEXT: movdqu 32(%rdi), %xmm2
1125 ; SSE42-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,1,2,1]
1126 ; SSE42-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,5,6]
1127 ; SSE42-NEXT: movdqa %xmm1, %xmm4
1128 ; SSE42-NEXT: pblendw {{.*#+}} xmm4 = xmm4[0],xmm2[1],xmm4[2,3],xmm2[4],xmm4[5,6],xmm2[7]
1129 ; SSE42-NEXT: pshufb {{.*#+}} xmm4 = xmm4[14,15,8,9,2,3,12,13,6,7,0,1,u,u,u,u]
1130 ; SSE42-NEXT: pblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5],xmm3[6,7]
1131 ; SSE42-NEXT: movdqa %xmm0, %xmm3
1132 ; SSE42-NEXT: pshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u,u,u,u,14,15,8,9,2,3]
1133 ; SSE42-NEXT: movdqa %xmm2, %xmm5
1134 ; SSE42-NEXT: pblendw {{.*#+}} xmm5 = xmm5[0,1],xmm1[2],xmm5[3,4],xmm1[5],xmm5[6,7]
1135 ; SSE42-NEXT: pshufb {{.*#+}} xmm5 = xmm5[12,13,6,7,0,1,10,11,4,5,u,u,u,u,u,u]
1136 ; SSE42-NEXT: pblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4],xmm3[5,6,7]
1137 ; SSE42-NEXT: pshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,u,u,u,12,13,6,7,0,1]
1138 ; SSE42-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]
1139 ; SSE42-NEXT: pshufb {{.*#+}} xmm1 = xmm1[10,11,4,5,14,15,8,9,2,3,u,u,u,u,u,u]
1140 ; SSE42-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm0[5,6,7]
1141 ; SSE42-NEXT: movdqu %xmm4, (%rsi)
1142 ; SSE42-NEXT: movdqu %xmm5, (%rdx)
1143 ; SSE42-NEXT: movdqu %xmm1, (%rcx)
1144 ; SSE42-NEXT: retq
1145 ;
1146 ; AVX1-LABEL: interleave_24i16_out_reverse:
1147 ; AVX1: # %bb.0:
1148 ; AVX1-NEXT: vmovdqu (%rdi), %xmm0
1149 ; AVX1-NEXT: vmovdqu 16(%rdi), %xmm1
1150 ; AVX1-NEXT: vmovdqu 32(%rdi), %xmm2
1151 ; AVX1-NEXT: vpshufb {{.*#+}} xmm3 = xmm2[14,15,8,9,2,3,u,u,u,u,u,u,u,u,u,u]
1152 ; AVX1-NEXT: vpblendw {{.*#+}} xmm4 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]
1153 ; AVX1-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u,u,u,12,13,6,7,0,1,10,11,4,5]
1154 ; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[3,4,5,6,7]
1155 ; AVX1-NEXT: vpshufb {{.*#+}} xmm4 = xmm2[12,13,6,7,0,1,u,u,u,u,u,u,u,u,u,u]
1156 ; AVX1-NEXT: vpblendw {{.*#+}} xmm5 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
1157 ; AVX1-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u,u,u,10,11,4,5,14,15,8,9,2,3]
1158 ; AVX1-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2],xmm5[3,4,5,6,7]
1159 ; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,1,2,3]
1160 ; AVX1-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,2,2,3,4,5,6,7]
1161 ; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]
1162 ; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,14,15,8,9,2,3,12,13,6,7,0,1]
1163 ; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3,4,5,6,7]
1164 ; AVX1-NEXT: vmovdqu %xmm3, (%rsi)
1165 ; AVX1-NEXT: vmovdqu %xmm4, (%rdx)
1166 ; AVX1-NEXT: vmovdqu %xmm0, (%rcx)
1167 ; AVX1-NEXT: retq
1168 ;
1169 ; AVX2-LABEL: interleave_24i16_out_reverse:
1170 ; AVX2: # %bb.0:
1171 ; AVX2-NEXT: vmovdqu (%rdi), %xmm0
1172 ; AVX2-NEXT: vmovdqu 16(%rdi), %xmm1
1173 ; AVX2-NEXT: vmovdqu 32(%rdi), %xmm2
1174 ; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0],xmm2[1],xmm0[2,3],xmm2[4],xmm0[5,6],xmm2[7]
1175 ; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm3[1,2],xmm1[3],xmm3[4,5],xmm1[6],xmm3[7]
1176 ; AVX2-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[14,15,8,9,2,3,12,13,6,7,0,1,10,11,4,5]
1177 ; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1,2],xmm2[3],xmm0[4,5],xmm2[6],xmm0[7]
1178 ; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm1[2],xmm4[3,4],xmm1[5],xmm4[6,7]
1179 ; AVX2-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[12,13,6,7,0,1,10,11,4,5,14,15,8,9,2,3]
1180 ; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2],xmm0[3,4],xmm2[5],xmm0[6,7]
1181 ; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]
1182 ; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[10,11,4,5,14,15,8,9,2,3,12,13,6,7,0,1]
1183 ; AVX2-NEXT: vmovdqu %xmm3, (%rsi)
1184 ; AVX2-NEXT: vmovdqu %xmm4, (%rdx)
1185 ; AVX2-NEXT: vmovdqu %xmm0, (%rcx)
1186 ; AVX2-NEXT: retq
1187 ;
1188 ; XOP-LABEL: interleave_24i16_out_reverse:
1189 ; XOP: # %bb.0:
1190 ; XOP-NEXT: vmovdqu (%rdi), %xmm0
1191 ; XOP-NEXT: vmovdqu 16(%rdi), %xmm1
1192 ; XOP-NEXT: vmovdqu 32(%rdi), %xmm2
1193 ; XOP-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]
1194 ; XOP-NEXT: vpperm {{.*#+}} xmm3 = xmm2[14,15,8,9,2,3],xmm3[12,13,6,7,0,1,10,11,4,5]
1195 ; XOP-NEXT: vpblendw {{.*#+}} xmm4 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
1196 ; XOP-NEXT: vpperm {{.*#+}} xmm4 = xmm2[12,13,6,7,0,1],xmm4[10,11,4,5,14,15,8,9,2,3]
1197 ; XOP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]
1198 ; XOP-NEXT: vpperm {{.*#+}} xmm0 = xmm2[10,11,4,5],xmm0[14,15,8,9,2,3,12,13,6,7,0,1]
1199 ; XOP-NEXT: vmovdqu %xmm3, (%rsi)
1200 ; XOP-NEXT: vmovdqu %xmm4, (%rdx)
1201 ; XOP-NEXT: vmovdqu %xmm0, (%rcx)
1202 ; XOP-NEXT: retq
1203 %wide.vec.reverse = load <24 x i16>, <24 x i16>* %p, align 4
1204 %wide.vec = shufflevector <24 x i16> %wide.vec.reverse, <24 x i16> undef, <24 x i32> <i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
1205 %s1 = shufflevector <24 x i16> %wide.vec, <24 x i16> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
1206 %s2 = shufflevector <24 x i16> %wide.vec, <24 x i16> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
1207 %s3 = shufflevector <24 x i16> %wide.vec, <24 x i16> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
1208 store <8 x i16> %s1, <8 x i16>* %q1, align 4
1209 store <8 x i16> %s2, <8 x i16>* %q2, align 4
1210 store <8 x i16> %s3, <8 x i16>* %q3, align 4
1211 ret void
1212 }
1214 define void @interleave_24i16_in(<24 x i16>* %p, <8 x i16>* %q1, <8 x i16>* %q2, <8 x i16>* %q3) nounwind {
1215 ; SSE2-LABEL: interleave_24i16_in:
1216 ; SSE2: # %bb.0:
1217 ; SSE2-NEXT: movdqu (%rsi), %xmm3
1218 ; SSE2-NEXT: movdqu (%rdx), %xmm2
1219 ; SSE2-NEXT: movdqu (%rcx), %xmm1
1220 ; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[0,0,0,0]
1221 ; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [65535,65535,0,65535,65535,0,65535,65535]
1222 ; SSE2-NEXT: movdqa %xmm0, %xmm5
1223 ; SSE2-NEXT: pandn %xmm4, %xmm5
1224 ; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm3[3,3,3,3]
1225 ; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm3[1,1,2,2]
1226 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
1227 ; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,2,1]
1228 ; SSE2-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,2,2,4,5,6,7]
1229 ; SSE2-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,4,5]
1230 ; SSE2-NEXT: pand %xmm0, %xmm3
1231 ; SSE2-NEXT: por %xmm5, %xmm3
1232 ; SSE2-NEXT: movdqa %xmm0, %xmm5
1233 ; SSE2-NEXT: pandn %xmm4, %xmm5
1234 ; SSE2-NEXT: pshuflw {{.*#+}} xmm4 = xmm2[3,3,3,3,4,5,6,7]
1235 ; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
1236 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,1,2,3]
1237 ; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,3,2,0,4,5,6,7]
1238 ; SSE2-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,6,7]
1239 ; SSE2-NEXT: pand %xmm0, %xmm2
1240 ; SSE2-NEXT: por %xmm5, %xmm2
1241 ; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [65535,0,65535,65535,0,65535,65535,0]
1242 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,2,2]
1243 ; SSE2-NEXT: pand %xmm5, %xmm1
1244 ; SSE2-NEXT: pandn %xmm6, %xmm5
1245 ; SSE2-NEXT: por %xmm1, %xmm5
1246 ; SSE2-NEXT: pand %xmm0, %xmm5
1247 ; SSE2-NEXT: pshufhw {{.*#+}} xmm1 = xmm4[0,1,2,3,4,4,4,4]
1248 ; SSE2-NEXT: pandn %xmm1, %xmm0
1249 ; SSE2-NEXT: por %xmm5, %xmm0
1250 ; SSE2-NEXT: movdqu %xmm0, 16(%rdi)
1251 ; SSE2-NEXT: movdqu %xmm2, 32(%rdi)
1252 ; SSE2-NEXT: movdqu %xmm3, (%rdi)
1253 ; SSE2-NEXT: retq
1254 ;
1255 ; SSE42-LABEL: interleave_24i16_in:
1256 ; SSE42: # %bb.0:
1257 ; SSE42-NEXT: movdqu (%rsi), %xmm0
1258 ; SSE42-NEXT: movdqu (%rdx), %xmm1
1259 ; SSE42-NEXT: movdqu (%rcx), %xmm2
1260 ; SSE42-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,2,2]
1261 ; SSE42-NEXT: pshufd {{.*#+}} xmm4 = xmm0[3,3,3,3]
1262 ; SSE42-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1263 ; SSE42-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,u,u,4,5,6,7,u,u,8,9,10,11]
1264 ; SSE42-NEXT: pshufd {{.*#+}} xmm5 = xmm2[0,0,0,0]
1265 ; SSE42-NEXT: pblendw {{.*#+}} xmm5 = xmm0[0,1],xmm5[2],xmm0[3,4],xmm5[5],xmm0[6,7]
1266 ; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,2,2]
1267 ; SSE42-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2,3],xmm3[4],xmm0[5,6],xmm3[7]
1268 ; SSE42-NEXT: pshuflw {{.*#+}} xmm3 = xmm1[3,3,3,3,4,5,6,7]
1269 ; SSE42-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,4,4]
1270 ; SSE42-NEXT: pblendw {{.*#+}} xmm3 = xmm0[0,1],xmm3[2],xmm0[3,4],xmm3[5],xmm0[6,7]
1271 ; SSE42-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
1272 ; SSE42-NEXT: pshufb {{.*#+}} xmm1 = xmm1[4,5,6,7,u,u,8,9,10,11,u,u,12,13,14,15]
1273 ; SSE42-NEXT: pblendw {{.*#+}} xmm4 = xmm1[0,1],xmm4[2],xmm1[3,4],xmm4[5],xmm1[6,7]
1274 ; SSE42-NEXT: movdqu %xmm4, 32(%rdi)
1275 ; SSE42-NEXT: movdqu %xmm3, 16(%rdi)
1276 ; SSE42-NEXT: movdqu %xmm5, (%rdi)
1277 ; SSE42-NEXT: retq
1278 ;
1279 ; AVX1-LABEL: interleave_24i16_in:
1280 ; AVX1: # %bb.0:
1281 ; AVX1-NEXT: vmovdqu (%rsi), %xmm0
1282 ; AVX1-NEXT: vmovdqu (%rdx), %xmm1
1283 ; AVX1-NEXT: vmovdqu (%rcx), %xmm2
1284 ; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,2]
1285 ; AVX1-NEXT: vpshuflw {{.*#+}} xmm4 = xmm1[3,3,3,3,4,5,6,7]
1286 ; AVX1-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,4,4,4]
1287 ; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2],xmm3[3,4],xmm4[5],xmm3[6,7]
1288 ; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[1,1,2,2]
1289 ; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1,2],xmm4[3],xmm3[4,5],xmm4[6],xmm3[7]
1290 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1291 ; AVX1-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[4,5,u,u,10,11,8,9,u,u,14,15,12,13,u,u]
1292 ; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[2,2,3,3]
1293 ; AVX1-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm5[1],xmm4[2,3],xmm5[4],xmm4[5,6],xmm5[7]
1294 ; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1295 ; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,u,u,4,5,6,7,u,u,8,9,10,11]
1296 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[0,0,0,0]
1297 ; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
1298 ; AVX1-NEXT: vmovdqu %xmm0, (%rdi)
1299 ; AVX1-NEXT: vmovdqu %xmm4, 32(%rdi)
1300 ; AVX1-NEXT: vmovdqu %xmm3, 16(%rdi)
1301 ; AVX1-NEXT: retq
1302 ;
1303 ; AVX2-SLOW-LABEL: interleave_24i16_in:
1304 ; AVX2-SLOW: # %bb.0:
1305 ; AVX2-SLOW-NEXT: vmovdqu (%rsi), %xmm0
1306 ; AVX2-SLOW-NEXT: vmovdqu (%rdx), %xmm1
1307 ; AVX2-SLOW-NEXT: vmovdqu (%rcx), %xmm2
1308 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm3
1309 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm4 = ymm3[0,1,u,u,6,7,2,3,u,u,8,9,4,5,u,u,16,17,u,u,22,23,18,19,u,u,24,25,20,21,u,u]
1310 ; AVX2-SLOW-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,3,0,1]
1311 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[u,u,0,1,u,u,u,u,2,3,u,u,u,u,4,5,u,u,22,23,u,u,u,u,24,25,u,u,u,u,26,27]
1312 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5,6],ymm3[7],ymm4[8],ymm3[9],ymm4[10,11],ymm3[12],ymm4[13,14],ymm3[15]
1313 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm4 = <u,0,0,u,1,1,u,2>
1314 ; AVX2-SLOW-NEXT: vpermd %ymm2, %ymm4, %ymm4
1315 ; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm5 = [255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255]
1316 ; AVX2-SLOW-NEXT: vpblendvb %ymm5, %ymm3, %ymm4, %ymm3
1317 ; AVX2-SLOW-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1318 ; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,u,u,10,11,8,9,u,u,14,15,12,13,u,u]
1319 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]
1320 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]
1321 ; AVX2-SLOW-NEXT: vmovdqu %xmm0, 32(%rdi)
1322 ; AVX2-SLOW-NEXT: vmovdqu %ymm3, (%rdi)
1323 ; AVX2-SLOW-NEXT: vzeroupper
1324 ; AVX2-SLOW-NEXT: retq
1325 ;
1326 ; AVX2-FAST-ALL-LABEL: interleave_24i16_in:
1327 ; AVX2-FAST-ALL: # %bb.0:
1328 ; AVX2-FAST-ALL-NEXT: vmovdqu (%rsi), %xmm0
1329 ; AVX2-FAST-ALL-NEXT: vmovdqu (%rdx), %xmm1
1330 ; AVX2-FAST-ALL-NEXT: vmovdqu (%rcx), %xmm2
1331 ; AVX2-FAST-ALL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm3
1332 ; AVX2-FAST-ALL-NEXT: vmovdqa {{.*#+}} ymm4 = <u,0,0,u,1,1,u,2>
1333 ; AVX2-FAST-ALL-NEXT: vpermd %ymm2, %ymm4, %ymm4
1334 ; AVX2-FAST-ALL-NEXT: vmovdqa {{.*#+}} ymm5 = [0,4,1,5,1,5,2,6]
1335 ; AVX2-FAST-ALL-NEXT: vpermd %ymm3, %ymm5, %ymm3
1336 ; AVX2-FAST-ALL-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[0,1,4,5,u,u,2,3,6,7,u,u,8,9,12,13,u,u,18,19,22,23,u,u,24,25,28,29,u,u,26,27]
1337 ; AVX2-FAST-ALL-NEXT: vmovdqa {{.*#+}} ymm5 = [255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255]
1338 ; AVX2-FAST-ALL-NEXT: vpblendvb %ymm5, %ymm3, %ymm4, %ymm3
1339 ; AVX2-FAST-ALL-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1340 ; AVX2-FAST-ALL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,u,u,10,11,8,9,u,u,14,15,12,13,u,u]
1341 ; AVX2-FAST-ALL-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]
1342 ; AVX2-FAST-ALL-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]
1343 ; AVX2-FAST-ALL-NEXT: vmovdqu %xmm0, 32(%rdi)
1344 ; AVX2-FAST-ALL-NEXT: vmovdqu %ymm3, (%rdi)
1345 ; AVX2-FAST-ALL-NEXT: vzeroupper
1346 ; AVX2-FAST-ALL-NEXT: retq
1347 ;
1348 ; AVX2-FAST-PERLANE-LABEL: interleave_24i16_in:
1349 ; AVX2-FAST-PERLANE: # %bb.0:
1350 ; AVX2-FAST-PERLANE-NEXT: vmovdqu (%rsi), %xmm0
1351 ; AVX2-FAST-PERLANE-NEXT: vmovdqu (%rdx), %xmm1
1352 ; AVX2-FAST-PERLANE-NEXT: vmovdqu (%rcx), %xmm2
1353 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm3
1354 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} ymm4 = ymm3[0,1,u,u,6,7,2,3,u,u,8,9,4,5,u,u,16,17,u,u,22,23,18,19,u,u,24,25,20,21,u,u]
1355 ; AVX2-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,3,0,1]
1356 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[u,u,0,1,u,u,u,u,2,3,u,u,u,u,4,5,u,u,22,23,u,u,u,u,24,25,u,u,u,u,26,27]
1357 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5,6],ymm3[7],ymm4[8],ymm3[9],ymm4[10,11],ymm3[12],ymm4[13,14],ymm3[15]
1358 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} ymm4 = <u,0,0,u,1,1,u,2>
1359 ; AVX2-FAST-PERLANE-NEXT: vpermd %ymm2, %ymm4, %ymm4
1360 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} ymm5 = [255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255]
1361 ; AVX2-FAST-PERLANE-NEXT: vpblendvb %ymm5, %ymm3, %ymm4, %ymm3
1362 ; AVX2-FAST-PERLANE-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1363 ; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,u,u,10,11,8,9,u,u,14,15,12,13,u,u]
1364 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]
1365 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]
1366 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %xmm0, 32(%rdi)
1367 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm3, (%rdi)
1368 ; AVX2-FAST-PERLANE-NEXT: vzeroupper
1369 ; AVX2-FAST-PERLANE-NEXT: retq
1370 ;
1371 ; XOP-LABEL: interleave_24i16_in:
1372 ; XOP: # %bb.0:
1373 ; XOP-NEXT: vmovdqu (%rsi), %xmm0
1374 ; XOP-NEXT: vmovdqu (%rdx), %xmm1
1375 ; XOP-NEXT: vmovdqu (%rcx), %xmm2
1376 ; XOP-NEXT: vpperm {{.*#+}} xmm3 = xmm0[u,u,6,7],xmm1[6,7],xmm0[u,u,8,9],xmm1[8,9],xmm0[u,u,10,11]
1377 ; XOP-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[1,1,2,2]
1378 ; XOP-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1,2],xmm4[3],xmm3[4,5],xmm4[6],xmm3[7]
1379 ; XOP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
1380 ; XOP-NEXT: vpperm {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm2[0,1],xmm4[4,5,6,7],xmm2[2,3],xmm4[8,9,10,11]
1381 ; XOP-NEXT: vinsertf128 $1, %xmm3, %ymm4, %ymm3
1382 ; XOP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
1383 ; XOP-NEXT: vpperm {{.*#+}} xmm0 = xmm0[4,5],xmm2[10,11],xmm0[10,11,8,9],xmm2[12,13],xmm0[14,15,12,13],xmm2[14,15]
1384 ; XOP-NEXT: vmovdqu %xmm0, 32(%rdi)
1385 ; XOP-NEXT: vmovups %ymm3, (%rdi)
1386 ; XOP-NEXT: vzeroupper
1387 ; XOP-NEXT: retq
1388 %s1 = load <8 x i16>, <8 x i16>* %q1, align 4
1389 %s2 = load <8 x i16>, <8 x i16>* %q2, align 4
1390 %s3 = load <8 x i16>, <8 x i16>* %q3, align 4
1391 %t1 = shufflevector <8 x i16> %s1, <8 x i16> %s2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
1392 %t2 = shufflevector <8 x i16> %s3, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1393 %interleaved = shufflevector <16 x i16> %t1, <16 x i16> %t2, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
1394 store <24 x i16> %interleaved, <24 x i16>* %p, align 4
1395 ret void
1396 }
1398 define void @interleave_24i32_out(<24 x i32>* %p, <8 x i32>* %q1, <8 x i32>* %q2, <8 x i32>* %q3) nounwind {
1399 ; SSE2-LABEL: interleave_24i32_out:
1400 ; SSE2: # %bb.0:
1401 ; SSE2-NEXT: movups 80(%rdi), %xmm8
1402 ; SSE2-NEXT: movups 64(%rdi), %xmm3
1403 ; SSE2-NEXT: movdqu (%rdi), %xmm1
1404 ; SSE2-NEXT: movups 16(%rdi), %xmm5
1405 ; SSE2-NEXT: movups 32(%rdi), %xmm10
1406 ; SSE2-NEXT: movdqu 48(%rdi), %xmm2
1407 ; SSE2-NEXT: movdqa %xmm1, %xmm11
1408 ; SSE2-NEXT: movaps %xmm10, %xmm7
1409 ; SSE2-NEXT: shufps {{.*#+}} xmm7 = xmm7[2,1],xmm5[3,3]
1410 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
1411 ; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,0],xmm5[0,0]
1412 ; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm5[1,1,1,1]
1413 ; SSE2-NEXT: shufps {{.*#+}} xmm5 = xmm5[2,3],xmm10[1,1]
1414 ; SSE2-NEXT: shufps {{.*#+}} xmm11 = xmm11[0,3],xmm5[0,2]
1415 ; SSE2-NEXT: movdqa %xmm2, %xmm5
1416 ; SSE2-NEXT: movaps %xmm8, %xmm4
1417 ; SSE2-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,1],xmm3[3,3]
1418 ; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm2[2,3,2,3]
1419 ; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,0],xmm3[0,0]
1420 ; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm3[1,1,1,1]
1421 ; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,3],xmm8[1,1]
1422 ; SSE2-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,3],xmm3[0,2]
1423 ; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm4[2,0]
1424 ; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,2],xmm7[2,0]
1425 ; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm9[0],xmm0[1],xmm9[1]
1426 ; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm10[0,3]
1427 ; SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm12[0],xmm6[1],xmm12[1]
1428 ; SSE2-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,1],xmm8[0,3]
1429 ; SSE2-NEXT: movups %xmm5, 16(%rsi)
1430 ; SSE2-NEXT: movups %xmm11, (%rsi)
1431 ; SSE2-NEXT: movups %xmm2, 16(%rdx)
1432 ; SSE2-NEXT: movups %xmm1, (%rdx)
1433 ; SSE2-NEXT: movups %xmm6, 16(%rcx)
1434 ; SSE2-NEXT: movups %xmm0, (%rcx)
1435 ; SSE2-NEXT: retq
1436 ;
1437 ; SSE42-LABEL: interleave_24i32_out:
1438 ; SSE42: # %bb.0:
1439 ; SSE42-NEXT: movups 80(%rdi), %xmm8
1440 ; SSE42-NEXT: movdqu 64(%rdi), %xmm9
1441 ; SSE42-NEXT: movdqu (%rdi), %xmm3
1442 ; SSE42-NEXT: movdqu 16(%rdi), %xmm2
1443 ; SSE42-NEXT: movups 32(%rdi), %xmm10
1444 ; SSE42-NEXT: movdqu 48(%rdi), %xmm5
1445 ; SSE42-NEXT: movdqa %xmm2, %xmm6
1446 ; SSE42-NEXT: pblendw {{.*#+}} xmm6 = xmm6[0,1],xmm3[2,3],xmm6[4,5,6,7]
1447 ; SSE42-NEXT: pshufd {{.*#+}} xmm7 = xmm3[2,3,2,3]
1448 ; SSE42-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,3],xmm2[2,3]
1449 ; SSE42-NEXT: insertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm10[1]
1450 ; SSE42-NEXT: movdqa %xmm9, %xmm1
1451 ; SSE42-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm5[2,3],xmm1[4,5,6,7]
1452 ; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm5[2,3,2,3]
1453 ; SSE42-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,3],xmm9[2,3]
1454 ; SSE42-NEXT: insertps {{.*#+}} xmm5 = xmm5[0,1,2],xmm8[1]
1455 ; SSE42-NEXT: pshufd {{.*#+}} xmm4 = xmm10[2,2,2,2]
1456 ; SSE42-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,0,3,3]
1457 ; SSE42-NEXT: pblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,5],xmm4[6,7]
1458 ; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,0,3,3]
1459 ; SSE42-NEXT: pshufd {{.*#+}} xmm4 = xmm8[2,2,2,2]
1460 ; SSE42-NEXT: pblendw {{.*#+}} xmm4 = xmm1[0,1,2,3,4,5],xmm4[6,7]
1461 ; SSE42-NEXT: pblendw {{.*#+}} xmm7 = xmm7[0,1],xmm2[2,3],xmm7[4,5,6,7]
1462 ; SSE42-NEXT: shufps {{.*#+}} xmm7 = xmm7[0,1],xmm10[0,3]
1463 ; SSE42-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm9[2,3],xmm0[4,5,6,7]
1464 ; SSE42-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm8[0,3]
1465 ; SSE42-NEXT: movups %xmm5, 16(%rsi)
1466 ; SSE42-NEXT: movups %xmm3, (%rsi)
1467 ; SSE42-NEXT: movdqu %xmm4, 16(%rdx)
1468 ; SSE42-NEXT: movdqu %xmm6, (%rdx)
1469 ; SSE42-NEXT: movups %xmm0, 16(%rcx)
1470 ; SSE42-NEXT: movups %xmm7, (%rcx)
1471 ; SSE42-NEXT: retq
1472 ;
1473 ; AVX1-LABEL: interleave_24i32_out:
1474 ; AVX1: # %bb.0:
1475 ; AVX1-NEXT: vmovups (%rdi), %ymm0
1476 ; AVX1-NEXT: vmovups 32(%rdi), %ymm1
1477 ; AVX1-NEXT: vmovups 80(%rdi), %xmm2
1478 ; AVX1-NEXT: vmovups 64(%rdi), %xmm3
1479 ; AVX1-NEXT: vinsertps {{.*#+}} xmm4 = zero,zero,xmm3[2],xmm2[1]
1480 ; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4
1481 ; AVX1-NEXT: vblendps {{.*#+}} ymm5 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
1482 ; AVX1-NEXT: vextractf128 $1, %ymm5, %xmm6
1483 ; AVX1-NEXT: vblendps {{.*#+}} xmm5 = xmm5[0,1],xmm6[2],xmm5[3]
1484 ; AVX1-NEXT: vpermilps {{.*#+}} xmm5 = xmm5[0,3,2,1]
1485 ; AVX1-NEXT: vpermilps {{.*#+}} xmm6 = xmm6[0,3,2,3]
1486 ; AVX1-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm5
1487 ; AVX1-NEXT: vblendps {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
1488 ; AVX1-NEXT: vblendps {{.*#+}} xmm5 = xmm3[0,1],xmm2[2],xmm3[3]
1489 ; AVX1-NEXT: vpermilps {{.*#+}} xmm5 = xmm5[0,0,3,2]
1490 ; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm5
1491 ; AVX1-NEXT: vblendps {{.*#+}} ymm6 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
1492 ; AVX1-NEXT: vextractf128 $1, %ymm6, %xmm7
1493 ; AVX1-NEXT: vblendps {{.*#+}} xmm6 = xmm7[0],xmm6[1,2],xmm7[3]
1494 ; AVX1-NEXT: vpermilps {{.*#+}} xmm6 = xmm6[1,0,3,2]
1495 ; AVX1-NEXT: vpermilps {{.*#+}} xmm7 = xmm7[1,1,3,3]
1496 ; AVX1-NEXT: vinsertf128 $1, %xmm7, %ymm6, %ymm6
1497 ; AVX1-NEXT: vblendps {{.*#+}} ymm5 = ymm6[0,1,2,3,4],ymm5[5,6,7]
1498 ; AVX1-NEXT: vshufps {{.*#+}} xmm2 = xmm3[0,1],xmm2[0,3]
1499 ; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2
1500 ; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
1501 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
1502 ; AVX1-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
1503 ; AVX1-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[2,1,0,3]
1504 ; AVX1-NEXT: vpermilps {{.*#+}} xmm1 = xmm1[2,3,0,1]
1505 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
1506 ; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
1507 ; AVX1-NEXT: vmovups %ymm4, (%rsi)
1508 ; AVX1-NEXT: vmovups %ymm5, (%rdx)
1509 ; AVX1-NEXT: vmovups %ymm0, (%rcx)
1510 ; AVX1-NEXT: vzeroupper
1511 ; AVX1-NEXT: retq
1512 ;
1513 ; AVX2-SLOW-LABEL: interleave_24i32_out:
1514 ; AVX2-SLOW: # %bb.0:
1515 ; AVX2-SLOW-NEXT: vmovups (%rdi), %ymm0
1516 ; AVX2-SLOW-NEXT: vmovups 32(%rdi), %ymm1
1517 ; AVX2-SLOW-NEXT: vmovups 64(%rdi), %ymm2
1518 ; AVX2-SLOW-NEXT: vbroadcastsd {{.*#+}} ymm3 = [21474836482,21474836482,21474836482,21474836482]
1519 ; AVX2-SLOW-NEXT: vpermps %ymm2, %ymm3, %ymm3
1520 ; AVX2-SLOW-NEXT: vblendps {{.*#+}} ymm4 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
1521 ; AVX2-SLOW-NEXT: vmovaps {{.*#+}} ymm5 = <0,3,6,1,4,7,u,u>
1522 ; AVX2-SLOW-NEXT: vpermps %ymm4, %ymm5, %ymm4
1523 ; AVX2-SLOW-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
1524 ; AVX2-SLOW-NEXT: vbroadcastf128 {{.*#+}} ymm4 = [0,0,3,6,0,0,3,6]
1525 ; AVX2-SLOW-NEXT: # ymm4 = mem[0,1,0,1]
1526 ; AVX2-SLOW-NEXT: vpermps %ymm2, %ymm4, %ymm4
1527 ; AVX2-SLOW-NEXT: vblendps {{.*#+}} ymm5 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
1528 ; AVX2-SLOW-NEXT: vmovaps {{.*#+}} ymm6 = <1,4,7,2,5,u,u,u>
1529 ; AVX2-SLOW-NEXT: vpermps %ymm5, %ymm6, %ymm5
1530 ; AVX2-SLOW-NEXT: vblendps {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5,6,7]
1531 ; AVX2-SLOW-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
1532 ; AVX2-SLOW-NEXT: vmovaps {{.*#+}} ymm1 = <2,5,0,3,6,u,u,u>
1533 ; AVX2-SLOW-NEXT: vpermps %ymm0, %ymm1, %ymm0
1534 ; AVX2-SLOW-NEXT: vpermilps {{.*#+}} ymm1 = ymm2[0,1,0,3,4,5,4,7]
1535 ; AVX2-SLOW-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
1536 ; AVX2-SLOW-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
1537 ; AVX2-SLOW-NEXT: vmovups %ymm3, (%rsi)
1538 ; AVX2-SLOW-NEXT: vmovups %ymm4, (%rdx)
1539 ; AVX2-SLOW-NEXT: vmovups %ymm0, (%rcx)
1540 ; AVX2-SLOW-NEXT: vzeroupper
1541 ; AVX2-SLOW-NEXT: retq
1542 ;
1543 ; AVX2-FAST-ALL-LABEL: interleave_24i32_out:
1544 ; AVX2-FAST-ALL: # %bb.0:
1545 ; AVX2-FAST-ALL-NEXT: vmovups (%rdi), %ymm0
1546 ; AVX2-FAST-ALL-NEXT: vmovups 32(%rdi), %ymm1
1547 ; AVX2-FAST-ALL-NEXT: vmovups 64(%rdi), %ymm2
1548 ; AVX2-FAST-ALL-NEXT: vbroadcastsd {{.*#+}} ymm3 = [21474836482,21474836482,21474836482,21474836482]
1549 ; AVX2-FAST-ALL-NEXT: vpermps %ymm2, %ymm3, %ymm3
1550 ; AVX2-FAST-ALL-NEXT: vblendps {{.*#+}} ymm4 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
1551 ; AVX2-FAST-ALL-NEXT: vmovaps {{.*#+}} ymm5 = <0,3,6,1,4,7,u,u>
1552 ; AVX2-FAST-ALL-NEXT: vpermps %ymm4, %ymm5, %ymm4
1553 ; AVX2-FAST-ALL-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
1554 ; AVX2-FAST-ALL-NEXT: vbroadcastf128 {{.*#+}} ymm4 = [0,0,3,6,0,0,3,6]
1555 ; AVX2-FAST-ALL-NEXT: # ymm4 = mem[0,1,0,1]
1556 ; AVX2-FAST-ALL-NEXT: vpermps %ymm2, %ymm4, %ymm4
1557 ; AVX2-FAST-ALL-NEXT: vblendps {{.*#+}} ymm5 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
1558 ; AVX2-FAST-ALL-NEXT: vmovaps {{.*#+}} ymm6 = <1,4,7,2,5,u,u,u>
1559 ; AVX2-FAST-ALL-NEXT: vpermps %ymm5, %ymm6, %ymm5
1560 ; AVX2-FAST-ALL-NEXT: vblendps {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5,6,7]
1561 ; AVX2-FAST-ALL-NEXT: vmovaps {{.*#+}} ymm5 = <u,u,u,u,u,1,4,7>
1562 ; AVX2-FAST-ALL-NEXT: vpermps %ymm2, %ymm5, %ymm2
1563 ; AVX2-FAST-ALL-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
1564 ; AVX2-FAST-ALL-NEXT: vmovaps {{.*#+}} ymm1 = <2,5,0,3,6,u,u,u>
1565 ; AVX2-FAST-ALL-NEXT: vpermps %ymm0, %ymm1, %ymm0
1566 ; AVX2-FAST-ALL-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
1567 ; AVX2-FAST-ALL-NEXT: vmovups %ymm3, (%rsi)
1568 ; AVX2-FAST-ALL-NEXT: vmovups %ymm4, (%rdx)
1569 ; AVX2-FAST-ALL-NEXT: vmovups %ymm0, (%rcx)
1570 ; AVX2-FAST-ALL-NEXT: vzeroupper
1571 ; AVX2-FAST-ALL-NEXT: retq
1572 ;
1573 ; AVX2-FAST-PERLANE-LABEL: interleave_24i32_out:
1574 ; AVX2-FAST-PERLANE: # %bb.0:
1575 ; AVX2-FAST-PERLANE-NEXT: vmovups (%rdi), %ymm0
1576 ; AVX2-FAST-PERLANE-NEXT: vmovups 32(%rdi), %ymm1
1577 ; AVX2-FAST-PERLANE-NEXT: vmovups 64(%rdi), %ymm2
1578 ; AVX2-FAST-PERLANE-NEXT: vbroadcastsd {{.*#+}} ymm3 = [21474836482,21474836482,21474836482,21474836482]
1579 ; AVX2-FAST-PERLANE-NEXT: vpermps %ymm2, %ymm3, %ymm3
1580 ; AVX2-FAST-PERLANE-NEXT: vblendps {{.*#+}} ymm4 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
1581 ; AVX2-FAST-PERLANE-NEXT: vmovaps {{.*#+}} ymm5 = <0,3,6,1,4,7,u,u>
1582 ; AVX2-FAST-PERLANE-NEXT: vpermps %ymm4, %ymm5, %ymm4
1583 ; AVX2-FAST-PERLANE-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
1584 ; AVX2-FAST-PERLANE-NEXT: vbroadcastf128 {{.*#+}} ymm4 = [0,0,3,6,0,0,3,6]
1585 ; AVX2-FAST-PERLANE-NEXT: # ymm4 = mem[0,1,0,1]
1586 ; AVX2-FAST-PERLANE-NEXT: vpermps %ymm2, %ymm4, %ymm4
1587 ; AVX2-FAST-PERLANE-NEXT: vblendps {{.*#+}} ymm5 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
1588 ; AVX2-FAST-PERLANE-NEXT: vmovaps {{.*#+}} ymm6 = <1,4,7,2,5,u,u,u>
1589 ; AVX2-FAST-PERLANE-NEXT: vpermps %ymm5, %ymm6, %ymm5
1590 ; AVX2-FAST-PERLANE-NEXT: vblendps {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5,6,7]
1591 ; AVX2-FAST-PERLANE-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
1592 ; AVX2-FAST-PERLANE-NEXT: vmovaps {{.*#+}} ymm1 = <2,5,0,3,6,u,u,u>
1593 ; AVX2-FAST-PERLANE-NEXT: vpermps %ymm0, %ymm1, %ymm0
1594 ; AVX2-FAST-PERLANE-NEXT: vpermilps {{.*#+}} ymm1 = ymm2[0,1,0,3,4,5,4,7]
1595 ; AVX2-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
1596 ; AVX2-FAST-PERLANE-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
1597 ; AVX2-FAST-PERLANE-NEXT: vmovups %ymm3, (%rsi)
1598 ; AVX2-FAST-PERLANE-NEXT: vmovups %ymm4, (%rdx)
1599 ; AVX2-FAST-PERLANE-NEXT: vmovups %ymm0, (%rcx)
1600 ; AVX2-FAST-PERLANE-NEXT: vzeroupper
1601 ; AVX2-FAST-PERLANE-NEXT: retq
1602 ;
1603 ; XOP-LABEL: interleave_24i32_out:
1604 ; XOP: # %bb.0:
1605 ; XOP-NEXT: vmovups (%rdi), %ymm0
1606 ; XOP-NEXT: vmovups 32(%rdi), %ymm1
1607 ; XOP-NEXT: vmovups 80(%rdi), %xmm2
1608 ; XOP-NEXT: vmovups 64(%rdi), %xmm3
1609 ; XOP-NEXT: vinsertps {{.*#+}} xmm4 = zero,zero,xmm3[2],xmm2[1]
1610 ; XOP-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4
1611 ; XOP-NEXT: vblendps {{.*#+}} ymm5 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
1612 ; XOP-NEXT: vextractf128 $1, %ymm5, %xmm6
1613 ; XOP-NEXT: vblendps {{.*#+}} xmm5 = xmm5[0,1],xmm6[2],xmm5[3]
1614 ; XOP-NEXT: vpermilps {{.*#+}} xmm5 = xmm5[0,3,2,1]
1615 ; XOP-NEXT: vpermilps {{.*#+}} xmm6 = xmm6[0,3,2,3]
1616 ; XOP-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm5
1617 ; XOP-NEXT: vblendps {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
1618 ; XOP-NEXT: vblendps {{.*#+}} xmm5 = xmm3[0,1],xmm2[2],xmm3[3]
1619 ; XOP-NEXT: vpermilps {{.*#+}} xmm5 = xmm5[0,0,3,2]
1620 ; XOP-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm5
1621 ; XOP-NEXT: vblendps {{.*#+}} ymm6 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
1622 ; XOP-NEXT: vextractf128 $1, %ymm6, %xmm7
1623 ; XOP-NEXT: vblendps {{.*#+}} xmm6 = xmm7[0],xmm6[1,2],xmm7[3]
1624 ; XOP-NEXT: vpermilps {{.*#+}} xmm6 = xmm6[1,0,3,2]
1625 ; XOP-NEXT: vpermilps {{.*#+}} xmm7 = xmm7[1,1,3,3]
1626 ; XOP-NEXT: vinsertf128 $1, %xmm7, %ymm6, %ymm6
1627 ; XOP-NEXT: vblendps {{.*#+}} ymm5 = ymm6[0,1,2,3,4],ymm5[5,6,7]
1628 ; XOP-NEXT: vshufps {{.*#+}} xmm2 = xmm3[0,1],xmm2[0,3]
1629 ; XOP-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2
1630 ; XOP-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
1631 ; XOP-NEXT: vextractf128 $1, %ymm0, %xmm1
1632 ; XOP-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
1633 ; XOP-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[2,1,0,3]
1634 ; XOP-NEXT: vpermilps {{.*#+}} xmm1 = xmm1[2,3,0,1]
1635 ; XOP-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
1636 ; XOP-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
1637 ; XOP-NEXT: vmovups %ymm4, (%rsi)
1638 ; XOP-NEXT: vmovups %ymm5, (%rdx)
1639 ; XOP-NEXT: vmovups %ymm0, (%rcx)
1640 ; XOP-NEXT: vzeroupper
1641 ; XOP-NEXT: retq
1642 %wide.vec = load <24 x i32>, <24 x i32>* %p, align 4
1643 %s1 = shufflevector <24 x i32> %wide.vec, <24 x i32> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
1644 %s2 = shufflevector <24 x i32> %wide.vec, <24 x i32> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
1645 %s3 = shufflevector <24 x i32> %wide.vec, <24 x i32> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
1646 store <8 x i32> %s1, <8 x i32>* %q1, align 4
1647 store <8 x i32> %s2, <8 x i32>* %q2, align 4
1648 store <8 x i32> %s3, <8 x i32>* %q3, align 4
1649 ret void
1650 }
1652 define void @interleave_24i32_in(<24 x i32>* %p, <8 x i32>* %q1, <8 x i32>* %q2, <8 x i32>* %q3) nounwind {
1653 ; SSE2-LABEL: interleave_24i32_in:
1654 ; SSE2: # %bb.0:
1655 ; SSE2-NEXT: movups (%rsi), %xmm1
1656 ; SSE2-NEXT: movups 16(%rsi), %xmm0
1657 ; SSE2-NEXT: movups (%rdx), %xmm8
1658 ; SSE2-NEXT: movups 16(%rdx), %xmm5
1659 ; SSE2-NEXT: movups (%rcx), %xmm3
1660 ; SSE2-NEXT: movups 16(%rcx), %xmm6
1661 ; SSE2-NEXT: movaps %xmm3, %xmm7
1662 ; SSE2-NEXT: shufps {{.*#+}} xmm7 = xmm7[0,1],xmm1[1,3]
1663 ; SSE2-NEXT: movaps %xmm1, %xmm9
1664 ; SSE2-NEXT: unpcklps {{.*#+}} xmm9 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
1665 ; SSE2-NEXT: shufps {{.*#+}} xmm9 = xmm9[0,1],xmm7[0,2]
1666 ; SSE2-NEXT: movaps %xmm5, %xmm7
1667 ; SSE2-NEXT: shufps {{.*#+}} xmm7 = xmm7[3,3],xmm6[3,3]
1668 ; SSE2-NEXT: movaps %xmm6, %xmm4
1669 ; SSE2-NEXT: unpckhps {{.*#+}} xmm4 = xmm4[2],xmm0[2],xmm4[3],xmm0[3]
1670 ; SSE2-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,3],xmm7[0,2]
1671 ; SSE2-NEXT: movaps %xmm0, %xmm7
1672 ; SSE2-NEXT: unpckhpd {{.*#+}} xmm7 = xmm7[1],xmm5[1]
1673 ; SSE2-NEXT: movaps %xmm6, %xmm2
1674 ; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,0],xmm5[1,0]
1675 ; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm7[0,2]
1676 ; SSE2-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,1],xmm0[1,3]
1677 ; SSE2-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
1678 ; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm6[0,2]
1679 ; SSE2-NEXT: movaps %xmm8, %xmm5
1680 ; SSE2-NEXT: shufps {{.*#+}} xmm5 = xmm5[3,3],xmm3[3,3]
1681 ; SSE2-NEXT: movaps %xmm3, %xmm6
1682 ; SSE2-NEXT: unpckhps {{.*#+}} xmm6 = xmm6[2],xmm1[2],xmm6[3],xmm1[3]
1683 ; SSE2-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,3],xmm5[0,2]
1684 ; SSE2-NEXT: unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm8[1]
1685 ; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,0],xmm8[1,0]
1686 ; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm1[0,2]
1687 ; SSE2-NEXT: movups %xmm3, 16(%rdi)
1688 ; SSE2-NEXT: movups %xmm6, 32(%rdi)
1689 ; SSE2-NEXT: movups %xmm0, 48(%rdi)
1690 ; SSE2-NEXT: movups %xmm2, 64(%rdi)
1691 ; SSE2-NEXT: movups %xmm4, 80(%rdi)
1692 ; SSE2-NEXT: movups %xmm9, (%rdi)
1693 ; SSE2-NEXT: retq
1694 ;
1695 ; SSE42-LABEL: interleave_24i32_in:
1696 ; SSE42: # %bb.0:
1697 ; SSE42-NEXT: movdqu (%rsi), %xmm8
1698 ; SSE42-NEXT: movdqu 16(%rsi), %xmm4
1699 ; SSE42-NEXT: movdqu (%rdx), %xmm2
1700 ; SSE42-NEXT: movdqu 16(%rdx), %xmm5
1701 ; SSE42-NEXT: movdqu (%rcx), %xmm3
1702 ; SSE42-NEXT: movdqu 16(%rcx), %xmm6
1703 ; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,0,1,1]
1704 ; SSE42-NEXT: pshufd {{.*#+}} xmm7 = xmm8[0,1,0,1]
1705 ; SSE42-NEXT: pblendw {{.*#+}} xmm7 = xmm7[0,1],xmm1[2,3],xmm7[4,5,6,7]
1706 ; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,1,0,1]
1707 ; SSE42-NEXT: pblendw {{.*#+}} xmm1 = xmm7[0,1,2,3],xmm1[4,5],xmm7[6,7]
1708 ; SSE42-NEXT: pshufd {{.*#+}} xmm7 = xmm4[2,3,2,3]
1709 ; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm6[2,3,2,3]
1710 ; SSE42-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm7[2,3],xmm0[4,5,6,7]
1711 ; SSE42-NEXT: pshufd {{.*#+}} xmm7 = xmm5[2,2,3,3]
1712 ; SSE42-NEXT: pblendw {{.*#+}} xmm7 = xmm0[0,1,2,3],xmm7[4,5],xmm0[6,7]
1713 ; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm5[1,1,2,2]
1714 ; SSE42-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm6[2,3],xmm0[4,5,6,7]
1715 ; SSE42-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm4[4,5],xmm0[6,7]
1716 ; SSE42-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,0,1,1]
1717 ; SSE42-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]
1718 ; SSE42-NEXT: pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5,6,7]
1719 ; SSE42-NEXT: pshufd {{.*#+}} xmm5 = xmm6[0,1,0,1]
1720 ; SSE42-NEXT: pblendw {{.*#+}} xmm5 = xmm4[0,1,2,3],xmm5[4,5],xmm4[6,7]
1721 ; SSE42-NEXT: pshufd {{.*#+}} xmm4 = xmm8[2,3,2,3]
1722 ; SSE42-NEXT: pshufd {{.*#+}} xmm6 = xmm3[2,3,2,3]
1723 ; SSE42-NEXT: pblendw {{.*#+}} xmm6 = xmm6[0,1],xmm4[2,3],xmm6[4,5,6,7]
1724 ; SSE42-NEXT: pshufd {{.*#+}} xmm4 = xmm2[2,2,3,3]
1725 ; SSE42-NEXT: pblendw {{.*#+}} xmm4 = xmm6[0,1,2,3],xmm4[4,5],xmm6[6,7]
1726 ; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,2,2]
1727 ; SSE42-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5,6,7]
1728 ; SSE42-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm8[4,5],xmm2[6,7]
1729 ; SSE42-NEXT: movdqu %xmm2, 16(%rdi)
1730 ; SSE42-NEXT: movdqu %xmm4, 32(%rdi)
1731 ; SSE42-NEXT: movdqu %xmm5, 48(%rdi)
1732 ; SSE42-NEXT: movdqu %xmm0, 64(%rdi)
1733 ; SSE42-NEXT: movdqu %xmm7, 80(%rdi)
1734 ; SSE42-NEXT: movdqu %xmm1, (%rdi)
1735 ; SSE42-NEXT: retq
1736 ;
1737 ; AVX1-LABEL: interleave_24i32_in:
1738 ; AVX1: # %bb.0:
1739 ; AVX1-NEXT: vmovups (%rdx), %xmm0
1740 ; AVX1-NEXT: vmovups (%rsi), %xmm1
1741 ; AVX1-NEXT: vunpckhpd {{.*#+}} xmm2 = xmm1[1],xmm0[1]
1742 ; AVX1-NEXT: vshufps {{.*#+}} xmm2 = xmm0[1,1],xmm2[0,2]
1743 ; AVX1-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1744 ; AVX1-NEXT: vshufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[2,1]
1745 ; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
1746 ; AVX1-NEXT: vbroadcastsd (%rcx), %ymm1
1747 ; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
1748 ; AVX1-NEXT: vmovups 16(%rcx), %xmm1
1749 ; AVX1-NEXT: vmovups 16(%rdx), %xmm2
1750 ; AVX1-NEXT: vshufps {{.*#+}} xmm3 = xmm2[3,0],xmm1[3,0]
1751 ; AVX1-NEXT: vshufps {{.*#+}} xmm3 = xmm1[2,1],xmm3[0,2]
1752 ; AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm1[1,0],xmm2[1,0]
1753 ; AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[2,2]
1754 ; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1
1755 ; AVX1-NEXT: vbroadcastsd 24(%rsi), %ymm2
1756 ; AVX1-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]
1757 ; AVX1-NEXT: vpermilpd {{.*#+}} ymm2 = mem[1,0,2,2]
1758 ; AVX1-NEXT: vpermilpd {{.*#+}} ymm3 = mem[1,1,2,2]
1759 ; AVX1-NEXT: vblendps {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6],ymm2[7]
1760 ; AVX1-NEXT: vpermilps {{.*#+}} ymm3 = mem[0,0,3,3,4,4,7,7]
1761 ; AVX1-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]
1762 ; AVX1-NEXT: vmovups %ymm2, 32(%rdi)
1763 ; AVX1-NEXT: vmovups %ymm1, 64(%rdi)
1764 ; AVX1-NEXT: vmovups %ymm0, (%rdi)
1765 ; AVX1-NEXT: vzeroupper
1766 ; AVX1-NEXT: retq
1767 ;
1768 ; AVX2-SLOW-LABEL: interleave_24i32_in:
1769 ; AVX2-SLOW: # %bb.0:
1770 ; AVX2-SLOW-NEXT: vmovups (%rsi), %ymm0
1771 ; AVX2-SLOW-NEXT: vmovups (%rdx), %ymm1
1772 ; AVX2-SLOW-NEXT: vmovups (%rcx), %ymm2
1773 ; AVX2-SLOW-NEXT: vpermilps {{.*#+}} xmm3 = mem[1,0,2,2]
1774 ; AVX2-SLOW-NEXT: vpermpd {{.*#+}} ymm3 = ymm3[0,1,0,1]
1775 ; AVX2-SLOW-NEXT: vpermpd {{.*#+}} ymm4 = ymm0[0,0,2,1]
1776 ; AVX2-SLOW-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5,6],ymm3[7]
1777 ; AVX2-SLOW-NEXT: vbroadcastsd (%rcx), %ymm4
1778 ; AVX2-SLOW-NEXT: vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]
1779 ; AVX2-SLOW-NEXT: vpermpd {{.*#+}} ymm4 = ymm2[2,1,3,3]
1780 ; AVX2-SLOW-NEXT: vpermilps {{.*#+}} ymm5 = ymm1[1,2,3,3,5,6,7,7]
1781 ; AVX2-SLOW-NEXT: vpermpd {{.*#+}} ymm5 = ymm5[2,2,2,3]
1782 ; AVX2-SLOW-NEXT: vblendps {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5,6],ymm4[7]
1783 ; AVX2-SLOW-NEXT: vbroadcastsd 24(%rsi), %ymm5
1784 ; AVX2-SLOW-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7]
1785 ; AVX2-SLOW-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,1,2,2]
1786 ; AVX2-SLOW-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[1,1,2,2]
1787 ; AVX2-SLOW-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7]
1788 ; AVX2-SLOW-NEXT: vpermilps {{.*#+}} ymm1 = ymm1[0,0,3,3,4,4,7,7]
1789 ; AVX2-SLOW-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
1790 ; AVX2-SLOW-NEXT: vmovups %ymm0, 32(%rdi)
1791 ; AVX2-SLOW-NEXT: vmovups %ymm4, 64(%rdi)
1792 ; AVX2-SLOW-NEXT: vmovups %ymm3, (%rdi)
1793 ; AVX2-SLOW-NEXT: vzeroupper
1794 ; AVX2-SLOW-NEXT: retq
1795 ;
1796 ; AVX2-FAST-ALL-LABEL: interleave_24i32_in:
1797 ; AVX2-FAST-ALL: # %bb.0:
1798 ; AVX2-FAST-ALL-NEXT: vmovups (%rsi), %ymm0
1799 ; AVX2-FAST-ALL-NEXT: vmovups (%rdx), %ymm1
1800 ; AVX2-FAST-ALL-NEXT: vmovups (%rcx), %ymm2
1801 ; AVX2-FAST-ALL-NEXT: vbroadcastf128 {{.*#+}} ymm3 = [1,0,2,2,1,0,2,2]
1802 ; AVX2-FAST-ALL-NEXT: # ymm3 = mem[0,1,0,1]
1803 ; AVX2-FAST-ALL-NEXT: vpermps %ymm1, %ymm3, %ymm3
1804 ; AVX2-FAST-ALL-NEXT: vpermpd {{.*#+}} ymm4 = ymm0[0,0,2,1]
1805 ; AVX2-FAST-ALL-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5,6],ymm3[7]
1806 ; AVX2-FAST-ALL-NEXT: vbroadcastsd (%rcx), %ymm4
1807 ; AVX2-FAST-ALL-NEXT: vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]
1808 ; AVX2-FAST-ALL-NEXT: vmovaps {{.*#+}} ymm4 = <5,u,u,6,u,u,7,u>
1809 ; AVX2-FAST-ALL-NEXT: vpermps %ymm1, %ymm4, %ymm4
1810 ; AVX2-FAST-ALL-NEXT: vpermpd {{.*#+}} ymm5 = ymm2[2,1,3,3]
1811 ; AVX2-FAST-ALL-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0],ymm5[1],ymm4[2,3],ymm5[4],ymm4[5,6],ymm5[7]
1812 ; AVX2-FAST-ALL-NEXT: vbroadcastsd 24(%rsi), %ymm5
1813 ; AVX2-FAST-ALL-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7]
1814 ; AVX2-FAST-ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,1,2,2]
1815 ; AVX2-FAST-ALL-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[1,1,2,2]
1816 ; AVX2-FAST-ALL-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7]
1817 ; AVX2-FAST-ALL-NEXT: vpermilps {{.*#+}} ymm1 = ymm1[0,0,3,3,4,4,7,7]
1818 ; AVX2-FAST-ALL-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
1819 ; AVX2-FAST-ALL-NEXT: vmovups %ymm0, 32(%rdi)
1820 ; AVX2-FAST-ALL-NEXT: vmovups %ymm4, 64(%rdi)
1821 ; AVX2-FAST-ALL-NEXT: vmovups %ymm3, (%rdi)
1822 ; AVX2-FAST-ALL-NEXT: vzeroupper
1823 ; AVX2-FAST-ALL-NEXT: retq
1824 ;
1825 ; AVX2-FAST-PERLANE-LABEL: interleave_24i32_in:
1826 ; AVX2-FAST-PERLANE: # %bb.0:
1827 ; AVX2-FAST-PERLANE-NEXT: vmovups (%rsi), %ymm0
1828 ; AVX2-FAST-PERLANE-NEXT: vmovups (%rdx), %ymm1
1829 ; AVX2-FAST-PERLANE-NEXT: vmovups (%rcx), %ymm2
1830 ; AVX2-FAST-PERLANE-NEXT: vpermilps {{.*#+}} xmm3 = mem[1,0,2,2]
1831 ; AVX2-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm3 = ymm3[0,1,0,1]
1832 ; AVX2-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm4 = ymm0[0,0,2,1]
1833 ; AVX2-FAST-PERLANE-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5,6],ymm3[7]
1834 ; AVX2-FAST-PERLANE-NEXT: vbroadcastsd (%rcx), %ymm4
1835 ; AVX2-FAST-PERLANE-NEXT: vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]
1836 ; AVX2-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm4 = ymm2[2,1,3,3]
1837 ; AVX2-FAST-PERLANE-NEXT: vpermilps {{.*#+}} ymm5 = ymm1[1,2,3,3,5,6,7,7]
1838 ; AVX2-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm5 = ymm5[2,2,2,3]
1839 ; AVX2-FAST-PERLANE-NEXT: vblendps {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5,6],ymm4[7]
1840 ; AVX2-FAST-PERLANE-NEXT: vbroadcastsd 24(%rsi), %ymm5
1841 ; AVX2-FAST-PERLANE-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7]
1842 ; AVX2-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,1,2,2]
1843 ; AVX2-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[1,1,2,2]
1844 ; AVX2-FAST-PERLANE-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7]
1845 ; AVX2-FAST-PERLANE-NEXT: vpermilps {{.*#+}} ymm1 = ymm1[0,0,3,3,4,4,7,7]
1846 ; AVX2-FAST-PERLANE-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
1847 ; AVX2-FAST-PERLANE-NEXT: vmovups %ymm0, 32(%rdi)
1848 ; AVX2-FAST-PERLANE-NEXT: vmovups %ymm4, 64(%rdi)
1849 ; AVX2-FAST-PERLANE-NEXT: vmovups %ymm3, (%rdi)
1850 ; AVX2-FAST-PERLANE-NEXT: vzeroupper
1851 ; AVX2-FAST-PERLANE-NEXT: retq
1852 ;
1853 ; XOP-LABEL: interleave_24i32_in:
1854 ; XOP: # %bb.0:
1855 ; XOP-NEXT: vmovups (%rsi), %ymm0
1856 ; XOP-NEXT: vmovups (%rcx), %ymm1
1857 ; XOP-NEXT: vpermil2ps {{.*#+}} ymm0 = ymm1[2],ymm0[3],ymm1[u,3],ymm0[4],ymm1[u,4],ymm0[5]
1858 ; XOP-NEXT: vmovups (%rdx), %xmm1
1859 ; XOP-NEXT: vmovups (%rsi), %xmm2
1860 ; XOP-NEXT: vunpckhpd {{.*#+}} xmm3 = xmm2[1],xmm1[1]
1861 ; XOP-NEXT: vshufps {{.*#+}} xmm3 = xmm1[1,1],xmm3[0,2]
1862 ; XOP-NEXT: vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
1863 ; XOP-NEXT: vshufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[2,1]
1864 ; XOP-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1
1865 ; XOP-NEXT: vbroadcastsd (%rcx), %ymm2
1866 ; XOP-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]
1867 ; XOP-NEXT: vmovups 16(%rcx), %xmm2
1868 ; XOP-NEXT: vmovups 16(%rdx), %xmm3
1869 ; XOP-NEXT: vshufps {{.*#+}} xmm4 = xmm3[3,0],xmm2[3,0]
1870 ; XOP-NEXT: vshufps {{.*#+}} xmm4 = xmm2[2,1],xmm4[0,2]
1871 ; XOP-NEXT: vshufps {{.*#+}} xmm2 = xmm2[1,0],xmm3[1,0]
1872 ; XOP-NEXT: vshufps {{.*#+}} xmm2 = xmm2[2,0],xmm3[2,2]
1873 ; XOP-NEXT: vinsertf128 $1, %xmm4, %ymm2, %ymm2
1874 ; XOP-NEXT: vbroadcastsd 24(%rsi), %ymm3
1875 ; XOP-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]
1876 ; XOP-NEXT: vpermilps {{.*#+}} ymm3 = mem[0,0,3,3,4,4,7,7]
1877 ; XOP-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm3[2],ymm0[3,4],ymm3[5],ymm0[6,7]
1878 ; XOP-NEXT: vmovups %ymm0, 32(%rdi)
1879 ; XOP-NEXT: vmovups %ymm2, 64(%rdi)
1880 ; XOP-NEXT: vmovups %ymm1, (%rdi)
1881 ; XOP-NEXT: vzeroupper
1882 ; XOP-NEXT: retq
1883 %s1 = load <8 x i32>, <8 x i32>* %q1, align 4
1884 %s2 = load <8 x i32>, <8 x i32>* %q2, align 4
1885 %s3 = load <8 x i32>, <8 x i32>* %q3, align 4
1886 %t1 = shufflevector <8 x i32> %s1, <8 x i32> %s2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
1887 %t2 = shufflevector <8 x i32> %s3, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1888 %interleaved = shufflevector <16 x i32> %t1, <16 x i32> %t2, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
1889 store <24 x i32> %interleaved, <24 x i32>* %p, align 4
1890 ret void
1891 }
1893 ; Repeat each element x 3 of <16 x i8> a0 + a1 to create a <96 x i8>.
1894 define void @splat3_128(<16 x i8> %a0, <16 x i8> %a1, <96 x i8> *%a2) {
1895 ; SSE2-LABEL: splat3_128:
1896 ; SSE2: # %bb.0:
1897 ; SSE2-NEXT: pxor %xmm4, %xmm4
1898 ; SSE2-NEXT: movdqa %xmm0, %xmm3
1899 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
1900 ; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm3[2,3,3,3,4,5,6,7]
1901 ; SSE2-NEXT: pshufhw {{.*#+}} xmm5 = xmm2[0,1,2,3,4,4,4,5]
1902 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,1,0,1]
1903 ; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,1,4,5,6,7]
1904 ; SSE2-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,6,6]
1905 ; SSE2-NEXT: packuswb %xmm5, %xmm2
1906 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15]
1907 ; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,1,0,1]
1908 ; SSE2-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[0,0,0,1,4,5,6,7]
1909 ; SSE2-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,6,6]
1910 ; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
1911 ; SSE2-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[1,1,2,2,4,5,6,7]
1912 ; SSE2-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,7,7,7]
1913 ; SSE2-NEXT: packuswb %xmm5, %xmm3
1914 ; SSE2-NEXT: pshuflw {{.*#+}} xmm5 = xmm0[2,3,3,3,4,5,6,7]
1915 ; SSE2-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,4,4,5]
1916 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
1917 ; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,1,2,2,4,5,6,7]
1918 ; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,7,7,7]
1919 ; SSE2-NEXT: packuswb %xmm0, %xmm5
1920 ; SSE2-NEXT: movdqa %xmm1, %xmm0
1921 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
1922 ; SSE2-NEXT: pshuflw {{.*#+}} xmm6 = xmm0[2,3,3,3,4,5,6,7]
1923 ; SSE2-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,4,4,5]
1924 ; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm0[0,1,0,1]
1925 ; SSE2-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,0,0,1,4,5,6,7]
1926 ; SSE2-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,5,5,6,6]
1927 ; SSE2-NEXT: packuswb %xmm6, %xmm7
1928 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm4[8],xmm1[9],xmm4[9],xmm1[10],xmm4[10],xmm1[11],xmm4[11],xmm1[12],xmm4[12],xmm1[13],xmm4[13],xmm1[14],xmm4[14],xmm1[15],xmm4[15]
1929 ; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[0,1,0,1]
1930 ; SSE2-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,0,0,1,4,5,6,7]
1931 ; SSE2-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,5,6,6]
1932 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
1933 ; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,1,2,2,4,5,6,7]
1934 ; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,7,7,7]
1935 ; SSE2-NEXT: packuswb %xmm4, %xmm0
1936 ; SSE2-NEXT: pshuflw {{.*#+}} xmm4 = xmm1[2,3,3,3,4,5,6,7]
1937 ; SSE2-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,4,4,5]
1938 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
1939 ; SSE2-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,1,2,2,4,5,6,7]
1940 ; SSE2-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,7,7,7]
1941 ; SSE2-NEXT: packuswb %xmm1, %xmm4
1942 ; SSE2-NEXT: movdqa %xmm4, 80(%rdi)
1943 ; SSE2-NEXT: movdqa %xmm0, 64(%rdi)
1944 ; SSE2-NEXT: movdqa %xmm7, 48(%rdi)
1945 ; SSE2-NEXT: movdqa %xmm5, 32(%rdi)
1946 ; SSE2-NEXT: movdqa %xmm3, 16(%rdi)
1947 ; SSE2-NEXT: movdqa %xmm2, (%rdi)
1948 ; SSE2-NEXT: retq
1949 ;
1950 ; SSE42-LABEL: splat3_128:
1951 ; SSE42: # %bb.0:
1952 ; SSE42-NEXT: movdqa {{.*#+}} xmm2 = [0,0,0,1,1,1,2,2,2,3,3,3,4,4,4,5]
1953 ; SSE42-NEXT: movdqa %xmm0, %xmm3
1954 ; SSE42-NEXT: pshufb %xmm2, %xmm3
1955 ; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [5,5,6,6,6,7,7,7,8,8,8,9,9,9,10,10]
1956 ; SSE42-NEXT: movdqa %xmm0, %xmm5
1957 ; SSE42-NEXT: pshufb %xmm4, %xmm5
1958 ; SSE42-NEXT: movdqa {{.*#+}} xmm6 = [10,11,11,11,12,12,12,13,13,13,14,14,14,15,15,15]
1959 ; SSE42-NEXT: pshufb %xmm6, %xmm0
1960 ; SSE42-NEXT: movdqa %xmm1, %xmm7
1961 ; SSE42-NEXT: pshufb %xmm2, %xmm7
1962 ; SSE42-NEXT: movdqa %xmm1, %xmm2
1963 ; SSE42-NEXT: pshufb %xmm4, %xmm2
1964 ; SSE42-NEXT: pshufb %xmm6, %xmm1
1965 ; SSE42-NEXT: movdqa %xmm1, 80(%rdi)
1966 ; SSE42-NEXT: movdqa %xmm2, 64(%rdi)
1967 ; SSE42-NEXT: movdqa %xmm7, 48(%rdi)
1968 ; SSE42-NEXT: movdqa %xmm0, 32(%rdi)
1969 ; SSE42-NEXT: movdqa %xmm5, 16(%rdi)
1970 ; SSE42-NEXT: movdqa %xmm3, (%rdi)
1971 ; SSE42-NEXT: retq
1972 ;
1973 ; AVX1-LABEL: splat3_128:
1974 ; AVX1: # %bb.0:
1975 ; AVX1-NEXT: vpalignr {{.*#+}} xmm2 = xmm1[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
1976 ; AVX1-NEXT: vpalignr {{.*#+}} xmm3 = xmm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
1977 ; AVX1-NEXT: vpalignr {{.*#+}} xmm4 = xmm1[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
1978 ; AVX1-NEXT: vpalignr {{.*#+}} xmm5 = xmm0[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
1979 ; AVX1-NEXT: vpalignr {{.*#+}} xmm6 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
1980 ; AVX1-NEXT: vpalignr {{.*#+}} xmm7 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
1981 ; AVX1-NEXT: vpalignr {{.*#+}} xmm3 = xmm5[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
1982 ; AVX1-NEXT: vpalignr {{.*#+}} xmm2 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
1983 ; AVX1-NEXT: vpalignr {{.*#+}} xmm5 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm5[0,1,2,3,4]
1984 ; AVX1-NEXT: vpalignr {{.*#+}} xmm4 = xmm1[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
1985 ; AVX1-NEXT: vpalignr {{.*#+}} xmm1 = xmm7[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
1986 ; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm6[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
1987 ; AVX1-NEXT: vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
1988 ; AVX1-NEXT: vpalignr {{.*#+}} xmm3 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm5[0,1,2,3,4]
1989 ; AVX1-NEXT: vpalignr {{.*#+}} xmm4 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm7[0,1,2,3,4]
1990 ; AVX1-NEXT: vpalignr {{.*#+}} xmm5 = xmm5[5,6,7,8,9,10,11,12,13,14,15],xmm6[0,1,2,3,4]
1991 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm6 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
1992 ; AVX1-NEXT: vpshufb %xmm6, %xmm3, %xmm3
1993 ; AVX1-NEXT: vpshufb %xmm6, %xmm0, %xmm0
1994 ; AVX1-NEXT: vpshufb %xmm6, %xmm5, %xmm5
1995 ; AVX1-NEXT: vpshufb %xmm6, %xmm1, %xmm1
1996 ; AVX1-NEXT: vpshufb %xmm6, %xmm2, %xmm2
1997 ; AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm4
1998 ; AVX1-NEXT: vmovdqa %xmm4, 80(%rdi)
1999 ; AVX1-NEXT: vmovdqa %xmm2, 64(%rdi)
2000 ; AVX1-NEXT: vmovdqa %xmm1, 48(%rdi)
2001 ; AVX1-NEXT: vmovdqa %xmm5, 32(%rdi)
2002 ; AVX1-NEXT: vmovdqa %xmm3, 16(%rdi)
2003 ; AVX1-NEXT: vmovdqa %xmm0, (%rdi)
2004 ; AVX1-NEXT: retq
2005 ;
2006 ; AVX2-LABEL: splat3_128:
2007 ; AVX2: # %bb.0:
2008 ; AVX2-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
2009 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
2010 ; AVX2-NEXT: vpalignr {{.*#+}} ymm1 = ymm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,22,23,24,25,26,27,28,29,30,31,16,17,18,19,20,21]
2011 ; AVX2-NEXT: vpalignr {{.*#+}} ymm2 = ymm0[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25,26]
2012 ; AVX2-NEXT: vpalignr {{.*#+}} ymm3 = ymm1[5,6,7,8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4],ymm1[21,22,23,24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20]
2013 ; AVX2-NEXT: vpalignr {{.*#+}} ymm1 = ymm2[5,6,7,8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4],ymm2[21,22,23,24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20]
2014 ; AVX2-NEXT: vpalignr {{.*#+}} ymm2 = ymm0[5,6,7,8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4],ymm0[21,22,23,24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20]
2015 ; AVX2-NEXT: vpalignr {{.*#+}} ymm0 = ymm3[5,6,7,8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4],ymm3[21,22,23,24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20]
2016 ; AVX2-NEXT: vpalignr {{.*#+}} ymm1 = ymm1[5,6,7,8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4],ymm1[21,22,23,24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20]
2017 ; AVX2-NEXT: vpalignr {{.*#+}} ymm2 = ymm2[5,6,7,8,9,10,11,12,13,14,15],ymm3[0,1,2,3,4],ymm2[21,22,23,24,25,26,27,28,29,30,31],ymm3[16,17,18,19,20]
2018 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm3
2019 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
2020 ; AVX2-NEXT: vpshufb %ymm4, %ymm3, %ymm3
2021 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
2022 ; AVX2-NEXT: vpshufb %ymm4, %ymm0, %ymm0
2023 ; AVX2-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm1[2,3],ymm2[2,3]
2024 ; AVX2-NEXT: vpshufb %ymm4, %ymm1, %ymm1
2025 ; AVX2-NEXT: vmovdqa %ymm1, 64(%rdi)
2026 ; AVX2-NEXT: vmovdqa %ymm0, 32(%rdi)
2027 ; AVX2-NEXT: vmovdqa %ymm3, (%rdi)
2028 ; AVX2-NEXT: vzeroupper
2029 ; AVX2-NEXT: retq
2030 ;
2031 ; XOP-LABEL: splat3_128:
2032 ; XOP: # %bb.0:
2033 ; XOP-NEXT: vpalignr {{.*#+}} xmm2 = xmm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
2034 ; XOP-NEXT: vpalignr {{.*#+}} xmm3 = xmm1[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
2035 ; XOP-NEXT: vpalignr {{.*#+}} xmm4 = xmm0[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
2036 ; XOP-NEXT: vpalignr {{.*#+}} xmm5 = xmm1[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
2037 ; XOP-NEXT: vpalignr {{.*#+}} xmm6 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
2038 ; XOP-NEXT: vpalignr {{.*#+}} xmm7 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
2039 ; XOP-NEXT: vpalignr {{.*#+}} xmm8 = xmm5[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
2040 ; XOP-NEXT: vpalignr {{.*#+}} xmm2 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
2041 ; XOP-NEXT: vpalignr {{.*#+}} xmm5 = xmm1[5,6,7,8,9,10,11,12,13,14,15],xmm5[0,1,2,3,4]
2042 ; XOP-NEXT: vpalignr {{.*#+}} xmm4 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
2043 ; XOP-NEXT: vmovdqa {{.*#+}} xmm3 = [5,16,11,6,17,12,7,18,13,8,19,14,9,20,15,10]
2044 ; XOP-NEXT: vpperm %xmm3, %xmm4, %xmm2, %xmm2
2045 ; XOP-NEXT: vpperm %xmm3, %xmm0, %xmm7, %xmm0
2046 ; XOP-NEXT: vpperm %xmm3, %xmm7, %xmm4, %xmm4
2047 ; XOP-NEXT: vpperm %xmm3, %xmm1, %xmm6, %xmm1
2048 ; XOP-NEXT: vpperm %xmm3, %xmm5, %xmm8, %xmm7
2049 ; XOP-NEXT: vpperm %xmm3, %xmm6, %xmm5, %xmm3
2050 ; XOP-NEXT: vmovdqa %xmm3, 80(%rdi)
2051 ; XOP-NEXT: vmovdqa %xmm7, 64(%rdi)
2052 ; XOP-NEXT: vmovdqa %xmm1, 48(%rdi)
2053 ; XOP-NEXT: vmovdqa %xmm4, 32(%rdi)
2054 ; XOP-NEXT: vmovdqa %xmm2, 16(%rdi)
2055 ; XOP-NEXT: vmovdqa %xmm0, (%rdi)
2056 ; XOP-NEXT: retq
2057 %1 = shufflevector <16 x i8> %a0, <16 x i8> %a1, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
2058 %2 = shufflevector <16 x i8> %a0, <16 x i8> %a1, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
2059 %3 = shufflevector <64 x i8> %1, <64 x i8> %2, <96 x i32> <i32 0, i32 32, i32 64, i32 1, i32 33, i32 65, i32 2, i32 34, i32 66, i32 3, i32 35, i32 67, i32 4, i32 36, i32 68, i32 5, i32 37, i32 69, i32 6, i32 38, i32 70, i32 7, i32 39, i32 71, i32 8, i32 40, i32 72, i32 9, i32 41, i32 73, i32 10, i32 42, i32 74, i32 11, i32 43, i32 75, i32 12, i32 44, i32 76, i32 13, i32 45, i32 77, i32 14, i32 46, i32 78, i32 15, i32 47, i32 79, i32 16, i32 48, i32 80, i32 17, i32 49, i32 81, i32 18, i32 50, i32 82, i32 19, i32 51, i32 83, i32 20, i32 52, i32 84, i32 21, i32 53, i32 85, i32 22, i32 54, i32 86, i32 23, i32 55, i32 87, i32 24, i32 56, i32 88, i32 25, i32 57, i32 89, i32 26, i32 58, i32 90, i32 27, i32 59, i32 91, i32 28, i32 60, i32 92, i32 29, i32 61, i32 93, i32 30, i32 62, i32 94, i32 31, i32 63, i32 95>
2060 store <96 x i8> %3, <96 x i8>* %a2
2061 ret void
2062 }
2064 ; Repeat each element x 3 of <32 x i8> a0 to create a <96 x i8>.
2065 define void @splat3_256(<32 x i8> %a0, <96 x i8> *%a1) {
2066 ; SSE2-LABEL: splat3_256:
2067 ; SSE2: # %bb.0:
2068 ; SSE2-NEXT: pxor %xmm4, %xmm4
2069 ; SSE2-NEXT: movdqa %xmm0, %xmm3
2070 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
2071 ; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm3[2,3,3,3,4,5,6,7]
2072 ; SSE2-NEXT: pshufhw {{.*#+}} xmm5 = xmm2[0,1,2,3,4,4,4,5]
2073 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,1,0,1]
2074 ; SSE2-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,1,4,5,6,7]
2075 ; SSE2-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,6,6]
2076 ; SSE2-NEXT: packuswb %xmm5, %xmm2
2077 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15]
2078 ; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,1,0,1]
2079 ; SSE2-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[0,0,0,1,4,5,6,7]
2080 ; SSE2-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,6,6]
2081 ; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
2082 ; SSE2-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[1,1,2,2,4,5,6,7]
2083 ; SSE2-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,7,7,7]
2084 ; SSE2-NEXT: packuswb %xmm5, %xmm3
2085 ; SSE2-NEXT: pshuflw {{.*#+}} xmm5 = xmm0[2,3,3,3,4,5,6,7]
2086 ; SSE2-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,4,4,5]
2087 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
2088 ; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,1,2,2,4,5,6,7]
2089 ; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,7,7,7]
2090 ; SSE2-NEXT: packuswb %xmm0, %xmm5
2091 ; SSE2-NEXT: movdqa %xmm1, %xmm0
2092 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
2093 ; SSE2-NEXT: pshuflw {{.*#+}} xmm6 = xmm0[2,3,3,3,4,5,6,7]
2094 ; SSE2-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,4,4,5]
2095 ; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm0[0,1,0,1]
2096 ; SSE2-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,0,0,1,4,5,6,7]
2097 ; SSE2-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,5,5,6,6]
2098 ; SSE2-NEXT: packuswb %xmm6, %xmm7
2099 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm4[8],xmm1[9],xmm4[9],xmm1[10],xmm4[10],xmm1[11],xmm4[11],xmm1[12],xmm4[12],xmm1[13],xmm4[13],xmm1[14],xmm4[14],xmm1[15],xmm4[15]
2100 ; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[0,1,0,1]
2101 ; SSE2-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,0,0,1,4,5,6,7]
2102 ; SSE2-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,5,6,6]
2103 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
2104 ; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,1,2,2,4,5,6,7]
2105 ; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,7,7,7]
2106 ; SSE2-NEXT: packuswb %xmm4, %xmm0
2107 ; SSE2-NEXT: pshuflw {{.*#+}} xmm4 = xmm1[2,3,3,3,4,5,6,7]
2108 ; SSE2-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,4,4,5]
2109 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
2110 ; SSE2-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,1,2,2,4,5,6,7]
2111 ; SSE2-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,7,7,7]
2112 ; SSE2-NEXT: packuswb %xmm1, %xmm4
2113 ; SSE2-NEXT: movdqa %xmm4, 80(%rdi)
2114 ; SSE2-NEXT: movdqa %xmm0, 64(%rdi)
2115 ; SSE2-NEXT: movdqa %xmm7, 48(%rdi)
2116 ; SSE2-NEXT: movdqa %xmm5, 32(%rdi)
2117 ; SSE2-NEXT: movdqa %xmm3, 16(%rdi)
2118 ; SSE2-NEXT: movdqa %xmm2, (%rdi)
2119 ; SSE2-NEXT: retq
2120 ;
2121 ; SSE42-LABEL: splat3_256:
2122 ; SSE42: # %bb.0:
2123 ; SSE42-NEXT: movdqa {{.*#+}} xmm2 = [0,0,0,1,1,1,2,2,2,3,3,3,4,4,4,5]
2124 ; SSE42-NEXT: movdqa %xmm0, %xmm3
2125 ; SSE42-NEXT: pshufb %xmm2, %xmm3
2126 ; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [5,5,6,6,6,7,7,7,8,8,8,9,9,9,10,10]
2127 ; SSE42-NEXT: movdqa %xmm0, %xmm5
2128 ; SSE42-NEXT: pshufb %xmm4, %xmm5
2129 ; SSE42-NEXT: movdqa {{.*#+}} xmm6 = [10,11,11,11,12,12,12,13,13,13,14,14,14,15,15,15]
2130 ; SSE42-NEXT: pshufb %xmm6, %xmm0
2131 ; SSE42-NEXT: movdqa %xmm1, %xmm7
2132 ; SSE42-NEXT: pshufb %xmm2, %xmm7
2133 ; SSE42-NEXT: movdqa %xmm1, %xmm2
2134 ; SSE42-NEXT: pshufb %xmm4, %xmm2
2135 ; SSE42-NEXT: pshufb %xmm6, %xmm1
2136 ; SSE42-NEXT: movdqa %xmm1, 80(%rdi)
2137 ; SSE42-NEXT: movdqa %xmm2, 64(%rdi)
2138 ; SSE42-NEXT: movdqa %xmm7, 48(%rdi)
2139 ; SSE42-NEXT: movdqa %xmm0, 32(%rdi)
2140 ; SSE42-NEXT: movdqa %xmm5, 16(%rdi)
2141 ; SSE42-NEXT: movdqa %xmm3, (%rdi)
2142 ; SSE42-NEXT: retq
2143 ;
2144 ; AVX1-LABEL: splat3_256:
2145 ; AVX1: # %bb.0:
2146 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
2147 ; AVX1-NEXT: vpalignr {{.*#+}} xmm2 = xmm1[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
2148 ; AVX1-NEXT: vpalignr {{.*#+}} xmm3 = xmm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
2149 ; AVX1-NEXT: vpalignr {{.*#+}} xmm4 = xmm1[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
2150 ; AVX1-NEXT: vpalignr {{.*#+}} xmm5 = xmm0[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
2151 ; AVX1-NEXT: vpalignr {{.*#+}} xmm6 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
2152 ; AVX1-NEXT: vpalignr {{.*#+}} xmm7 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
2153 ; AVX1-NEXT: vpalignr {{.*#+}} xmm3 = xmm5[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
2154 ; AVX1-NEXT: vpalignr {{.*#+}} xmm2 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
2155 ; AVX1-NEXT: vpalignr {{.*#+}} xmm5 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm5[0,1,2,3,4]
2156 ; AVX1-NEXT: vpalignr {{.*#+}} xmm4 = xmm1[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
2157 ; AVX1-NEXT: vpalignr {{.*#+}} xmm1 = xmm7[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
2158 ; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm6[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
2159 ; AVX1-NEXT: vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
2160 ; AVX1-NEXT: vpalignr {{.*#+}} xmm3 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm5[0,1,2,3,4]
2161 ; AVX1-NEXT: vpalignr {{.*#+}} xmm4 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm7[0,1,2,3,4]
2162 ; AVX1-NEXT: vpalignr {{.*#+}} xmm5 = xmm5[5,6,7,8,9,10,11,12,13,14,15],xmm6[0,1,2,3,4]
2163 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm6 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
2164 ; AVX1-NEXT: vpshufb %xmm6, %xmm3, %xmm3
2165 ; AVX1-NEXT: vpshufb %xmm6, %xmm0, %xmm0
2166 ; AVX1-NEXT: vpshufb %xmm6, %xmm5, %xmm5
2167 ; AVX1-NEXT: vpshufb %xmm6, %xmm1, %xmm1
2168 ; AVX1-NEXT: vpshufb %xmm6, %xmm2, %xmm2
2169 ; AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm4
2170 ; AVX1-NEXT: vmovdqa %xmm4, 80(%rdi)
2171 ; AVX1-NEXT: vmovdqa %xmm2, 64(%rdi)
2172 ; AVX1-NEXT: vmovdqa %xmm1, 48(%rdi)
2173 ; AVX1-NEXT: vmovdqa %xmm5, 32(%rdi)
2174 ; AVX1-NEXT: vmovdqa %xmm3, 16(%rdi)
2175 ; AVX1-NEXT: vmovdqa %xmm0, (%rdi)
2176 ; AVX1-NEXT: vzeroupper
2177 ; AVX1-NEXT: retq
2178 ;
2179 ; AVX2-LABEL: splat3_256:
2180 ; AVX2: # %bb.0:
2181 ; AVX2-NEXT: vpalignr {{.*#+}} ymm1 = ymm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,22,23,24,25,26,27,28,29,30,31,16,17,18,19,20,21]
2182 ; AVX2-NEXT: vpalignr {{.*#+}} ymm2 = ymm0[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25,26]
2183 ; AVX2-NEXT: vpalignr {{.*#+}} ymm3 = ymm1[5,6,7,8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4],ymm1[21,22,23,24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20]
2184 ; AVX2-NEXT: vpalignr {{.*#+}} ymm1 = ymm2[5,6,7,8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4],ymm2[21,22,23,24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20]
2185 ; AVX2-NEXT: vpalignr {{.*#+}} ymm2 = ymm0[5,6,7,8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4],ymm0[21,22,23,24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20]
2186 ; AVX2-NEXT: vpalignr {{.*#+}} ymm0 = ymm3[5,6,7,8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4],ymm3[21,22,23,24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20]
2187 ; AVX2-NEXT: vpalignr {{.*#+}} ymm1 = ymm1[5,6,7,8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4],ymm1[21,22,23,24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20]
2188 ; AVX2-NEXT: vpalignr {{.*#+}} ymm2 = ymm2[5,6,7,8,9,10,11,12,13,14,15],ymm3[0,1,2,3,4],ymm2[21,22,23,24,25,26,27,28,29,30,31],ymm3[16,17,18,19,20]
2189 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm3
2190 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
2191 ; AVX2-NEXT: vpshufb %ymm4, %ymm3, %ymm3
2192 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
2193 ; AVX2-NEXT: vpshufb %ymm4, %ymm0, %ymm0
2194 ; AVX2-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm1[2,3],ymm2[2,3]
2195 ; AVX2-NEXT: vpshufb %ymm4, %ymm1, %ymm1
2196 ; AVX2-NEXT: vmovdqa %ymm1, 64(%rdi)
2197 ; AVX2-NEXT: vmovdqa %ymm0, 32(%rdi)
2198 ; AVX2-NEXT: vmovdqa %ymm3, (%rdi)
2199 ; AVX2-NEXT: vzeroupper
2200 ; AVX2-NEXT: retq
2201 ;
2202 ; XOP-LABEL: splat3_256:
2203 ; XOP: # %bb.0:
2204 ; XOP-NEXT: vpalignr {{.*#+}} xmm1 = xmm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
2205 ; XOP-NEXT: vextractf128 $1, %ymm0, %xmm2
2206 ; XOP-NEXT: vpalignr {{.*#+}} xmm3 = xmm2[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
2207 ; XOP-NEXT: vpalignr {{.*#+}} xmm4 = xmm0[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
2208 ; XOP-NEXT: vpalignr {{.*#+}} xmm5 = xmm2[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
2209 ; XOP-NEXT: vpalignr {{.*#+}} xmm6 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
2210 ; XOP-NEXT: vpalignr {{.*#+}} xmm7 = xmm1[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
2211 ; XOP-NEXT: vpalignr {{.*#+}} xmm8 = xmm5[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
2212 ; XOP-NEXT: vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
2213 ; XOP-NEXT: vpalignr {{.*#+}} xmm5 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm5[0,1,2,3,4]
2214 ; XOP-NEXT: vpalignr {{.*#+}} xmm4 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
2215 ; XOP-NEXT: vmovdqa {{.*#+}} xmm3 = [5,16,11,6,17,12,7,18,13,8,19,14,9,20,15,10]
2216 ; XOP-NEXT: vpperm %xmm3, %xmm4, %xmm1, %xmm1
2217 ; XOP-NEXT: vpperm %xmm3, %xmm0, %xmm7, %xmm0
2218 ; XOP-NEXT: vpperm %xmm3, %xmm7, %xmm4, %xmm4
2219 ; XOP-NEXT: vpperm %xmm3, %xmm2, %xmm6, %xmm2
2220 ; XOP-NEXT: vpperm %xmm3, %xmm5, %xmm8, %xmm7
2221 ; XOP-NEXT: vpperm %xmm3, %xmm6, %xmm5, %xmm3
2222 ; XOP-NEXT: vmovdqa %xmm3, 80(%rdi)
2223 ; XOP-NEXT: vmovdqa %xmm7, 64(%rdi)
2224 ; XOP-NEXT: vmovdqa %xmm2, 48(%rdi)
2225 ; XOP-NEXT: vmovdqa %xmm4, 32(%rdi)
2226 ; XOP-NEXT: vmovdqa %xmm1, 16(%rdi)
2227 ; XOP-NEXT: vmovdqa %xmm0, (%rdi)
2228 ; XOP-NEXT: vzeroupper
2229 ; XOP-NEXT: retq
2230 %1 = shufflevector <32 x i8> %a0, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
2231 %2 = shufflevector <32 x i8> %a0, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
2232 %3 = shufflevector <64 x i8> %1, <64 x i8> %2, <96 x i32> <i32 0, i32 32, i32 64, i32 1, i32 33, i32 65, i32 2, i32 34, i32 66, i32 3, i32 35, i32 67, i32 4, i32 36, i32 68, i32 5, i32 37, i32 69, i32 6, i32 38, i32 70, i32 7, i32 39, i32 71, i32 8, i32 40, i32 72, i32 9, i32 41, i32 73, i32 10, i32 42, i32 74, i32 11, i32 43, i32 75, i32 12, i32 44, i32 76, i32 13, i32 45, i32 77, i32 14, i32 46, i32 78, i32 15, i32 47, i32 79, i32 16, i32 48, i32 80, i32 17, i32 49, i32 81, i32 18, i32 50, i32 82, i32 19, i32 51, i32 83, i32 20, i32 52, i32 84, i32 21, i32 53, i32 85, i32 22, i32 54, i32 86, i32 23, i32 55, i32 87, i32 24, i32 56, i32 88, i32 25, i32 57, i32 89, i32 26, i32 58, i32 90, i32 27, i32 59, i32 91, i32 28, i32 60, i32 92, i32 29, i32 61, i32 93, i32 30, i32 62, i32 94, i32 31, i32 63, i32 95>
2233 store <96 x i8> %3, <96 x i8>* %a1
2234 ret void
2235 }
2237 ; D79987
2238 define <16 x i32> @splat_v3i32(<3 x i32>* %ptr) {
2239 ; SSE2-LABEL: splat_v3i32:
2240 ; SSE2: # %bb.0:
2241 ; SSE2-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero
2242 ; SSE2-NEXT: xorps %xmm1, %xmm1
2243 ; SSE2-NEXT: movss {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
2244 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,0,1]
2245 ; SSE2-NEXT: andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
2246 ; SSE2-NEXT: xorps %xmm1, %xmm1
2247 ; SSE2-NEXT: xorps %xmm3, %xmm3
2248 ; SSE2-NEXT: retq
2249 ;
2250 ; SSE42-LABEL: splat_v3i32:
2251 ; SSE42: # %bb.0:
2252 ; SSE42-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
2253 ; SSE42-NEXT: pxor %xmm1, %xmm1
2254 ; SSE42-NEXT: pxor %xmm2, %xmm2
2255 ; SSE42-NEXT: pblendw {{.*#+}} xmm2 = xmm0[0,1],xmm2[2,3,4,5,6,7]
2256 ; SSE42-NEXT: pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5,6,7]
2257 ; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,0,1]
2258 ; SSE42-NEXT: pxor %xmm1, %xmm1
2259 ; SSE42-NEXT: xorps %xmm3, %xmm3
2260 ; SSE42-NEXT: retq
2261 ;
2262 ; AVX1-LABEL: splat_v3i32:
2263 ; AVX1: # %bb.0:
2264 ; AVX1-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
2265 ; AVX1-NEXT: vpinsrd $2, 8(%rdi), %xmm0, %xmm1
2266 ; AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
2267 ; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3,4,5,6,7]
2268 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
2269 ; AVX1-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4,5,6,7]
2270 ; AVX1-NEXT: retq
2271 ;
2272 ; AVX2-SLOW-LABEL: splat_v3i32:
2273 ; AVX2-SLOW: # %bb.0:
2274 ; AVX2-SLOW-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero
2275 ; AVX2-SLOW-NEXT: vxorps %xmm2, %xmm2, %xmm2
2276 ; AVX2-SLOW-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0],ymm1[1],ymm2[2,3,4,5,6,7]
2277 ; AVX2-SLOW-NEXT: vbroadcastss %xmm1, %xmm1
2278 ; AVX2-SLOW-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4,5,6,7]
2279 ; AVX2-SLOW-NEXT: retq
2280 ;
2281 ; AVX2-FAST-LABEL: splat_v3i32:
2282 ; AVX2-FAST: # %bb.0:
2283 ; AVX2-FAST-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
2284 ; AVX2-FAST-NEXT: vpxor %xmm0, %xmm0, %xmm0
2285 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6,7]
2286 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,ymm1[0,1,2,3],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
2287 ; AVX2-FAST-NEXT: retq
2288 ;
2289 ; XOP-LABEL: splat_v3i32:
2290 ; XOP: # %bb.0:
2291 ; XOP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
2292 ; XOP-NEXT: vpinsrd $2, 8(%rdi), %xmm0, %xmm1
2293 ; XOP-NEXT: vxorps %xmm2, %xmm2, %xmm2
2294 ; XOP-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3,4,5,6,7]
2295 ; XOP-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
2296 ; XOP-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4,5,6,7]
2297 ; XOP-NEXT: retq
2298 %1 = load <3 x i32>, <3 x i32>* %ptr, align 1
2299 %2 = shufflevector <3 x i32> %1, <3 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
2300 %3 = shufflevector <16 x i32> <i32 0, i32 undef, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 undef, i32 0, i32 0, i32 0, i32 0, i32 0>, <16 x i32> %2, <16 x i32> <i32 0, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 11, i32 12, i32 13, i32 14, i32 15>
2301 ret <16 x i32 > %3
2302 }
2304 define <2 x double> @wrongorder(<4 x double> %A, <8 x double>* %P) #0 {
2305 ; SSE2-LABEL: wrongorder:
2306 ; SSE2: # %bb.0:
2307 ; SSE2-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0,0]
2308 ; SSE2-NEXT: movaps %xmm0, 48(%rdi)
2309 ; SSE2-NEXT: movaps %xmm0, 32(%rdi)
2310 ; SSE2-NEXT: movaps %xmm0, 16(%rdi)
2311 ; SSE2-NEXT: movaps %xmm0, (%rdi)
2312 ; SSE2-NEXT: retq
2313 ;
2314 ; SSE42-LABEL: wrongorder:
2315 ; SSE42: # %bb.0:
2316 ; SSE42-NEXT: movddup {{.*#+}} xmm0 = xmm0[0,0]
2317 ; SSE42-NEXT: movapd %xmm0, 48(%rdi)
2318 ; SSE42-NEXT: movapd %xmm0, 32(%rdi)
2319 ; SSE42-NEXT: movapd %xmm0, 16(%rdi)
2320 ; SSE42-NEXT: movapd %xmm0, (%rdi)
2321 ; SSE42-NEXT: retq
2322 ;
2323 ; AVX1-LABEL: wrongorder:
2324 ; AVX1: # %bb.0:
2325 ; AVX1-NEXT: vmovddup {{.*#+}} xmm0 = xmm0[0,0]
2326 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm1
2327 ; AVX1-NEXT: vmovaps %ymm1, 32(%rdi)
2328 ; AVX1-NEXT: vmovaps %ymm1, (%rdi)
2329 ; AVX1-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
2330 ; AVX1-NEXT: vzeroupper
2331 ; AVX1-NEXT: retq
2332 ;
2333 ; AVX2-LABEL: wrongorder:
2334 ; AVX2: # %bb.0:
2335 ; AVX2-NEXT: vbroadcastsd %xmm0, %ymm0
2336 ; AVX2-NEXT: vmovaps %ymm0, 32(%rdi)
2337 ; AVX2-NEXT: vmovaps %ymm0, (%rdi)
2338 ; AVX2-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
2339 ; AVX2-NEXT: vzeroupper
2340 ; AVX2-NEXT: retq
2341 ;
2342 ; XOP-LABEL: wrongorder:
2343 ; XOP: # %bb.0:
2344 ; XOP-NEXT: vmovddup {{.*#+}} xmm0 = xmm0[0,0]
2345 ; XOP-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm1
2346 ; XOP-NEXT: vmovaps %ymm1, 32(%rdi)
2347 ; XOP-NEXT: vmovaps %ymm1, (%rdi)
2348 ; XOP-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
2349 ; XOP-NEXT: vzeroupper
2350 ; XOP-NEXT: retq
2351 %shuffle = shufflevector <4 x double> %A, <4 x double> %A, <8 x i32> zeroinitializer
2352 store <8 x double> %shuffle, <8 x double>* %P, align 64
2353 %m2 = load <8 x double>, <8 x double>* %P, align 64
2354 store <8 x double> %m2, <8 x double>* %P, align 64
2355 %m3 = load <8 x double>, <8 x double>* %P, align 64
2356 %m4 = shufflevector <8 x double> %m3, <8 x double> undef, <2 x i32> <i32 2, i32 0>
2357 ret <2 x double> %m4
2358 }
2360 define void @PR41097() {
2361 ; SSE2-LABEL: PR41097:
2362 ; SSE2: # %bb.0:
2363 ; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2364 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
2365 ; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
2366 ; SSE2-NEXT: psrad $24, %xmm0
2367 ; SSE2-NEXT: pxor %xmm1, %xmm1
2368 ; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
2369 ; SSE2-NEXT: movdqu %xmm0, (%rax)
2370 ; SSE2-NEXT: retq
2371 ;
2372 ; SSE42-LABEL: PR41097:
2373 ; SSE42: # %bb.0:
2374 ; SSE42-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2375 ; SSE42-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0,3,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
2376 ; SSE42-NEXT: pmovsxbd %xmm0, %xmm0
2377 ; SSE42-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
2378 ; SSE42-NEXT: movdqu %xmm0, (%rax)
2379 ; SSE42-NEXT: retq
2380 ;
2381 ; AVX-LABEL: PR41097:
2382 ; AVX: # %bb.0:
2383 ; AVX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2384 ; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,3,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
2385 ; AVX-NEXT: vpmovsxbd %xmm0, %xmm0
2386 ; AVX-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
2387 ; AVX-NEXT: vmovdqu %xmm0, (%rax)
2388 ; AVX-NEXT: retq
2389 ;
2390 ; XOP-LABEL: PR41097:
2391 ; XOP: # %bb.0:
2392 ; XOP-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
2393 ; XOP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,3,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
2394 ; XOP-NEXT: vpmovsxbd %xmm0, %xmm0
2395 ; XOP-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
2396 ; XOP-NEXT: vmovdqu %xmm0, (%rax)
2397 ; XOP-NEXT: retq
2398 %wide.vec = load <6 x i8>, <6 x i8>* undef, align 1
2399 %strided.vec = shufflevector <6 x i8> %wide.vec, <6 x i8> undef, <2 x i32> <i32 0, i32 3>
2400 %tmp = sext <2 x i8> %strided.vec to <2 x i32>
2401 %tmp7 = zext <2 x i32> %tmp to <2 x i64>
2402 store <2 x i64> %tmp7, <2 x i64>* undef, align 8
2403 ret void
2404 }
2406 define void @D107009(<64 x i32>* %input, <64 x i32>* %output) {
2407 ; SSE-LABEL: D107009:
2408 ; SSE: # %bb.0:
2409 ; SSE-NEXT: movdqa 16(%rdi), %xmm0
2410 ; SSE-NEXT: movdqa 80(%rdi), %xmm1
2411 ; SSE-NEXT: movdqa 144(%rdi), %xmm2
2412 ; SSE-NEXT: movdqa 208(%rdi), %xmm3
2413 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
2414 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
2415 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
2416 ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[1],mem[1]
2417 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[1],mem[1]
2418 ; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
2419 ; SSE-NEXT: psrld $16, %xmm2
2420 ; SSE-NEXT: psrld $16, %xmm0
2421 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
2422 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
2423 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm0[3,3,3,3]
2424 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
2425 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm2[2,3,2,3]
2426 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm2[3,3,3,3]
2427 ; SSE-NEXT: movdqa %xmm0, 128(%rsi)
2428 ; SSE-NEXT: movdqa %xmm2, 144(%rsi)
2429 ; SSE-NEXT: movdqa %xmm0, 16(%rsi)
2430 ; SSE-NEXT: movdqa %xmm7, 240(%rsi)
2431 ; SSE-NEXT: movdqa %xmm6, 208(%rsi)
2432 ; SSE-NEXT: movdqa %xmm5, 176(%rsi)
2433 ; SSE-NEXT: movdqa %xmm4, 112(%rsi)
2434 ; SSE-NEXT: movdqa %xmm3, 80(%rsi)
2435 ; SSE-NEXT: movdqa %xmm1, 48(%rsi)
2436 ; SSE-NEXT: retq
2437 ;
2438 ; AVX1-LABEL: D107009:
2439 ; AVX1: # %bb.0:
2440 ; AVX1-NEXT: vmovups 96(%rdi), %ymm0
2441 ; AVX1-NEXT: vmovups (%rdi), %ymm1
2442 ; AVX1-NEXT: vmovups 128(%rdi), %ymm2
2443 ; AVX1-NEXT: vmovups 224(%rdi), %ymm3
2444 ; AVX1-NEXT: vunpcklpd {{.*#+}} ymm3 = ymm3[0],mem[0],ymm3[2],mem[2]
2445 ; AVX1-NEXT: vunpcklps {{.*#+}} ymm2 = ymm2[0],mem[0],ymm2[1],mem[1],ymm2[4],mem[4],ymm2[5],mem[5]
2446 ; AVX1-NEXT: vshufps {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,0],ymm2[4,5],ymm3[6,4]
2447 ; AVX1-NEXT: vunpcklps {{.*#+}} ymm1 = ymm1[0],mem[0],ymm1[1],mem[1],ymm1[4],mem[4],ymm1[5],mem[5]
2448 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
2449 ; AVX1-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[2],mem[2]
2450 ; AVX1-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,1,2,0,4,5,6,4]
2451 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
2452 ; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
2453 ; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0
2454 ; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm1
2455 ; AVX1-NEXT: vpsrld $16, %xmm1, %xmm1
2456 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
2457 ; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,1,1,3]
2458 ; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,1,3,3]
2459 ; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2
2460 ; AVX1-NEXT: vmovshdup {{.*#+}} ymm3 = ymm1[1,1,3,3,5,5,7,7]
2461 ; AVX1-NEXT: vpermilps {{.*#+}} ymm4 = ymm1[3,3,3,3,7,7,7,7]
2462 ; AVX1-NEXT: vpermilpd {{.*#+}} ymm5 = ymm1[0,0,3,2]
2463 ; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm0[1,1,1,1]
2464 ; AVX1-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6
2465 ; AVX1-NEXT: vpshufd {{.*#+}} xmm7 = xmm0[3,3,3,3]
2466 ; AVX1-NEXT: vinsertf128 $1, %xmm7, %ymm0, %ymm7
2467 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
2468 ; AVX1-NEXT: vmovups %ymm0, (%rsi)
2469 ; AVX1-NEXT: vmovups %ymm1, 128(%rsi)
2470 ; AVX1-NEXT: vmovups %ymm7, 96(%rsi)
2471 ; AVX1-NEXT: vmovups %ymm6, 32(%rsi)
2472 ; AVX1-NEXT: vmovupd %ymm5, 192(%rsi)
2473 ; AVX1-NEXT: vmovups %ymm4, 224(%rsi)
2474 ; AVX1-NEXT: vmovups %ymm3, 160(%rsi)
2475 ; AVX1-NEXT: vmovups %ymm2, 64(%rsi)
2476 ; AVX1-NEXT: vzeroupper
2477 ; AVX1-NEXT: retq
2478 ;
2479 ; AVX2-LABEL: D107009:
2480 ; AVX2: # %bb.0:
2481 ; AVX2-NEXT: vmovdqu 64(%rdi), %ymm0
2482 ; AVX2-NEXT: vmovdqu 128(%rdi), %ymm1
2483 ; AVX2-NEXT: vmovdqu 192(%rdi), %ymm2
2484 ; AVX2-NEXT: vpunpckldq {{.*#+}} ymm2 = ymm2[0],mem[0],ymm2[1],mem[1],ymm2[4],mem[4],ymm2[5],mem[5]
2485 ; AVX2-NEXT: vpunpckldq {{.*#+}} ymm1 = ymm1[0],mem[0],ymm1[1],mem[1],ymm1[4],mem[4],ymm1[5],mem[5]
2486 ; AVX2-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[2],ymm2[2]
2487 ; AVX2-NEXT: vpbroadcastd 48(%rdi), %xmm2
2488 ; AVX2-NEXT: vpbroadcastd 16(%rdi), %ymm3
2489 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
2490 ; AVX2-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5]
2491 ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]
2492 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
2493 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
2494 ; AVX2-NEXT: vpsrld $16, %ymm0, %ymm0
2495 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
2496 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
2497 ; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
2498 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
2499 ; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
2500 ; AVX2-NEXT: vpbroadcastd %xmm0, %ymm4
2501 ; AVX2-NEXT: vpshufd {{.*#+}} ymm5 = ymm0[3,3,3,3,7,7,7,7]
2502 ; AVX2-NEXT: vpshufd {{.*#+}} ymm6 = ymm0[2,3,2,3,6,7,6,7]
2503 ; AVX2-NEXT: vpshufd {{.*#+}} ymm7 = ymm0[1,1,1,1,5,5,5,5]
2504 ; AVX2-NEXT: vmovdqu %ymm0, 128(%rsi)
2505 ; AVX2-NEXT: vmovdqu %ymm7, 160(%rsi)
2506 ; AVX2-NEXT: vmovdqu %ymm6, 192(%rsi)
2507 ; AVX2-NEXT: vmovdqu %ymm5, 224(%rsi)
2508 ; AVX2-NEXT: vmovdqu %ymm4, (%rsi)
2509 ; AVX2-NEXT: vmovdqu %ymm3, 64(%rsi)
2510 ; AVX2-NEXT: vmovdqu %ymm2, 32(%rsi)
2511 ; AVX2-NEXT: vmovdqu %ymm1, 96(%rsi)
2512 ; AVX2-NEXT: vzeroupper
2513 ; AVX2-NEXT: retq
2514 ;
2515 ; XOP-LABEL: D107009:
2516 ; XOP: # %bb.0:
2517 ; XOP-NEXT: vmovups 96(%rdi), %ymm0
2518 ; XOP-NEXT: vmovups (%rdi), %ymm1
2519 ; XOP-NEXT: vmovups 128(%rdi), %ymm2
2520 ; XOP-NEXT: vmovups 224(%rdi), %ymm3
2521 ; XOP-NEXT: vunpcklpd {{.*#+}} ymm3 = ymm3[0],mem[0],ymm3[2],mem[2]
2522 ; XOP-NEXT: vunpcklps {{.*#+}} ymm2 = ymm2[0],mem[0],ymm2[1],mem[1],ymm2[4],mem[4],ymm2[5],mem[5]
2523 ; XOP-NEXT: vshufps {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,0],ymm2[4,5],ymm3[6,4]
2524 ; XOP-NEXT: vunpcklps {{.*#+}} ymm1 = ymm1[0],mem[0],ymm1[1],mem[1],ymm1[4],mem[4],ymm1[5],mem[5]
2525 ; XOP-NEXT: vextractf128 $1, %ymm1, %xmm1
2526 ; XOP-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[2],mem[2]
2527 ; XOP-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,1,2,0,4,5,6,4]
2528 ; XOP-NEXT: vextractf128 $1, %ymm0, %xmm0
2529 ; XOP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
2530 ; XOP-NEXT: vpsrld $16, %xmm0, %xmm0
2531 ; XOP-NEXT: vextractf128 $1, %ymm2, %xmm1
2532 ; XOP-NEXT: vpsrld $16, %xmm1, %xmm1
2533 ; XOP-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
2534 ; XOP-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,1,1,3]
2535 ; XOP-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,1,3,3]
2536 ; XOP-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2
2537 ; XOP-NEXT: vmovshdup {{.*#+}} ymm3 = ymm1[1,1,3,3,5,5,7,7]
2538 ; XOP-NEXT: vpermilps {{.*#+}} ymm4 = ymm1[3,3,3,3,7,7,7,7]
2539 ; XOP-NEXT: vpermilpd {{.*#+}} ymm5 = ymm1[0,0,3,2]
2540 ; XOP-NEXT: vpshufd {{.*#+}} xmm6 = xmm0[1,1,1,1]
2541 ; XOP-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6
2542 ; XOP-NEXT: vpshufd {{.*#+}} xmm7 = xmm0[3,3,3,3]
2543 ; XOP-NEXT: vinsertf128 $1, %xmm7, %ymm0, %ymm7
2544 ; XOP-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
2545 ; XOP-NEXT: vmovups %ymm0, (%rsi)
2546 ; XOP-NEXT: vmovups %ymm1, 128(%rsi)
2547 ; XOP-NEXT: vmovups %ymm7, 96(%rsi)
2548 ; XOP-NEXT: vmovups %ymm6, 32(%rsi)
2549 ; XOP-NEXT: vmovupd %ymm5, 192(%rsi)
2550 ; XOP-NEXT: vmovups %ymm4, 224(%rsi)
2551 ; XOP-NEXT: vmovups %ymm3, 160(%rsi)
2552 ; XOP-NEXT: vmovups %ymm2, 64(%rsi)
2553 ; XOP-NEXT: vzeroupper
2554 ; XOP-NEXT: retq
2555 %i = load <64 x i32>, <64 x i32>* %input, align 16
2556 %i2 = shufflevector <64 x i32> %i, <64 x i32> poison, <8 x i32> <i32 4, i32 12, i32 20, i32 28, i32 36, i32 44, i32 52, i32 60>
2557 %i3 = lshr <8 x i32> %i2, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
2558 %i4 = add <8 x i32> zeroinitializer, %i3
2559 %i5 = shufflevector <8 x i32> %i4, <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
2560 %i6 = shufflevector <16 x i32> %i5, <16 x i32> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
2561 %i7 = shufflevector <32 x i32> poison, <32 x i32> %i6, <64 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 40, i32 48, i32 56, i32 1, i32 9, i32 17, i32 25, i32 33, i32 41, i32 49, i32 57, i32 2, i32 10, i32 18, i32 26, i32 34, i32 42, i32 50, i32 58, i32 3, i32 11, i32 19, i32 27, i32 35, i32 43, i32 51, i32 59, i32 4, i32 12, i32 20, i32 28, i32 36, i32 44, i32 52, i32 60, i32 5, i32 13, i32 21, i32 29, i32 37, i32 45, i32 53, i32 61, i32 6, i32 14, i32 22, i32 30, i32 38, i32 46, i32 54, i32 62, i32 7, i32 15, i32 23, i32 31, i32 39, i32 47, i32 55, i32 63>
2562 store <64 x i32> %i7, <64 x i32>* %output, align 16
2563 ret void
2564 }