| blob | bff39467c1eb2ff134bf8c4d2cbaef235a0e1982 |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
3 ; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2OR512,AVX2
4 ; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=+avx512f -mattr=+avx512bw | FileCheck %s --check-prefixes=AVX,AVX2OR512,AVX512
6 define <4 x double> @load_factorf64_4(<16 x double>* %ptr) {
7 ; AVX-LABEL: load_factorf64_4:
8 ; AVX: # %bb.0:
9 ; AVX-NEXT: vmovupd (%rdi), %ymm0
10 ; AVX-NEXT: vmovupd 32(%rdi), %ymm1
11 ; AVX-NEXT: vmovupd 64(%rdi), %ymm2
12 ; AVX-NEXT: vmovupd 96(%rdi), %ymm3
13 ; AVX-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm0[0,1],ymm2[0,1]
14 ; AVX-NEXT: vperm2f128 {{.*#+}} ymm5 = ymm1[0,1],ymm3[0,1]
15 ; AVX-NEXT: vhaddpd %ymm5, %ymm4, %ymm4
16 ; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3]
17 ; AVX-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm1[2,3],ymm3[2,3]
18 ; AVX-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
19 ; AVX-NEXT: vaddpd %ymm2, %ymm4, %ymm2
20 ; AVX-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
21 ; AVX-NEXT: vaddpd %ymm0, %ymm2, %ymm0
22 ; AVX-NEXT: retq
23 %wide.vec = load <16 x double>, <16 x double>* %ptr, align 16
24 %strided.v0 = shufflevector <16 x double> %wide.vec, <16 x double> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
25 %strided.v1 = shufflevector <16 x double> %wide.vec, <16 x double> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
26 %strided.v2 = shufflevector <16 x double> %wide.vec, <16 x double> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
27 %strided.v3 = shufflevector <16 x double> %wide.vec, <16 x double> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
28 %add1 = fadd <4 x double> %strided.v0, %strided.v1
29 %add2 = fadd <4 x double> %add1, %strided.v2
30 %add3 = fadd <4 x double> %add2, %strided.v3
31 ret <4 x double> %add3
32 }
34 define <4 x double> @load_factorf64_2(<16 x double>* %ptr) {
35 ; AVX-LABEL: load_factorf64_2:
36 ; AVX: # %bb.0:
37 ; AVX-NEXT: vmovupd (%rdi), %ymm0
38 ; AVX-NEXT: vmovupd 32(%rdi), %ymm1
39 ; AVX-NEXT: vmovupd 64(%rdi), %ymm2
40 ; AVX-NEXT: vmovupd 96(%rdi), %ymm3
41 ; AVX-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm0[0,1],ymm2[0,1]
42 ; AVX-NEXT: vperm2f128 {{.*#+}} ymm5 = ymm1[0,1],ymm3[0,1]
43 ; AVX-NEXT: vunpcklpd {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[2],ymm5[2]
44 ; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3]
45 ; AVX-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm1[2,3],ymm3[2,3]
46 ; AVX-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
47 ; AVX-NEXT: vmulpd %ymm0, %ymm4, %ymm0
48 ; AVX-NEXT: retq
49 %wide.vec = load <16 x double>, <16 x double>* %ptr, align 16
50 %strided.v0 = shufflevector <16 x double> %wide.vec, <16 x double> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
51 %strided.v3 = shufflevector <16 x double> %wide.vec, <16 x double> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
52 %mul = fmul <4 x double> %strided.v0, %strided.v3
53 ret <4 x double> %mul
54 }
56 define <4 x double> @load_factorf64_1(<16 x double>* %ptr) {
57 ; AVX-LABEL: load_factorf64_1:
58 ; AVX: # %bb.0:
59 ; AVX-NEXT: vmovupd (%rdi), %ymm0
60 ; AVX-NEXT: vmovupd 32(%rdi), %ymm1
61 ; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],mem[0,1]
62 ; AVX-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm1[0,1],mem[0,1]
63 ; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
64 ; AVX-NEXT: vmulpd %ymm0, %ymm0, %ymm0
65 ; AVX-NEXT: retq
66 %wide.vec = load <16 x double>, <16 x double>* %ptr, align 16
67 %strided.v0 = shufflevector <16 x double> %wide.vec, <16 x double> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
68 %strided.v3 = shufflevector <16 x double> %wide.vec, <16 x double> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
69 %mul = fmul <4 x double> %strided.v0, %strided.v3
70 ret <4 x double> %mul
71 }
73 define <4 x i64> @load_factori64_4(<16 x i64>* %ptr) {
74 ; AVX1-LABEL: load_factori64_4:
75 ; AVX1: # %bb.0:
76 ; AVX1-NEXT: vmovups (%rdi), %ymm0
77 ; AVX1-NEXT: vmovups 32(%rdi), %ymm1
78 ; AVX1-NEXT: vmovups 64(%rdi), %ymm2
79 ; AVX1-NEXT: vmovups 96(%rdi), %ymm3
80 ; AVX1-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm0[0,1],ymm2[0,1]
81 ; AVX1-NEXT: vperm2f128 {{.*#+}} ymm5 = ymm1[0,1],ymm3[0,1]
82 ; AVX1-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3]
83 ; AVX1-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm1[2,3],ymm3[2,3]
84 ; AVX1-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm4[0],ymm5[0],ymm4[2],ymm5[2]
85 ; AVX1-NEXT: vunpcklpd {{.*#+}} ymm3 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
86 ; AVX1-NEXT: vunpckhpd {{.*#+}} ymm4 = ymm4[1],ymm5[1],ymm4[3],ymm5[3]
87 ; AVX1-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
88 ; AVX1-NEXT: vextractf128 $1, %ymm4, %xmm1
89 ; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm5
90 ; AVX1-NEXT: vpaddq %xmm3, %xmm4, %xmm4
91 ; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm3
92 ; AVX1-NEXT: vpaddq %xmm3, %xmm1, %xmm1
93 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
94 ; AVX1-NEXT: vpaddq %xmm3, %xmm1, %xmm1
95 ; AVX1-NEXT: vpaddq %xmm1, %xmm5, %xmm1
96 ; AVX1-NEXT: vpaddq %xmm0, %xmm4, %xmm0
97 ; AVX1-NEXT: vpaddq %xmm0, %xmm2, %xmm0
98 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
99 ; AVX1-NEXT: retq
100 ;
101 ; AVX2OR512-LABEL: load_factori64_4:
102 ; AVX2OR512: # %bb.0:
103 ; AVX2OR512-NEXT: vmovdqu (%rdi), %ymm0
104 ; AVX2OR512-NEXT: vmovdqu 32(%rdi), %ymm1
105 ; AVX2OR512-NEXT: vmovdqu 64(%rdi), %ymm2
106 ; AVX2OR512-NEXT: vmovdqu 96(%rdi), %ymm3
107 ; AVX2OR512-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm0[0,1],ymm2[0,1]
108 ; AVX2OR512-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm1[0,1],ymm3[0,1]
109 ; AVX2OR512-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3]
110 ; AVX2OR512-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm1[2,3],ymm3[2,3]
111 ; AVX2OR512-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm4[0],ymm5[0],ymm4[2],ymm5[2]
112 ; AVX2OR512-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
113 ; AVX2OR512-NEXT: vpunpckhqdq {{.*#+}} ymm4 = ymm4[1],ymm5[1],ymm4[3],ymm5[3]
114 ; AVX2OR512-NEXT: vpaddq %ymm3, %ymm4, %ymm3
115 ; AVX2OR512-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
116 ; AVX2OR512-NEXT: vpaddq %ymm0, %ymm3, %ymm0
117 ; AVX2OR512-NEXT: vpaddq %ymm0, %ymm2, %ymm0
118 ; AVX2OR512-NEXT: retq
119 %wide.vec = load <16 x i64>, <16 x i64>* %ptr, align 16
120 %strided.v0 = shufflevector <16 x i64> %wide.vec, <16 x i64> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
121 %strided.v1 = shufflevector <16 x i64> %wide.vec, <16 x i64> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
122 %strided.v2 = shufflevector <16 x i64> %wide.vec, <16 x i64> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
123 %strided.v3 = shufflevector <16 x i64> %wide.vec, <16 x i64> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
124 %add1 = add <4 x i64> %strided.v0, %strided.v1
125 %add2 = add <4 x i64> %add1, %strided.v2
126 %add3 = add <4 x i64> %add2, %strided.v3
127 ret <4 x i64> %add3
128 }
130 define void @store_factorf64_4(<16 x double>* %ptr, <4 x double> %v0, <4 x double> %v1, <4 x double> %v2, <4 x double> %v3) {
131 ; AVX1-LABEL: store_factorf64_4:
132 ; AVX1: # %bb.0:
133 ; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm4
134 ; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm5
135 ; AVX1-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3]
136 ; AVX1-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm1[2,3],ymm3[2,3]
137 ; AVX1-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm4[0],ymm5[0],ymm4[2],ymm5[2]
138 ; AVX1-NEXT: vunpcklpd {{.*#+}} ymm3 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
139 ; AVX1-NEXT: vunpckhpd {{.*#+}} ymm4 = ymm4[1],ymm5[1],ymm4[3],ymm5[3]
140 ; AVX1-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
141 ; AVX1-NEXT: vmovups %ymm0, 96(%rdi)
142 ; AVX1-NEXT: vmovups %ymm3, 64(%rdi)
143 ; AVX1-NEXT: vmovups %ymm4, 32(%rdi)
144 ; AVX1-NEXT: vmovups %ymm2, (%rdi)
145 ; AVX1-NEXT: vzeroupper
146 ; AVX1-NEXT: retq
147 ;
148 ; AVX2-LABEL: store_factorf64_4:
149 ; AVX2: # %bb.0:
150 ; AVX2-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm4
151 ; AVX2-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm5
152 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3]
153 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm1[2,3],ymm3[2,3]
154 ; AVX2-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm4[0],ymm5[0],ymm4[2],ymm5[2]
155 ; AVX2-NEXT: vunpcklpd {{.*#+}} ymm3 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
156 ; AVX2-NEXT: vunpckhpd {{.*#+}} ymm4 = ymm4[1],ymm5[1],ymm4[3],ymm5[3]
157 ; AVX2-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
158 ; AVX2-NEXT: vmovups %ymm0, 96(%rdi)
159 ; AVX2-NEXT: vmovups %ymm3, 64(%rdi)
160 ; AVX2-NEXT: vmovups %ymm4, 32(%rdi)
161 ; AVX2-NEXT: vmovups %ymm2, (%rdi)
162 ; AVX2-NEXT: vzeroupper
163 ; AVX2-NEXT: retq
164 ;
165 ; AVX512-LABEL: store_factorf64_4:
166 ; AVX512: # %bb.0:
167 ; AVX512-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm4
168 ; AVX512-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm5
169 ; AVX512-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3]
170 ; AVX512-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm1[2,3],ymm3[2,3]
171 ; AVX512-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm4[0],ymm5[0],ymm4[2],ymm5[2]
172 ; AVX512-NEXT: vunpcklpd {{.*#+}} ymm3 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
173 ; AVX512-NEXT: vunpckhpd {{.*#+}} ymm4 = ymm4[1],ymm5[1],ymm4[3],ymm5[3]
174 ; AVX512-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
175 ; AVX512-NEXT: vinsertf64x4 $1, %ymm4, %zmm2, %zmm1
176 ; AVX512-NEXT: vinsertf64x4 $1, %ymm0, %zmm3, %zmm0
177 ; AVX512-NEXT: vmovups %zmm0, 64(%rdi)
178 ; AVX512-NEXT: vmovups %zmm1, (%rdi)
179 ; AVX512-NEXT: vzeroupper
180 ; AVX512-NEXT: retq
181 %s0 = shufflevector <4 x double> %v0, <4 x double> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
182 %s1 = shufflevector <4 x double> %v2, <4 x double> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
183 %interleaved.vec = shufflevector <8 x double> %s0, <8 x double> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
184 store <16 x double> %interleaved.vec, <16 x double>* %ptr, align 16
185 ret void
186 }
188 define void @store_factori64_4(<16 x i64>* %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3) {
189 ; AVX1-LABEL: store_factori64_4:
190 ; AVX1: # %bb.0:
191 ; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm4
192 ; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm5
193 ; AVX1-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3]
194 ; AVX1-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm1[2,3],ymm3[2,3]
195 ; AVX1-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm4[0],ymm5[0],ymm4[2],ymm5[2]
196 ; AVX1-NEXT: vunpcklpd {{.*#+}} ymm3 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
197 ; AVX1-NEXT: vunpckhpd {{.*#+}} ymm4 = ymm4[1],ymm5[1],ymm4[3],ymm5[3]
198 ; AVX1-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
199 ; AVX1-NEXT: vmovups %ymm0, 96(%rdi)
200 ; AVX1-NEXT: vmovups %ymm3, 64(%rdi)
201 ; AVX1-NEXT: vmovups %ymm4, 32(%rdi)
202 ; AVX1-NEXT: vmovups %ymm2, (%rdi)
203 ; AVX1-NEXT: vzeroupper
204 ; AVX1-NEXT: retq
205 ;
206 ; AVX2-LABEL: store_factori64_4:
207 ; AVX2: # %bb.0:
208 ; AVX2-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm4
209 ; AVX2-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm5
210 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3]
211 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm1[2,3],ymm3[2,3]
212 ; AVX2-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm4[0],ymm5[0],ymm4[2],ymm5[2]
213 ; AVX2-NEXT: vunpcklpd {{.*#+}} ymm3 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
214 ; AVX2-NEXT: vunpckhpd {{.*#+}} ymm4 = ymm4[1],ymm5[1],ymm4[3],ymm5[3]
215 ; AVX2-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
216 ; AVX2-NEXT: vmovups %ymm0, 96(%rdi)
217 ; AVX2-NEXT: vmovups %ymm3, 64(%rdi)
218 ; AVX2-NEXT: vmovups %ymm4, 32(%rdi)
219 ; AVX2-NEXT: vmovups %ymm2, (%rdi)
220 ; AVX2-NEXT: vzeroupper
221 ; AVX2-NEXT: retq
222 ;
223 ; AVX512-LABEL: store_factori64_4:
224 ; AVX512: # %bb.0:
225 ; AVX512-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm4
226 ; AVX512-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm5
227 ; AVX512-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3]
228 ; AVX512-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm1[2,3],ymm3[2,3]
229 ; AVX512-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm4[0],ymm5[0],ymm4[2],ymm5[2]
230 ; AVX512-NEXT: vunpcklpd {{.*#+}} ymm3 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
231 ; AVX512-NEXT: vunpckhpd {{.*#+}} ymm4 = ymm4[1],ymm5[1],ymm4[3],ymm5[3]
232 ; AVX512-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
233 ; AVX512-NEXT: vinsertf64x4 $1, %ymm4, %zmm2, %zmm1
234 ; AVX512-NEXT: vinsertf64x4 $1, %ymm0, %zmm3, %zmm0
235 ; AVX512-NEXT: vmovups %zmm0, 64(%rdi)
236 ; AVX512-NEXT: vmovups %zmm1, (%rdi)
237 ; AVX512-NEXT: vzeroupper
238 ; AVX512-NEXT: retq
239 %s0 = shufflevector <4 x i64> %v0, <4 x i64> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
240 %s1 = shufflevector <4 x i64> %v2, <4 x i64> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
241 %interleaved.vec = shufflevector <8 x i64> %s0, <8 x i64> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
242 store <16 x i64> %interleaved.vec, <16 x i64>* %ptr, align 16
243 ret void
244 }
247 define void @interleaved_store_vf32_i8_stride4(<32 x i8> %x1, <32 x i8> %x2, <32 x i8> %x3, <32 x i8> %x4, <128 x i8>* %p) {
248 ; AVX1-LABEL: interleaved_store_vf32_i8_stride4:
249 ; AVX1: # %bb.0:
250 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm4 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
251 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
252 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm6
253 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm7 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
254 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm8 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
255 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm9 = xmm6[8],xmm5[8],xmm6[9],xmm5[9],xmm6[10],xmm5[10],xmm6[11],xmm5[11],xmm6[12],xmm5[12],xmm6[13],xmm5[13],xmm6[14],xmm5[14],xmm6[15],xmm5[15]
256 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm5 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
257 ; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm6
258 ; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm0
259 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3],xmm0[4],xmm6[4],xmm0[5],xmm6[5],xmm0[6],xmm6[6],xmm0[7],xmm6[7]
260 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
261 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm6[8],xmm0[9],xmm6[9],xmm0[10],xmm6[10],xmm0[11],xmm6[11],xmm0[12],xmm6[12],xmm0[13],xmm6[13],xmm0[14],xmm6[14],xmm0[15],xmm6[15]
262 ; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm7[0],xmm1[0],xmm7[1],xmm1[1],xmm7[2],xmm1[2],xmm7[3],xmm1[3]
263 ; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
264 ; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm6, %ymm3
265 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm7[4],xmm1[4],xmm7[5],xmm1[5],xmm7[6],xmm1[6],xmm7[7],xmm1[7]
266 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
267 ; AVX1-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm3[2,3],ymm1[2,3]
268 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]
269 ; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm0[0],xmm9[1],xmm0[1],xmm9[2],xmm0[2],xmm9[3],xmm0[3]
270 ; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm8[0],xmm2[0],xmm8[1],xmm2[1],xmm8[2],xmm2[2],xmm8[3],xmm2[3]
271 ; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm5, %ymm4
272 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm9[4],xmm0[4],xmm9[5],xmm0[5],xmm9[6],xmm0[6],xmm9[7],xmm0[7]
273 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
274 ; AVX1-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm4[2,3],ymm0[2,3]
275 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm8[4],xmm2[4],xmm8[5],xmm2[5],xmm8[6],xmm2[6],xmm8[7],xmm2[7]
276 ; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm6, %ymm3
277 ; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm5, %ymm2
278 ; AVX1-NEXT: vmovaps %ymm2, 32(%rdi)
279 ; AVX1-NEXT: vmovaps %ymm3, (%rdi)
280 ; AVX1-NEXT: vmovaps %ymm0, 96(%rdi)
281 ; AVX1-NEXT: vmovaps %ymm1, 64(%rdi)
282 ; AVX1-NEXT: vzeroupper
283 ; AVX1-NEXT: retq
284 ;
285 ; AVX2-LABEL: interleaved_store_vf32_i8_stride4:
286 ; AVX2: # %bb.0:
287 ; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm4 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
288 ; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm0 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
289 ; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[4],ymm3[4],ymm2[5],ymm3[5],ymm2[6],ymm3[6],ymm2[7],ymm3[7],ymm2[16],ymm3[16],ymm2[17],ymm3[17],ymm2[18],ymm3[18],ymm2[19],ymm3[19],ymm2[20],ymm3[20],ymm2[21],ymm3[21],ymm2[22],ymm3[22],ymm2[23],ymm3[23]
290 ; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15],ymm2[24],ymm3[24],ymm2[25],ymm3[25],ymm2[26],ymm3[26],ymm2[27],ymm3[27],ymm2[28],ymm3[28],ymm2[29],ymm3[29],ymm2[30],ymm3[30],ymm2[31],ymm3[31]
291 ; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm3 = ymm4[0],ymm1[0],ymm4[1],ymm1[1],ymm4[2],ymm1[2],ymm4[3],ymm1[3],ymm4[8],ymm1[8],ymm4[9],ymm1[9],ymm4[10],ymm1[10],ymm4[11],ymm1[11]
292 ; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm1 = ymm4[4],ymm1[4],ymm4[5],ymm1[5],ymm4[6],ymm1[6],ymm4[7],ymm1[7],ymm4[12],ymm1[12],ymm4[13],ymm1[13],ymm4[14],ymm1[14],ymm4[15],ymm1[15]
293 ; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm4 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11]
294 ; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15]
295 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm3, %ymm2
296 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm4, %ymm5
297 ; AVX2-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm3[2,3],ymm1[2,3]
298 ; AVX2-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm4[2,3],ymm0[2,3]
299 ; AVX2-NEXT: vmovdqa %ymm0, 96(%rdi)
300 ; AVX2-NEXT: vmovdqa %ymm1, 64(%rdi)
301 ; AVX2-NEXT: vmovdqa %ymm5, 32(%rdi)
302 ; AVX2-NEXT: vmovdqa %ymm2, (%rdi)
303 ; AVX2-NEXT: vzeroupper
304 ; AVX2-NEXT: retq
305 ;
306 ; AVX512-LABEL: interleaved_store_vf32_i8_stride4:
307 ; AVX512: # %bb.0:
308 ; AVX512-NEXT: vpunpcklbw {{.*#+}} ymm4 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
309 ; AVX512-NEXT: vpunpckhbw {{.*#+}} ymm0 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
310 ; AVX512-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[4],ymm3[4],ymm2[5],ymm3[5],ymm2[6],ymm3[6],ymm2[7],ymm3[7],ymm2[16],ymm3[16],ymm2[17],ymm3[17],ymm2[18],ymm3[18],ymm2[19],ymm3[19],ymm2[20],ymm3[20],ymm2[21],ymm3[21],ymm2[22],ymm3[22],ymm2[23],ymm3[23]
311 ; AVX512-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15],ymm2[24],ymm3[24],ymm2[25],ymm3[25],ymm2[26],ymm3[26],ymm2[27],ymm3[27],ymm2[28],ymm3[28],ymm2[29],ymm3[29],ymm2[30],ymm3[30],ymm2[31],ymm3[31]
312 ; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm3 = ymm4[0],ymm1[0],ymm4[1],ymm1[1],ymm4[2],ymm1[2],ymm4[3],ymm1[3],ymm4[8],ymm1[8],ymm4[9],ymm1[9],ymm4[10],ymm1[10],ymm4[11],ymm1[11]
313 ; AVX512-NEXT: vpunpckhwd {{.*#+}} ymm1 = ymm4[4],ymm1[4],ymm4[5],ymm1[5],ymm4[6],ymm1[6],ymm4[7],ymm1[7],ymm4[12],ymm1[12],ymm4[13],ymm1[13],ymm4[14],ymm1[14],ymm4[15],ymm1[15]
314 ; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm4 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11]
315 ; AVX512-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15]
316 ; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm3, %ymm2
317 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm4, %ymm5
318 ; AVX512-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm3[2,3],ymm1[2,3]
319 ; AVX512-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm4[2,3],ymm0[2,3]
320 ; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm2, %zmm2
321 ; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
322 ; AVX512-NEXT: vmovdqa64 %zmm0, 64(%rdi)
323 ; AVX512-NEXT: vmovdqa64 %zmm2, (%rdi)
324 ; AVX512-NEXT: vzeroupper
325 ; AVX512-NEXT: retq
326 %v1 = shufflevector <32 x i8> %x1, <32 x i8> %x2, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
327 %v2 = shufflevector <32 x i8> %x3, <32 x i8> %x4, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
328 %interleaved.vec = shufflevector <64 x i8> %v1, <64 x i8> %v2, <128 x i32> <i32 0, i32 32, i32 64, i32 96, i32 1, i32 33, i32 65, i32 97, i32 2, i32 34, i32 66, i32 98, i32 3, i32 35, i32 67, i32 99, i32 4, i32 36, i32 68, i32 100, i32 5, i32 37, i32 69, i32 101, i32 6, i32 38, i32 70, i32 102, i32 7, i32 39, i32 71, i32 103, i32 8, i32 40, i32 72, i32 104, i32 9, i32 41, i32 73, i32 105, i32 10, i32 42, i32 74, i32 106, i32 11, i32 43, i32 75, i32 107, i32 12, i32 44, i32 76, i32 108, i32 13, i32 45, i32 77, i32 109, i32 14, i32 46, i32 78, i32 110, i32 15, i32 47, i32 79, i32 111, i32 16, i32 48, i32 80, i32 112, i32 17, i32 49, i32 81, i32 113, i32 18, i32 50, i32 82, i32 114, i32 19, i32 51, i32 83, i32 115, i32 20, i32 52, i32 84, i32 116, i32 21, i32 53, i32 85, i32 117, i32 22, i32 54, i32 86, i32 118, i32 23, i32 55, i32 87, i32 119, i32 24, i32 56, i32 88, i32 120, i32 25, i32 57, i32 89, i32 121, i32 26, i32 58, i32 90, i32 122, i32 27, i32 59, i32 91, i32 123, i32 28, i32 60, i32 92, i32 124, i32 29, i32 61, i32 93, i32 125, i32 30, i32 62, i32 94, i32 126, i32 31, i32 63, i32 95, i32 127>
329 store <128 x i8> %interleaved.vec, <128 x i8>* %p
330 ret void
331 }
333 define void @interleaved_store_vf16_i8_stride4(<16 x i8> %x1, <16 x i8> %x2, <16 x i8> %x3, <16 x i8> %x4, <64 x i8>* %p) {
334 ; AVX1-LABEL: interleaved_store_vf16_i8_stride4:
335 ; AVX1: # %bb.0:
336 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm4 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
337 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
338 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
339 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
340 ; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
341 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
342 ; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
343 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
344 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm1
345 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm4, %ymm0
346 ; AVX1-NEXT: vmovaps %ymm0, 32(%rdi)
347 ; AVX1-NEXT: vmovaps %ymm1, (%rdi)
348 ; AVX1-NEXT: vzeroupper
349 ; AVX1-NEXT: retq
350 ;
351 ; AVX2-LABEL: interleaved_store_vf16_i8_stride4:
352 ; AVX2: # %bb.0:
353 ; AVX2-NEXT: vpunpcklbw {{.*#+}} xmm4 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
354 ; AVX2-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
355 ; AVX2-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
356 ; AVX2-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
357 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
358 ; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
359 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
360 ; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
361 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm3, %ymm1
362 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm4, %ymm0
363 ; AVX2-NEXT: vmovdqa %ymm0, 32(%rdi)
364 ; AVX2-NEXT: vmovdqa %ymm1, (%rdi)
365 ; AVX2-NEXT: vzeroupper
366 ; AVX2-NEXT: retq
367 ;
368 ; AVX512-LABEL: interleaved_store_vf16_i8_stride4:
369 ; AVX512: # %bb.0:
370 ; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm4 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
371 ; AVX512-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
372 ; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
373 ; AVX512-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
374 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
375 ; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
376 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
377 ; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
378 ; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm3, %ymm1
379 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm4, %ymm0
380 ; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
381 ; AVX512-NEXT: vmovdqa64 %zmm0, (%rdi)
382 ; AVX512-NEXT: vzeroupper
383 ; AVX512-NEXT: retq
384 %v1 = shufflevector <16 x i8> %x1, <16 x i8> %x2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
385 %v2 = shufflevector <16 x i8> %x3, <16 x i8> %x4, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
386 %interleaved.vec = shufflevector <32 x i8> %v1, <32 x i8> %v2, <64 x i32> <i32 0,i32 16,i32 32,i32 48,i32 1,i32 17,i32 33,i32 49,i32 2,i32 18,i32 34,i32 50,i32 3,i32 19,i32 35,i32 51,i32 4,i32 20,i32 36,i32 52,i32 5,i32 21,i32 37,i32 53,i32 6,i32 22,i32 38,i32 54,i32 7,i32 23,i32 39,i32 55,i32 8,i32 24,i32 40,i32 56,i32 9,i32 25,i32 41,i32 57,i32 10,i32 26,i32 42,i32 58,i32 11,i32 27,i32 43,i32 59,i32 12,i32 28,i32 44,i32 60,i32 13,i32 29,i32 45,i32 61,i32 14,i32 30,i32 46,i32 62,i32 15,i32 31,i32 47,i32 63>
387 store <64 x i8> %interleaved.vec, <64 x i8>* %p
388 ret void
389 }
391 define <8 x i8> @interleaved_load_vf8_i8_stride4(<32 x i8>* %ptr) {
392 ; AVX-LABEL: interleaved_load_vf8_i8_stride4:
393 ; AVX: # %bb.0:
394 ; AVX-NEXT: vmovdqa {{.*#+}} xmm0 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
395 ; AVX-NEXT: vmovdqa (%rdi), %xmm1
396 ; AVX-NEXT: vmovdqa 16(%rdi), %xmm2
397 ; AVX-NEXT: vpshufb %xmm0, %xmm2, %xmm3
398 ; AVX-NEXT: vpshufb %xmm0, %xmm1, %xmm0
399 ; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm0[0],xmm3[0]
400 ; AVX-NEXT: vmovdqa {{.*#+}} xmm5 = [1,1,3,3,5,5,7,7,7,7,3,3,6,6,7,7]
401 ; AVX-NEXT: vpshufb %xmm5, %xmm3, %xmm3
402 ; AVX-NEXT: vpshufb %xmm5, %xmm0, %xmm0
403 ; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
404 ; AVX-NEXT: vpaddw %xmm0, %xmm4, %xmm0
405 ; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [6,7,2,3,14,15,10,11,14,15,10,11,12,13,14,15]
406 ; AVX-NEXT: vpshufb %xmm3, %xmm2, %xmm2
407 ; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm2[1,0,3,2,4,5,6,7]
408 ; AVX-NEXT: vpshufb %xmm3, %xmm1, %xmm1
409 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm1[1,0,3,2,4,5,6,7]
410 ; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
411 ; AVX-NEXT: vmovdqa {{.*#+}} xmm4 = [3,3,1,1,7,7,5,5,1,1,5,5,0,0,1,1]
412 ; AVX-NEXT: vpshufb %xmm4, %xmm2, %xmm2
413 ; AVX-NEXT: vpshufb %xmm4, %xmm1, %xmm1
414 ; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
415 ; AVX-NEXT: vpaddw %xmm3, %xmm1, %xmm1
416 ; AVX-NEXT: vpmullw %xmm1, %xmm0, %xmm0
417 ; AVX-NEXT: retq
418 %wide.vec = load <32 x i8>, <32 x i8>* %ptr, align 16
419 %v1 = shufflevector <32 x i8> %wide.vec, <32 x i8> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
420 %v2 = shufflevector <32 x i8> %wide.vec, <32 x i8> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
421 %v3 = shufflevector <32 x i8> %wide.vec, <32 x i8> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
422 %v4 = shufflevector <32 x i8> %wide.vec, <32 x i8> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
424 %add1 = add <8 x i8> %v1, %v2
425 %add2 = add <8 x i8> %v4, %v3
426 %add3 = mul <8 x i8> %add1, %add2
427 ret <8 x i8> %add3
428 }
430 define <16 x i1> @interleaved_load_vf16_i8_stride4(<64 x i8>* %ptr) {
431 ; AVX1-LABEL: interleaved_load_vf16_i8_stride4:
432 ; AVX1: # %bb.0:
433 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = <u,u,u,u,0,4,8,12,u,u,u,u,u,u,u,u>
434 ; AVX1-NEXT: vmovdqa (%rdi), %xmm0
435 ; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1
436 ; AVX1-NEXT: vmovdqa 32(%rdi), %xmm3
437 ; AVX1-NEXT: vmovdqa 48(%rdi), %xmm4
438 ; AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm5
439 ; AVX1-NEXT: vpshufb %xmm2, %xmm3, %xmm2
440 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
441 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm5 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
442 ; AVX1-NEXT: vpshufb %xmm5, %xmm1, %xmm6
443 ; AVX1-NEXT: vpshufb %xmm5, %xmm0, %xmm5
444 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
445 ; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm5[0,1,2,3],xmm2[4,5,6,7]
446 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm5 = <u,u,u,u,1,5,9,13,u,u,u,u,u,u,u,u>
447 ; AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm6
448 ; AVX1-NEXT: vpshufb %xmm5, %xmm3, %xmm5
449 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
450 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm6 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u>
451 ; AVX1-NEXT: vpshufb %xmm6, %xmm1, %xmm7
452 ; AVX1-NEXT: vpshufb %xmm6, %xmm0, %xmm6
453 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
454 ; AVX1-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1,2,3],xmm5[4,5,6,7]
455 ; AVX1-NEXT: vpcmpeqb %xmm5, %xmm2, %xmm2
456 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm5 = <u,u,u,u,2,6,10,14,u,u,u,u,u,u,u,u>
457 ; AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm6
458 ; AVX1-NEXT: vpshufb %xmm5, %xmm3, %xmm5
459 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
460 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm6 = <2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u>
461 ; AVX1-NEXT: vpshufb %xmm6, %xmm1, %xmm7
462 ; AVX1-NEXT: vpshufb %xmm6, %xmm0, %xmm6
463 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
464 ; AVX1-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1,2,3],xmm5[4,5,6,7]
465 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm6 = <u,u,u,u,3,7,11,15,u,u,u,u,u,u,u,u>
466 ; AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm4
467 ; AVX1-NEXT: vpshufb %xmm6, %xmm3, %xmm3
468 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
469 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = <3,7,11,15,u,u,u,u,u,u,u,u,u,u,u,u>
470 ; AVX1-NEXT: vpshufb %xmm4, %xmm1, %xmm1
471 ; AVX1-NEXT: vpshufb %xmm4, %xmm0, %xmm0
472 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
473 ; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7]
474 ; AVX1-NEXT: vpcmpeqb %xmm0, %xmm5, %xmm0
475 ; AVX1-NEXT: vpxor %xmm0, %xmm2, %xmm0
476 ; AVX1-NEXT: vpxor {{.*}}(%rip), %xmm0, %xmm0
477 ; AVX1-NEXT: retq
478 ;
479 ; AVX2-LABEL: interleaved_load_vf16_i8_stride4:
480 ; AVX2: # %bb.0:
481 ; AVX2-NEXT: vmovdqa (%rdi), %xmm0
482 ; AVX2-NEXT: vmovdqa 16(%rdi), %xmm1
483 ; AVX2-NEXT: vmovdqa 32(%rdi), %xmm2
484 ; AVX2-NEXT: vmovdqa 48(%rdi), %xmm3
485 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm4 = <u,u,u,u,0,4,8,12,u,u,u,u,u,u,u,u>
486 ; AVX2-NEXT: vpshufb %xmm4, %xmm3, %xmm5
487 ; AVX2-NEXT: vpshufb %xmm4, %xmm2, %xmm4
488 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
489 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm5 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
490 ; AVX2-NEXT: vpshufb %xmm5, %xmm1, %xmm6
491 ; AVX2-NEXT: vpshufb %xmm5, %xmm0, %xmm5
492 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
493 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
494 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm5 = <u,u,u,u,1,5,9,13,u,u,u,u,u,u,u,u>
495 ; AVX2-NEXT: vpshufb %xmm5, %xmm3, %xmm6
496 ; AVX2-NEXT: vpshufb %xmm5, %xmm2, %xmm5
497 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
498 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm6 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u>
499 ; AVX2-NEXT: vpshufb %xmm6, %xmm1, %xmm7
500 ; AVX2-NEXT: vpshufb %xmm6, %xmm0, %xmm6
501 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
502 ; AVX2-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]
503 ; AVX2-NEXT: vpcmpeqb %xmm5, %xmm4, %xmm4
504 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm5 = <u,u,u,u,2,6,10,14,u,u,u,u,u,u,u,u>
505 ; AVX2-NEXT: vpshufb %xmm5, %xmm3, %xmm6
506 ; AVX2-NEXT: vpshufb %xmm5, %xmm2, %xmm5
507 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
508 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm6 = <2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u>
509 ; AVX2-NEXT: vpshufb %xmm6, %xmm1, %xmm7
510 ; AVX2-NEXT: vpshufb %xmm6, %xmm0, %xmm6
511 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
512 ; AVX2-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]
513 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm6 = <u,u,u,u,3,7,11,15,u,u,u,u,u,u,u,u>
514 ; AVX2-NEXT: vpshufb %xmm6, %xmm3, %xmm3
515 ; AVX2-NEXT: vpshufb %xmm6, %xmm2, %xmm2
516 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
517 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = <3,7,11,15,u,u,u,u,u,u,u,u,u,u,u,u>
518 ; AVX2-NEXT: vpshufb %xmm3, %xmm1, %xmm1
519 ; AVX2-NEXT: vpshufb %xmm3, %xmm0, %xmm0
520 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
521 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3]
522 ; AVX2-NEXT: vpcmpeqb %xmm0, %xmm5, %xmm0
523 ; AVX2-NEXT: vpxor %xmm0, %xmm4, %xmm0
524 ; AVX2-NEXT: vpxor {{.*}}(%rip), %xmm0, %xmm0
525 ; AVX2-NEXT: retq
526 ;
527 ; AVX512-LABEL: interleaved_load_vf16_i8_stride4:
528 ; AVX512: # %bb.0:
529 ; AVX512-NEXT: vmovdqa (%rdi), %xmm0
530 ; AVX512-NEXT: vmovdqa 16(%rdi), %xmm1
531 ; AVX512-NEXT: vmovdqa 32(%rdi), %xmm2
532 ; AVX512-NEXT: vmovdqa 48(%rdi), %xmm3
533 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm4 = <u,u,u,u,0,4,8,12,u,u,u,u,u,u,u,u>
534 ; AVX512-NEXT: vpshufb %xmm4, %xmm3, %xmm5
535 ; AVX512-NEXT: vpshufb %xmm4, %xmm2, %xmm4
536 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
537 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm5 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
538 ; AVX512-NEXT: vpshufb %xmm5, %xmm1, %xmm6
539 ; AVX512-NEXT: vpshufb %xmm5, %xmm0, %xmm5
540 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
541 ; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm5[0,1],xmm4[2,3]
542 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm5 = <u,u,u,u,1,5,9,13,u,u,u,u,u,u,u,u>
543 ; AVX512-NEXT: vpshufb %xmm5, %xmm3, %xmm6
544 ; AVX512-NEXT: vpshufb %xmm5, %xmm2, %xmm5
545 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
546 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm6 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u>
547 ; AVX512-NEXT: vpshufb %xmm6, %xmm1, %xmm7
548 ; AVX512-NEXT: vpshufb %xmm6, %xmm0, %xmm6
549 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
550 ; AVX512-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]
551 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm6 = <u,u,u,u,2,6,10,14,u,u,u,u,u,u,u,u>
552 ; AVX512-NEXT: vpshufb %xmm6, %xmm3, %xmm7
553 ; AVX512-NEXT: vpshufb %xmm6, %xmm2, %xmm6
554 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
555 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm7 = <2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u>
556 ; AVX512-NEXT: vpshufb %xmm7, %xmm1, %xmm4
557 ; AVX512-NEXT: vpshufb %xmm7, %xmm0, %xmm7
558 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm7[0],xmm4[0],xmm7[1],xmm4[1]
559 ; AVX512-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0,1],xmm6[2,3]
560 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm6 = <u,u,u,u,3,7,11,15,u,u,u,u,u,u,u,u>
561 ; AVX512-NEXT: vpshufb %xmm6, %xmm3, %xmm3
562 ; AVX512-NEXT: vpshufb %xmm6, %xmm2, %xmm2
563 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
564 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm3 = <3,7,11,15,u,u,u,u,u,u,u,u,u,u,u,u>
565 ; AVX512-NEXT: vpshufb %xmm3, %xmm1, %xmm1
566 ; AVX512-NEXT: vpshufb %xmm3, %xmm0, %xmm0
567 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
568 ; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3]
569 ; AVX512-NEXT: vpcmpeqb %zmm5, %zmm8, %k0
570 ; AVX512-NEXT: vpcmpeqb %zmm0, %zmm4, %k1
571 ; AVX512-NEXT: kxnorw %k1, %k0, %k0
572 ; AVX512-NEXT: vpmovm2b %k0, %zmm0
573 ; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0
574 ; AVX512-NEXT: vzeroupper
575 ; AVX512-NEXT: retq
576 %wide.vec = load <64 x i8>, <64 x i8>* %ptr
577 %v1 = shufflevector <64 x i8> %wide.vec, <64 x i8> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
578 %v2 = shufflevector <64 x i8> %wide.vec, <64 x i8> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
579 %v3 = shufflevector <64 x i8> %wide.vec, <64 x i8> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
580 %v4 = shufflevector <64 x i8> %wide.vec, <64 x i8> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
582 %cmp1 = icmp eq <16 x i8> %v1, %v2
583 %cmp2 = icmp eq <16 x i8> %v3, %v4
584 %res = icmp eq <16 x i1> %cmp1, %cmp2
586 ret <16 x i1> %res
587 }
589 define <32 x i1> @interleaved_load_vf32_i8_stride4(<128 x i8>* %ptr) {
590 ; AVX1-LABEL: interleaved_load_vf32_i8_stride4:
591 ; AVX1: # %bb.0:
592 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm0 = <u,u,u,u,0,4,8,12,u,u,u,u,u,u,u,u>
593 ; AVX1-NEXT: vmovdqa 112(%rdi), %xmm11
594 ; AVX1-NEXT: vpshufb %xmm0, %xmm11, %xmm1
595 ; AVX1-NEXT: vmovdqa 96(%rdi), %xmm12
596 ; AVX1-NEXT: vpshufb %xmm0, %xmm12, %xmm3
597 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
598 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
599 ; AVX1-NEXT: vmovdqa 80(%rdi), %xmm14
600 ; AVX1-NEXT: vpshufb %xmm2, %xmm14, %xmm4
601 ; AVX1-NEXT: vmovdqa 64(%rdi), %xmm6
602 ; AVX1-NEXT: vpshufb %xmm2, %xmm6, %xmm5
603 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
604 ; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm4[0,1,2,3],xmm1[4,5,6,7]
605 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm8
606 ; AVX1-NEXT: vmovdqa (%rdi), %xmm13
607 ; AVX1-NEXT: vmovdqa 16(%rdi), %xmm15
608 ; AVX1-NEXT: vmovdqa 32(%rdi), %xmm7
609 ; AVX1-NEXT: vmovdqa 48(%rdi), %xmm5
610 ; AVX1-NEXT: vpshufb %xmm0, %xmm5, %xmm1
611 ; AVX1-NEXT: vpshufb %xmm0, %xmm7, %xmm0
612 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
613 ; AVX1-NEXT: vpshufb %xmm2, %xmm15, %xmm1
614 ; AVX1-NEXT: vpshufb %xmm2, %xmm13, %xmm2
615 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
616 ; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
617 ; AVX1-NEXT: vblendps {{.*#+}} ymm8 = ymm0[0,1,2,3],ymm8[4,5,6,7]
618 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm0 = <u,u,u,u,1,5,9,13,u,u,u,u,u,u,u,u>
619 ; AVX1-NEXT: vpshufb %xmm0, %xmm11, %xmm1
620 ; AVX1-NEXT: vpshufb %xmm0, %xmm12, %xmm2
621 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
622 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u>
623 ; AVX1-NEXT: vpshufb %xmm2, %xmm14, %xmm3
624 ; AVX1-NEXT: vpshufb %xmm2, %xmm6, %xmm4
625 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
626 ; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
627 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
628 ; AVX1-NEXT: vpshufb %xmm0, %xmm5, %xmm3
629 ; AVX1-NEXT: vpshufb %xmm0, %xmm7, %xmm0
630 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
631 ; AVX1-NEXT: vpshufb %xmm2, %xmm15, %xmm3
632 ; AVX1-NEXT: vpshufb %xmm2, %xmm13, %xmm2
633 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
634 ; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
635 ; AVX1-NEXT: vblendps {{.*#+}} ymm9 = ymm0[0,1,2,3],ymm1[4,5,6,7]
636 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm0 = <u,u,u,u,2,6,10,14,u,u,u,u,u,u,u,u>
637 ; AVX1-NEXT: vpshufb %xmm0, %xmm11, %xmm1
638 ; AVX1-NEXT: vpshufb %xmm0, %xmm12, %xmm2
639 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
640 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = <2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u>
641 ; AVX1-NEXT: vpshufb %xmm2, %xmm14, %xmm3
642 ; AVX1-NEXT: vpshufb %xmm2, %xmm6, %xmm4
643 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
644 ; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
645 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
646 ; AVX1-NEXT: vpshufb %xmm0, %xmm5, %xmm3
647 ; AVX1-NEXT: vpshufb %xmm0, %xmm7, %xmm0
648 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
649 ; AVX1-NEXT: vpshufb %xmm2, %xmm15, %xmm3
650 ; AVX1-NEXT: vpshufb %xmm2, %xmm13, %xmm2
651 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
652 ; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
653 ; AVX1-NEXT: vblendps {{.*#+}} ymm10 = ymm0[0,1,2,3],ymm1[4,5,6,7]
654 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm0 = <u,u,u,u,3,7,11,15,u,u,u,u,u,u,u,u>
655 ; AVX1-NEXT: vpshufb %xmm0, %xmm11, %xmm1
656 ; AVX1-NEXT: vpshufb %xmm0, %xmm12, %xmm2
657 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
658 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = <3,7,11,15,u,u,u,u,u,u,u,u,u,u,u,u>
659 ; AVX1-NEXT: vpshufb %xmm2, %xmm14, %xmm3
660 ; AVX1-NEXT: vpshufb %xmm2, %xmm6, %xmm4
661 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
662 ; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
663 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
664 ; AVX1-NEXT: vpshufb %xmm0, %xmm5, %xmm3
665 ; AVX1-NEXT: vpshufb %xmm0, %xmm7, %xmm0
666 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
667 ; AVX1-NEXT: vpshufb %xmm2, %xmm15, %xmm3
668 ; AVX1-NEXT: vpshufb %xmm2, %xmm13, %xmm2
669 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
670 ; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
671 ; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
672 ; AVX1-NEXT: vextractf128 $1, %ymm9, %xmm1
673 ; AVX1-NEXT: vextractf128 $1, %ymm8, %xmm2
674 ; AVX1-NEXT: vpcmpeqb %xmm1, %xmm2, %xmm1
675 ; AVX1-NEXT: vpcmpeqb %xmm9, %xmm8, %xmm2
676 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1
677 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
678 ; AVX1-NEXT: vextractf128 $1, %ymm10, %xmm3
679 ; AVX1-NEXT: vpcmpeqb %xmm2, %xmm3, %xmm2
680 ; AVX1-NEXT: vpcmpeqb %xmm0, %xmm10, %xmm0
681 ; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
682 ; AVX1-NEXT: vxorps %ymm0, %ymm1, %ymm0
683 ; AVX1-NEXT: vxorps {{.*}}(%rip), %ymm0, %ymm0
684 ; AVX1-NEXT: retq
685 ;
686 ; AVX2-LABEL: interleaved_load_vf32_i8_stride4:
687 ; AVX2: # %bb.0:
688 ; AVX2-NEXT: vmovdqa (%rdi), %xmm9
689 ; AVX2-NEXT: vmovdqa 16(%rdi), %xmm11
690 ; AVX2-NEXT: vmovdqa 32(%rdi), %xmm12
691 ; AVX2-NEXT: vmovdqa 48(%rdi), %xmm13
692 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm6 = <u,u,u,u,0,4,8,12,u,u,u,u,u,u,u,u>
693 ; AVX2-NEXT: vpshufb %xmm6, %xmm13, %xmm4
694 ; AVX2-NEXT: vpshufb %xmm6, %xmm12, %xmm5
695 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
696 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm0 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
697 ; AVX2-NEXT: vpshufb %xmm0, %xmm11, %xmm5
698 ; AVX2-NEXT: vpshufb %xmm0, %xmm9, %xmm7
699 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
700 ; AVX2-NEXT: vpblendd {{.*#+}} xmm8 = xmm5[0,1],xmm4[2,3]
701 ; AVX2-NEXT: vmovdqa 112(%rdi), %xmm14
702 ; AVX2-NEXT: vpshufb %xmm6, %xmm14, %xmm7
703 ; AVX2-NEXT: vpermq {{.*#+}} ymm5 = mem[2,3,0,1]
704 ; AVX2-NEXT: vextracti128 $1, %ymm5, %xmm5
705 ; AVX2-NEXT: vpshufb %xmm6, %xmm5, %xmm6
706 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
707 ; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm10
708 ; AVX2-NEXT: vmovdqa 80(%rdi), %xmm6
709 ; AVX2-NEXT: vpshufb %xmm0, %xmm6, %xmm1
710 ; AVX2-NEXT: vpermq {{.*#+}} ymm7 = mem[2,3,0,1]
711 ; AVX2-NEXT: vextracti128 $1, %ymm7, %xmm7
712 ; AVX2-NEXT: vpshufb %xmm0, %xmm7, %xmm0
713 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
714 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
715 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm10[6,7]
716 ; AVX2-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm0[4,5,6,7]
717 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm1 = <u,u,u,u,1,5,9,13,u,u,u,u,u,u,u,u>
718 ; AVX2-NEXT: vpshufb %xmm1, %xmm13, %xmm0
719 ; AVX2-NEXT: vpshufb %xmm1, %xmm12, %xmm2
720 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
721 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u>
722 ; AVX2-NEXT: vpshufb %xmm2, %xmm11, %xmm3
723 ; AVX2-NEXT: vpshufb %xmm2, %xmm9, %xmm4
724 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
725 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
726 ; AVX2-NEXT: vpshufb %xmm1, %xmm14, %xmm3
727 ; AVX2-NEXT: vpshufb %xmm1, %xmm5, %xmm1
728 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
729 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
730 ; AVX2-NEXT: vpshufb %xmm2, %xmm6, %xmm3
731 ; AVX2-NEXT: vpshufb %xmm2, %xmm7, %xmm2
732 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
733 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
734 ; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
735 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
736 ; AVX2-NEXT: vpcmpeqb %ymm0, %ymm8, %ymm8
737 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm0 = <u,u,u,u,2,6,10,14,u,u,u,u,u,u,u,u>
738 ; AVX2-NEXT: vpshufb %xmm0, %xmm13, %xmm1
739 ; AVX2-NEXT: vpshufb %xmm0, %xmm12, %xmm2
740 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
741 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = <2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u>
742 ; AVX2-NEXT: vpshufb %xmm2, %xmm11, %xmm3
743 ; AVX2-NEXT: vpshufb %xmm2, %xmm9, %xmm4
744 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
745 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
746 ; AVX2-NEXT: vpshufb %xmm0, %xmm14, %xmm3
747 ; AVX2-NEXT: vpshufb %xmm0, %xmm5, %xmm0
748 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
749 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
750 ; AVX2-NEXT: vpshufb %xmm2, %xmm6, %xmm3
751 ; AVX2-NEXT: vpshufb %xmm2, %xmm7, %xmm2
752 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
753 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
754 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
755 ; AVX2-NEXT: vpblendd {{.*#+}} ymm10 = ymm1[0,1,2,3],ymm0[4,5,6,7]
756 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm1 = <u,u,u,u,3,7,11,15,u,u,u,u,u,u,u,u>
757 ; AVX2-NEXT: vpshufb %xmm1, %xmm13, %xmm2
758 ; AVX2-NEXT: vpshufb %xmm1, %xmm12, %xmm3
759 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
760 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = <3,7,11,15,u,u,u,u,u,u,u,u,u,u,u,u>
761 ; AVX2-NEXT: vpshufb %xmm3, %xmm11, %xmm4
762 ; AVX2-NEXT: vpshufb %xmm3, %xmm9, %xmm0
763 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
764 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3]
765 ; AVX2-NEXT: vpshufb %xmm1, %xmm14, %xmm2
766 ; AVX2-NEXT: vpshufb %xmm1, %xmm5, %xmm1
767 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
768 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
769 ; AVX2-NEXT: vpshufb %xmm3, %xmm6, %xmm2
770 ; AVX2-NEXT: vpshufb %xmm3, %xmm7, %xmm3
771 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
772 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
773 ; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
774 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
775 ; AVX2-NEXT: vpcmpeqb %ymm0, %ymm10, %ymm0
776 ; AVX2-NEXT: vpxor %ymm0, %ymm8, %ymm0
777 ; AVX2-NEXT: vpxor {{.*}}(%rip), %ymm0, %ymm0
778 ; AVX2-NEXT: retq
779 ;
780 ; AVX512-LABEL: interleaved_load_vf32_i8_stride4:
781 ; AVX512: # %bb.0:
782 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm6 = <u,u,u,u,0,4,8,12,u,u,u,u,u,u,u,u>
783 ; AVX512-NEXT: vmovdqa (%rdi), %xmm10
784 ; AVX512-NEXT: vmovdqa 16(%rdi), %xmm11
785 ; AVX512-NEXT: vmovdqa 32(%rdi), %xmm12
786 ; AVX512-NEXT: vmovdqa 48(%rdi), %xmm13
787 ; AVX512-NEXT: vpshufb %xmm6, %xmm13, %xmm4
788 ; AVX512-NEXT: vpshufb %xmm6, %xmm12, %xmm5
789 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
790 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm0 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
791 ; AVX512-NEXT: vpshufb %xmm0, %xmm11, %xmm5
792 ; AVX512-NEXT: vpshufb %xmm0, %xmm10, %xmm7
793 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
794 ; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm5[0,1],xmm4[2,3]
795 ; AVX512-NEXT: vmovdqa 112(%rdi), %xmm14
796 ; AVX512-NEXT: vpshufb %xmm6, %xmm14, %xmm7
797 ; AVX512-NEXT: vpermq {{.*#+}} ymm5 = mem[2,3,0,1]
798 ; AVX512-NEXT: vextracti128 $1, %ymm5, %xmm5
799 ; AVX512-NEXT: vpshufb %xmm6, %xmm5, %xmm6
800 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
801 ; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm9
802 ; AVX512-NEXT: vmovdqa 80(%rdi), %xmm6
803 ; AVX512-NEXT: vpshufb %xmm0, %xmm6, %xmm1
804 ; AVX512-NEXT: vpermq {{.*#+}} ymm7 = mem[2,3,0,1]
805 ; AVX512-NEXT: vextracti128 $1, %ymm7, %xmm7
806 ; AVX512-NEXT: vpshufb %xmm0, %xmm7, %xmm0
807 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
808 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
809 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm9[6,7]
810 ; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm0[4,5,6,7]
811 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm0 = <u,u,u,u,1,5,9,13,u,u,u,u,u,u,u,u>
812 ; AVX512-NEXT: vpshufb %xmm0, %xmm13, %xmm1
813 ; AVX512-NEXT: vpshufb %xmm0, %xmm12, %xmm2
814 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
815 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm2 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u>
816 ; AVX512-NEXT: vpshufb %xmm2, %xmm11, %xmm3
817 ; AVX512-NEXT: vpshufb %xmm2, %xmm10, %xmm4
818 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
819 ; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
820 ; AVX512-NEXT: vpshufb %xmm0, %xmm14, %xmm3
821 ; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm0
822 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
823 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
824 ; AVX512-NEXT: vpshufb %xmm2, %xmm6, %xmm3
825 ; AVX512-NEXT: vpshufb %xmm2, %xmm7, %xmm2
826 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
827 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
828 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
829 ; AVX512-NEXT: vpblendd {{.*#+}} ymm9 = ymm1[0,1,2,3],ymm0[4,5,6,7]
830 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm0 = <u,u,u,u,2,6,10,14,u,u,u,u,u,u,u,u>
831 ; AVX512-NEXT: vpshufb %xmm0, %xmm13, %xmm1
832 ; AVX512-NEXT: vpshufb %xmm0, %xmm12, %xmm2
833 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
834 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm2 = <2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u>
835 ; AVX512-NEXT: vpshufb %xmm2, %xmm11, %xmm3
836 ; AVX512-NEXT: vpshufb %xmm2, %xmm10, %xmm4
837 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
838 ; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
839 ; AVX512-NEXT: vpshufb %xmm0, %xmm14, %xmm3
840 ; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm0
841 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
842 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
843 ; AVX512-NEXT: vpshufb %xmm2, %xmm6, %xmm3
844 ; AVX512-NEXT: vpshufb %xmm2, %xmm7, %xmm2
845 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
846 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
847 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
848 ; AVX512-NEXT: vpblendd {{.*#+}} ymm15 = ymm1[0,1,2,3],ymm0[4,5,6,7]
849 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm1 = <u,u,u,u,3,7,11,15,u,u,u,u,u,u,u,u>
850 ; AVX512-NEXT: vpshufb %xmm1, %xmm13, %xmm2
851 ; AVX512-NEXT: vpshufb %xmm1, %xmm12, %xmm3
852 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
853 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm3 = <3,7,11,15,u,u,u,u,u,u,u,u,u,u,u,u>
854 ; AVX512-NEXT: vpshufb %xmm3, %xmm11, %xmm4
855 ; AVX512-NEXT: vpshufb %xmm3, %xmm10, %xmm0
856 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
857 ; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3]
858 ; AVX512-NEXT: vpshufb %xmm1, %xmm14, %xmm2
859 ; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm1
860 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
861 ; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
862 ; AVX512-NEXT: vpshufb %xmm3, %xmm6, %xmm2
863 ; AVX512-NEXT: vpshufb %xmm3, %xmm7, %xmm3
864 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
865 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
866 ; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
867 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
868 ; AVX512-NEXT: vpcmpeqb %zmm9, %zmm8, %k0
869 ; AVX512-NEXT: vpcmpeqb %zmm0, %zmm15, %k1
870 ; AVX512-NEXT: kxnord %k1, %k0, %k0
871 ; AVX512-NEXT: vpmovm2b %k0, %zmm0
872 ; AVX512-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
873 ; AVX512-NEXT: retq
874 %wide.vec = load <128 x i8>, <128 x i8>* %ptr
875 %v1 = shufflevector <128 x i8> %wide.vec, <128 x i8> undef, <32 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60, i32 64, i32 68, i32 72, i32 76, i32 80, i32 84, i32 88, i32 92, i32 96, i32 100, i32 104, i32 108, i32 112, i32 116, i32 120, i32 124>
877 %v2 = shufflevector <128 x i8> %wide.vec, <128 x i8> undef, <32 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61, i32 65, i32 69, i32 73, i32 77, i32 81, i32 85, i32 89, i32 93, i32 97, i32 101, i32 105, i32 109, i32 113, i32 117, i32 121, i32 125>
879 %v3 = shufflevector <128 x i8> %wide.vec, <128 x i8> undef, <32 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62, i32 66, i32 70, i32 74, i32 78, i32 82, i32 86, i32 90, i32 94, i32 98, i32 102, i32 106, i32 110, i32 114, i32 118, i32 122, i32 126>
881 %v4 = shufflevector <128 x i8> %wide.vec, <128 x i8> undef, <32 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63, i32 67, i32 71, i32 75, i32 79, i32 83, i32 87, i32 91, i32 95, i32 99, i32 103, i32 107, i32 111, i32 115, i32 119, i32 123, i32 127>
883 %cmp1 = icmp eq <32 x i8> %v1, %v2
884 %cmp2 = icmp eq <32 x i8> %v3, %v4
885 %res = icmp eq <32 x i1> %cmp1, %cmp2
887 ret <32 x i1> %res
888 }
890 define void @interleaved_store_vf8_i8_stride4(<8 x i8> %x1, <8 x i8> %x2, <8 x i8> %x3, <8 x i8> %x4, <32 x i8>* %p) {
891 ; AVX1-LABEL: interleaved_store_vf8_i8_stride4:
892 ; AVX1: # %bb.0:
893 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
894 ; AVX1-NEXT: vpshufb %xmm4, %xmm1, %xmm1
895 ; AVX1-NEXT: vpshufb %xmm4, %xmm0, %xmm0
896 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
897 ; AVX1-NEXT: vpshufb %xmm4, %xmm3, %xmm1
898 ; AVX1-NEXT: vpshufb %xmm4, %xmm2, %xmm2
899 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
900 ; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
901 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
902 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0
903 ; AVX1-NEXT: vmovaps %ymm0, (%rdi)
904 ; AVX1-NEXT: vzeroupper
905 ; AVX1-NEXT: retq
906 ;
907 ; AVX2OR512-LABEL: interleaved_store_vf8_i8_stride4:
908 ; AVX2OR512: # %bb.0:
909 ; AVX2OR512-NEXT: vmovdqa {{.*#+}} xmm4 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
910 ; AVX2OR512-NEXT: vpshufb %xmm4, %xmm1, %xmm1
911 ; AVX2OR512-NEXT: vpshufb %xmm4, %xmm0, %xmm0
912 ; AVX2OR512-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
913 ; AVX2OR512-NEXT: vpshufb %xmm4, %xmm3, %xmm1
914 ; AVX2OR512-NEXT: vpshufb %xmm4, %xmm2, %xmm2
915 ; AVX2OR512-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
916 ; AVX2OR512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
917 ; AVX2OR512-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
918 ; AVX2OR512-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm0
919 ; AVX2OR512-NEXT: vmovdqa %ymm0, (%rdi)
920 ; AVX2OR512-NEXT: vzeroupper
921 ; AVX2OR512-NEXT: retq
922 %v1 = shufflevector <8 x i8> %x1, <8 x i8> %x2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
923 %v2 = shufflevector <8 x i8> %x3, <8 x i8> %x4, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
924 %interleaved.vec = shufflevector <16 x i8> %v1, <16 x i8> %v2, <32 x i32> <i32 0,i32 8,i32 16,i32 24,i32 1,i32 9,i32 17,i32 25,i32 2,i32 10,i32 18,i32 26,i32 3,i32 11,i32 19,i32 27,i32 4,i32 12,i32 20,i32 28,i32 5,i32 13,i32 21,i32 29,i32 6,i32 14,i32 22,i32 30,i32 7,i32 15,i32 23,i32 31>
925 store <32 x i8> %interleaved.vec, <32 x i8>* %p
926 ret void
927 }
929 define <32 x i8> @interleaved_load_vf32_i8_stride3(<96 x i8>* %ptr){
930 ; AVX1-LABEL: interleaved_load_vf32_i8_stride3:
931 ; AVX1: # %bb.0:
932 ; AVX1-NEXT: vmovdqa (%rdi), %xmm0
933 ; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1
934 ; AVX1-NEXT: vmovdqa 32(%rdi), %xmm2
935 ; AVX1-NEXT: vmovdqa 48(%rdi), %xmm3
936 ; AVX1-NEXT: vmovdqa 64(%rdi), %xmm4
937 ; AVX1-NEXT: vmovdqa 80(%rdi), %xmm5
938 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm6 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]
939 ; AVX1-NEXT: vpshufb %xmm6, %xmm0, %xmm0
940 ; AVX1-NEXT: vpshufb %xmm6, %xmm3, %xmm3
941 ; AVX1-NEXT: vpshufb %xmm6, %xmm1, %xmm1
942 ; AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm4
943 ; AVX1-NEXT: vpshufb %xmm6, %xmm2, %xmm2
944 ; AVX1-NEXT: vpshufb %xmm6, %xmm5, %xmm5
945 ; AVX1-NEXT: vpalignr {{.*#+}} xmm6 = xmm5[11,12,13,14,15],xmm3[0,1,2,3,4,5,6,7,8,9,10]
946 ; AVX1-NEXT: vpalignr {{.*#+}} xmm7 = xmm2[11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10]
947 ; AVX1-NEXT: vpalignr {{.*#+}} xmm3 = xmm3[11,12,13,14,15],xmm4[0,1,2,3,4,5,6,7,8,9,10]
948 ; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]
949 ; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm8
950 ; AVX1-NEXT: vpalignr {{.*#+}} xmm4 = xmm4[11,12,13,14,15],xmm5[0,1,2,3,4,5,6,7,8,9,10]
951 ; AVX1-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[11,12,13,14,15],xmm2[0,1,2,3,4,5,6,7,8,9,10]
952 ; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm1, %ymm2
953 ; AVX1-NEXT: vpalignr {{.*#+}} xmm1 = xmm7[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]
954 ; AVX1-NEXT: vpalignr {{.*#+}} xmm4 = xmm6[11,12,13,14,15],xmm4[0,1,2,3,4,5,6,7,8,9,10]
955 ; AVX1-NEXT: vbroadcastf128 {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]
956 ; AVX1-NEXT: # ymm5 = mem[0,1,0,1]
957 ; AVX1-NEXT: vandnps %ymm2, %ymm5, %ymm2
958 ; AVX1-NEXT: vandps %ymm5, %ymm8, %ymm5
959 ; AVX1-NEXT: vorps %ymm2, %ymm5, %ymm2
960 ; AVX1-NEXT: vpalignr {{.*#+}} xmm3 = xmm3[11,12,13,14,15],xmm6[0,1,2,3,4,5,6,7,8,9,10]
961 ; AVX1-NEXT: vpalignr {{.*#+}} xmm3 = xmm3[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]
962 ; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm7[0,1,2,3,4,5,6,7,8,9,10]
963 ; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]
964 ; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm5
965 ; AVX1-NEXT: vpaddb %xmm4, %xmm5, %xmm4
966 ; AVX1-NEXT: vpaddb %xmm4, %xmm3, %xmm3
967 ; AVX1-NEXT: vpaddb %xmm1, %xmm2, %xmm1
968 ; AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
969 ; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0
970 ; AVX1-NEXT: retq
971 ;
972 ; AVX2OR512-LABEL: interleaved_load_vf32_i8_stride3:
973 ; AVX2OR512: # %bb.0:
974 ; AVX2OR512-NEXT: vmovdqa (%rdi), %xmm0
975 ; AVX2OR512-NEXT: vmovdqa 16(%rdi), %xmm1
976 ; AVX2OR512-NEXT: vmovdqa 32(%rdi), %xmm2
977 ; AVX2OR512-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm0
978 ; AVX2OR512-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm1
979 ; AVX2OR512-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm2
980 ; AVX2OR512-NEXT: vmovdqa {{.*#+}} ymm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]
981 ; AVX2OR512-NEXT: vpshufb %ymm3, %ymm0, %ymm0
982 ; AVX2OR512-NEXT: vpshufb %ymm3, %ymm1, %ymm1
983 ; AVX2OR512-NEXT: vpshufb %ymm3, %ymm2, %ymm2
984 ; AVX2OR512-NEXT: vpalignr {{.*#+}} ymm3 = ymm2[11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7,8,9,10],ymm2[27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23,24,25,26]
985 ; AVX2OR512-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]
986 ; AVX2OR512-NEXT: vpalignr {{.*#+}} ymm1 = ymm1[11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7,8,9,10],ymm1[27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23,24,25,26]
987 ; AVX2OR512-NEXT: vpalignr {{.*#+}} ymm2 = ymm3[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm3[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]
988 ; AVX2OR512-NEXT: vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]
989 ; AVX2OR512-NEXT: vpblendvb %ymm4, %ymm0, %ymm1, %ymm1
990 ; AVX2OR512-NEXT: vpaddb %ymm2, %ymm1, %ymm1
991 ; AVX2OR512-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23,24,25,26]
992 ; AVX2OR512-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]
993 ; AVX2OR512-NEXT: vpaddb %ymm1, %ymm0, %ymm0
994 ; AVX2OR512-NEXT: retq
995 %wide.vec = load <96 x i8>, <96 x i8>* %ptr
996 %v1 = shufflevector <96 x i8> %wide.vec, <96 x i8> undef,<32 x i32> <i32 0,i32 3,i32 6,i32 9,i32 12,i32 15,i32 18,i32 21,i32 24,i32 27,i32 30,i32 33,i32 36,i32 39,i32 42,i32 45,i32 48,i32 51,i32 54,i32 57,i32 60,i32 63,i32 66,i32 69,i32 72,i32 75,i32 78,i32 81,i32 84,i32 87,i32 90,i32 93>
997 %v2 = shufflevector <96 x i8> %wide.vec, <96 x i8> undef,<32 x i32> <i32 1,i32 4,i32 7,i32 10,i32 13,i32 16,i32 19,i32 22,i32 25,i32 28,i32 31,i32 34,i32 37,i32 40,i32 43,i32 46,i32 49,i32 52,i32 55,i32 58,i32 61,i32 64,i32 67,i32 70,i32 73,i32 76,i32 79,i32 82,i32 85,i32 88,i32 91,i32 94>
998 %v3 = shufflevector <96 x i8> %wide.vec, <96 x i8> undef,<32 x i32> <i32 2,i32 5,i32 8,i32 11,i32 14,i32 17,i32 20,i32 23,i32 26,i32 29,i32 32,i32 35,i32 38,i32 41,i32 44,i32 47,i32 50,i32 53,i32 56,i32 59,i32 62,i32 65,i32 68,i32 71,i32 74,i32 77,i32 80,i32 83,i32 86,i32 89,i32 92,i32 95>
999 %add1 = add <32 x i8> %v1, %v2
1000 %add2 = add <32 x i8> %v3, %add1
1001 ret <32 x i8> %add2
1002 }
1004 define <16 x i8> @interleaved_load_vf16_i8_stride3(<48 x i8>* %ptr){
1005 ; AVX-LABEL: interleaved_load_vf16_i8_stride3:
1006 ; AVX: # %bb.0:
1007 ; AVX-NEXT: vmovdqa (%rdi), %xmm0
1008 ; AVX-NEXT: vmovdqa 16(%rdi), %xmm1
1009 ; AVX-NEXT: vmovdqa 32(%rdi), %xmm2
1010 ; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]
1011 ; AVX-NEXT: vpshufb %xmm3, %xmm0, %xmm0
1012 ; AVX-NEXT: vpshufb %xmm3, %xmm1, %xmm1
1013 ; AVX-NEXT: vpshufb %xmm3, %xmm2, %xmm2
1014 ; AVX-NEXT: vpalignr {{.*#+}} xmm3 = xmm2[11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10]
1015 ; AVX-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]
1016 ; AVX-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[11,12,13,14,15],xmm2[0,1,2,3,4,5,6,7,8,9,10]
1017 ; AVX-NEXT: vpalignr {{.*#+}} xmm2 = xmm3[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]
1018 ; AVX-NEXT: vmovdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]
1019 ; AVX-NEXT: vpblendvb %xmm4, %xmm0, %xmm1, %xmm1
1020 ; AVX-NEXT: vpaddb %xmm2, %xmm1, %xmm1
1021 ; AVX-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm3[0,1,2,3,4,5,6,7,8,9,10]
1022 ; AVX-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]
1023 ; AVX-NEXT: vpaddb %xmm1, %xmm0, %xmm0
1024 ; AVX-NEXT: retq
1025 %wide.vec = load <48 x i8>, <48 x i8>* %ptr
1026 %v1 = shufflevector <48 x i8> %wide.vec, <48 x i8> undef,<16 x i32> <i32 0,i32 3,i32 6,i32 9,i32 12,i32 15,i32 18,i32 21,i32 24,i32 27,i32 30,i32 33,i32 36,i32 39,i32 42 ,i32 45>
1027 %v2 = shufflevector <48 x i8> %wide.vec, <48 x i8> undef,<16 x i32> <i32 1,i32 4,i32 7,i32 10,i32 13,i32 16,i32 19,i32 22,i32 25,i32 28,i32 31,i32 34,i32 37,i32 40,i32 43,i32 46>
1028 %v3 = shufflevector <48 x i8> %wide.vec, <48 x i8> undef,<16 x i32> <i32 2,i32 5,i32 8,i32 11,i32 14,i32 17,i32 20,i32 23,i32 26,i32 29,i32 32,i32 35,i32 38,i32 41,i32 44,i32 47>
1029 %add1 = add <16 x i8> %v1, %v2
1030 %add2 = add <16 x i8> %v3, %add1
1031 ret <16 x i8> %add2
1032 }
1034 define <8 x i8> @interleaved_load_vf8_i8_stride3(<24 x i8>* %ptr){
1035 ; AVX-LABEL: interleaved_load_vf8_i8_stride3:
1036 ; AVX: # %bb.0:
1037 ; AVX-NEXT: vmovdqa (%rdi), %xmm0
1038 ; AVX-NEXT: vmovdqa 16(%rdi), %xmm1
1039 ; AVX-NEXT: vpshufb {{.*#+}} xmm2 = zero,xmm1[u],zero,xmm1[u],zero,xmm1[u],zero,xmm1[u],zero,xmm1[u],zero,xmm1[u,2,u,5,u]
1040 ; AVX-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[0,u,3,u,6,u,9,u,12,u,15,u],zero,xmm0[u],zero,xmm0[u]
1041 ; AVX-NEXT: vpor %xmm2, %xmm3, %xmm2
1042 ; AVX-NEXT: vpshufb {{.*#+}} xmm3 = zero,xmm1[u],zero,xmm1[u],zero,xmm1[u],zero,xmm1[u],zero,xmm1[u,0,u,3,u,6,u]
1043 ; AVX-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[1,u,4,u,7,u,10,u,13,u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u]
1044 ; AVX-NEXT: vpor %xmm3, %xmm4, %xmm3
1045 ; AVX-NEXT: vpshufb {{.*#+}} xmm1 = zero,xmm1[u],zero,xmm1[u],zero,xmm1[u],zero,xmm1[u],zero,xmm1[u,1,u,4,u,7,u]
1046 ; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,u,5,u,8,u,11,u,14,u],zero,xmm0[u],zero,xmm0[u],zero,xmm0[u]
1047 ; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0
1048 ; AVX-NEXT: vpaddw %xmm0, %xmm3, %xmm0
1049 ; AVX-NEXT: vpaddw %xmm0, %xmm2, %xmm0
1050 ; AVX-NEXT: retq
1051 %wide.vec = load <24 x i8>, <24 x i8>* %ptr
1052 %v1 = shufflevector <24 x i8> %wide.vec, <24 x i8> undef,<8 x i32> <i32 0,i32 3,i32 6,i32 9,i32 12,i32 15,i32 18,i32 21>
1053 %v2 = shufflevector <24 x i8> %wide.vec, <24 x i8> undef,<8 x i32> <i32 1,i32 4,i32 7,i32 10,i32 13,i32 16,i32 19,i32 22>
1054 %v3 = shufflevector <24 x i8> %wide.vec, <24 x i8> undef,<8 x i32> <i32 2,i32 5,i32 8,i32 11,i32 14,i32 17,i32 20,i32 23>
1055 %add1 = add <8 x i8> %v1, %v2
1056 %add2 = add <8 x i8> %v3, %add1
1057 ret <8 x i8> %add2
1058 }
1060 define void @interleaved_store_vf8_i8_stride3(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <24 x i8>* %p) {
1061 ; AVX-LABEL: interleaved_store_vf8_i8_stride3:
1062 ; AVX: # %bb.0:
1063 ; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
1064 ; AVX-NEXT: vpshufb %xmm3, %xmm1, %xmm1
1065 ; AVX-NEXT: vpshufb %xmm3, %xmm0, %xmm0
1066 ; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
1067 ; AVX-NEXT: vpshufb %xmm3, %xmm2, %xmm1
1068 ; AVX-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[0,8],zero,xmm0[1,9],zero,xmm0[2,10],zero,xmm0[3,11],zero,xmm0[4,12],zero,xmm0[5]
1069 ; AVX-NEXT: vpshufb {{.*#+}} xmm3 = zero,zero,xmm1[0],zero,zero,xmm1[1],zero,zero,xmm1[2],zero,zero,xmm1[3],zero,zero,xmm1[4],zero
1070 ; AVX-NEXT: vpor %xmm3, %xmm2, %xmm2
1071 ; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[13],zero,xmm0[6,14],zero,xmm0[7,15],zero,xmm0[u,u,u,u,u,u,u,u]
1072 ; AVX-NEXT: vpshufb {{.*#+}} xmm1 = zero,xmm1[5],zero,zero,xmm1[6],zero,zero,xmm1[7,u,u,u,u,u,u,u,u]
1073 ; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0
1074 ; AVX-NEXT: vmovq %xmm0, 16(%rdi)
1075 ; AVX-NEXT: vmovdqu %xmm2, (%rdi)
1076 ; AVX-NEXT: retq
1077 %1 = shufflevector <8 x i8> %a, <8 x i8> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
1078 %2 = shufflevector <8 x i8> %c, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1079 %interleaved.vec = shufflevector <16 x i8> %1, <16 x i8> %2, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
1080 store <24 x i8> %interleaved.vec, <24 x i8>* %p, align 1
1081 ret void
1082 }
1084 define void @interleaved_store_vf16_i8_stride3(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <48 x i8>* %p) {
1085 ; AVX1-LABEL: interleaved_store_vf16_i8_stride3:
1086 ; AVX1: # %bb.0:
1087 ; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
1088 ; AVX1-NEXT: vpalignr {{.*#+}} xmm3 = xmm1[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
1089 ; AVX1-NEXT: vpalignr {{.*#+}} xmm4 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
1090 ; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
1091 ; AVX1-NEXT: vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
1092 ; AVX1-NEXT: vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
1093 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
1094 ; AVX1-NEXT: vpshufb %xmm3, %xmm1, %xmm1
1095 ; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
1096 ; AVX1-NEXT: vpshufb %xmm3, %xmm0, %xmm0
1097 ; AVX1-NEXT: vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
1098 ; AVX1-NEXT: vpshufb %xmm3, %xmm2, %xmm2
1099 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
1100 ; AVX1-NEXT: vmovdqu %xmm2, 32(%rdi)
1101 ; AVX1-NEXT: vmovups %ymm0, (%rdi)
1102 ; AVX1-NEXT: vzeroupper
1103 ; AVX1-NEXT: retq
1104 ;
1105 ; AVX2-LABEL: interleaved_store_vf16_i8_stride3:
1106 ; AVX2: # %bb.0:
1107 ; AVX2-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
1108 ; AVX2-NEXT: vpalignr {{.*#+}} xmm3 = xmm1[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
1109 ; AVX2-NEXT: vpalignr {{.*#+}} xmm4 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
1110 ; AVX2-NEXT: vpalignr {{.*#+}} xmm0 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
1111 ; AVX2-NEXT: vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
1112 ; AVX2-NEXT: vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
1113 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
1114 ; AVX2-NEXT: vpshufb %xmm3, %xmm1, %xmm1
1115 ; AVX2-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
1116 ; AVX2-NEXT: vpshufb %xmm3, %xmm0, %xmm0
1117 ; AVX2-NEXT: vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
1118 ; AVX2-NEXT: vpshufb %xmm3, %xmm2, %xmm2
1119 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
1120 ; AVX2-NEXT: vmovdqu %xmm2, 32(%rdi)
1121 ; AVX2-NEXT: vmovdqu %ymm0, (%rdi)
1122 ; AVX2-NEXT: vzeroupper
1123 ; AVX2-NEXT: retq
1124 ;
1125 ; AVX512-LABEL: interleaved_store_vf16_i8_stride3:
1126 ; AVX512: # %bb.0:
1127 ; AVX512-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
1128 ; AVX512-NEXT: vpalignr {{.*#+}} xmm3 = xmm1[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
1129 ; AVX512-NEXT: vpalignr {{.*#+}} xmm4 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
1130 ; AVX512-NEXT: vpalignr {{.*#+}} xmm0 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
1131 ; AVX512-NEXT: vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
1132 ; AVX512-NEXT: vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
1133 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
1134 ; AVX512-NEXT: vpshufb %xmm3, %xmm1, %xmm1
1135 ; AVX512-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
1136 ; AVX512-NEXT: vpshufb %xmm3, %xmm0, %xmm0
1137 ; AVX512-NEXT: vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
1138 ; AVX512-NEXT: vpshufb %xmm3, %xmm2, %xmm2
1139 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
1140 ; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm1
1141 ; AVX512-NEXT: vmovdqu %ymm0, (%rdi)
1142 ; AVX512-NEXT: vextracti32x4 $2, %zmm1, 32(%rdi)
1143 ; AVX512-NEXT: vzeroupper
1144 ; AVX512-NEXT: retq
1145 %1 = shufflevector <16 x i8> %a, <16 x i8> %b, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
1146 %2 = shufflevector <16 x i8> %c, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1147 %interleaved.vec = shufflevector <32 x i8> %1, <32 x i8> %2, <48 x i32> <i32 0, i32 16, i32 32, i32 1, i32 17, i32 33, i32 2, i32 18, i32 34, i32 3, i32 19, i32 35, i32 4, i32 20, i32 36, i32 5, i32 21, i32 37, i32 6, i32 22, i32 38, i32 7, i32 23, i32 39, i32 8, i32 24, i32 40, i32 9, i32 25, i32 41, i32 10, i32 26, i32 42, i32 11, i32 27, i32 43, i32 12, i32 28, i32 44, i32 13, i32 29, i32 45, i32 14, i32 30, i32 46, i32 15, i32 31, i32 47>
1148 store <48 x i8> %interleaved.vec, <48 x i8>* %p, align 1
1149 ret void
1150 }
1152 define void @interleaved_store_vf32_i8_stride3(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, <96 x i8>* %p) {
1153 ; AVX1-LABEL: interleaved_store_vf32_i8_stride3:
1154 ; AVX1: # %bb.0:
1155 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
1156 ; AVX1-NEXT: vpalignr {{.*#+}} xmm8 = xmm3[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
1157 ; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
1158 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm9
1159 ; AVX1-NEXT: vpalignr {{.*#+}} xmm5 = xmm9[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
1160 ; AVX1-NEXT: vpalignr {{.*#+}} xmm6 = xmm1[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
1161 ; AVX1-NEXT: vpalignr {{.*#+}} xmm7 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
1162 ; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
1163 ; AVX1-NEXT: vpalignr {{.*#+}} xmm4 = xmm8[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
1164 ; AVX1-NEXT: vpalignr {{.*#+}} xmm10 = xmm6[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
1165 ; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm5[5,6,7,8,9,10,11,12,13,14,15],xmm8[0,1,2,3,4]
1166 ; AVX1-NEXT: vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm6[0,1,2,3,4]
1167 ; AVX1-NEXT: vpalignr {{.*#+}} xmm3 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm5[0,1,2,3,4]
1168 ; AVX1-NEXT: vpalignr {{.*#+}} xmm5 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm9[0,1,2,3,4]
1169 ; AVX1-NEXT: vpalignr {{.*#+}} xmm1 = xmm7[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
1170 ; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
1171 ; AVX1-NEXT: vpalignr {{.*#+}} xmm6 = xmm10[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
1172 ; AVX1-NEXT: vpalignr {{.*#+}} xmm3 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
1173 ; AVX1-NEXT: vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm7[0,1,2,3,4]
1174 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
1175 ; AVX1-NEXT: vpshufb %xmm4, %xmm6, %xmm6
1176 ; AVX1-NEXT: vpshufb %xmm4, %xmm1, %xmm1
1177 ; AVX1-NEXT: vinsertf128 $1, %xmm6, %ymm1, %ymm1
1178 ; AVX1-NEXT: vpshufb %xmm4, %xmm2, %xmm2
1179 ; AVX1-NEXT: vpshufb %xmm4, %xmm5, %xmm5
1180 ; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm2, %ymm2
1181 ; AVX1-NEXT: vpshufb %xmm4, %xmm3, %xmm3
1182 ; AVX1-NEXT: vpshufb %xmm4, %xmm0, %xmm0
1183 ; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0
1184 ; AVX1-NEXT: vmovups %ymm0, 64(%rdi)
1185 ; AVX1-NEXT: vmovups %ymm2, 32(%rdi)
1186 ; AVX1-NEXT: vmovups %ymm1, (%rdi)
1187 ; AVX1-NEXT: vzeroupper
1188 ; AVX1-NEXT: retq
1189 ;
1190 ; AVX2-LABEL: interleaved_store_vf32_i8_stride3:
1191 ; AVX2: # %bb.0:
1192 ; AVX2-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,22,23,24,25,26,27,28,29,30,31,16,17,18,19,20,21]
1193 ; AVX2-NEXT: vpalignr {{.*#+}} ymm3 = ymm1[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25,26]
1194 ; AVX2-NEXT: vpalignr {{.*#+}} ymm4 = ymm0[5,6,7,8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4],ymm0[21,22,23,24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20]
1195 ; AVX2-NEXT: vpalignr {{.*#+}} ymm0 = ymm3[5,6,7,8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4],ymm3[21,22,23,24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20]
1196 ; AVX2-NEXT: vpalignr {{.*#+}} ymm2 = ymm2[5,6,7,8,9,10,11,12,13,14,15],ymm3[0,1,2,3,4],ymm2[21,22,23,24,25,26,27,28,29,30,31],ymm3[16,17,18,19,20]
1197 ; AVX2-NEXT: vpalignr {{.*#+}} ymm1 = ymm4[5,6,7,8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4],ymm4[21,22,23,24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20]
1198 ; AVX2-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[5,6,7,8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4],ymm0[21,22,23,24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20]
1199 ; AVX2-NEXT: vpalignr {{.*#+}} ymm2 = ymm2[5,6,7,8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4],ymm2[21,22,23,24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20]
1200 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm3
1201 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
1202 ; AVX2-NEXT: vpshufb %ymm4, %ymm3, %ymm3
1203 ; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
1204 ; AVX2-NEXT: vpshufb %ymm4, %ymm1, %ymm1
1205 ; AVX2-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3]
1206 ; AVX2-NEXT: vpshufb %ymm4, %ymm0, %ymm0
1207 ; AVX2-NEXT: vmovdqu %ymm0, 64(%rdi)
1208 ; AVX2-NEXT: vmovdqu %ymm1, 32(%rdi)
1209 ; AVX2-NEXT: vmovdqu %ymm3, (%rdi)
1210 ; AVX2-NEXT: vzeroupper
1211 ; AVX2-NEXT: retq
1212 ;
1213 ; AVX512-LABEL: interleaved_store_vf32_i8_stride3:
1214 ; AVX512: # %bb.0:
1215 ; AVX512-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,22,23,24,25,26,27,28,29,30,31,16,17,18,19,20,21]
1216 ; AVX512-NEXT: vpalignr {{.*#+}} ymm3 = ymm1[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25,26]
1217 ; AVX512-NEXT: vpalignr {{.*#+}} ymm4 = ymm0[5,6,7,8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4],ymm0[21,22,23,24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20]
1218 ; AVX512-NEXT: vpalignr {{.*#+}} ymm0 = ymm3[5,6,7,8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4],ymm3[21,22,23,24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20]
1219 ; AVX512-NEXT: vpalignr {{.*#+}} ymm2 = ymm2[5,6,7,8,9,10,11,12,13,14,15],ymm3[0,1,2,3,4],ymm2[21,22,23,24,25,26,27,28,29,30,31],ymm3[16,17,18,19,20]
1220 ; AVX512-NEXT: vpalignr {{.*#+}} ymm1 = ymm4[5,6,7,8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4],ymm4[21,22,23,24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20]
1221 ; AVX512-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[5,6,7,8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4],ymm0[21,22,23,24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20]
1222 ; AVX512-NEXT: vpalignr {{.*#+}} ymm2 = ymm2[5,6,7,8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4],ymm2[21,22,23,24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20]
1223 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm3
1224 ; AVX512-NEXT: vmovdqa {{.*#+}} ymm4 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
1225 ; AVX512-NEXT: vpshufb %ymm4, %ymm3, %ymm3
1226 ; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
1227 ; AVX512-NEXT: vpshufb %ymm4, %ymm1, %ymm1
1228 ; AVX512-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3]
1229 ; AVX512-NEXT: vpshufb %ymm4, %ymm0, %ymm0
1230 ; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1
1231 ; AVX512-NEXT: vmovdqu %ymm0, 64(%rdi)
1232 ; AVX512-NEXT: vmovdqu64 %zmm1, (%rdi)
1233 ; AVX512-NEXT: vzeroupper
1234 ; AVX512-NEXT: retq
1235 %1 = shufflevector <32 x i8> %a, <32 x i8> %b, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
1236 %2 = shufflevector <32 x i8> %c, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1237 %interleaved.vec = shufflevector <64 x i8> %1, <64 x i8> %2, <96 x i32> <i32 0, i32 32, i32 64, i32 1, i32 33, i32 65, i32 2, i32 34, i32 66, i32 3, i32 35, i32 67, i32 4, i32 36, i32 68, i32 5, i32 37, i32 69, i32 6, i32 38, i32 70, i32 7, i32 39, i32 71, i32 8, i32 40, i32 72, i32 9, i32 41, i32 73, i32 10, i32 42, i32 74, i32 11, i32 43, i32 75, i32 12, i32 44, i32 76, i32 13, i32 45, i32 77, i32 14, i32 46, i32 78, i32 15, i32 47, i32 79, i32 16, i32 48, i32 80, i32 17, i32 49, i32 81, i32 18, i32 50, i32 82, i32 19, i32 51, i32 83, i32 20, i32 52, i32 84, i32 21, i32 53, i32 85, i32 22, i32 54, i32 86, i32 23, i32 55, i32 87, i32 24, i32 56, i32 88, i32 25, i32 57, i32 89, i32 26, i32 58, i32 90, i32 27, i32 59, i32 91, i32 28, i32 60, i32 92, i32 29, i32 61, i32 93, i32 30, i32 62, i32 94, i32 31, i32 63, i32 95>
1238 store <96 x i8> %interleaved.vec, <96 x i8>* %p, align 1
1239 ret void
1240 }
1242 define void @interleaved_store_vf64_i8_stride3(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c, <192 x i8>* %p) {
1243 ; AVX1-LABEL: interleaved_store_vf64_i8_stride3:
1244 ; AVX1: # %bb.0:
1245 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm6
1246 ; AVX1-NEXT: vpalignr {{.*#+}} xmm8 = xmm6[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
1247 ; AVX1-NEXT: vpalignr {{.*#+}} xmm9 = xmm1[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
1248 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm7
1249 ; AVX1-NEXT: vpalignr {{.*#+}} xmm12 = xmm7[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
1250 ; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
1251 ; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm10
1252 ; AVX1-NEXT: vpalignr {{.*#+}} xmm13 = xmm10[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
1253 ; AVX1-NEXT: vpalignr {{.*#+}} xmm15 = xmm3[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
1254 ; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm11
1255 ; AVX1-NEXT: vpalignr {{.*#+}} xmm7 = xmm11[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
1256 ; AVX1-NEXT: vpalignr {{.*#+}} xmm1 = xmm2[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
1257 ; AVX1-NEXT: vpalignr {{.*#+}} xmm14 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
1258 ; AVX1-NEXT: vextractf128 $1, %ymm4, %xmm6
1259 ; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm1[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
1260 ; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1261 ; AVX1-NEXT: vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
1262 ; AVX1-NEXT: vpalignr {{.*#+}} xmm4 = xmm12[5,6,7,8,9,10,11,12,13,14,15],xmm6[0,1,2,3,4]
1263 ; AVX1-NEXT: vpalignr {{.*#+}} xmm12 = xmm7[5,6,7,8,9,10,11,12,13,14,15],xmm12[0,1,2,3,4]
1264 ; AVX1-NEXT: vpalignr {{.*#+}} xmm6 = xmm6[5,6,7,8,9,10,11,12,13,14,15],xmm7[0,1,2,3,4]
1265 ; AVX1-NEXT: vpalignr {{.*#+}} xmm7 = xmm9[5,6,7,8,9,10,11,12,13,14,15],xmm5[0,1,2,3,4]
1266 ; AVX1-NEXT: vpalignr {{.*#+}} xmm9 = xmm15[5,6,7,8,9,10,11,12,13,14,15],xmm9[0,1,2,3,4]
1267 ; AVX1-NEXT: vpalignr {{.*#+}} xmm15 = xmm5[5,6,7,8,9,10,11,12,13,14,15],xmm15[0,1,2,3,4]
1268 ; AVX1-NEXT: vextractf128 $1, %ymm5, %xmm5
1269 ; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm8[5,6,7,8,9,10,11,12,13,14,15],xmm5[0,1,2,3,4]
1270 ; AVX1-NEXT: vpalignr {{.*#+}} xmm8 = xmm13[5,6,7,8,9,10,11,12,13,14,15],xmm8[0,1,2,3,4]
1271 ; AVX1-NEXT: vpalignr {{.*#+}} xmm5 = xmm5[5,6,7,8,9,10,11,12,13,14,15],xmm13[0,1,2,3,4]
1272 ; AVX1-NEXT: vpalignr {{.*#+}} xmm10 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm10[0,1,2,3,4]
1273 ; AVX1-NEXT: vpalignr {{.*#+}} xmm8 = xmm8[5,6,7,8,9,10,11,12,13,14,15],xmm5[0,1,2,3,4]
1274 ; AVX1-NEXT: vpalignr {{.*#+}} xmm13 = xmm5[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
1275 ; AVX1-NEXT: vpalignr {{.*#+}} xmm3 = xmm7[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
1276 ; AVX1-NEXT: vpalignr {{.*#+}} xmm5 = xmm9[5,6,7,8,9,10,11,12,13,14,15],xmm15[0,1,2,3,4]
1277 ; AVX1-NEXT: vpalignr {{.*#+}} xmm9 = xmm15[5,6,7,8,9,10,11,12,13,14,15],xmm7[0,1,2,3,4]
1278 ; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm11[0,1,2,3,4]
1279 ; AVX1-NEXT: vpalignr {{.*#+}} xmm11 = xmm12[5,6,7,8,9,10,11,12,13,14,15],xmm6[0,1,2,3,4]
1280 ; AVX1-NEXT: vpalignr {{.*#+}} xmm4 = xmm6[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
1281 ; AVX1-NEXT: vpalignr {{.*#+}} xmm2 = xmm14[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
1282 ; AVX1-NEXT: vpalignr $5, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm6 # 16-byte Folded Reload
1283 ; AVX1-NEXT: # xmm6 = mem[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
1284 ; AVX1-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[5,6,7,8,9,10,11,12,13,14,15],xmm14[0,1,2,3,4]
1285 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm7 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
1286 ; AVX1-NEXT: vpshufb %xmm7, %xmm6, %xmm6
1287 ; AVX1-NEXT: vpshufb %xmm7, %xmm2, %xmm2
1288 ; AVX1-NEXT: vinsertf128 $1, %xmm6, %ymm2, %ymm2
1289 ; AVX1-NEXT: vpshufb %xmm7, %xmm1, %xmm1
1290 ; AVX1-NEXT: vpshufb %xmm7, %xmm0, %xmm0
1291 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
1292 ; AVX1-NEXT: vpshufb %xmm7, %xmm4, %xmm1
1293 ; AVX1-NEXT: vpshufb %xmm7, %xmm11, %xmm4
1294 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm4, %ymm1
1295 ; AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm4
1296 ; AVX1-NEXT: vpshufb %xmm7, %xmm3, %xmm3
1297 ; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3
1298 ; AVX1-NEXT: vpshufb %xmm7, %xmm9, %xmm4
1299 ; AVX1-NEXT: vpshufb %xmm7, %xmm10, %xmm5
1300 ; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm4, %ymm4
1301 ; AVX1-NEXT: vpshufb %xmm7, %xmm13, %xmm5
1302 ; AVX1-NEXT: vpshufb %xmm7, %xmm8, %xmm6
1303 ; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm6, %ymm5
1304 ; AVX1-NEXT: vmovups %ymm5, 160(%rdi)
1305 ; AVX1-NEXT: vmovups %ymm4, 128(%rdi)
1306 ; AVX1-NEXT: vmovups %ymm3, 96(%rdi)
1307 ; AVX1-NEXT: vmovups %ymm1, 64(%rdi)
1308 ; AVX1-NEXT: vmovups %ymm0, 32(%rdi)
1309 ; AVX1-NEXT: vmovups %ymm2, (%rdi)
1310 ; AVX1-NEXT: vzeroupper
1311 ; AVX1-NEXT: retq
1312 ;
1313 ; AVX2-LABEL: interleaved_store_vf64_i8_stride3:
1314 ; AVX2: # %bb.0:
1315 ; AVX2-NEXT: vpalignr {{.*#+}} ymm1 = ymm1[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,22,23,24,25,26,27,28,29,30,31,16,17,18,19,20,21]
1316 ; AVX2-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,22,23,24,25,26,27,28,29,30,31,16,17,18,19,20,21]
1317 ; AVX2-NEXT: vpalignr {{.*#+}} ymm6 = ymm3[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25,26]
1318 ; AVX2-NEXT: vpalignr {{.*#+}} ymm7 = ymm2[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25,26]
1319 ; AVX2-NEXT: vpalignr {{.*#+}} ymm8 = ymm0[5,6,7,8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4],ymm0[21,22,23,24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20]
1320 ; AVX2-NEXT: vpalignr {{.*#+}} ymm9 = ymm1[5,6,7,8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4],ymm1[21,22,23,24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20]
1321 ; AVX2-NEXT: vpalignr {{.*#+}} ymm0 = ymm7[5,6,7,8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4],ymm7[21,22,23,24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20]
1322 ; AVX2-NEXT: vpalignr {{.*#+}} ymm1 = ymm6[5,6,7,8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4],ymm6[21,22,23,24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20]
1323 ; AVX2-NEXT: vpalignr {{.*#+}} ymm4 = ymm4[5,6,7,8,9,10,11,12,13,14,15],ymm7[0,1,2,3,4],ymm4[21,22,23,24,25,26,27,28,29,30,31],ymm7[16,17,18,19,20]
1324 ; AVX2-NEXT: vpalignr {{.*#+}} ymm5 = ymm5[5,6,7,8,9,10,11,12,13,14,15],ymm6[0,1,2,3,4],ymm5[21,22,23,24,25,26,27,28,29,30,31],ymm6[16,17,18,19,20]
1325 ; AVX2-NEXT: vpalignr {{.*#+}} ymm3 = ymm9[5,6,7,8,9,10,11,12,13,14,15],ymm3[0,1,2,3,4],ymm9[21,22,23,24,25,26,27,28,29,30,31],ymm3[16,17,18,19,20]
1326 ; AVX2-NEXT: vpalignr {{.*#+}} ymm2 = ymm8[5,6,7,8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4],ymm8[21,22,23,24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20]
1327 ; AVX2-NEXT: vpalignr {{.*#+}} ymm1 = ymm1[5,6,7,8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4],ymm1[21,22,23,24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20]
1328 ; AVX2-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[5,6,7,8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4],ymm0[21,22,23,24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20]
1329 ; AVX2-NEXT: vpalignr {{.*#+}} ymm5 = ymm5[5,6,7,8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4],ymm5[21,22,23,24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20]
1330 ; AVX2-NEXT: vpalignr {{.*#+}} ymm4 = ymm4[5,6,7,8,9,10,11,12,13,14,15],ymm8[0,1,2,3,4],ymm4[21,22,23,24,25,26,27,28,29,30,31],ymm8[16,17,18,19,20]
1331 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm6
1332 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm7 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
1333 ; AVX2-NEXT: vpshufb %ymm7, %ymm6, %ymm6
1334 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
1335 ; AVX2-NEXT: vpshufb %ymm7, %ymm2, %ymm2
1336 ; AVX2-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm4[2,3]
1337 ; AVX2-NEXT: vpshufb %ymm7, %ymm0, %ymm0
1338 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm3, %ymm4
1339 ; AVX2-NEXT: vpshufb %ymm7, %ymm4, %ymm4
1340 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3],ymm3[4,5,6,7]
1341 ; AVX2-NEXT: vpshufb %ymm7, %ymm3, %ymm3
1342 ; AVX2-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm1[2,3],ymm5[2,3]
1343 ; AVX2-NEXT: vpshufb %ymm7, %ymm1, %ymm1
1344 ; AVX2-NEXT: vmovdqu %ymm1, 160(%rdi)
1345 ; AVX2-NEXT: vmovdqu %ymm3, 128(%rdi)
1346 ; AVX2-NEXT: vmovdqu %ymm0, 64(%rdi)
1347 ; AVX2-NEXT: vmovdqu %ymm2, 32(%rdi)
1348 ; AVX2-NEXT: vmovdqu %ymm4, 96(%rdi)
1349 ; AVX2-NEXT: vmovdqu %ymm6, (%rdi)
1350 ; AVX2-NEXT: vzeroupper
1351 ; AVX2-NEXT: retq
1352 ;
1353 ; AVX512-LABEL: interleaved_store_vf64_i8_stride3:
1354 ; AVX512: # %bb.0:
1355 ; AVX512-NEXT: vpalignr {{.*#+}} zmm0 = zmm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,22,23,24,25,26,27,28,29,30,31,16,17,18,19,20,21,38,39,40,41,42,43,44,45,46,47,32,33,34,35,36,37,54,55,56,57,58,59,60,61,62,63,48,49,50,51,52,53]
1356 ; AVX512-NEXT: vpalignr {{.*#+}} zmm1 = zmm1[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25,26,43,44,45,46,47,32,33,34,35,36,37,38,39,40,41,42,59,60,61,62,63,48,49,50,51,52,53,54,55,56,57,58]
1357 ; AVX512-NEXT: vpalignr {{.*#+}} zmm3 = zmm0[5,6,7,8,9,10,11,12,13,14,15],zmm2[0,1,2,3,4],zmm0[21,22,23,24,25,26,27,28,29,30,31],zmm2[16,17,18,19,20],zmm0[37,38,39,40,41,42,43,44,45,46,47],zmm2[32,33,34,35,36],zmm0[53,54,55,56,57,58,59,60,61,62,63],zmm2[48,49,50,51,52]
1358 ; AVX512-NEXT: vpalignr {{.*#+}} zmm0 = zmm1[5,6,7,8,9,10,11,12,13,14,15],zmm0[0,1,2,3,4],zmm1[21,22,23,24,25,26,27,28,29,30,31],zmm0[16,17,18,19,20],zmm1[37,38,39,40,41,42,43,44,45,46,47],zmm0[32,33,34,35,36],zmm1[53,54,55,56,57,58,59,60,61,62,63],zmm0[48,49,50,51,52]
1359 ; AVX512-NEXT: vpalignr {{.*#+}} zmm1 = zmm2[5,6,7,8,9,10,11,12,13,14,15],zmm1[0,1,2,3,4],zmm2[21,22,23,24,25,26,27,28,29,30,31],zmm1[16,17,18,19,20],zmm2[37,38,39,40,41,42,43,44,45,46,47],zmm1[32,33,34,35,36],zmm2[53,54,55,56,57,58,59,60,61,62,63],zmm1[48,49,50,51,52]
1360 ; AVX512-NEXT: vpalignr {{.*#+}} zmm2 = zmm3[5,6,7,8,9,10,11,12,13,14,15],zmm0[0,1,2,3,4],zmm3[21,22,23,24,25,26,27,28,29,30,31],zmm0[16,17,18,19,20],zmm3[37,38,39,40,41,42,43,44,45,46,47],zmm0[32,33,34,35,36],zmm3[53,54,55,56,57,58,59,60,61,62,63],zmm0[48,49,50,51,52]
1361 ; AVX512-NEXT: vpalignr {{.*#+}} zmm0 = zmm0[5,6,7,8,9,10,11,12,13,14,15],zmm1[0,1,2,3,4],zmm0[21,22,23,24,25,26,27,28,29,30,31],zmm1[16,17,18,19,20],zmm0[37,38,39,40,41,42,43,44,45,46,47],zmm1[32,33,34,35,36],zmm0[53,54,55,56,57,58,59,60,61,62,63],zmm1[48,49,50,51,52]
1362 ; AVX512-NEXT: vpalignr {{.*#+}} zmm1 = zmm1[5,6,7,8,9,10,11,12,13,14,15],zmm3[0,1,2,3,4],zmm1[21,22,23,24,25,26,27,28,29,30,31],zmm3[16,17,18,19,20],zmm1[37,38,39,40,41,42,43,44,45,46,47],zmm3[32,33,34,35,36],zmm1[53,54,55,56,57,58,59,60,61,62,63],zmm3[48,49,50,51,52]
1363 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm3
1364 ; AVX512-NEXT: vmovdqa {{.*#+}} ymm4 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
1365 ; AVX512-NEXT: vpshufb %ymm4, %ymm3, %ymm3
1366 ; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm1[0,1,2,3],ymm2[4,5,6,7]
1367 ; AVX512-NEXT: vpshufb %ymm4, %ymm5, %ymm5
1368 ; AVX512-NEXT: vperm2i128 {{.*#+}} ymm6 = ymm0[2,3],ymm1[2,3]
1369 ; AVX512-NEXT: vpshufb %ymm4, %ymm6, %ymm6
1370 ; AVX512-NEXT: vextracti64x4 $1, %zmm2, %ymm2
1371 ; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm0
1372 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm7
1373 ; AVX512-NEXT: vpshufb %ymm4, %ymm7, %ymm7
1374 ; AVX512-NEXT: vextracti64x4 $1, %zmm1, %ymm1
1375 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm2[4,5,6,7]
1376 ; AVX512-NEXT: vpshufb %ymm4, %ymm2, %ymm2
1377 ; AVX512-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3]
1378 ; AVX512-NEXT: vpshufb %ymm4, %ymm0, %ymm0
1379 ; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm3, %zmm1
1380 ; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm6, %zmm3
1381 ; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0
1382 ; AVX512-NEXT: vmovdqu64 %zmm0, 128(%rdi)
1383 ; AVX512-NEXT: vmovdqu64 %zmm3, 64(%rdi)
1384 ; AVX512-NEXT: vmovdqu64 %zmm1, (%rdi)
1385 ; AVX512-NEXT: vzeroupper
1386 ; AVX512-NEXT: retq
1387 %1 = shufflevector <64 x i8> %a, <64 x i8> %b, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
1388 %2 = shufflevector <64 x i8> %c, <64 x i8> undef, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1389 %3 = shufflevector <128 x i8> %1, <128 x i8> %2, <192 x i32> <i32 0, i32 64, i32 128, i32 1, i32 65, i32 129, i32 2, i32 66, i32 130, i32 3, i32 67, i32 131, i32 4, i32 68, i32 132, i32 5, i32 69, i32 133, i32 6, i32 70, i32 134, i32 7, i32 71, i32 135, i32 8, i32 72, i32 136, i32 9, i32 73, i32 137, i32 10, i32 74, i32 138, i32 11, i32 75, i32 139, i32 12, i32 76, i32 140, i32 13, i32 77, i32 141, i32 14, i32 78, i32 142, i32 15, i32 79, i32 143, i32 16, i32 80, i32 144, i32 17, i32 81, i32 145, i32 18, i32 82, i32 146, i32 19, i32 83, i32 147, i32 20, i32 84, i32 148, i32 21, i32 85, i32 149, i32 22, i32 86, i32 150, i32 23, i32 87, i32 151, i32 24, i32 88, i32 152, i32 25, i32 89, i32 153, i32 26, i32 90, i32 154, i32 27, i32 91, i32 155, i32 28, i32 92, i32 156, i32 29, i32 93, i32 157, i32 30, i32 94, i32 158, i32 31, i32 95, i32 159, i32 32, i32 96, i32 160, i32 33, i32 97, i32 161, i32 34, i32 98, i32 162, i32 35, i32 99, i32 163, i32 36, i32 100, i32 164, i32 37, i32 101, i32 165, i32 38, i32 102, i32 166, i32 39, i32 103, i32 167, i32 40, i32 104, i32 168, i32 41, i32 105, i32 169, i32 42, i32 106, i32 170, i32 43, i32 107, i32 171, i32 44, i32 108, i32 172, i32 45, i32 109, i32 173, i32 46, i32 110, i32 174, i32 47, i32 111, i32 175, i32 48, i32 112, i32 176, i32 49, i32 113, i32 177, i32 50, i32 114, i32 178, i32 51, i32 115, i32 179, i32 52, i32 116, i32 180, i32 53, i32 117, i32 181, i32 54, i32 118, i32 182, i32 55, i32 119, i32 183, i32 56, i32 120, i32 184, i32 57, i32 121, i32 185, i32 58, i32 122, i32 186, i32 59, i32 123, i32 187, i32 60, i32 124, i32 188, i32 61, i32 125, i32 189, i32 62, i32 126, i32 190, i32 63, i32 127, i32 191>
1390 store <192 x i8> %3, <192 x i8>* %p, align 1
1391 ret void
1392 }
1394 define <64 x i8> @interleaved_load_vf64_i8_stride3(<192 x i8>* %ptr){
1395 ; AVX1-LABEL: interleaved_load_vf64_i8_stride3:
1396 ; AVX1: # %bb.0:
1397 ; AVX1-NEXT: vmovdqu (%rdi), %xmm11
1398 ; AVX1-NEXT: vmovdqu 16(%rdi), %xmm10
1399 ; AVX1-NEXT: vmovdqu 32(%rdi), %xmm8
1400 ; AVX1-NEXT: vmovdqu 48(%rdi), %xmm3
1401 ; AVX1-NEXT: vmovdqu 64(%rdi), %xmm12
1402 ; AVX1-NEXT: vmovdqu 80(%rdi), %xmm9
1403 ; AVX1-NEXT: vmovdqu 96(%rdi), %xmm6
1404 ; AVX1-NEXT: vmovdqu 112(%rdi), %xmm14
1405 ; AVX1-NEXT: vmovdqu 128(%rdi), %xmm13
1406 ; AVX1-NEXT: vmovdqu 144(%rdi), %xmm5
1407 ; AVX1-NEXT: vmovdqu 160(%rdi), %xmm1
1408 ; AVX1-NEXT: vmovdqu 176(%rdi), %xmm15
1409 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]
1410 ; AVX1-NEXT: vpshufb %xmm4, %xmm6, %xmm6
1411 ; AVX1-NEXT: vpshufb %xmm4, %xmm5, %xmm5
1412 ; AVX1-NEXT: vpshufb %xmm4, %xmm11, %xmm11
1413 ; AVX1-NEXT: vpshufb %xmm4, %xmm3, %xmm3
1414 ; AVX1-NEXT: vpshufb %xmm4, %xmm10, %xmm10
1415 ; AVX1-NEXT: vpshufb %xmm4, %xmm12, %xmm12
1416 ; AVX1-NEXT: vpshufb %xmm4, %xmm14, %xmm14
1417 ; AVX1-NEXT: vpshufb %xmm4, %xmm1, %xmm1
1418 ; AVX1-NEXT: vpshufb %xmm4, %xmm13, %xmm2
1419 ; AVX1-NEXT: vpshufb %xmm4, %xmm15, %xmm0
1420 ; AVX1-NEXT: vpshufb %xmm4, %xmm8, %xmm7
1421 ; AVX1-NEXT: vpshufb %xmm4, %xmm9, %xmm4
1422 ; AVX1-NEXT: vpalignr {{.*#+}} xmm13 = xmm4[11,12,13,14,15],xmm3[0,1,2,3,4,5,6,7,8,9,10]
1423 ; AVX1-NEXT: vpalignr {{.*#+}} xmm15 = xmm7[11,12,13,14,15],xmm11[0,1,2,3,4,5,6,7,8,9,10]
1424 ; AVX1-NEXT: vpalignr {{.*#+}} xmm9 = xmm0[11,12,13,14,15],xmm5[0,1,2,3,4,5,6,7,8,9,10]
1425 ; AVX1-NEXT: vpalignr {{.*#+}} xmm8 = xmm2[11,12,13,14,15],xmm6[0,1,2,3,4,5,6,7,8,9,10]
1426 ; AVX1-NEXT: vpalignr {{.*#+}} xmm5 = xmm5[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]
1427 ; AVX1-NEXT: vpalignr {{.*#+}} xmm6 = xmm6[11,12,13,14,15],xmm14[0,1,2,3,4,5,6,7,8,9,10]
1428 ; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm1[11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10]
1429 ; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm6, %ymm1
1430 ; AVX1-NEXT: vpalignr {{.*#+}} xmm3 = xmm3[11,12,13,14,15],xmm12[0,1,2,3,4,5,6,7,8,9,10]
1431 ; AVX1-NEXT: vpalignr {{.*#+}} xmm11 = xmm11[11,12,13,14,15],xmm10[0,1,2,3,4,5,6,7,8,9,10]
1432 ; AVX1-NEXT: vpalignr {{.*#+}} xmm2 = xmm14[11,12,13,14,15],xmm2[0,1,2,3,4,5,6,7,8,9,10]
1433 ; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm11, %ymm14
1434 ; AVX1-NEXT: vpalignr {{.*#+}} xmm4 = xmm12[11,12,13,14,15],xmm4[0,1,2,3,4,5,6,7,8,9,10]
1435 ; AVX1-NEXT: vpalignr {{.*#+}} xmm7 = xmm10[11,12,13,14,15],xmm7[0,1,2,3,4,5,6,7,8,9,10]
1436 ; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm7, %ymm10
1437 ; AVX1-NEXT: vbroadcastf128 {{.*#+}} ymm12 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]
1438 ; AVX1-NEXT: # ymm12 = mem[0,1,0,1]
1439 ; AVX1-NEXT: vandnps %ymm10, %ymm12, %ymm10
1440 ; AVX1-NEXT: vandps %ymm12, %ymm14, %ymm14
1441 ; AVX1-NEXT: vorps %ymm10, %ymm14, %ymm10
1442 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm14
1443 ; AVX1-NEXT: vandnps %ymm14, %ymm12, %ymm14
1444 ; AVX1-NEXT: vandps %ymm12, %ymm1, %ymm1
1445 ; AVX1-NEXT: vorps %ymm14, %ymm1, %ymm1
1446 ; AVX1-NEXT: vpalignr {{.*#+}} xmm4 = xmm13[11,12,13,14,15],xmm4[0,1,2,3,4,5,6,7,8,9,10]
1447 ; AVX1-NEXT: vpalignr {{.*#+}} xmm3 = xmm3[11,12,13,14,15],xmm13[0,1,2,3,4,5,6,7,8,9,10]
1448 ; AVX1-NEXT: vpalignr {{.*#+}} xmm12 = xmm15[11,12,13,14,15],xmm7[0,1,2,3,4,5,6,7,8,9,10]
1449 ; AVX1-NEXT: vpalignr {{.*#+}} xmm7 = xmm11[11,12,13,14,15],xmm15[0,1,2,3,4,5,6,7,8,9,10]
1450 ; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm9[11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10]
1451 ; AVX1-NEXT: vpalignr {{.*#+}} xmm5 = xmm5[11,12,13,14,15],xmm9[0,1,2,3,4,5,6,7,8,9,10]
1452 ; AVX1-NEXT: vpalignr {{.*#+}} xmm9 = xmm8[11,12,13,14,15],xmm2[0,1,2,3,4,5,6,7,8,9,10]
1453 ; AVX1-NEXT: vpalignr {{.*#+}} xmm6 = xmm6[11,12,13,14,15],xmm8[0,1,2,3,4,5,6,7,8,9,10]
1454 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
1455 ; AVX1-NEXT: vpaddb %xmm0, %xmm2, %xmm0
1456 ; AVX1-NEXT: vpalignr {{.*#+}} xmm2 = xmm5[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]
1457 ; AVX1-NEXT: vpaddb %xmm0, %xmm2, %xmm2
1458 ; AVX1-NEXT: vextractf128 $1, %ymm10, %xmm0
1459 ; AVX1-NEXT: vpaddb %xmm4, %xmm0, %xmm0
1460 ; AVX1-NEXT: vpalignr {{.*#+}} xmm3 = xmm3[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]
1461 ; AVX1-NEXT: vpaddb %xmm0, %xmm3, %xmm0
1462 ; AVX1-NEXT: vpaddb %xmm12, %xmm10, %xmm3
1463 ; AVX1-NEXT: vpalignr {{.*#+}} xmm4 = xmm7[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]
1464 ; AVX1-NEXT: vpaddb %xmm3, %xmm4, %xmm3
1465 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm3, %ymm0
1466 ; AVX1-NEXT: vpaddb %xmm9, %xmm1, %xmm1
1467 ; AVX1-NEXT: vpalignr {{.*#+}} xmm3 = xmm6[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]
1468 ; AVX1-NEXT: vpaddb %xmm1, %xmm3, %xmm1
1469 ; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1
1470 ; AVX1-NEXT: retq
1471 ;
1472 ; AVX2-LABEL: interleaved_load_vf64_i8_stride3:
1473 ; AVX2: # %bb.0:
1474 ; AVX2-NEXT: vmovdqu (%rdi), %xmm0
1475 ; AVX2-NEXT: vmovdqu 16(%rdi), %xmm1
1476 ; AVX2-NEXT: vmovdqu 32(%rdi), %xmm2
1477 ; AVX2-NEXT: vmovdqu 96(%rdi), %xmm3
1478 ; AVX2-NEXT: vmovdqu 112(%rdi), %xmm4
1479 ; AVX2-NEXT: vmovdqu 128(%rdi), %xmm5
1480 ; AVX2-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm0
1481 ; AVX2-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm1
1482 ; AVX2-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm2
1483 ; AVX2-NEXT: vinserti128 $1, 144(%rdi), %ymm3, %ymm3
1484 ; AVX2-NEXT: vinserti128 $1, 160(%rdi), %ymm4, %ymm4
1485 ; AVX2-NEXT: vinserti128 $1, 176(%rdi), %ymm5, %ymm5
1486 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm6 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]
1487 ; AVX2-NEXT: vpshufb %ymm6, %ymm3, %ymm3
1488 ; AVX2-NEXT: vpshufb %ymm6, %ymm0, %ymm0
1489 ; AVX2-NEXT: vpshufb %ymm6, %ymm1, %ymm1
1490 ; AVX2-NEXT: vpshufb %ymm6, %ymm4, %ymm4
1491 ; AVX2-NEXT: vpshufb %ymm6, %ymm5, %ymm5
1492 ; AVX2-NEXT: vpshufb %ymm6, %ymm2, %ymm2
1493 ; AVX2-NEXT: vpalignr {{.*#+}} ymm6 = ymm2[11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7,8,9,10],ymm2[27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23,24,25,26]
1494 ; AVX2-NEXT: vpalignr {{.*#+}} ymm7 = ymm5[11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7,8,9,10],ymm5[27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23,24,25,26]
1495 ; AVX2-NEXT: vpalignr {{.*#+}} ymm3 = ymm3[11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7,8,9,10],ymm3[27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23,24,25,26]
1496 ; AVX2-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]
1497 ; AVX2-NEXT: vpalignr {{.*#+}} ymm1 = ymm1[11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7,8,9,10],ymm1[27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23,24,25,26]
1498 ; AVX2-NEXT: vpalignr {{.*#+}} ymm2 = ymm4[11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7,8,9,10],ymm4[27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23,24,25,26]
1499 ; AVX2-NEXT: vpalignr {{.*#+}} ymm4 = ymm7[11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7,8,9,10],ymm7[27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23,24,25,26]
1500 ; AVX2-NEXT: vpalignr {{.*#+}} ymm5 = ymm6[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm6[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]
1501 ; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm8 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]
1502 ; AVX2-NEXT: # ymm8 = mem[0,1,0,1]
1503 ; AVX2-NEXT: vpblendvb %ymm8, %ymm0, %ymm1, %ymm1
1504 ; AVX2-NEXT: vpaddb %ymm5, %ymm1, %ymm1
1505 ; AVX2-NEXT: vpblendvb %ymm8, %ymm3, %ymm2, %ymm2
1506 ; AVX2-NEXT: vpaddb %ymm4, %ymm2, %ymm2
1507 ; AVX2-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm6[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm6[16,17,18,19,20,21,22,23,24,25,26]
1508 ; AVX2-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]
1509 ; AVX2-NEXT: vpaddb %ymm1, %ymm0, %ymm0
1510 ; AVX2-NEXT: vpalignr {{.*#+}} ymm1 = ymm3[11,12,13,14,15],ymm7[0,1,2,3,4,5,6,7,8,9,10],ymm3[27,28,29,30,31],ymm7[16,17,18,19,20,21,22,23,24,25,26]
1511 ; AVX2-NEXT: vpalignr {{.*#+}} ymm1 = ymm1[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]
1512 ; AVX2-NEXT: vpaddb %ymm2, %ymm1, %ymm1
1513 ; AVX2-NEXT: retq
1514 ;
1515 ; AVX512-LABEL: interleaved_load_vf64_i8_stride3:
1516 ; AVX512: # %bb.0:
1517 ; AVX512-NEXT: vmovdqu (%rdi), %xmm0
1518 ; AVX512-NEXT: vmovdqu 16(%rdi), %xmm1
1519 ; AVX512-NEXT: vmovdqu 32(%rdi), %xmm2
1520 ; AVX512-NEXT: vmovdqu 96(%rdi), %xmm3
1521 ; AVX512-NEXT: vmovdqu 112(%rdi), %xmm4
1522 ; AVX512-NEXT: vmovdqu 128(%rdi), %xmm5
1523 ; AVX512-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm0
1524 ; AVX512-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm1
1525 ; AVX512-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm2
1526 ; AVX512-NEXT: vinserti128 $1, 144(%rdi), %ymm3, %ymm3
1527 ; AVX512-NEXT: vinserti128 $1, 160(%rdi), %ymm4, %ymm4
1528 ; AVX512-NEXT: vinserti128 $1, 176(%rdi), %ymm5, %ymm5
1529 ; AVX512-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0
1530 ; AVX512-NEXT: vinserti64x4 $1, %ymm4, %zmm1, %zmm1
1531 ; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm2, %zmm2
1532 ; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]
1533 ; AVX512-NEXT: vpshufb %zmm3, %zmm0, %zmm0
1534 ; AVX512-NEXT: vpshufb %zmm3, %zmm1, %zmm1
1535 ; AVX512-NEXT: vpshufb %zmm3, %zmm2, %zmm2
1536 ; AVX512-NEXT: vpalignr {{.*#+}} zmm3 = zmm2[11,12,13,14,15],zmm0[0,1,2,3,4,5,6,7,8,9,10],zmm2[27,28,29,30,31],zmm0[16,17,18,19,20,21,22,23,24,25,26],zmm2[43,44,45,46,47],zmm0[32,33,34,35,36,37,38,39,40,41,42],zmm2[59,60,61,62,63],zmm0[48,49,50,51,52,53,54,55,56,57,58]
1537 ; AVX512-NEXT: vpalignr {{.*#+}} zmm0 = zmm0[11,12,13,14,15],zmm1[0,1,2,3,4,5,6,7,8,9,10],zmm0[27,28,29,30,31],zmm1[16,17,18,19,20,21,22,23,24,25,26],zmm0[43,44,45,46,47],zmm1[32,33,34,35,36,37,38,39,40,41,42],zmm0[59,60,61,62,63],zmm1[48,49,50,51,52,53,54,55,56,57,58]
1538 ; AVX512-NEXT: movabsq $-576188069258921984, %rax # imm = 0xF800F800F800F800
1539 ; AVX512-NEXT: kmovq %rax, %k1
1540 ; AVX512-NEXT: vpalignr {{.*#+}} ymm4 = ymm0[11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23,24,25,26]
1541 ; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm5
1542 ; AVX512-NEXT: vpalignr {{.*#+}} zmm0 {%k1} = zmm1[11,12,13,14,15],zmm2[0,1,2,3,4,5,6,7,8,9,10],zmm1[27,28,29,30,31],zmm2[16,17,18,19,20,21,22,23,24,25,26],zmm1[43,44,45,46,47],zmm2[32,33,34,35,36,37,38,39,40,41,42],zmm1[59,60,61,62,63],zmm2[48,49,50,51,52,53,54,55,56,57,58]
1543 ; AVX512-NEXT: vpalignr {{.*#+}} zmm1 = zmm1[11,12,13,14,15],zmm2[0,1,2,3,4,5,6,7,8,9,10],zmm1[27,28,29,30,31],zmm2[16,17,18,19,20,21,22,23,24,25,26],zmm1[43,44,45,46,47],zmm2[32,33,34,35,36,37,38,39,40,41,42],zmm1[59,60,61,62,63],zmm2[48,49,50,51,52,53,54,55,56,57,58]
1544 ; AVX512-NEXT: vpalignr {{.*#+}} zmm1 = zmm3[11,12,13,14,15],zmm1[0,1,2,3,4,5,6,7,8,9,10],zmm3[27,28,29,30,31],zmm1[16,17,18,19,20,21,22,23,24,25,26],zmm3[43,44,45,46,47],zmm1[32,33,34,35,36,37,38,39,40,41,42],zmm3[59,60,61,62,63],zmm1[48,49,50,51,52,53,54,55,56,57,58]
1545 ; AVX512-NEXT: vpaddb %zmm1, %zmm0, %zmm0
1546 ; AVX512-NEXT: vpalignr {{.*#+}} ymm1 = ymm4[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]
1547 ; AVX512-NEXT: vextracti64x4 $1, %zmm3, %ymm2
1548 ; AVX512-NEXT: vpalignr {{.*#+}} ymm2 = ymm5[11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7,8,9,10],ymm5[27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23,24,25,26]
1549 ; AVX512-NEXT: vpalignr {{.*#+}} ymm2 = ymm2[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]
1550 ; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1
1551 ; AVX512-NEXT: vpaddb %zmm0, %zmm1, %zmm0
1552 ; AVX512-NEXT: retq
1553 %wide.vec = load <192 x i8>, <192 x i8>* %ptr, align 1
1554 %v1 = shufflevector <192 x i8> %wide.vec, <192 x i8> undef, <64 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45, i32 48, i32 51, i32 54, i32 57, i32 60, i32 63, i32 66, i32 69, i32 72, i32 75, i32 78, i32 81, i32 84, i32 87, i32 90, i32 93, i32 96, i32 99, i32 102, i32 105, i32 108, i32 111, i32 114, i32 117, i32 120, i32 123, i32 126, i32 129, i32 132, i32 135, i32 138, i32 141, i32 144, i32 147, i32 150, i32 153, i32 156, i32 159, i32 162, i32 165, i32 168, i32 171, i32 174, i32 177, i32 180, i32 183, i32 186, i32 189>
1555 %v2 = shufflevector <192 x i8> %wide.vec, <192 x i8> undef, <64 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46, i32 49, i32 52, i32 55, i32 58, i32 61, i32 64, i32 67, i32 70, i32 73, i32 76, i32 79, i32 82, i32 85, i32 88, i32 91, i32 94, i32 97, i32 100, i32 103, i32 106, i32 109, i32 112, i32 115, i32 118, i32 121, i32 124, i32 127, i32 130, i32 133, i32 136, i32 139, i32 142, i32 145, i32 148, i32 151, i32 154, i32 157, i32 160, i32 163, i32 166, i32 169, i32 172, i32 175, i32 178, i32 181, i32 184, i32 187, i32 190>
1556 %v3 = shufflevector <192 x i8> %wide.vec, <192 x i8> undef, <64 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47, i32 50, i32 53, i32 56, i32 59, i32 62, i32 65, i32 68, i32 71, i32 74, i32 77, i32 80, i32 83, i32 86, i32 89, i32 92, i32 95, i32 98, i32 101, i32 104, i32 107, i32 110, i32 113, i32 116, i32 119, i32 122, i32 125, i32 128, i32 131, i32 134, i32 137, i32 140, i32 143, i32 146, i32 149, i32 152, i32 155, i32 158, i32 161, i32 164, i32 167, i32 170, i32 173, i32 176, i32 179, i32 182, i32 185, i32 188, i32 191>
1557 %add1 = add <64 x i8> %v1, %v2
1558 %add2 = add <64 x i8> %v3, %add1
1559 ret <64 x i8> %add2
1560 }
1562 define void @interleaved_store_vf64_i8_stride4(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c,<64 x i8> %d, <256 x i8>* %p) {
1563 ; AVX1-LABEL: interleaved_store_vf64_i8_stride4:
1564 ; AVX1: # %bb.0:
1565 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm8 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
1566 ; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm11
1567 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm12
1568 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm9 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3],xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7]
1569 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm10 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
1570 ; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm13
1571 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm14
1572 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm15 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3],xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]
1573 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
1574 ; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1575 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm11 = xmm12[8],xmm11[8],xmm12[9],xmm11[9],xmm12[10],xmm11[10],xmm12[11],xmm11[11],xmm12[12],xmm11[12],xmm12[13],xmm11[13],xmm12[14],xmm11[14],xmm12[15],xmm11[15]
1576 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm12 = xmm1[8],xmm3[8],xmm1[9],xmm3[9],xmm1[10],xmm3[10],xmm1[11],xmm3[11],xmm1[12],xmm3[12],xmm1[13],xmm3[13],xmm1[14],xmm3[14],xmm1[15],xmm3[15]
1577 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm13 = xmm14[8],xmm13[8],xmm14[9],xmm13[9],xmm14[10],xmm13[10],xmm14[11],xmm13[11],xmm14[12],xmm13[12],xmm14[13],xmm13[13],xmm14[14],xmm13[14],xmm14[15],xmm13[15]
1578 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm14 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3],xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7]
1579 ; AVX1-NEXT: vextractf128 $1, %ymm6, %xmm1
1580 ; AVX1-NEXT: vextractf128 $1, %ymm4, %xmm3
1581 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
1582 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm4[8],xmm6[8],xmm4[9],xmm6[9],xmm4[10],xmm6[10],xmm4[11],xmm6[11],xmm4[12],xmm6[12],xmm4[13],xmm6[13],xmm4[14],xmm6[14],xmm4[15],xmm6[15]
1583 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm6 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3],xmm5[4],xmm7[4],xmm5[5],xmm7[5],xmm5[6],xmm7[6],xmm5[7],xmm7[7]
1584 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
1585 ; AVX1-NEXT: vextractf128 $1, %ymm7, %xmm3
1586 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm5[8],xmm7[8],xmm5[9],xmm7[9],xmm5[10],xmm7[10],xmm5[11],xmm7[11],xmm5[12],xmm7[12],xmm5[13],xmm7[13],xmm5[14],xmm7[14],xmm5[15],xmm7[15]
1587 ; AVX1-NEXT: vextractf128 $1, %ymm5, %xmm5
1588 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3],xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
1589 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15]
1590 ; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
1591 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm15[4],xmm0[4],xmm15[5],xmm0[5],xmm15[6],xmm0[6],xmm15[7],xmm0[7]
1592 ; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3]
1593 ; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm15, %ymm5
1594 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
1595 ; AVX1-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm5[2,3],ymm0[2,3]
1596 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm10[4],xmm6[4],xmm10[5],xmm6[5],xmm10[6],xmm6[6],xmm10[7],xmm6[7]
1597 ; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm9[0],xmm2[0],xmm9[1],xmm2[1],xmm9[2],xmm2[2],xmm9[3],xmm2[3]
1598 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm9[4],xmm2[4],xmm9[5],xmm2[5],xmm9[6],xmm2[6],xmm9[7],xmm2[7]
1599 ; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm8[0],xmm14[0],xmm8[1],xmm14[1],xmm8[2],xmm14[2],xmm8[3],xmm14[3]
1600 ; AVX1-NEXT: vinsertf128 $1, %xmm6, %ymm9, %ymm6
1601 ; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2
1602 ; AVX1-NEXT: vperm2f128 {{.*#+}} ymm10 = ymm6[2,3],ymm2[2,3]
1603 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm14 = xmm8[4],xmm14[4],xmm8[5],xmm14[5],xmm8[6],xmm14[6],xmm8[7],xmm14[7]
1604 ; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm13[0],xmm3[0],xmm13[1],xmm3[1],xmm13[2],xmm3[2],xmm13[3],xmm3[3]
1605 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm13[4],xmm3[4],xmm13[5],xmm3[5],xmm13[6],xmm3[6],xmm13[7],xmm3[7]
1606 ; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm12[0],xmm7[0],xmm12[1],xmm7[1],xmm12[2],xmm7[2],xmm12[3],xmm7[3]
1607 ; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm8, %ymm2
1608 ; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3
1609 ; AVX1-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm2[2,3],ymm3[2,3]
1610 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm12[4],xmm7[4],xmm12[5],xmm7[5],xmm12[6],xmm7[6],xmm12[7],xmm7[7]
1611 ; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm11[0],xmm1[0],xmm11[1],xmm1[1],xmm11[2],xmm1[2],xmm11[3],xmm1[3]
1612 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm11[4],xmm1[4],xmm11[5],xmm1[5],xmm11[6],xmm1[6],xmm11[7],xmm1[7]
1613 ; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
1614 ; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
1615 ; AVX1-NEXT: vinsertf128 $1, %xmm7, %ymm11, %ymm7
1616 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
1617 ; AVX1-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm7[2,3],ymm1[2,3]
1618 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
1619 ; AVX1-NEXT: vinsertf128 $1, %xmm14, %ymm9, %ymm6
1620 ; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm11, %ymm4
1621 ; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm15, %ymm5
1622 ; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm8, %ymm3
1623 ; AVX1-NEXT: vmovaps %ymm3, 160(%rdi)
1624 ; AVX1-NEXT: vmovaps %ymm5, 128(%rdi)
1625 ; AVX1-NEXT: vmovaps %ymm4, 32(%rdi)
1626 ; AVX1-NEXT: vmovaps %ymm6, (%rdi)
1627 ; AVX1-NEXT: vmovaps %ymm2, 224(%rdi)
1628 ; AVX1-NEXT: vmovaps %ymm0, 192(%rdi)
1629 ; AVX1-NEXT: vmovaps %ymm1, 96(%rdi)
1630 ; AVX1-NEXT: vmovaps %ymm10, 64(%rdi)
1631 ; AVX1-NEXT: vzeroupper
1632 ; AVX1-NEXT: retq
1633 ;
1634 ; AVX2-LABEL: interleaved_store_vf64_i8_stride4:
1635 ; AVX2: # %bb.0:
1636 ; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm8 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
1637 ; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm9 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[4],ymm3[4],ymm1[5],ymm3[5],ymm1[6],ymm3[6],ymm1[7],ymm3[7],ymm1[16],ymm3[16],ymm1[17],ymm3[17],ymm1[18],ymm3[18],ymm1[19],ymm3[19],ymm1[20],ymm3[20],ymm1[21],ymm3[21],ymm1[22],ymm3[22],ymm1[23],ymm3[23]
1638 ; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm0 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]
1639 ; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm1 = ymm1[8],ymm3[8],ymm1[9],ymm3[9],ymm1[10],ymm3[10],ymm1[11],ymm3[11],ymm1[12],ymm3[12],ymm1[13],ymm3[13],ymm1[14],ymm3[14],ymm1[15],ymm3[15],ymm1[24],ymm3[24],ymm1[25],ymm3[25],ymm1[26],ymm3[26],ymm1[27],ymm3[27],ymm1[28],ymm3[28],ymm1[29],ymm3[29],ymm1[30],ymm3[30],ymm1[31],ymm3[31]
1640 ; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm4[0],ymm6[0],ymm4[1],ymm6[1],ymm4[2],ymm6[2],ymm4[3],ymm6[3],ymm4[4],ymm6[4],ymm4[5],ymm6[5],ymm4[6],ymm6[6],ymm4[7],ymm6[7],ymm4[16],ymm6[16],ymm4[17],ymm6[17],ymm4[18],ymm6[18],ymm4[19],ymm6[19],ymm4[20],ymm6[20],ymm4[21],ymm6[21],ymm4[22],ymm6[22],ymm4[23],ymm6[23]
1641 ; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm3 = ymm5[0],ymm7[0],ymm5[1],ymm7[1],ymm5[2],ymm7[2],ymm5[3],ymm7[3],ymm5[4],ymm7[4],ymm5[5],ymm7[5],ymm5[6],ymm7[6],ymm5[7],ymm7[7],ymm5[16],ymm7[16],ymm5[17],ymm7[17],ymm5[18],ymm7[18],ymm5[19],ymm7[19],ymm5[20],ymm7[20],ymm5[21],ymm7[21],ymm5[22],ymm7[22],ymm5[23],ymm7[23]
1642 ; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm4[8],ymm6[8],ymm4[9],ymm6[9],ymm4[10],ymm6[10],ymm4[11],ymm6[11],ymm4[12],ymm6[12],ymm4[13],ymm6[13],ymm4[14],ymm6[14],ymm4[15],ymm6[15],ymm4[24],ymm6[24],ymm4[25],ymm6[25],ymm4[26],ymm6[26],ymm4[27],ymm6[27],ymm4[28],ymm6[28],ymm4[29],ymm6[29],ymm4[30],ymm6[30],ymm4[31],ymm6[31]
1643 ; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm5 = ymm5[8],ymm7[8],ymm5[9],ymm7[9],ymm5[10],ymm7[10],ymm5[11],ymm7[11],ymm5[12],ymm7[12],ymm5[13],ymm7[13],ymm5[14],ymm7[14],ymm5[15],ymm7[15],ymm5[24],ymm7[24],ymm5[25],ymm7[25],ymm5[26],ymm7[26],ymm5[27],ymm7[27],ymm5[28],ymm7[28],ymm5[29],ymm7[29],ymm5[30],ymm7[30],ymm5[31],ymm7[31]
1644 ; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm6 = ymm9[0],ymm3[0],ymm9[1],ymm3[1],ymm9[2],ymm3[2],ymm9[3],ymm3[3],ymm9[8],ymm3[8],ymm9[9],ymm3[9],ymm9[10],ymm3[10],ymm9[11],ymm3[11]
1645 ; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm7 = ymm8[0],ymm2[0],ymm8[1],ymm2[1],ymm8[2],ymm2[2],ymm8[3],ymm2[3],ymm8[8],ymm2[8],ymm8[9],ymm2[9],ymm8[10],ymm2[10],ymm8[11],ymm2[11]
1646 ; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm9[4],ymm3[4],ymm9[5],ymm3[5],ymm9[6],ymm3[6],ymm9[7],ymm3[7],ymm9[12],ymm3[12],ymm9[13],ymm3[13],ymm9[14],ymm3[14],ymm9[15],ymm3[15]
1647 ; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm2 = ymm8[4],ymm2[4],ymm8[5],ymm2[5],ymm8[6],ymm2[6],ymm8[7],ymm2[7],ymm8[12],ymm2[12],ymm8[13],ymm2[13],ymm8[14],ymm2[14],ymm8[15],ymm2[15]
1648 ; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm8 = ymm1[0],ymm5[0],ymm1[1],ymm5[1],ymm1[2],ymm5[2],ymm1[3],ymm5[3],ymm1[8],ymm5[8],ymm1[9],ymm5[9],ymm1[10],ymm5[10],ymm1[11],ymm5[11]
1649 ; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm9 = ymm0[0],ymm4[0],ymm0[1],ymm4[1],ymm0[2],ymm4[2],ymm0[3],ymm4[3],ymm0[8],ymm4[8],ymm0[9],ymm4[9],ymm0[10],ymm4[10],ymm0[11],ymm4[11]
1650 ; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm5[4],ymm1[5],ymm5[5],ymm1[6],ymm5[6],ymm1[7],ymm5[7],ymm1[12],ymm5[12],ymm1[13],ymm5[13],ymm1[14],ymm5[14],ymm1[15],ymm5[15]
1651 ; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm4[4],ymm0[5],ymm4[5],ymm0[6],ymm4[6],ymm0[7],ymm4[7],ymm0[12],ymm4[12],ymm0[13],ymm4[13],ymm0[14],ymm4[14],ymm0[15],ymm4[15]
1652 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm7, %ymm4
1653 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm9, %ymm5
1654 ; AVX2-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
1655 ; AVX2-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm9[2,3],ymm0[2,3]
1656 ; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm6, %ymm7
1657 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm8, %ymm9
1658 ; AVX2-NEXT: vperm2i128 {{.*#+}} ymm3 = ymm6[2,3],ymm3[2,3]
1659 ; AVX2-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm8[2,3],ymm1[2,3]
1660 ; AVX2-NEXT: vmovdqa %ymm1, 224(%rdi)
1661 ; AVX2-NEXT: vmovdqa %ymm3, 192(%rdi)
1662 ; AVX2-NEXT: vmovdqa %ymm0, 96(%rdi)
1663 ; AVX2-NEXT: vmovdqa %ymm2, 64(%rdi)
1664 ; AVX2-NEXT: vmovdqa %ymm9, 160(%rdi)
1665 ; AVX2-NEXT: vmovdqa %ymm7, 128(%rdi)
1666 ; AVX2-NEXT: vmovdqa %ymm5, 32(%rdi)
1667 ; AVX2-NEXT: vmovdqa %ymm4, (%rdi)
1668 ; AVX2-NEXT: vzeroupper
1669 ; AVX2-NEXT: retq
1670 ;
1671 ; AVX512-LABEL: interleaved_store_vf64_i8_stride4:
1672 ; AVX512: # %bb.0:
1673 ; AVX512-NEXT: vpunpcklbw {{.*#+}} zmm4 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[16],zmm1[16],zmm0[17],zmm1[17],zmm0[18],zmm1[18],zmm0[19],zmm1[19],zmm0[20],zmm1[20],zmm0[21],zmm1[21],zmm0[22],zmm1[22],zmm0[23],zmm1[23],zmm0[32],zmm1[32],zmm0[33],zmm1[33],zmm0[34],zmm1[34],zmm0[35],zmm1[35],zmm0[36],zmm1[36],zmm0[37],zmm1[37],zmm0[38],zmm1[38],zmm0[39],zmm1[39],zmm0[48],zmm1[48],zmm0[49],zmm1[49],zmm0[50],zmm1[50],zmm0[51],zmm1[51],zmm0[52],zmm1[52],zmm0[53],zmm1[53],zmm0[54],zmm1[54],zmm0[55],zmm1[55]
1674 ; AVX512-NEXT: vpunpckhbw {{.*#+}} zmm0 = zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[12],zmm1[12],zmm0[13],zmm1[13],zmm0[14],zmm1[14],zmm0[15],zmm1[15],zmm0[24],zmm1[24],zmm0[25],zmm1[25],zmm0[26],zmm1[26],zmm0[27],zmm1[27],zmm0[28],zmm1[28],zmm0[29],zmm1[29],zmm0[30],zmm1[30],zmm0[31],zmm1[31],zmm0[40],zmm1[40],zmm0[41],zmm1[41],zmm0[42],zmm1[42],zmm0[43],zmm1[43],zmm0[44],zmm1[44],zmm0[45],zmm1[45],zmm0[46],zmm1[46],zmm0[47],zmm1[47],zmm0[56],zmm1[56],zmm0[57],zmm1[57],zmm0[58],zmm1[58],zmm0[59],zmm1[59],zmm0[60],zmm1[60],zmm0[61],zmm1[61],zmm0[62],zmm1[62],zmm0[63],zmm1[63]
1675 ; AVX512-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm2[0],zmm3[0],zmm2[1],zmm3[1],zmm2[2],zmm3[2],zmm2[3],zmm3[3],zmm2[4],zmm3[4],zmm2[5],zmm3[5],zmm2[6],zmm3[6],zmm2[7],zmm3[7],zmm2[16],zmm3[16],zmm2[17],zmm3[17],zmm2[18],zmm3[18],zmm2[19],zmm3[19],zmm2[20],zmm3[20],zmm2[21],zmm3[21],zmm2[22],zmm3[22],zmm2[23],zmm3[23],zmm2[32],zmm3[32],zmm2[33],zmm3[33],zmm2[34],zmm3[34],zmm2[35],zmm3[35],zmm2[36],zmm3[36],zmm2[37],zmm3[37],zmm2[38],zmm3[38],zmm2[39],zmm3[39],zmm2[48],zmm3[48],zmm2[49],zmm3[49],zmm2[50],zmm3[50],zmm2[51],zmm3[51],zmm2[52],zmm3[52],zmm2[53],zmm3[53],zmm2[54],zmm3[54],zmm2[55],zmm3[55]
1676 ; AVX512-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm2[8],zmm3[8],zmm2[9],zmm3[9],zmm2[10],zmm3[10],zmm2[11],zmm3[11],zmm2[12],zmm3[12],zmm2[13],zmm3[13],zmm2[14],zmm3[14],zmm2[15],zmm3[15],zmm2[24],zmm3[24],zmm2[25],zmm3[25],zmm2[26],zmm3[26],zmm2[27],zmm3[27],zmm2[28],zmm3[28],zmm2[29],zmm3[29],zmm2[30],zmm3[30],zmm2[31],zmm3[31],zmm2[40],zmm3[40],zmm2[41],zmm3[41],zmm2[42],zmm3[42],zmm2[43],zmm3[43],zmm2[44],zmm3[44],zmm2[45],zmm3[45],zmm2[46],zmm3[46],zmm2[47],zmm3[47],zmm2[56],zmm3[56],zmm2[57],zmm3[57],zmm2[58],zmm3[58],zmm2[59],zmm3[59],zmm2[60],zmm3[60],zmm2[61],zmm3[61],zmm2[62],zmm3[62],zmm2[63],zmm3[63]
1677 ; AVX512-NEXT: vpunpcklwd {{.*#+}} zmm3 = zmm4[0],zmm1[0],zmm4[1],zmm1[1],zmm4[2],zmm1[2],zmm4[3],zmm1[3],zmm4[8],zmm1[8],zmm4[9],zmm1[9],zmm4[10],zmm1[10],zmm4[11],zmm1[11],zmm4[16],zmm1[16],zmm4[17],zmm1[17],zmm4[18],zmm1[18],zmm4[19],zmm1[19],zmm4[24],zmm1[24],zmm4[25],zmm1[25],zmm4[26],zmm1[26],zmm4[27],zmm1[27]
1678 ; AVX512-NEXT: vpunpckhwd {{.*#+}} zmm1 = zmm4[4],zmm1[4],zmm4[5],zmm1[5],zmm4[6],zmm1[6],zmm4[7],zmm1[7],zmm4[12],zmm1[12],zmm4[13],zmm1[13],zmm4[14],zmm1[14],zmm4[15],zmm1[15],zmm4[20],zmm1[20],zmm4[21],zmm1[21],zmm4[22],zmm1[22],zmm4[23],zmm1[23],zmm4[28],zmm1[28],zmm4[29],zmm1[29],zmm4[30],zmm1[30],zmm4[31],zmm1[31]
1679 ; AVX512-NEXT: vpunpcklwd {{.*#+}} zmm4 = zmm0[0],zmm2[0],zmm0[1],zmm2[1],zmm0[2],zmm2[2],zmm0[3],zmm2[3],zmm0[8],zmm2[8],zmm0[9],zmm2[9],zmm0[10],zmm2[10],zmm0[11],zmm2[11],zmm0[16],zmm2[16],zmm0[17],zmm2[17],zmm0[18],zmm2[18],zmm0[19],zmm2[19],zmm0[24],zmm2[24],zmm0[25],zmm2[25],zmm0[26],zmm2[26],zmm0[27],zmm2[27]
1680 ; AVX512-NEXT: vpunpckhwd {{.*#+}} zmm0 = zmm0[4],zmm2[4],zmm0[5],zmm2[5],zmm0[6],zmm2[6],zmm0[7],zmm2[7],zmm0[12],zmm2[12],zmm0[13],zmm2[13],zmm0[14],zmm2[14],zmm0[15],zmm2[15],zmm0[20],zmm2[20],zmm0[21],zmm2[21],zmm0[22],zmm2[22],zmm0[23],zmm2[23],zmm0[28],zmm2[28],zmm0[29],zmm2[29],zmm0[30],zmm2[30],zmm0[31],zmm2[31]
1681 ; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm3, %ymm2
1682 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm4, %ymm5
1683 ; AVX512-NEXT: vperm2i128 {{.*#+}} ymm6 = ymm3[2,3],ymm1[2,3]
1684 ; AVX512-NEXT: vperm2i128 {{.*#+}} ymm7 = ymm4[2,3],ymm0[2,3]
1685 ; AVX512-NEXT: vextracti64x4 $1, %zmm3, %ymm3
1686 ; AVX512-NEXT: vextracti64x4 $1, %zmm1, %ymm1
1687 ; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm3, %ymm8
1688 ; AVX512-NEXT: vextracti64x4 $1, %zmm4, %ymm4
1689 ; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm0
1690 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm4, %ymm9
1691 ; AVX512-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm3[2,3],ymm1[2,3]
1692 ; AVX512-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm4[2,3],ymm0[2,3]
1693 ; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm2, %zmm2
1694 ; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm6, %zmm3
1695 ; AVX512-NEXT: vinserti64x4 $1, %ymm9, %zmm8, %zmm4
1696 ; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
1697 ; AVX512-NEXT: vmovdqa64 %zmm0, 192(%rdi)
1698 ; AVX512-NEXT: vmovdqa64 %zmm3, 64(%rdi)
1699 ; AVX512-NEXT: vmovdqa64 %zmm4, 128(%rdi)
1700 ; AVX512-NEXT: vmovdqa64 %zmm2, (%rdi)
1701 ; AVX512-NEXT: vzeroupper
1702 ; AVX512-NEXT: retq
1703 %1 = shufflevector <64 x i8> %a, <64 x i8> %b, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
1704 %2 = shufflevector <64 x i8> %c, <64 x i8> %d, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
1705 %interleaved = shufflevector <128 x i8> %1, <128 x i8> %2, <256 x i32> <i32 0, i32 64, i32 128, i32 192, i32 1, i32 65, i32 129, i32 193, i32 2, i32 66, i32 130, i32 194, i32 3, i32 67, i32 131, i32 195, i32 4, i32 68, i32 132, i32 196, i32 5, i32 69, i32 133, i32 197, i32 6, i32 70, i32 134, i32 198, i32 7, i32 71, i32 135, i32 199, i32 8, i32 72, i32 136, i32 200, i32 9, i32 73, i32 137, i32 201, i32 10, i32 74, i32 138, i32 202, i32 11, i32 75, i32 139, i32 203, i32 12, i32 76, i32 140, i32 204, i32 13, i32 77, i32 141, i32 205, i32 14, i32 78, i32 142, i32 206, i32 15, i32 79, i32 143, i32 207, i32 16, i32 80, i32 144, i32 208, i32 17, i32 81, i32 145, i32 209, i32 18, i32 82, i32 146, i32 210, i32 19, i32 83, i32 147, i32 211, i32 20, i32 84, i32 148, i32 212, i32 21, i32 85, i32 149, i32 213, i32 22, i32 86, i32 150, i32 214, i32 23, i32 87, i32 151, i32 215, i32 24, i32 88, i32 152, i32 216, i32 25, i32 89, i32 153, i32 217, i32 26, i32 90, i32 154, i32 218, i32 27, i32 91, i32 155, i32 219, i32 28, i32 92, i32 156, i32 220, i32 29, i32 93, i32 157, i32 221, i32 30, i32 94, i32 158, i32 222, i32 31, i32 95, i32 159, i32 223, i32 32, i32 96, i32 160, i32 224, i32 33, i32 97, i32 161, i32 225, i32 34, i32 98, i32 162, i32 226, i32 35, i32 99, i32 163, i32 227, i32 36, i32 100, i32 164, i32 228, i32 37, i32 101, i32 165, i32 229, i32 38, i32 102, i32 166, i32 230, i32 39, i32 103, i32 167, i32 231, i32 40, i32 104, i32 168, i32 232, i32 41, i32 105, i32 169, i32 233, i32 42, i32 106, i32 170, i32 234, i32 43, i32 107, i32 171, i32 235, i32 44, i32 108, i32 172, i32 236, i32 45, i32 109, i32 173, i32 237, i32 46, i32 110, i32 174, i32 238, i32 47, i32 111, i32 175, i32 239, i32 48, i32 112, i32 176, i32 240, i32 49, i32 113, i32 177, i32 241, i32 50, i32 114, i32 178, i32 242, i32 51, i32 115, i32 179, i32 243, i32 52, i32 116, i32 180, i32 244, i32 53, i32 117, i32 181, i32 245, i32 54, i32 118, i32 182, i32 246, i32 55, i32 119, i32 183, i32 247, i32 56, i32 120, i32 184, i32 248, i32 57, i32 121, i32 185, i32 249, i32 58, i32 122, i32 186, i32 250, i32 59, i32 123, i32 187, i32 251, i32 60, i32 124, i32 188, i32 252, i32 61, i32 125, i32 189, i32 253, i32 62, i32 126, i32 190, i32 254, i32 63, i32 127, i32 191, i32 255>
1706 store <256 x i8> %interleaved, <256 x i8>* %p
1707 ret void
1708 }