| blob | f24abb5684000998bd395972512e2fa20df7964e |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc -mtriple=arm64-apple-ios -mattr=+sve -o - %s | FileCheck %s
3 ; RUN: llc -mtriple=aarch64_be-unknown-linux -mattr=+sve -o - %s | FileCheck --check-prefix=CHECK-BE %s
5 ; CHECK-LABEL: lCPI0_0:
6 ; CHECK-NEXT: .byte 0 ; 0x0
7 ; CHECK-NEXT: .byte 255 ; 0xff
8 ; CHECK-NEXT: .byte 255 ; 0xff
9 ; CHECK-NEXT: .byte 255 ; 0xff
10 ; CHECK-NEXT: .byte 1 ; 0x1
11 ; CHECK-NEXT: .byte 255 ; 0xff
12 ; CHECK-NEXT: .byte 255 ; 0xff
13 ; CHECK-NEXT: .byte 255 ; 0xff
14 ; CHECK-NEXT: .byte 2 ; 0x2
15 ; CHECK-NEXT: .byte 255 ; 0xff
16 ; CHECK-NEXT: .byte 255 ; 0xff
17 ; CHECK-NEXT: .byte 255 ; 0xff
18 ; CHECK-NEXT: .byte 3 ; 0x3
19 ; CHECK-NEXT: .byte 255 ; 0xff
20 ; CHECK-NEXT: .byte 255 ; 0xff
21 ; CHECK-NEXT: .byte 255 ; 0xff
22 ; CHECK-NEXT:lCPI0_1:
23 ; CHECK-NEXT: .byte 4 ; 0x4
24 ; CHECK-NEXT: .byte 255 ; 0xff
25 ; CHECK-NEXT: .byte 255 ; 0xff
26 ; CHECK-NEXT: .byte 255 ; 0xff
27 ; CHECK-NEXT: .byte 5 ; 0x5
28 ; CHECK-NEXT: .byte 255 ; 0xff
29 ; CHECK-NEXT: .byte 255 ; 0xff
30 ; CHECK-NEXT: .byte 255 ; 0xff
31 ; CHECK-NEXT: .byte 6 ; 0x6
32 ; CHECK-NEXT: .byte 255 ; 0xff
33 ; CHECK-NEXT: .byte 255 ; 0xff
34 ; CHECK-NEXT: .byte 255 ; 0xff
35 ; CHECK-NEXT: .byte 7 ; 0x7
36 ; CHECK-NEXT: .byte 255 ; 0xff
37 ; CHECK-NEXT: .byte 255 ; 0xff
38 ; CHECK-NEXT: .byte 255 ; 0xff
39 ; CHECK-NEXT:lCPI0_2:
40 ; CHECK-NEXT: .byte 8 ; 0x8
41 ; CHECK-NEXT: .byte 255 ; 0xff
42 ; CHECK-NEXT: .byte 255 ; 0xff
43 ; CHECK-NEXT: .byte 255 ; 0xff
44 ; CHECK-NEXT: .byte 9 ; 0x9
45 ; CHECK-NEXT: .byte 255 ; 0xff
46 ; CHECK-NEXT: .byte 255 ; 0xff
47 ; CHECK-NEXT: .byte 255 ; 0xff
48 ; CHECK-NEXT: .byte 10 ; 0xa
49 ; CHECK-NEXT: .byte 255 ; 0xff
50 ; CHECK-NEXT: .byte 255 ; 0xff
51 ; CHECK-NEXT: .byte 255 ; 0xff
52 ; CHECK-NEXT: .byte 11 ; 0xb
53 ; CHECK-NEXT: .byte 255 ; 0xff
54 ; CHECK-NEXT: .byte 255 ; 0xff
55 ; CHECK-NEXT: .byte 255 ; 0xff
56 ; CHECK-NEXT:lCPI0_3:
57 ; CHECK-NEXT: .byte 12 ; 0xc
58 ; CHECK-NEXT: .byte 255 ; 0xff
59 ; CHECK-NEXT: .byte 255 ; 0xff
60 ; CHECK-NEXT: .byte 255 ; 0xff
61 ; CHECK-NEXT: .byte 13 ; 0xd
62 ; CHECK-NEXT: .byte 255 ; 0xff
63 ; CHECK-NEXT: .byte 255 ; 0xff
64 ; CHECK-NEXT: .byte 255 ; 0xff
65 ; CHECK-NEXT: .byte 14 ; 0xe
66 ; CHECK-NEXT: .byte 255 ; 0xff
67 ; CHECK-NEXT: .byte 255 ; 0xff
68 ; CHECK-NEXT: .byte 255 ; 0xff
69 ; CHECK-NEXT: .byte 15 ; 0xf
70 ; CHECK-NEXT: .byte 255 ; 0xff
71 ; CHECK-NEXT: .byte 255 ; 0xff
72 ; CHECK-NEXT: .byte 255 ; 0xff
74 ; CHECK-BE: .LCPI0_0:
75 ; CHECK-BE-NEXT: .byte 255 // 0xff
76 ; CHECK-BE-NEXT: .byte 255 // 0xff
77 ; CHECK-BE-NEXT: .byte 255 // 0xff
78 ; CHECK-BE-NEXT: .byte 0 // 0x0
79 ; CHECK-BE-NEXT: .byte 255 // 0xff
80 ; CHECK-BE-NEXT: .byte 255 // 0xff
81 ; CHECK-BE-NEXT: .byte 255 // 0xff
82 ; CHECK-BE-NEXT: .byte 1 // 0x1
83 ; CHECK-BE-NEXT: .byte 255 // 0xff
84 ; CHECK-BE-NEXT: .byte 255 // 0xff
85 ; CHECK-BE-NEXT: .byte 255 // 0xff
86 ; CHECK-BE-NEXT: .byte 2 // 0x2
87 ; CHECK-BE-NEXT: .byte 255 // 0xff
88 ; CHECK-BE-NEXT: .byte 255 // 0xff
89 ; CHECK-BE-NEXT: .byte 255 // 0xff
90 ; CHECK-BE-NEXT: .byte 3 // 0x3
91 ; CHECK-BE-NEXT: .LCPI0_1:
92 ; CHECK-BE-NEXT: .byte 255 // 0xff
93 ; CHECK-BE-NEXT: .byte 255 // 0xff
94 ; CHECK-BE-NEXT: .byte 255 // 0xff
95 ; CHECK-BE-NEXT: .byte 4 // 0x4
96 ; CHECK-BE-NEXT: .byte 255 // 0xff
97 ; CHECK-BE-NEXT: .byte 255 // 0xff
98 ; CHECK-BE-NEXT: .byte 255 // 0xff
99 ; CHECK-BE-NEXT: .byte 5 // 0x5
100 ; CHECK-BE-NEXT: .byte 255 // 0xff
101 ; CHECK-BE-NEXT: .byte 255 // 0xff
102 ; CHECK-BE-NEXT: .byte 255 // 0xff
103 ; CHECK-BE-NEXT: .byte 6 // 0x6
104 ; CHECK-BE-NEXT: .byte 255 // 0xff
105 ; CHECK-BE-NEXT: .byte 255 // 0xff
106 ; CHECK-BE-NEXT: .byte 255 // 0xff
107 ; CHECK-BE-NEXT: .byte 7 // 0x7
108 ; CHECK-BE-NEXT: .LCPI0_2:
109 ; CHECK-BE-NEXT: .byte 255 // 0xff
110 ; CHECK-BE-NEXT: .byte 255 // 0xff
111 ; CHECK-BE-NEXT: .byte 255 // 0xff
112 ; CHECK-BE-NEXT: .byte 8 // 0x8
113 ; CHECK-BE-NEXT: .byte 255 // 0xff
114 ; CHECK-BE-NEXT: .byte 255 // 0xff
115 ; CHECK-BE-NEXT: .byte 255 // 0xff
116 ; CHECK-BE-NEXT: .byte 9 // 0x9
117 ; CHECK-BE-NEXT: .byte 255 // 0xff
118 ; CHECK-BE-NEXT: .byte 255 // 0xff
119 ; CHECK-BE-NEXT: .byte 255 // 0xff
120 ; CHECK-BE-NEXT: .byte 10 // 0xa
121 ; CHECK-BE-NEXT: .byte 255 // 0xff
122 ; CHECK-BE-NEXT: .byte 255 // 0xff
123 ; CHECK-BE-NEXT: .byte 255 // 0xff
124 ; CHECK-BE-NEXT: .byte 11 // 0xb
125 ; CHECK-BE-NEXT: .LCPI0_3:
126 ; CHECK-BE-NEXT: .byte 255 // 0xff
127 ; CHECK-BE-NEXT: .byte 255 // 0xff
128 ; CHECK-BE-NEXT: .byte 255 // 0xff
129 ; CHECK-BE-NEXT: .byte 12 // 0xc
130 ; CHECK-BE-NEXT: .byte 255 // 0xff
131 ; CHECK-BE-NEXT: .byte 255 // 0xff
132 ; CHECK-BE-NEXT: .byte 255 // 0xff
133 ; CHECK-BE-NEXT: .byte 13 // 0xd
134 ; CHECK-BE-NEXT: .byte 255 // 0xff
135 ; CHECK-BE-NEXT: .byte 255 // 0xff
136 ; CHECK-BE-NEXT: .byte 255 // 0xff
137 ; CHECK-BE-NEXT: .byte 14 // 0xe
138 ; CHECK-BE-NEXT: .byte 255 // 0xff
139 ; CHECK-BE-NEXT: .byte 255 // 0xff
140 ; CHECK-BE-NEXT: .byte 255 // 0xff
141 ; CHECK-BE-NEXT: .byte 15 // 0xf
143 ; It's profitable to convert the zext to a shuffle, which in turn will be
144 ; lowered to 4 tbl instructions. The masks are materialized outside the loop.
145 define void @zext_v16i8_to_v16i32_in_loop(ptr %src, ptr %dst) {
146 ; CHECK-LABEL: zext_v16i8_to_v16i32_in_loop:
147 ; CHECK: ; %bb.0: ; %entry
148 ; CHECK-NEXT: Lloh0:
149 ; CHECK-NEXT: adrp x8, lCPI0_0@PAGE
150 ; CHECK-NEXT: Lloh1:
151 ; CHECK-NEXT: adrp x9, lCPI0_1@PAGE
152 ; CHECK-NEXT: Lloh2:
153 ; CHECK-NEXT: adrp x10, lCPI0_2@PAGE
154 ; CHECK-NEXT: Lloh3:
155 ; CHECK-NEXT: ldr q0, [x8, lCPI0_0@PAGEOFF]
156 ; CHECK-NEXT: Lloh4:
157 ; CHECK-NEXT: adrp x8, lCPI0_3@PAGE
158 ; CHECK-NEXT: Lloh5:
159 ; CHECK-NEXT: ldr q1, [x9, lCPI0_1@PAGEOFF]
160 ; CHECK-NEXT: Lloh6:
161 ; CHECK-NEXT: ldr q2, [x10, lCPI0_2@PAGEOFF]
162 ; CHECK-NEXT: Lloh7:
163 ; CHECK-NEXT: ldr q3, [x8, lCPI0_3@PAGEOFF]
164 ; CHECK-NEXT: mov x8, xzr
165 ; CHECK-NEXT: LBB0_1: ; %loop
166 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
167 ; CHECK-NEXT: ldr q4, [x0, x8]
168 ; CHECK-NEXT: add x8, x8, #16
169 ; CHECK-NEXT: cmp x8, #128
170 ; CHECK-NEXT: tbl.16b v5, { v4 }, v3
171 ; CHECK-NEXT: tbl.16b v6, { v4 }, v2
172 ; CHECK-NEXT: tbl.16b v7, { v4 }, v1
173 ; CHECK-NEXT: tbl.16b v4, { v4 }, v0
174 ; CHECK-NEXT: stp q6, q5, [x1, #32]
175 ; CHECK-NEXT: stp q4, q7, [x1], #64
176 ; CHECK-NEXT: b.ne LBB0_1
177 ; CHECK-NEXT: ; %bb.2: ; %exit
178 ; CHECK-NEXT: ret
179 ; CHECK-NEXT: .loh AdrpLdr Lloh4, Lloh7
180 ; CHECK-NEXT: .loh AdrpLdr Lloh2, Lloh6
181 ; CHECK-NEXT: .loh AdrpLdr Lloh1, Lloh5
182 ; CHECK-NEXT: .loh AdrpAdrp Lloh0, Lloh4
183 ; CHECK-NEXT: .loh AdrpLdr Lloh0, Lloh3
184 ;
185 ; CHECK-BE-LABEL: zext_v16i8_to_v16i32_in_loop:
186 ; CHECK-BE: // %bb.0: // %entry
187 ; CHECK-BE-NEXT: adrp x8, .LCPI0_0
188 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI0_0
189 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]
190 ; CHECK-BE-NEXT: adrp x8, .LCPI0_1
191 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI0_1
192 ; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
193 ; CHECK-BE-NEXT: adrp x8, .LCPI0_2
194 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI0_2
195 ; CHECK-BE-NEXT: ld1 { v2.16b }, [x8]
196 ; CHECK-BE-NEXT: adrp x8, .LCPI0_3
197 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI0_3
198 ; CHECK-BE-NEXT: ld1 { v3.16b }, [x8]
199 ; CHECK-BE-NEXT: mov x8, xzr
200 ; CHECK-BE-NEXT: .LBB0_1: // %loop
201 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
202 ; CHECK-BE-NEXT: add x9, x0, x8
203 ; CHECK-BE-NEXT: add x8, x8, #16
204 ; CHECK-BE-NEXT: ld1 { v4.16b }, [x9]
205 ; CHECK-BE-NEXT: add x9, x1, #48
206 ; CHECK-BE-NEXT: cmp x8, #128
207 ; CHECK-BE-NEXT: tbl v5.16b, { v4.16b }, v3.16b
208 ; CHECK-BE-NEXT: tbl v6.16b, { v4.16b }, v2.16b
209 ; CHECK-BE-NEXT: tbl v7.16b, { v4.16b }, v1.16b
210 ; CHECK-BE-NEXT: tbl v4.16b, { v4.16b }, v0.16b
211 ; CHECK-BE-NEXT: st1 { v5.16b }, [x9]
212 ; CHECK-BE-NEXT: add x9, x1, #32
213 ; CHECK-BE-NEXT: st1 { v6.16b }, [x9]
214 ; CHECK-BE-NEXT: add x9, x1, #16
215 ; CHECK-BE-NEXT: st1 { v4.16b }, [x1]
216 ; CHECK-BE-NEXT: add x1, x1, #64
217 ; CHECK-BE-NEXT: st1 { v7.16b }, [x9]
218 ; CHECK-BE-NEXT: b.ne .LBB0_1
219 ; CHECK-BE-NEXT: // %bb.2: // %exit
220 ; CHECK-BE-NEXT: ret
221 entry:
222 br label %loop
224 loop:
225 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
226 %src.gep = getelementptr i8, ptr %src, i64 %iv
227 %load = load <16 x i8>, ptr %src.gep
228 %ext = zext <16 x i8> %load to <16 x i32>
229 %dst.gep = getelementptr i32, ptr %dst, i64 %iv
230 store <16 x i32> %ext, ptr %dst.gep
231 %iv.next = add nuw i64 %iv, 16
232 %ec = icmp eq i64 %iv.next, 128
233 br i1 %ec, label %exit, label %loop
235 exit:
236 ret void
237 }
239 define void @zext_v16i8_to_v16i32_in_loop_not_header(ptr %src, ptr %dst, i1 %c) {
240 ; CHECK-LABEL: zext_v16i8_to_v16i32_in_loop_not_header:
241 ; CHECK: ; %bb.0: ; %entry
242 ; CHECK-NEXT: mov x8, xzr
243 ; CHECK-NEXT: b LBB1_2
244 ; CHECK-NEXT: LBB1_1: ; %loop.latch
245 ; CHECK-NEXT: ; in Loop: Header=BB1_2 Depth=1
246 ; CHECK-NEXT: add x8, x8, #16
247 ; CHECK-NEXT: add x1, x1, #64
248 ; CHECK-NEXT: cmp x8, #128
249 ; CHECK-NEXT: b.eq LBB1_4
250 ; CHECK-NEXT: LBB1_2: ; %loop
251 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
252 ; CHECK-NEXT: tbz w2, #0, LBB1_1
253 ; CHECK-NEXT: ; %bb.3: ; %then
254 ; CHECK-NEXT: ; in Loop: Header=BB1_2 Depth=1
255 ; CHECK-NEXT: ldr q0, [x0, x8]
256 ; CHECK-NEXT: ushll2.8h v1, v0, #0
257 ; CHECK-NEXT: ushll.8h v0, v0, #0
258 ; CHECK-NEXT: ushll2.4s v2, v1, #0
259 ; CHECK-NEXT: ushll.4s v1, v1, #0
260 ; CHECK-NEXT: ushll2.4s v3, v0, #0
261 ; CHECK-NEXT: ushll.4s v0, v0, #0
262 ; CHECK-NEXT: stp q1, q2, [x1, #32]
263 ; CHECK-NEXT: stp q0, q3, [x1]
264 ; CHECK-NEXT: b LBB1_1
265 ; CHECK-NEXT: LBB1_4: ; %exit
266 ; CHECK-NEXT: ret
267 ;
268 ; CHECK-BE-LABEL: zext_v16i8_to_v16i32_in_loop_not_header:
269 ; CHECK-BE: // %bb.0: // %entry
270 ; CHECK-BE-NEXT: mov x8, xzr
271 ; CHECK-BE-NEXT: b .LBB1_2
272 ; CHECK-BE-NEXT: .LBB1_1: // %loop.latch
273 ; CHECK-BE-NEXT: // in Loop: Header=BB1_2 Depth=1
274 ; CHECK-BE-NEXT: add x8, x8, #16
275 ; CHECK-BE-NEXT: add x1, x1, #64
276 ; CHECK-BE-NEXT: cmp x8, #128
277 ; CHECK-BE-NEXT: b.eq .LBB1_4
278 ; CHECK-BE-NEXT: .LBB1_2: // %loop
279 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
280 ; CHECK-BE-NEXT: tbz w2, #0, .LBB1_1
281 ; CHECK-BE-NEXT: // %bb.3: // %then
282 ; CHECK-BE-NEXT: // in Loop: Header=BB1_2 Depth=1
283 ; CHECK-BE-NEXT: add x9, x0, x8
284 ; CHECK-BE-NEXT: add x10, x1, #32
285 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x9]
286 ; CHECK-BE-NEXT: add x9, x1, #48
287 ; CHECK-BE-NEXT: ushll2 v1.8h, v0.16b, #0
288 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
289 ; CHECK-BE-NEXT: ushll2 v2.4s, v1.8h, #0
290 ; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #0
291 ; CHECK-BE-NEXT: ushll2 v3.4s, v0.8h, #0
292 ; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
293 ; CHECK-BE-NEXT: st1 { v2.4s }, [x9]
294 ; CHECK-BE-NEXT: add x9, x1, #16
295 ; CHECK-BE-NEXT: st1 { v1.4s }, [x10]
296 ; CHECK-BE-NEXT: st1 { v3.4s }, [x9]
297 ; CHECK-BE-NEXT: st1 { v0.4s }, [x1]
298 ; CHECK-BE-NEXT: b .LBB1_1
299 ; CHECK-BE-NEXT: .LBB1_4: // %exit
300 ; CHECK-BE-NEXT: ret
301 entry:
302 br label %loop
304 loop:
305 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
306 br i1 %c, label %then, label %loop.latch
308 then:
309 %src.gep = getelementptr i8, ptr %src, i64 %iv
310 %load = load <16 x i8>, ptr %src.gep
311 %ext = zext <16 x i8> %load to <16 x i32>
312 %dst.gep = getelementptr i32, ptr %dst, i64 %iv
313 store <16 x i32> %ext, ptr %dst.gep
314 br label %loop.latch
316 loop.latch:
317 %iv.next = add nuw i64 %iv, 16
318 %ec = icmp eq i64 %iv.next, 128
319 br i1 %ec, label %exit, label %loop
321 exit:
322 ret void
323 }
325 ; Not profitable to use shuffle/tbl, as 4 tbls + materializing the masks
326 ; require more instructions than lowering zext directly.
327 define void @zext_v16i8_to_v16i32_no_loop(ptr %src, ptr %dst) {
328 ; CHECK-LABEL: zext_v16i8_to_v16i32_no_loop:
329 ; CHECK: ; %bb.0: ; %entry
330 ; CHECK-NEXT: ldr q0, [x0]
331 ; CHECK-NEXT: ushll2.8h v1, v0, #0
332 ; CHECK-NEXT: ushll.8h v0, v0, #0
333 ; CHECK-NEXT: ushll2.4s v2, v1, #0
334 ; CHECK-NEXT: ushll.4s v1, v1, #0
335 ; CHECK-NEXT: ushll2.4s v3, v0, #0
336 ; CHECK-NEXT: ushll.4s v0, v0, #0
337 ; CHECK-NEXT: stp q1, q2, [x1, #32]
338 ; CHECK-NEXT: stp q0, q3, [x1]
339 ; CHECK-NEXT: ret
340 ;
341 ; CHECK-BE-LABEL: zext_v16i8_to_v16i32_no_loop:
342 ; CHECK-BE: // %bb.0: // %entry
343 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x0]
344 ; CHECK-BE-NEXT: add x8, x1, #48
345 ; CHECK-BE-NEXT: ushll2 v1.8h, v0.16b, #0
346 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
347 ; CHECK-BE-NEXT: ushll2 v2.4s, v1.8h, #0
348 ; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #0
349 ; CHECK-BE-NEXT: ushll2 v3.4s, v0.8h, #0
350 ; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
351 ; CHECK-BE-NEXT: st1 { v2.4s }, [x8]
352 ; CHECK-BE-NEXT: add x8, x1, #32
353 ; CHECK-BE-NEXT: st1 { v1.4s }, [x8]
354 ; CHECK-BE-NEXT: add x8, x1, #16
355 ; CHECK-BE-NEXT: st1 { v3.4s }, [x8]
356 ; CHECK-BE-NEXT: st1 { v0.4s }, [x1]
357 ; CHECK-BE-NEXT: ret
358 entry:
359 %load = load <16 x i8>, ptr %src
360 %ext = zext <16 x i8> %load to <16 x i32>
361 store <16 x i32> %ext, ptr %dst
362 ret void
363 }
365 ; Avoid using tbl when optimizing for size.
366 define void @zext_v16i8_to_v16i32_in_loop_optsize(ptr %src, ptr %dst) optsize {
367 ; CHECK-LABEL: zext_v16i8_to_v16i32_in_loop_optsize:
368 ; CHECK: ; %bb.0: ; %entry
369 ; CHECK-NEXT: mov x8, xzr
370 ; CHECK-NEXT: LBB3_1: ; %loop
371 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
372 ; CHECK-NEXT: ldr q0, [x0, x8]
373 ; CHECK-NEXT: add x8, x8, #16
374 ; CHECK-NEXT: cmp x8, #128
375 ; CHECK-NEXT: ushll2.8h v1, v0, #0
376 ; CHECK-NEXT: ushll.8h v0, v0, #0
377 ; CHECK-NEXT: ushll2.4s v2, v1, #0
378 ; CHECK-NEXT: ushll.4s v1, v1, #0
379 ; CHECK-NEXT: ushll2.4s v3, v0, #0
380 ; CHECK-NEXT: ushll.4s v0, v0, #0
381 ; CHECK-NEXT: stp q1, q2, [x1, #32]
382 ; CHECK-NEXT: stp q0, q3, [x1], #64
383 ; CHECK-NEXT: b.ne LBB3_1
384 ; CHECK-NEXT: ; %bb.2: ; %exit
385 ; CHECK-NEXT: ret
386 ;
387 ; CHECK-BE-LABEL: zext_v16i8_to_v16i32_in_loop_optsize:
388 ; CHECK-BE: // %bb.0: // %entry
389 ; CHECK-BE-NEXT: mov x8, xzr
390 ; CHECK-BE-NEXT: .LBB3_1: // %loop
391 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
392 ; CHECK-BE-NEXT: add x9, x0, x8
393 ; CHECK-BE-NEXT: add x8, x8, #16
394 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x9]
395 ; CHECK-BE-NEXT: add x9, x1, #48
396 ; CHECK-BE-NEXT: cmp x8, #128
397 ; CHECK-BE-NEXT: ushll2 v1.8h, v0.16b, #0
398 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
399 ; CHECK-BE-NEXT: ushll2 v2.4s, v1.8h, #0
400 ; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #0
401 ; CHECK-BE-NEXT: ushll2 v3.4s, v0.8h, #0
402 ; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
403 ; CHECK-BE-NEXT: st1 { v2.4s }, [x9]
404 ; CHECK-BE-NEXT: add x9, x1, #32
405 ; CHECK-BE-NEXT: st1 { v1.4s }, [x9]
406 ; CHECK-BE-NEXT: add x9, x1, #16
407 ; CHECK-BE-NEXT: st1 { v0.4s }, [x1]
408 ; CHECK-BE-NEXT: add x1, x1, #64
409 ; CHECK-BE-NEXT: st1 { v3.4s }, [x9]
410 ; CHECK-BE-NEXT: b.ne .LBB3_1
411 ; CHECK-BE-NEXT: // %bb.2: // %exit
412 ; CHECK-BE-NEXT: ret
413 entry:
414 br label %loop
416 loop:
417 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
418 %src.gep = getelementptr i8, ptr %src, i64 %iv
419 %load = load <16 x i8>, ptr %src.gep
420 %ext = zext <16 x i8> %load to <16 x i32>
421 %dst.gep = getelementptr i32, ptr %dst, i64 %iv
422 store <16 x i32> %ext, ptr %dst.gep
423 %iv.next = add nuw i64 %iv, 16
424 %ec = icmp eq i64 %iv.next, 128
425 br i1 %ec, label %exit, label %loop
427 exit:
428 ret void
429 }
431 ; Avoid using tbl when optimizing for size.
432 define void @zext_v16i8_to_v16i32_in_loop_minsize(ptr %src, ptr %dst) minsize {
433 ; CHECK-LABEL: zext_v16i8_to_v16i32_in_loop_minsize:
434 ; CHECK: ; %bb.0: ; %entry
435 ; CHECK-NEXT: mov x8, xzr
436 ; CHECK-NEXT: LBB4_1: ; %loop
437 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
438 ; CHECK-NEXT: ldr q0, [x0, x8]
439 ; CHECK-NEXT: add x8, x8, #16
440 ; CHECK-NEXT: cmp x8, #128
441 ; CHECK-NEXT: ushll2.8h v1, v0, #0
442 ; CHECK-NEXT: ushll.8h v0, v0, #0
443 ; CHECK-NEXT: ushll2.4s v2, v1, #0
444 ; CHECK-NEXT: ushll.4s v1, v1, #0
445 ; CHECK-NEXT: ushll2.4s v3, v0, #0
446 ; CHECK-NEXT: ushll.4s v0, v0, #0
447 ; CHECK-NEXT: stp q1, q2, [x1, #32]
448 ; CHECK-NEXT: stp q0, q3, [x1], #64
449 ; CHECK-NEXT: b.ne LBB4_1
450 ; CHECK-NEXT: ; %bb.2: ; %exit
451 ; CHECK-NEXT: ret
452 ;
453 ; CHECK-BE-LABEL: zext_v16i8_to_v16i32_in_loop_minsize:
454 ; CHECK-BE: // %bb.0: // %entry
455 ; CHECK-BE-NEXT: mov x8, xzr
456 ; CHECK-BE-NEXT: .LBB4_1: // %loop
457 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
458 ; CHECK-BE-NEXT: add x9, x0, x8
459 ; CHECK-BE-NEXT: add x8, x8, #16
460 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x9]
461 ; CHECK-BE-NEXT: add x9, x1, #48
462 ; CHECK-BE-NEXT: cmp x8, #128
463 ; CHECK-BE-NEXT: ushll2 v1.8h, v0.16b, #0
464 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
465 ; CHECK-BE-NEXT: ushll2 v2.4s, v1.8h, #0
466 ; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #0
467 ; CHECK-BE-NEXT: ushll2 v3.4s, v0.8h, #0
468 ; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
469 ; CHECK-BE-NEXT: st1 { v2.4s }, [x9]
470 ; CHECK-BE-NEXT: add x9, x1, #32
471 ; CHECK-BE-NEXT: st1 { v1.4s }, [x9]
472 ; CHECK-BE-NEXT: add x9, x1, #16
473 ; CHECK-BE-NEXT: st1 { v0.4s }, [x1]
474 ; CHECK-BE-NEXT: add x1, x1, #64
475 ; CHECK-BE-NEXT: st1 { v3.4s }, [x9]
476 ; CHECK-BE-NEXT: b.ne .LBB4_1
477 ; CHECK-BE-NEXT: // %bb.2: // %exit
478 ; CHECK-BE-NEXT: ret
479 entry:
480 br label %loop
482 loop:
483 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
484 %src.gep = getelementptr i8, ptr %src, i64 %iv
485 %load = load <16 x i8>, ptr %src.gep
486 %ext = zext <16 x i8> %load to <16 x i32>
487 %dst.gep = getelementptr i32, ptr %dst, i64 %iv
488 store <16 x i32> %ext, ptr %dst.gep
489 %iv.next = add nuw i64 %iv, 16
490 %ec = icmp eq i64 %iv.next, 128
491 br i1 %ec, label %exit, label %loop
493 exit:
494 ret void
495 }
497 define void @zext_v16i8_to_v16i16_in_loop(ptr %src, ptr %dst) {
498 ; CHECK-LABEL: zext_v16i8_to_v16i16_in_loop:
499 ; CHECK: ; %bb.0: ; %entry
500 ; CHECK-NEXT: mov x8, xzr
501 ; CHECK-NEXT: LBB5_1: ; %loop
502 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
503 ; CHECK-NEXT: ldr q0, [x0, x8]
504 ; CHECK-NEXT: add x8, x8, #16
505 ; CHECK-NEXT: cmp x8, #128
506 ; CHECK-NEXT: ushll2.8h v1, v0, #0
507 ; CHECK-NEXT: ushll.8h v0, v0, #0
508 ; CHECK-NEXT: stp q0, q1, [x1], #32
509 ; CHECK-NEXT: b.ne LBB5_1
510 ; CHECK-NEXT: ; %bb.2: ; %exit
511 ; CHECK-NEXT: ret
512 ;
513 ; CHECK-BE-LABEL: zext_v16i8_to_v16i16_in_loop:
514 ; CHECK-BE: // %bb.0: // %entry
515 ; CHECK-BE-NEXT: mov x8, xzr
516 ; CHECK-BE-NEXT: .LBB5_1: // %loop
517 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
518 ; CHECK-BE-NEXT: add x9, x0, x8
519 ; CHECK-BE-NEXT: add x8, x8, #16
520 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x9]
521 ; CHECK-BE-NEXT: add x9, x1, #16
522 ; CHECK-BE-NEXT: cmp x8, #128
523 ; CHECK-BE-NEXT: ushll2 v1.8h, v0.16b, #0
524 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
525 ; CHECK-BE-NEXT: st1 { v0.8h }, [x1]
526 ; CHECK-BE-NEXT: add x1, x1, #32
527 ; CHECK-BE-NEXT: st1 { v1.8h }, [x9]
528 ; CHECK-BE-NEXT: b.ne .LBB5_1
529 ; CHECK-BE-NEXT: // %bb.2: // %exit
530 ; CHECK-BE-NEXT: ret
533 entry:
534 br label %loop
536 loop:
537 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
538 %src.gep = getelementptr i8, ptr %src, i64 %iv
539 %load = load <16 x i8>, ptr %src.gep
540 %ext = zext <16 x i8> %load to <16 x i16>
541 %dst.gep = getelementptr i16, ptr %dst, i64 %iv
542 store <16 x i16> %ext, ptr %dst.gep
543 %iv.next = add nuw i64 %iv, 16
544 %ec = icmp eq i64 %iv.next, 128
545 br i1 %ec, label %exit, label %loop
547 exit:
548 ret void
549 }
551 ; CHECK-LABEL: lCPI6_0:
552 ; CHECK-NEXT: .byte 0 ; 0x0
553 ; CHECK-NEXT: .byte 255 ; 0xff
554 ; CHECK-NEXT: .byte 255 ; 0xff
555 ; CHECK-NEXT: .byte 255 ; 0xff
556 ; CHECK-NEXT: .byte 1 ; 0x1
557 ; CHECK-NEXT: .byte 255 ; 0xff
558 ; CHECK-NEXT: .byte 255 ; 0xff
559 ; CHECK-NEXT: .byte 255 ; 0xff
560 ; CHECK-NEXT: .byte 2 ; 0x2
561 ; CHECK-NEXT: .byte 255 ; 0xff
562 ; CHECK-NEXT: .byte 255 ; 0xff
563 ; CHECK-NEXT: .byte 255 ; 0xff
564 ; CHECK-NEXT: .byte 3 ; 0x3
565 ; CHECK-NEXT: .byte 255 ; 0xff
566 ; CHECK-NEXT: .byte 255 ; 0xff
567 ; CHECK-NEXT: .byte 255 ; 0xff
568 ; CHECK-NEXT: lCPI6_1:
569 ; CHECK-NEXT: .byte 4 ; 0x4
570 ; CHECK-NEXT: .byte 255 ; 0xff
571 ; CHECK-NEXT: .byte 255 ; 0xff
572 ; CHECK-NEXT: .byte 255 ; 0xff
573 ; CHECK-NEXT: .byte 5 ; 0x5
574 ; CHECK-NEXT: .byte 255 ; 0xff
575 ; CHECK-NEXT: .byte 255 ; 0xff
576 ; CHECK-NEXT: .byte 255 ; 0xff
577 ; CHECK-NEXT: .byte 6 ; 0x6
578 ; CHECK-NEXT: .byte 255 ; 0xff
579 ; CHECK-NEXT: .byte 255 ; 0xff
580 ; CHECK-NEXT: .byte 255 ; 0xff
581 ; CHECK-NEXT: .byte 7 ; 0x7
582 ; CHECK-NEXT: .byte 255 ; 0xff
583 ; CHECK-NEXT: .byte 255 ; 0xff
584 ; CHECK-NEXT: .byte 255 ; 0xff
586 ; CHECK-BE: .LCPI6_0:
587 ; CHECK-BE-NEXT: .byte 255 // 0xff
588 ; CHECK-BE-NEXT: .byte 255 // 0xff
589 ; CHECK-BE-NEXT: .byte 255 // 0xff
590 ; CHECK-BE-NEXT: .byte 0 // 0x0
591 ; CHECK-BE-NEXT: .byte 255 // 0xff
592 ; CHECK-BE-NEXT: .byte 255 // 0xff
593 ; CHECK-BE-NEXT: .byte 255 // 0xff
594 ; CHECK-BE-NEXT: .byte 1 // 0x1
595 ; CHECK-BE-NEXT: .byte 255 // 0xff
596 ; CHECK-BE-NEXT: .byte 255 // 0xff
597 ; CHECK-BE-NEXT: .byte 255 // 0xff
598 ; CHECK-BE-NEXT: .byte 2 // 0x2
599 ; CHECK-BE-NEXT: .byte 255 // 0xff
600 ; CHECK-BE-NEXT: .byte 255 // 0xff
601 ; CHECK-BE-NEXT: .byte 255 // 0xff
602 ; CHECK-BE-NEXT: .byte 3 // 0x3
603 ; CHECK-BE-NEXT: .LCPI6_1:
604 ; CHECK-BE-NEXT: .byte 255 // 0xff
605 ; CHECK-BE-NEXT: .byte 255 // 0xff
606 ; CHECK-BE-NEXT: .byte 255 // 0xff
607 ; CHECK-BE-NEXT: .byte 4 // 0x4
608 ; CHECK-BE-NEXT: .byte 255 // 0xff
609 ; CHECK-BE-NEXT: .byte 255 // 0xff
610 ; CHECK-BE-NEXT: .byte 255 // 0xff
611 ; CHECK-BE-NEXT: .byte 5 // 0x5
612 ; CHECK-BE-NEXT: .byte 255 // 0xff
613 ; CHECK-BE-NEXT: .byte 255 // 0xff
614 ; CHECK-BE-NEXT: .byte 255 // 0xff
615 ; CHECK-BE-NEXT: .byte 6 // 0x6
616 ; CHECK-BE-NEXT: .byte 255 // 0xff
617 ; CHECK-BE-NEXT: .byte 255 // 0xff
618 ; CHECK-BE-NEXT: .byte 255 // 0xff
619 ; CHECK-BE-NEXT: .byte 7 // 0x7
621 define void @zext_v8i8_to_v8i32_in_loop(ptr %src, ptr %dst) {
622 ; CHECK-LABEL: zext_v8i8_to_v8i32_in_loop:
623 ; CHECK: ; %bb.0: ; %entry
624 ; CHECK-NEXT: Lloh8:
625 ; CHECK-NEXT: adrp x8, lCPI6_0@PAGE
626 ; CHECK-NEXT: Lloh9:
627 ; CHECK-NEXT: adrp x9, lCPI6_1@PAGE
628 ; CHECK-NEXT: Lloh10:
629 ; CHECK-NEXT: ldr q0, [x8, lCPI6_0@PAGEOFF]
630 ; CHECK-NEXT: Lloh11:
631 ; CHECK-NEXT: ldr q1, [x9, lCPI6_1@PAGEOFF]
632 ; CHECK-NEXT: mov x8, xzr
633 ; CHECK-NEXT: LBB6_1: ; %loop
634 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
635 ; CHECK-NEXT: ldr d2, [x0, x8]
636 ; CHECK-NEXT: add x8, x8, #16
637 ; CHECK-NEXT: cmp x8, #128
638 ; CHECK-NEXT: tbl.16b v3, { v2 }, v1
639 ; CHECK-NEXT: tbl.16b v2, { v2 }, v0
640 ; CHECK-NEXT: stp q2, q3, [x1], #64
641 ; CHECK-NEXT: b.ne LBB6_1
642 ; CHECK-NEXT: ; %bb.2: ; %exit
643 ; CHECK-NEXT: ret
644 ; CHECK-NEXT: .loh AdrpLdr Lloh9, Lloh11
645 ; CHECK-NEXT: .loh AdrpLdr Lloh8, Lloh10
646 ;
647 ; CHECK-BE-LABEL: zext_v8i8_to_v8i32_in_loop:
648 ; CHECK-BE: // %bb.0: // %entry
649 ; CHECK-BE-NEXT: adrp x8, .LCPI6_0
650 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI6_0
651 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]
652 ; CHECK-BE-NEXT: adrp x8, .LCPI6_1
653 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI6_1
654 ; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
655 ; CHECK-BE-NEXT: mov x8, xzr
656 ; CHECK-BE-NEXT: .LBB6_1: // %loop
657 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
658 ; CHECK-BE-NEXT: add x9, x0, x8
659 ; CHECK-BE-NEXT: add x8, x8, #16
660 ; CHECK-BE-NEXT: ld1 { v2.8b }, [x9]
661 ; CHECK-BE-NEXT: add x9, x1, #16
662 ; CHECK-BE-NEXT: cmp x8, #128
663 ; CHECK-BE-NEXT: tbl v3.16b, { v2.16b }, v1.16b
664 ; CHECK-BE-NEXT: tbl v2.16b, { v2.16b }, v0.16b
665 ; CHECK-BE-NEXT: st1 { v2.16b }, [x1]
666 ; CHECK-BE-NEXT: add x1, x1, #64
667 ; CHECK-BE-NEXT: st1 { v3.16b }, [x9]
668 ; CHECK-BE-NEXT: b.ne .LBB6_1
669 ; CHECK-BE-NEXT: // %bb.2: // %exit
670 ; CHECK-BE-NEXT: ret
671 entry:
672 br label %loop
674 loop:
675 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
676 %src.gep = getelementptr i8, ptr %src, i64 %iv
677 %load = load <8 x i8>, ptr %src.gep
678 %ext = zext <8 x i8> %load to <8 x i32>
679 %dst.gep = getelementptr i32, ptr %dst, i64 %iv
680 store <8 x i32> %ext, ptr %dst.gep
681 %iv.next = add nuw i64 %iv, 16
682 %ec = icmp eq i64 %iv.next, 128
683 br i1 %ec, label %exit, label %loop
685 exit:
686 ret void
687 }
689 define void @zext_v16i8_to_v16i64_in_loop(ptr %src, ptr %dst) {
690 ; CHECK-LABEL: zext_v16i8_to_v16i64_in_loop:
691 ; CHECK: ; %bb.0: ; %entry
692 ; CHECK-NEXT: mov x8, xzr
693 ; CHECK-NEXT: LBB7_1: ; %loop
694 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
695 ; CHECK-NEXT: ldr q0, [x0, x8]
696 ; CHECK-NEXT: add x8, x8, #16
697 ; CHECK-NEXT: cmp x8, #128
698 ; CHECK-NEXT: ushll2.8h v1, v0, #0
699 ; CHECK-NEXT: ushll.8h v0, v0, #0
700 ; CHECK-NEXT: ushll2.4s v2, v1, #0
701 ; CHECK-NEXT: ushll.4s v1, v1, #0
702 ; CHECK-NEXT: ushll2.4s v4, v0, #0
703 ; CHECK-NEXT: ushll.4s v0, v0, #0
704 ; CHECK-NEXT: ushll2.2d v3, v2, #0
705 ; CHECK-NEXT: ushll.2d v2, v2, #0
706 ; CHECK-NEXT: ushll2.2d v5, v1, #0
707 ; CHECK-NEXT: ushll.2d v1, v1, #0
708 ; CHECK-NEXT: stp q2, q3, [x1, #96]
709 ; CHECK-NEXT: ushll2.2d v3, v4, #0
710 ; CHECK-NEXT: ushll.2d v2, v4, #0
711 ; CHECK-NEXT: ushll2.2d v4, v0, #0
712 ; CHECK-NEXT: ushll.2d v0, v0, #0
713 ; CHECK-NEXT: stp q1, q5, [x1, #64]
714 ; CHECK-NEXT: stp q2, q3, [x1, #32]
715 ; CHECK-NEXT: stp q0, q4, [x1], #128
716 ; CHECK-NEXT: b.ne LBB7_1
717 ; CHECK-NEXT: ; %bb.2: ; %exit
718 ; CHECK-NEXT: ret
719 ;
720 ; CHECK-BE-LABEL: zext_v16i8_to_v16i64_in_loop:
721 ; CHECK-BE: // %bb.0: // %entry
722 ; CHECK-BE-NEXT: mov x8, xzr
723 ; CHECK-BE-NEXT: .LBB7_1: // %loop
724 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
725 ; CHECK-BE-NEXT: add x9, x0, x8
726 ; CHECK-BE-NEXT: add x8, x8, #16
727 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x9]
728 ; CHECK-BE-NEXT: add x9, x1, #112
729 ; CHECK-BE-NEXT: cmp x8, #128
730 ; CHECK-BE-NEXT: ushll2 v1.8h, v0.16b, #0
731 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
732 ; CHECK-BE-NEXT: ushll2 v2.4s, v1.8h, #0
733 ; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #0
734 ; CHECK-BE-NEXT: ushll2 v4.4s, v0.8h, #0
735 ; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
736 ; CHECK-BE-NEXT: ushll2 v3.2d, v2.4s, #0
737 ; CHECK-BE-NEXT: ushll v2.2d, v2.2s, #0
738 ; CHECK-BE-NEXT: ushll2 v5.2d, v1.4s, #0
739 ; CHECK-BE-NEXT: ushll v1.2d, v1.2s, #0
740 ; CHECK-BE-NEXT: st1 { v3.2d }, [x9]
741 ; CHECK-BE-NEXT: add x9, x1, #96
742 ; CHECK-BE-NEXT: ushll2 v3.2d, v4.4s, #0
743 ; CHECK-BE-NEXT: st1 { v2.2d }, [x9]
744 ; CHECK-BE-NEXT: add x9, x1, #80
745 ; CHECK-BE-NEXT: ushll v2.2d, v4.2s, #0
746 ; CHECK-BE-NEXT: st1 { v5.2d }, [x9]
747 ; CHECK-BE-NEXT: add x9, x1, #48
748 ; CHECK-BE-NEXT: st1 { v3.2d }, [x9]
749 ; CHECK-BE-NEXT: ushll2 v3.2d, v0.4s, #0
750 ; CHECK-BE-NEXT: ushll v0.2d, v0.2s, #0
751 ; CHECK-BE-NEXT: add x9, x1, #64
752 ; CHECK-BE-NEXT: st1 { v1.2d }, [x9]
753 ; CHECK-BE-NEXT: add x9, x1, #32
754 ; CHECK-BE-NEXT: st1 { v2.2d }, [x9]
755 ; CHECK-BE-NEXT: add x9, x1, #16
756 ; CHECK-BE-NEXT: st1 { v0.2d }, [x1]
757 ; CHECK-BE-NEXT: add x1, x1, #128
758 ; CHECK-BE-NEXT: st1 { v3.2d }, [x9]
759 ; CHECK-BE-NEXT: b.ne .LBB7_1
760 ; CHECK-BE-NEXT: // %bb.2: // %exit
761 ; CHECK-BE-NEXT: ret
764 entry:
765 br label %loop
767 loop:
768 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
769 %src.gep = getelementptr i8, ptr %src, i64 %iv
770 %load = load <16 x i8>, ptr %src.gep
771 %ext = zext <16 x i8> %load to <16 x i64>
772 %dst.gep = getelementptr i64, ptr %dst, i64 %iv
773 store <16 x i64> %ext, ptr %dst.gep
774 %iv.next = add nuw i64 %iv, 16
775 %ec = icmp eq i64 %iv.next, 128
776 br i1 %ec, label %exit, label %loop
778 exit:
779 ret void
780 }
782 define void @zext_v8i8_to_v8i64_in_loop(ptr %src, ptr %dst) {
783 ; CHECK-LABEL: zext_v8i8_to_v8i64_in_loop:
784 ; CHECK: ; %bb.0: ; %entry
785 ; CHECK-NEXT: mov x8, xzr
786 ; CHECK-NEXT: LBB8_1: ; %loop
787 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
788 ; CHECK-NEXT: ldr d0, [x0, x8]
789 ; CHECK-NEXT: add x8, x8, #16
790 ; CHECK-NEXT: cmp x8, #128
791 ; CHECK-NEXT: ushll.8h v0, v0, #0
792 ; CHECK-NEXT: ushll2.4s v1, v0, #0
793 ; CHECK-NEXT: ushll.4s v0, v0, #0
794 ; CHECK-NEXT: ushll2.2d v2, v1, #0
795 ; CHECK-NEXT: ushll.2d v1, v1, #0
796 ; CHECK-NEXT: ushll2.2d v3, v0, #0
797 ; CHECK-NEXT: ushll.2d v0, v0, #0
798 ; CHECK-NEXT: stp q1, q2, [x1, #32]
799 ; CHECK-NEXT: stp q0, q3, [x1], #128
800 ; CHECK-NEXT: b.ne LBB8_1
801 ; CHECK-NEXT: ; %bb.2: ; %exit
802 ; CHECK-NEXT: ret
803 ;
804 ; CHECK-BE-LABEL: zext_v8i8_to_v8i64_in_loop:
805 ; CHECK-BE: // %bb.0: // %entry
806 ; CHECK-BE-NEXT: mov x8, xzr
807 ; CHECK-BE-NEXT: .LBB8_1: // %loop
808 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
809 ; CHECK-BE-NEXT: add x9, x0, x8
810 ; CHECK-BE-NEXT: add x8, x8, #16
811 ; CHECK-BE-NEXT: ld1 { v0.8b }, [x9]
812 ; CHECK-BE-NEXT: add x9, x1, #48
813 ; CHECK-BE-NEXT: cmp x8, #128
814 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
815 ; CHECK-BE-NEXT: ushll2 v1.4s, v0.8h, #0
816 ; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
817 ; CHECK-BE-NEXT: ushll2 v2.2d, v1.4s, #0
818 ; CHECK-BE-NEXT: ushll v1.2d, v1.2s, #0
819 ; CHECK-BE-NEXT: ushll2 v3.2d, v0.4s, #0
820 ; CHECK-BE-NEXT: ushll v0.2d, v0.2s, #0
821 ; CHECK-BE-NEXT: st1 { v2.2d }, [x9]
822 ; CHECK-BE-NEXT: add x9, x1, #32
823 ; CHECK-BE-NEXT: st1 { v1.2d }, [x9]
824 ; CHECK-BE-NEXT: add x9, x1, #16
825 ; CHECK-BE-NEXT: st1 { v0.2d }, [x1]
826 ; CHECK-BE-NEXT: add x1, x1, #128
827 ; CHECK-BE-NEXT: st1 { v3.2d }, [x9]
828 ; CHECK-BE-NEXT: b.ne .LBB8_1
829 ; CHECK-BE-NEXT: // %bb.2: // %exit
830 ; CHECK-BE-NEXT: ret
833 entry:
834 br label %loop
836 loop:
837 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
838 %src.gep = getelementptr i8, ptr %src, i64 %iv
839 %load = load <8 x i8>, ptr %src.gep
840 %ext = zext <8 x i8> %load to <8 x i64>
841 %dst.gep = getelementptr i64, ptr %dst, i64 %iv
842 store <8 x i64> %ext, ptr %dst.gep
843 %iv.next = add nuw i64 %iv, 16
844 %ec = icmp eq i64 %iv.next, 128
845 br i1 %ec, label %exit, label %loop
847 exit:
848 ret void
849 }
851 define void @zext_v8i8_to_v8i16_in_loop(ptr %src, ptr %dst) {
852 ; CHECK-LABEL: zext_v8i8_to_v8i16_in_loop:
853 ; CHECK: ; %bb.0: ; %entry
854 ; CHECK-NEXT: mov x8, xzr
855 ; CHECK-NEXT: LBB9_1: ; %loop
856 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
857 ; CHECK-NEXT: ldr d0, [x0, x8]
858 ; CHECK-NEXT: add x8, x8, #16
859 ; CHECK-NEXT: cmp x8, #128
860 ; CHECK-NEXT: ushll.8h v0, v0, #0
861 ; CHECK-NEXT: str q0, [x1], #32
862 ; CHECK-NEXT: b.ne LBB9_1
863 ; CHECK-NEXT: ; %bb.2: ; %exit
864 ; CHECK-NEXT: ret
865 ;
866 ; CHECK-BE-LABEL: zext_v8i8_to_v8i16_in_loop:
867 ; CHECK-BE: // %bb.0: // %entry
868 ; CHECK-BE-NEXT: mov x8, xzr
869 ; CHECK-BE-NEXT: .LBB9_1: // %loop
870 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
871 ; CHECK-BE-NEXT: add x9, x0, x8
872 ; CHECK-BE-NEXT: add x8, x8, #16
873 ; CHECK-BE-NEXT: ld1 { v0.8b }, [x9]
874 ; CHECK-BE-NEXT: cmp x8, #128
875 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
876 ; CHECK-BE-NEXT: st1 { v0.8h }, [x1]
877 ; CHECK-BE-NEXT: add x1, x1, #32
878 ; CHECK-BE-NEXT: b.ne .LBB9_1
879 ; CHECK-BE-NEXT: // %bb.2: // %exit
880 ; CHECK-BE-NEXT: ret
884 entry:
885 br label %loop
887 loop:
888 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
889 %src.gep = getelementptr i8, ptr %src, i64 %iv
890 %load = load <8 x i8>, ptr %src.gep
891 %ext = zext <8 x i8> %load to <8 x i16>
892 %dst.gep = getelementptr i16, ptr %dst, i64 %iv
893 store <8 x i16> %ext, ptr %dst.gep
894 %iv.next = add nuw i64 %iv, 16
895 %ec = icmp eq i64 %iv.next, 128
896 br i1 %ec, label %exit, label %loop
898 exit:
899 ret void
900 }
902 define void @zext_v8i8_to_v8i20_in_loop(ptr %src, ptr %dst) {
903 ; CHECK-LABEL: zext_v8i8_to_v8i20_in_loop:
904 ; CHECK: ; %bb.0: ; %entry
905 ; CHECK-NEXT: mov x8, xzr
906 ; CHECK-NEXT: LBB10_1: ; %loop
907 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
908 ; CHECK-NEXT: ldr d0, [x0, x8]
909 ; CHECK-NEXT: add x8, x8, #16
910 ; CHECK-NEXT: cmp x8, #128
911 ; CHECK-NEXT: ushll.8h v0, v0, #0
912 ; CHECK-NEXT: ushll2.4s v1, v0, #0
913 ; CHECK-NEXT: ushll.4s v0, v0, #0
914 ; CHECK-NEXT: mov.s w9, v1[1]
915 ; CHECK-NEXT: mov.s w11, v0[1]
916 ; CHECK-NEXT: fmov w10, s1
917 ; CHECK-NEXT: fmov w14, s0
918 ; CHECK-NEXT: mov.s w13, v1[2]
919 ; CHECK-NEXT: mov.s w16, v0[2]
920 ; CHECK-NEXT: mov.s w12, v1[3]
921 ; CHECK-NEXT: mov.s w15, v0[3]
922 ; CHECK-NEXT: orr x9, x10, x9, lsl #20
923 ; CHECK-NEXT: orr x10, x14, x11, lsl #20
924 ; CHECK-NEXT: orr x9, x9, x13, lsl #40
925 ; CHECK-NEXT: orr x10, x10, x16, lsl #40
926 ; CHECK-NEXT: lsr w11, w12, #4
927 ; CHECK-NEXT: lsr w13, w15, #4
928 ; CHECK-NEXT: orr x9, x9, x12, lsl #60
929 ; CHECK-NEXT: orr x10, x10, x15, lsl #60
930 ; CHECK-NEXT: strh w11, [x1, #18]
931 ; CHECK-NEXT: strh w13, [x1, #8]
932 ; CHECK-NEXT: stur x9, [x1, #10]
933 ; CHECK-NEXT: str x10, [x1], #64
934 ; CHECK-NEXT: b.ne LBB10_1
935 ; CHECK-NEXT: ; %bb.2: ; %exit
936 ; CHECK-NEXT: ret
937 ;
938 ; CHECK-BE-LABEL: zext_v8i8_to_v8i20_in_loop:
939 ; CHECK-BE: // %bb.0: // %entry
940 ; CHECK-BE-NEXT: mov x8, xzr
941 ; CHECK-BE-NEXT: .LBB10_1: // %loop
942 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
943 ; CHECK-BE-NEXT: add x9, x0, x8
944 ; CHECK-BE-NEXT: add x8, x8, #16
945 ; CHECK-BE-NEXT: ld1 { v0.8b }, [x9]
946 ; CHECK-BE-NEXT: cmp x8, #128
947 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
948 ; CHECK-BE-NEXT: ushll2 v1.4s, v0.8h, #0
949 ; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
950 ; CHECK-BE-NEXT: mov w9, v1.s[1]
951 ; CHECK-BE-NEXT: mov w10, v0.s[1]
952 ; CHECK-BE-NEXT: fmov w12, s1
953 ; CHECK-BE-NEXT: fmov w14, s0
954 ; CHECK-BE-NEXT: mov w11, v1.s[2]
955 ; CHECK-BE-NEXT: mov w13, v0.s[2]
956 ; CHECK-BE-NEXT: mov w15, v1.s[3]
957 ; CHECK-BE-NEXT: lsl x9, x9, #40
958 ; CHECK-BE-NEXT: lsl x10, x10, #40
959 ; CHECK-BE-NEXT: orr x9, x9, x12, lsl #60
960 ; CHECK-BE-NEXT: orr x10, x10, x14, lsl #60
961 ; CHECK-BE-NEXT: lsr x12, x12, #4
962 ; CHECK-BE-NEXT: strh w15, [x1, #18]
963 ; CHECK-BE-NEXT: orr x9, x9, x11, lsl #20
964 ; CHECK-BE-NEXT: orr x10, x10, x13, lsl #20
965 ; CHECK-BE-NEXT: mov w11, v0.s[3]
966 ; CHECK-BE-NEXT: lsr x13, x14, #4
967 ; CHECK-BE-NEXT: lsr x9, x9, #16
968 ; CHECK-BE-NEXT: lsr x10, x10, #16
969 ; CHECK-BE-NEXT: bfi x9, x12, #48, #4
970 ; CHECK-BE-NEXT: bfi x10, x13, #48, #4
971 ; CHECK-BE-NEXT: strh w11, [x1, #8]
972 ; CHECK-BE-NEXT: stur x9, [x1, #10]
973 ; CHECK-BE-NEXT: str x10, [x1], #64
974 ; CHECK-BE-NEXT: b.ne .LBB10_1
975 ; CHECK-BE-NEXT: // %bb.2: // %exit
976 ; CHECK-BE-NEXT: ret
979 entry:
980 br label %loop
982 loop:
983 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
984 %src.gep = getelementptr i8, ptr %src, i64 %iv
985 %load = load <8 x i8>, ptr %src.gep
986 %ext = zext <8 x i8> %load to <8 x i20>
987 %dst.gep = getelementptr i20, ptr %dst, i64 %iv
988 store <8 x i20> %ext, ptr %dst.gep
989 %iv.next = add nuw i64 %iv, 16
990 %ec = icmp eq i64 %iv.next, 128
991 br i1 %ec, label %exit, label %loop
993 exit:
994 ret void
995 }
997 define void @zext_v4i8_to_v4i32_in_loop(ptr %src, ptr %dst) {
998 ; CHECK-LABEL: zext_v4i8_to_v4i32_in_loop:
999 ; CHECK: ; %bb.0: ; %entry
1000 ; CHECK-NEXT: mov x8, xzr
1001 ; CHECK-NEXT: LBB11_1: ; %loop
1002 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
1003 ; CHECK-NEXT: ldr s0, [x0, x8]
1004 ; CHECK-NEXT: add x8, x8, #16
1005 ; CHECK-NEXT: cmp x8, #128
1006 ; CHECK-NEXT: ushll.8h v0, v0, #0
1007 ; CHECK-NEXT: ushll.4s v0, v0, #0
1008 ; CHECK-NEXT: str q0, [x1], #64
1009 ; CHECK-NEXT: b.ne LBB11_1
1010 ; CHECK-NEXT: ; %bb.2: ; %exit
1011 ; CHECK-NEXT: ret
1012 ;
1013 ; CHECK-BE-LABEL: zext_v4i8_to_v4i32_in_loop:
1014 ; CHECK-BE: // %bb.0: // %entry
1015 ; CHECK-BE-NEXT: adrp x8, .LCPI11_0
1016 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI11_0
1017 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]
1018 ; CHECK-BE-NEXT: mov x8, xzr
1019 ; CHECK-BE-NEXT: .LBB11_1: // %loop
1020 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
1021 ; CHECK-BE-NEXT: ldr s1, [x0, x8]
1022 ; CHECK-BE-NEXT: add x8, x8, #16
1023 ; CHECK-BE-NEXT: cmp x8, #128
1024 ; CHECK-BE-NEXT: rev32 v1.16b, v1.16b
1025 ; CHECK-BE-NEXT: tbl v1.16b, { v1.16b }, v0.16b
1026 ; CHECK-BE-NEXT: st1 { v1.16b }, [x1]
1027 ; CHECK-BE-NEXT: add x1, x1, #64
1028 ; CHECK-BE-NEXT: b.ne .LBB11_1
1029 ; CHECK-BE-NEXT: // %bb.2: // %exit
1030 ; CHECK-BE-NEXT: ret
1033 entry:
1034 br label %loop
1036 loop:
1037 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
1038 %src.gep = getelementptr i8, ptr %src, i64 %iv
1039 %load = load <4 x i8>, ptr %src.gep
1040 %ext = zext <4 x i8> %load to <4 x i32>
1041 %dst.gep = getelementptr i32, ptr %dst, i64 %iv
1042 store <4 x i32> %ext, ptr %dst.gep
1043 %iv.next = add nuw i64 %iv, 16
1044 %ec = icmp eq i64 %iv.next, 128
1045 br i1 %ec, label %exit, label %loop
1047 exit:
1048 ret void
1049 }
1051 ; CHECK-LABEL: lCPI12_0:
1052 ; CHECK-NEXT: .byte 0 ; 0x0
1053 ; CHECK-NEXT: .byte 255 ; 0xff
1054 ; CHECK-NEXT: .byte 255 ; 0xff
1055 ; CHECK-NEXT: .byte 255 ; 0xff
1056 ; CHECK-NEXT: .byte 1 ; 0x1
1057 ; CHECK-NEXT: .byte 255 ; 0xff
1058 ; CHECK-NEXT: .byte 255 ; 0xff
1059 ; CHECK-NEXT: .byte 255 ; 0xff
1060 ; CHECK-NEXT: .byte 2 ; 0x2
1061 ; CHECK-NEXT: .byte 255 ; 0xff
1062 ; CHECK-NEXT: .byte 255 ; 0xff
1063 ; CHECK-NEXT: .byte 255 ; 0xff
1064 ; CHECK-NEXT: .byte 3 ; 0x3
1065 ; CHECK-NEXT: .byte 255 ; 0xff
1066 ; CHECK-NEXT: .byte 255 ; 0xff
1067 ; CHECK-NEXT: .byte 255 ; 0xff
1068 ; CHECK-NEXT: lCPI12_1:
1069 ; CHECK-NEXT: .byte 4 ; 0x4
1070 ; CHECK-NEXT: .byte 255 ; 0xff
1071 ; CHECK-NEXT: .byte 255 ; 0xff
1072 ; CHECK-NEXT: .byte 255 ; 0xff
1073 ; CHECK-NEXT: .byte 5 ; 0x5
1074 ; CHECK-NEXT: .byte 255 ; 0xff
1075 ; CHECK-NEXT: .byte 255 ; 0xff
1076 ; CHECK-NEXT: .byte 255 ; 0xff
1077 ; CHECK-NEXT: .byte 6 ; 0x6
1078 ; CHECK-NEXT: .byte 255 ; 0xff
1079 ; CHECK-NEXT: .byte 255 ; 0xff
1080 ; CHECK-NEXT: .byte 255 ; 0xff
1081 ; CHECK-NEXT: .byte 7 ; 0x7
1082 ; CHECK-NEXT: .byte 255 ; 0xff
1083 ; CHECK-NEXT: .byte 255 ; 0xff
1084 ; CHECK-NEXT: .byte 255 ; 0xff
1085 ; CHECK-NEXT: lCPI12_2:
1086 ; CHECK-NEXT: .byte 8 ; 0x8
1087 ; CHECK-NEXT: .byte 255 ; 0xff
1088 ; CHECK-NEXT: .byte 255 ; 0xff
1089 ; CHECK-NEXT: .byte 255 ; 0xff
1090 ; CHECK-NEXT: .byte 9 ; 0x9
1091 ; CHECK-NEXT: .byte 255 ; 0xff
1092 ; CHECK-NEXT: .byte 255 ; 0xff
1093 ; CHECK-NEXT: .byte 255 ; 0xff
1094 ; CHECK-NEXT: .byte 10 ; 0xa
1095 ; CHECK-NEXT: .byte 255 ; 0xff
1096 ; CHECK-NEXT: .byte 255 ; 0xff
1097 ; CHECK-NEXT: .byte 255 ; 0xff
1098 ; CHECK-NEXT: .byte 11 ; 0xb
1099 ; CHECK-NEXT: .byte 255 ; 0xff
1100 ; CHECK-NEXT: .byte 255 ; 0xff
1101 ; CHECK-NEXT: .byte 255 ; 0xff
1103 ; CHECK-BE-LABEL: .LCPI12_0:
1104 ; CHECK-BE-NEXT: .byte 255 // 0xff
1105 ; CHECK-BE-NEXT: .byte 255 // 0xff
1106 ; CHECK-BE-NEXT: .byte 255 // 0xff
1107 ; CHECK-BE-NEXT: .byte 0 // 0x0
1108 ; CHECK-BE-NEXT: .byte 255 // 0xff
1109 ; CHECK-BE-NEXT: .byte 255 // 0xff
1110 ; CHECK-BE-NEXT: .byte 255 // 0xff
1111 ; CHECK-BE-NEXT: .byte 1 // 0x1
1112 ; CHECK-BE-NEXT: .byte 255 // 0xff
1113 ; CHECK-BE-NEXT: .byte 255 // 0xff
1114 ; CHECK-BE-NEXT: .byte 255 // 0xff
1115 ; CHECK-BE-NEXT: .byte 2 // 0x2
1116 ; CHECK-BE-NEXT: .byte 255 // 0xff
1117 ; CHECK-BE-NEXT: .byte 255 // 0xff
1118 ; CHECK-BE-NEXT: .byte 255 // 0xff
1119 ; CHECK-BE-NEXT: .byte 3 // 0x3
1120 ; CHECK-BE-NEXT: .LCPI12_1:
1121 ; CHECK-BE-NEXT: .byte 255 // 0xff
1122 ; CHECK-BE-NEXT: .byte 255 // 0xff
1123 ; CHECK-BE-NEXT: .byte 255 // 0xff
1124 ; CHECK-BE-NEXT: .byte 4 // 0x4
1125 ; CHECK-BE-NEXT: .byte 255 // 0xff
1126 ; CHECK-BE-NEXT: .byte 255 // 0xff
1127 ; CHECK-BE-NEXT: .byte 255 // 0xff
1128 ; CHECK-BE-NEXT: .byte 5 // 0x5
1129 ; CHECK-BE-NEXT: .byte 255 // 0xff
1130 ; CHECK-BE-NEXT: .byte 255 // 0xff
1131 ; CHECK-BE-NEXT: .byte 255 // 0xff
1132 ; CHECK-BE-NEXT: .byte 6 // 0x6
1133 ; CHECK-BE-NEXT: .byte 255 // 0xff
1134 ; CHECK-BE-NEXT: .byte 255 // 0xff
1135 ; CHECK-BE-NEXT: .byte 255 // 0xff
1136 ; CHECK-BE-NEXT: .byte 7 // 0x7
1137 ; CHECK-BE-NEXT: .LCPI12_2:
1138 ; CHECK-BE-NEXT: .byte 255 // 0xff
1139 ; CHECK-BE-NEXT: .byte 255 // 0xff
1140 ; CHECK-BE-NEXT: .byte 255 // 0xff
1141 ; CHECK-BE-NEXT: .byte 8 // 0x8
1142 ; CHECK-BE-NEXT: .byte 255 // 0xff
1143 ; CHECK-BE-NEXT: .byte 255 // 0xff
1144 ; CHECK-BE-NEXT: .byte 255 // 0xff
1145 ; CHECK-BE-NEXT: .byte 9 // 0x9
1146 ; CHECK-BE-NEXT: .byte 255 // 0xff
1147 ; CHECK-BE-NEXT: .byte 255 // 0xff
1148 ; CHECK-BE-NEXT: .byte 255 // 0xff
1149 ; CHECK-BE-NEXT: .byte 10 // 0xa
1150 ; CHECK-BE-NEXT: .byte 255 // 0xff
1151 ; CHECK-BE-NEXT: .byte 255 // 0xff
1152 ; CHECK-BE-NEXT: .byte 255 // 0xff
1153 ; CHECK-BE-NEXT: .byte 11 // 0xb
1155 define void @zext_v12i8_to_v12i32_in_loop(ptr %src, ptr %dst) {
1156 ; CHECK-LABEL: zext_v12i8_to_v12i32_in_loop:
1157 ; CHECK: ; %bb.0: ; %entry
1158 ; CHECK-NEXT: Lloh12:
1159 ; CHECK-NEXT: adrp x8, lCPI12_0@PAGE
1160 ; CHECK-NEXT: Lloh13:
1161 ; CHECK-NEXT: adrp x9, lCPI12_1@PAGE
1162 ; CHECK-NEXT: Lloh14:
1163 ; CHECK-NEXT: adrp x10, lCPI12_2@PAGE
1164 ; CHECK-NEXT: Lloh15:
1165 ; CHECK-NEXT: ldr q0, [x8, lCPI12_0@PAGEOFF]
1166 ; CHECK-NEXT: Lloh16:
1167 ; CHECK-NEXT: ldr q1, [x9, lCPI12_1@PAGEOFF]
1168 ; CHECK-NEXT: Lloh17:
1169 ; CHECK-NEXT: ldr q2, [x10, lCPI12_2@PAGEOFF]
1170 ; CHECK-NEXT: mov x8, xzr
1171 ; CHECK-NEXT: LBB12_1: ; %loop
1172 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
1173 ; CHECK-NEXT: ldr q3, [x0, x8]
1174 ; CHECK-NEXT: add x8, x8, #16
1175 ; CHECK-NEXT: cmp x8, #128
1176 ; CHECK-NEXT: tbl.16b v4, { v3 }, v2
1177 ; CHECK-NEXT: tbl.16b v5, { v3 }, v1
1178 ; CHECK-NEXT: tbl.16b v3, { v3 }, v0
1179 ; CHECK-NEXT: stp q5, q4, [x1, #16]
1180 ; CHECK-NEXT: str q3, [x1], #64
1181 ; CHECK-NEXT: b.ne LBB12_1
1182 ; CHECK-NEXT: ; %bb.2: ; %exit
1183 ; CHECK-NEXT: ret
1184 ; CHECK-NEXT: .loh AdrpLdr Lloh14, Lloh17
1185 ; CHECK-NEXT: .loh AdrpLdr Lloh13, Lloh16
1186 ; CHECK-NEXT: .loh AdrpLdr Lloh12, Lloh15
1187 ;
1188 ; CHECK-BE-LABEL: zext_v12i8_to_v12i32_in_loop:
1189 ; CHECK-BE: // %bb.0: // %entry
1190 ; CHECK-BE-NEXT: adrp x8, .LCPI12_0
1191 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI12_0
1192 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]
1193 ; CHECK-BE-NEXT: adrp x8, .LCPI12_1
1194 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI12_1
1195 ; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
1196 ; CHECK-BE-NEXT: adrp x8, .LCPI12_2
1197 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI12_2
1198 ; CHECK-BE-NEXT: ld1 { v2.16b }, [x8]
1199 ; CHECK-BE-NEXT: mov x8, xzr
1200 ; CHECK-BE-NEXT: .LBB12_1: // %loop
1201 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
1202 ; CHECK-BE-NEXT: add x9, x0, x8
1203 ; CHECK-BE-NEXT: add x8, x8, #16
1204 ; CHECK-BE-NEXT: add x10, x1, #16
1205 ; CHECK-BE-NEXT: ld1 { v3.16b }, [x9]
1206 ; CHECK-BE-NEXT: add x9, x1, #32
1207 ; CHECK-BE-NEXT: cmp x8, #128
1208 ; CHECK-BE-NEXT: tbl v4.16b, { v3.16b }, v2.16b
1209 ; CHECK-BE-NEXT: tbl v5.16b, { v3.16b }, v1.16b
1210 ; CHECK-BE-NEXT: tbl v3.16b, { v3.16b }, v0.16b
1211 ; CHECK-BE-NEXT: st1 { v3.16b }, [x1]
1212 ; CHECK-BE-NEXT: add x1, x1, #64
1213 ; CHECK-BE-NEXT: st1 { v4.16b }, [x9]
1214 ; CHECK-BE-NEXT: st1 { v5.16b }, [x10]
1215 ; CHECK-BE-NEXT: b.ne .LBB12_1
1216 ; CHECK-BE-NEXT: // %bb.2: // %exit
1217 ; CHECK-BE-NEXT: ret
1220 entry:
1221 br label %loop
1223 loop:
1224 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
1225 %src.gep = getelementptr i8, ptr %src, i64 %iv
1226 %load = load <12 x i8>, ptr %src.gep
1227 %ext = zext <12 x i8> %load to <12 x i32>
1228 %dst.gep = getelementptr i32, ptr %dst, i64 %iv
1229 store <12 x i32> %ext, ptr %dst.gep
1230 %iv.next = add nuw i64 %iv, 16
1231 %ec = icmp eq i64 %iv.next, 128
1232 br i1 %ec, label %exit, label %loop
1234 exit:
1235 ret void
1236 }
1238 define void @zext_v16i4_to_v16i32_in_loop(ptr %src, ptr %dst) {
1239 ; CHECK-LABEL: zext_v16i4_to_v16i32_in_loop:
1240 ; CHECK: ; %bb.0: ; %entry
1241 ; CHECK-NEXT: movi.4s v0, #15
1242 ; CHECK-NEXT: mov x8, xzr
1243 ; CHECK-NEXT: LBB13_1: ; %loop
1244 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
1245 ; CHECK-NEXT: ldr x9, [x0, x8]
1246 ; CHECK-NEXT: add x8, x8, #16
1247 ; CHECK-NEXT: cmp x8, #128
1248 ; CHECK-NEXT: and w11, w9, #0xf
1249 ; CHECK-NEXT: ubfx w10, w9, #4, #4
1250 ; CHECK-NEXT: fmov s1, w11
1251 ; CHECK-NEXT: mov.b v1[1], w10
1252 ; CHECK-NEXT: ubfx w10, w9, #8, #4
1253 ; CHECK-NEXT: mov.b v1[2], w10
1254 ; CHECK-NEXT: ubfx w10, w9, #12, #4
1255 ; CHECK-NEXT: mov.b v1[3], w10
1256 ; CHECK-NEXT: ubfx w10, w9, #16, #4
1257 ; CHECK-NEXT: mov.b v1[4], w10
1258 ; CHECK-NEXT: ubfx w10, w9, #20, #4
1259 ; CHECK-NEXT: mov.b v1[5], w10
1260 ; CHECK-NEXT: ubfx w10, w9, #24, #4
1261 ; CHECK-NEXT: mov.b v1[6], w10
1262 ; CHECK-NEXT: ubfx x10, x9, #28, #4
1263 ; CHECK-NEXT: mov.b v1[7], w10
1264 ; CHECK-NEXT: ubfx x10, x9, #32, #4
1265 ; CHECK-NEXT: mov.b v1[8], w10
1266 ; CHECK-NEXT: ubfx x10, x9, #36, #4
1267 ; CHECK-NEXT: mov.b v1[9], w10
1268 ; CHECK-NEXT: ubfx x10, x9, #40, #4
1269 ; CHECK-NEXT: mov.b v1[10], w10
1270 ; CHECK-NEXT: ubfx x10, x9, #44, #4
1271 ; CHECK-NEXT: mov.b v1[11], w10
1272 ; CHECK-NEXT: ubfx x10, x9, #48, #4
1273 ; CHECK-NEXT: mov.b v1[12], w10
1274 ; CHECK-NEXT: ubfx x10, x9, #52, #4
1275 ; CHECK-NEXT: mov.b v1[13], w10
1276 ; CHECK-NEXT: ubfx x10, x9, #56, #4
1277 ; CHECK-NEXT: lsr x9, x9, #60
1278 ; CHECK-NEXT: mov.b v1[14], w10
1279 ; CHECK-NEXT: mov.b v1[15], w9
1280 ; CHECK-NEXT: ext.16b v2, v1, v1, #8
1281 ; CHECK-NEXT: zip2.8b v3, v1, v0
1282 ; CHECK-NEXT: zip1.8b v1, v1, v0
1283 ; CHECK-NEXT: zip2.8b v4, v2, v0
1284 ; CHECK-NEXT: zip1.8b v2, v2, v0
1285 ; CHECK-NEXT: ushll.4s v3, v3, #0
1286 ; CHECK-NEXT: ushll.4s v1, v1, #0
1287 ; CHECK-NEXT: and.16b v3, v3, v0
1288 ; CHECK-NEXT: ushll.4s v4, v4, #0
1289 ; CHECK-NEXT: ushll.4s v2, v2, #0
1290 ; CHECK-NEXT: and.16b v1, v1, v0
1291 ; CHECK-NEXT: and.16b v4, v4, v0
1292 ; CHECK-NEXT: and.16b v2, v2, v0
1293 ; CHECK-NEXT: stp q1, q3, [x1]
1294 ; CHECK-NEXT: stp q2, q4, [x1, #32]
1295 ; CHECK-NEXT: add x1, x1, #64
1296 ; CHECK-NEXT: b.ne LBB13_1
1297 ; CHECK-NEXT: ; %bb.2: ; %exit
1298 ; CHECK-NEXT: ret
1299 ;
1300 ; CHECK-BE-LABEL: zext_v16i4_to_v16i32_in_loop:
1301 ; CHECK-BE: // %bb.0: // %entry
1302 ; CHECK-BE-NEXT: movi v0.4s, #15
1303 ; CHECK-BE-NEXT: mov x8, xzr
1304 ; CHECK-BE-NEXT: .LBB13_1: // %loop
1305 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
1306 ; CHECK-BE-NEXT: ldr x9, [x0, x8]
1307 ; CHECK-BE-NEXT: add x8, x8, #16
1308 ; CHECK-BE-NEXT: cmp x8, #128
1309 ; CHECK-BE-NEXT: lsr x10, x9, #60
1310 ; CHECK-BE-NEXT: ubfx x11, x9, #56, #4
1311 ; CHECK-BE-NEXT: fmov s1, w10
1312 ; CHECK-BE-NEXT: ubfx x10, x9, #52, #4
1313 ; CHECK-BE-NEXT: mov v1.b[1], w11
1314 ; CHECK-BE-NEXT: mov v1.b[2], w10
1315 ; CHECK-BE-NEXT: ubfx x10, x9, #48, #4
1316 ; CHECK-BE-NEXT: mov v1.b[3], w10
1317 ; CHECK-BE-NEXT: ubfx x10, x9, #44, #4
1318 ; CHECK-BE-NEXT: mov v1.b[4], w10
1319 ; CHECK-BE-NEXT: ubfx x10, x9, #40, #4
1320 ; CHECK-BE-NEXT: mov v1.b[5], w10
1321 ; CHECK-BE-NEXT: ubfx x10, x9, #36, #4
1322 ; CHECK-BE-NEXT: mov v1.b[6], w10
1323 ; CHECK-BE-NEXT: ubfx x10, x9, #32, #4
1324 ; CHECK-BE-NEXT: mov v1.b[7], w10
1325 ; CHECK-BE-NEXT: ubfx x10, x9, #28, #4
1326 ; CHECK-BE-NEXT: mov v1.b[8], w10
1327 ; CHECK-BE-NEXT: ubfx w10, w9, #24, #4
1328 ; CHECK-BE-NEXT: mov v1.b[9], w10
1329 ; CHECK-BE-NEXT: ubfx w10, w9, #20, #4
1330 ; CHECK-BE-NEXT: mov v1.b[10], w10
1331 ; CHECK-BE-NEXT: ubfx w10, w9, #16, #4
1332 ; CHECK-BE-NEXT: mov v1.b[11], w10
1333 ; CHECK-BE-NEXT: ubfx w10, w9, #12, #4
1334 ; CHECK-BE-NEXT: mov v1.b[12], w10
1335 ; CHECK-BE-NEXT: ubfx w10, w9, #8, #4
1336 ; CHECK-BE-NEXT: mov v1.b[13], w10
1337 ; CHECK-BE-NEXT: ubfx w10, w9, #4, #4
1338 ; CHECK-BE-NEXT: and w9, w9, #0xf
1339 ; CHECK-BE-NEXT: mov v1.b[14], w10
1340 ; CHECK-BE-NEXT: add x10, x1, #32
1341 ; CHECK-BE-NEXT: mov v1.b[15], w9
1342 ; CHECK-BE-NEXT: add x9, x1, #16
1343 ; CHECK-BE-NEXT: ext v2.16b, v1.16b, v1.16b, #8
1344 ; CHECK-BE-NEXT: zip2 v3.8b, v1.8b, v0.8b
1345 ; CHECK-BE-NEXT: zip1 v1.8b, v1.8b, v0.8b
1346 ; CHECK-BE-NEXT: zip2 v4.8b, v2.8b, v0.8b
1347 ; CHECK-BE-NEXT: zip1 v2.8b, v2.8b, v0.8b
1348 ; CHECK-BE-NEXT: rev16 v3.8b, v3.8b
1349 ; CHECK-BE-NEXT: rev16 v1.8b, v1.8b
1350 ; CHECK-BE-NEXT: rev16 v4.8b, v4.8b
1351 ; CHECK-BE-NEXT: rev16 v2.8b, v2.8b
1352 ; CHECK-BE-NEXT: ushll v3.4s, v3.4h, #0
1353 ; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #0
1354 ; CHECK-BE-NEXT: and v3.16b, v3.16b, v0.16b
1355 ; CHECK-BE-NEXT: ushll v4.4s, v4.4h, #0
1356 ; CHECK-BE-NEXT: ushll v2.4s, v2.4h, #0
1357 ; CHECK-BE-NEXT: and v1.16b, v1.16b, v0.16b
1358 ; CHECK-BE-NEXT: st1 { v3.4s }, [x9]
1359 ; CHECK-BE-NEXT: add x9, x1, #48
1360 ; CHECK-BE-NEXT: and v4.16b, v4.16b, v0.16b
1361 ; CHECK-BE-NEXT: and v2.16b, v2.16b, v0.16b
1362 ; CHECK-BE-NEXT: st1 { v1.4s }, [x1]
1363 ; CHECK-BE-NEXT: add x1, x1, #64
1364 ; CHECK-BE-NEXT: st1 { v4.4s }, [x9]
1365 ; CHECK-BE-NEXT: st1 { v2.4s }, [x10]
1366 ; CHECK-BE-NEXT: b.ne .LBB13_1
1367 ; CHECK-BE-NEXT: // %bb.2: // %exit
1368 ; CHECK-BE-NEXT: ret
1371 entry:
1372 br label %loop
1374 loop:
1375 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
1376 %src.gep = getelementptr i4, ptr %src, i64 %iv
1377 %load = load <16 x i4>, ptr %src.gep
1378 %ext = zext <16 x i4> %load to <16 x i32>
1379 %dst.gep = getelementptr i32, ptr %dst, i64 %iv
1380 store <16 x i32> %ext, ptr %dst.gep
1381 %iv.next = add nuw i64 %iv, 16
1382 %ec = icmp eq i64 %iv.next, 128
1383 br i1 %ec, label %exit, label %loop
1385 exit:
1386 ret void
1387 }
1389 define void @zext_v16i16_to_v16i64_in_loop(ptr %src, ptr %dst) {
1390 ; CHECK-LABEL: zext_v16i16_to_v16i64_in_loop:
1391 ; CHECK: ; %bb.0: ; %entry
1392 ; CHECK-NEXT: mov x8, xzr
1393 ; CHECK-NEXT: LBB14_1: ; %loop
1394 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
1395 ; CHECK-NEXT: add x9, x0, x8
1396 ; CHECK-NEXT: add x8, x8, #32
1397 ; CHECK-NEXT: ldp q1, q0, [x9]
1398 ; CHECK-NEXT: cmp x8, #256
1399 ; CHECK-NEXT: ushll2.4s v2, v0, #0
1400 ; CHECK-NEXT: ushll2.4s v3, v1, #0
1401 ; CHECK-NEXT: ushll.4s v0, v0, #0
1402 ; CHECK-NEXT: ushll.4s v1, v1, #0
1403 ; CHECK-NEXT: ushll2.2d v4, v2, #0
1404 ; CHECK-NEXT: ushll.2d v2, v2, #0
1405 ; CHECK-NEXT: ushll2.2d v5, v3, #0
1406 ; CHECK-NEXT: ushll.2d v3, v3, #0
1407 ; CHECK-NEXT: stp q2, q4, [x1, #96]
1408 ; CHECK-NEXT: ushll2.2d v4, v0, #0
1409 ; CHECK-NEXT: ushll.2d v0, v0, #0
1410 ; CHECK-NEXT: ushll2.2d v2, v1, #0
1411 ; CHECK-NEXT: ushll.2d v1, v1, #0
1412 ; CHECK-NEXT: stp q3, q5, [x1, #32]
1413 ; CHECK-NEXT: stp q0, q4, [x1, #64]
1414 ; CHECK-NEXT: stp q1, q2, [x1], #128
1415 ; CHECK-NEXT: b.ne LBB14_1
1416 ; CHECK-NEXT: ; %bb.2: ; %exit
1417 ; CHECK-NEXT: ret
1418 ;
1419 ; CHECK-BE-LABEL: zext_v16i16_to_v16i64_in_loop:
1420 ; CHECK-BE: // %bb.0: // %entry
1421 ; CHECK-BE-NEXT: mov x8, xzr
1422 ; CHECK-BE-NEXT: .LBB14_1: // %loop
1423 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
1424 ; CHECK-BE-NEXT: add x9, x0, x8
1425 ; CHECK-BE-NEXT: add x8, x8, #32
1426 ; CHECK-BE-NEXT: ld1 { v0.8h }, [x9]
1427 ; CHECK-BE-NEXT: add x9, x9, #16
1428 ; CHECK-BE-NEXT: cmp x8, #256
1429 ; CHECK-BE-NEXT: ld1 { v1.8h }, [x9]
1430 ; CHECK-BE-NEXT: add x9, x1, #48
1431 ; CHECK-BE-NEXT: ushll2 v2.4s, v0.8h, #0
1432 ; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
1433 ; CHECK-BE-NEXT: ushll2 v3.4s, v1.8h, #0
1434 ; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #0
1435 ; CHECK-BE-NEXT: ushll2 v4.2d, v2.4s, #0
1436 ; CHECK-BE-NEXT: ushll v2.2d, v2.2s, #0
1437 ; CHECK-BE-NEXT: ushll2 v5.2d, v0.4s, #0
1438 ; CHECK-BE-NEXT: ushll v0.2d, v0.2s, #0
1439 ; CHECK-BE-NEXT: st1 { v4.2d }, [x9]
1440 ; CHECK-BE-NEXT: add x9, x1, #32
1441 ; CHECK-BE-NEXT: ushll2 v4.2d, v3.4s, #0
1442 ; CHECK-BE-NEXT: st1 { v2.2d }, [x9]
1443 ; CHECK-BE-NEXT: add x9, x1, #16
1444 ; CHECK-BE-NEXT: ushll v2.2d, v3.2s, #0
1445 ; CHECK-BE-NEXT: st1 { v5.2d }, [x9]
1446 ; CHECK-BE-NEXT: add x9, x1, #112
1447 ; CHECK-BE-NEXT: ushll2 v3.2d, v1.4s, #0
1448 ; CHECK-BE-NEXT: st1 { v4.2d }, [x9]
1449 ; CHECK-BE-NEXT: add x9, x1, #96
1450 ; CHECK-BE-NEXT: ushll v1.2d, v1.2s, #0
1451 ; CHECK-BE-NEXT: st1 { v2.2d }, [x9]
1452 ; CHECK-BE-NEXT: add x9, x1, #80
1453 ; CHECK-BE-NEXT: st1 { v3.2d }, [x9]
1454 ; CHECK-BE-NEXT: add x9, x1, #64
1455 ; CHECK-BE-NEXT: st1 { v0.2d }, [x1]
1456 ; CHECK-BE-NEXT: add x1, x1, #128
1457 ; CHECK-BE-NEXT: st1 { v1.2d }, [x9]
1458 ; CHECK-BE-NEXT: b.ne .LBB14_1
1459 ; CHECK-BE-NEXT: // %bb.2: // %exit
1460 ; CHECK-BE-NEXT: ret
1463 entry:
1464 br label %loop
1466 loop:
1467 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
1468 %src.gep = getelementptr i16, ptr %src, i64 %iv
1469 %load = load <16 x i16>, ptr %src.gep
1470 %ext = zext <16 x i16> %load to <16 x i64>
1471 %dst.gep = getelementptr i64, ptr %dst, i64 %iv
1472 store <16 x i64> %ext, ptr %dst.gep
1473 %iv.next = add nuw i64 %iv, 16
1474 %ec = icmp eq i64 %iv.next, 128
1475 br i1 %ec, label %exit, label %loop
1477 exit:
1478 ret void
1479 }
1481 define void @zext_v16i32_to_v16i64_in_loop(ptr %src, ptr %dst) {
1482 ; CHECK-LABEL: zext_v16i32_to_v16i64_in_loop:
1483 ; CHECK: ; %bb.0: ; %entry
1484 ; CHECK-NEXT: mov x8, xzr
1485 ; CHECK-NEXT: LBB15_1: ; %loop
1486 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
1487 ; CHECK-NEXT: add x9, x0, x8
1488 ; CHECK-NEXT: add x8, x8, #64
1489 ; CHECK-NEXT: ldp q1, q0, [x9, #32]
1490 ; CHECK-NEXT: cmp x8, #512
1491 ; CHECK-NEXT: ldp q5, q4, [x9]
1492 ; CHECK-NEXT: ushll2.2d v2, v0, #0
1493 ; CHECK-NEXT: ushll.2d v0, v0, #0
1494 ; CHECK-NEXT: ushll2.2d v3, v1, #0
1495 ; CHECK-NEXT: ushll.2d v1, v1, #0
1496 ; CHECK-NEXT: stp q0, q2, [x1, #96]
1497 ; CHECK-NEXT: ushll2.2d v2, v4, #0
1498 ; CHECK-NEXT: ushll.2d v0, v4, #0
1499 ; CHECK-NEXT: stp q1, q3, [x1, #64]
1500 ; CHECK-NEXT: ushll2.2d v3, v5, #0
1501 ; CHECK-NEXT: ushll.2d v1, v5, #0
1502 ; CHECK-NEXT: stp q0, q2, [x1, #32]
1503 ; CHECK-NEXT: stp q1, q3, [x1], #128
1504 ; CHECK-NEXT: b.ne LBB15_1
1505 ; CHECK-NEXT: ; %bb.2: ; %exit
1506 ; CHECK-NEXT: ret
1507 ;
1508 ; CHECK-BE-LABEL: zext_v16i32_to_v16i64_in_loop:
1509 ; CHECK-BE: // %bb.0: // %entry
1510 ; CHECK-BE-NEXT: mov x8, xzr
1511 ; CHECK-BE-NEXT: .LBB15_1: // %loop
1512 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
1513 ; CHECK-BE-NEXT: add x9, x0, x8
1514 ; CHECK-BE-NEXT: add x8, x8, #64
1515 ; CHECK-BE-NEXT: ld1 { v0.4s }, [x9]
1516 ; CHECK-BE-NEXT: add x10, x9, #48
1517 ; CHECK-BE-NEXT: cmp x8, #512
1518 ; CHECK-BE-NEXT: ld1 { v1.4s }, [x10]
1519 ; CHECK-BE-NEXT: add x10, x9, #32
1520 ; CHECK-BE-NEXT: add x9, x9, #16
1521 ; CHECK-BE-NEXT: ld1 { v4.4s }, [x9]
1522 ; CHECK-BE-NEXT: ld1 { v2.4s }, [x10]
1523 ; CHECK-BE-NEXT: add x9, x1, #16
1524 ; CHECK-BE-NEXT: ushll2 v3.2d, v0.4s, #0
1525 ; CHECK-BE-NEXT: ushll v0.2d, v0.2s, #0
1526 ; CHECK-BE-NEXT: add x10, x1, #80
1527 ; CHECK-BE-NEXT: ushll2 v5.2d, v1.4s, #0
1528 ; CHECK-BE-NEXT: ushll2 v6.2d, v2.4s, #0
1529 ; CHECK-BE-NEXT: st1 { v3.2d }, [x9]
1530 ; CHECK-BE-NEXT: ushll2 v3.2d, v4.4s, #0
1531 ; CHECK-BE-NEXT: add x9, x1, #112
1532 ; CHECK-BE-NEXT: st1 { v0.2d }, [x1]
1533 ; CHECK-BE-NEXT: ushll v0.2d, v1.2s, #0
1534 ; CHECK-BE-NEXT: ushll v1.2d, v2.2s, #0
1535 ; CHECK-BE-NEXT: st1 { v5.2d }, [x9]
1536 ; CHECK-BE-NEXT: add x9, x1, #48
1537 ; CHECK-BE-NEXT: ushll v2.2d, v4.2s, #0
1538 ; CHECK-BE-NEXT: st1 { v3.2d }, [x9]
1539 ; CHECK-BE-NEXT: add x9, x1, #64
1540 ; CHECK-BE-NEXT: st1 { v6.2d }, [x10]
1541 ; CHECK-BE-NEXT: add x10, x1, #96
1542 ; CHECK-BE-NEXT: st1 { v1.2d }, [x9]
1543 ; CHECK-BE-NEXT: add x9, x1, #32
1544 ; CHECK-BE-NEXT: add x1, x1, #128
1545 ; CHECK-BE-NEXT: st1 { v0.2d }, [x10]
1546 ; CHECK-BE-NEXT: st1 { v2.2d }, [x9]
1547 ; CHECK-BE-NEXT: b.ne .LBB15_1
1548 ; CHECK-BE-NEXT: // %bb.2: // %exit
1549 ; CHECK-BE-NEXT: ret
1552 entry:
1553 br label %loop
1555 loop:
1556 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
1557 %src.gep = getelementptr i32, ptr %src, i64 %iv
1558 %load = load <16 x i32>, ptr %src.gep
1559 %ext = zext <16 x i32> %load to <16 x i64>
1560 %dst.gep = getelementptr i64, ptr %dst, i64 %iv
1561 store <16 x i64> %ext, ptr %dst.gep
1562 %iv.next = add nuw i64 %iv, 16
1563 %ec = icmp eq i64 %iv.next, 128
1564 br i1 %ec, label %exit, label %loop
1566 exit:
1567 ret void
1568 }
1570 define void @zext_v8i8_to_v8i128_in_loop(ptr %src, ptr %dst) {
1571 ; CHECK-LABEL: zext_v8i8_to_v8i128_in_loop:
1572 ; CHECK: ; %bb.0: ; %entry
1573 ; CHECK-NEXT: mov x8, xzr
1574 ; CHECK-NEXT: LBB16_1: ; %loop
1575 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
1576 ; CHECK-NEXT: ldr d0, [x0, x8]
1577 ; CHECK-NEXT: add x9, x1, #112
1578 ; CHECK-NEXT: add x8, x8, #16
1579 ; CHECK-NEXT: str xzr, [x1, #120]
1580 ; CHECK-NEXT: cmp x8, #128
1581 ; CHECK-NEXT: ushll.8h v0, v0, #0
1582 ; CHECK-NEXT: str xzr, [x1, #104]
1583 ; CHECK-NEXT: str xzr, [x1, #88]
1584 ; CHECK-NEXT: str xzr, [x1, #72]
1585 ; CHECK-NEXT: ushll2.4s v1, v0, #0
1586 ; CHECK-NEXT: ushll.4s v0, v0, #0
1587 ; CHECK-NEXT: str xzr, [x1, #56]
1588 ; CHECK-NEXT: str xzr, [x1, #40]
1589 ; CHECK-NEXT: str xzr, [x1, #24]
1590 ; CHECK-NEXT: ushll2.2d v2, v1, #0
1591 ; CHECK-NEXT: ushll.2d v1, v1, #0
1592 ; CHECK-NEXT: ushll2.2d v3, v0, #0
1593 ; CHECK-NEXT: ushll.2d v0, v0, #0
1594 ; CHECK-NEXT: str xzr, [x1, #8]
1595 ; CHECK-NEXT: st1.d { v2 }[1], [x9]
1596 ; CHECK-NEXT: add x9, x1, #80
1597 ; CHECK-NEXT: st1.d { v1 }[1], [x9]
1598 ; CHECK-NEXT: add x9, x1, #48
1599 ; CHECK-NEXT: str d2, [x1, #96]
1600 ; CHECK-NEXT: st1.d { v3 }[1], [x9]
1601 ; CHECK-NEXT: add x9, x1, #16
1602 ; CHECK-NEXT: str d1, [x1, #64]
1603 ; CHECK-NEXT: str d3, [x1, #32]
1604 ; CHECK-NEXT: str d0, [x1]
1605 ; CHECK-NEXT: add x1, x1, #256
1606 ; CHECK-NEXT: st1.d { v0 }[1], [x9]
1607 ; CHECK-NEXT: b.ne LBB16_1
1608 ; CHECK-NEXT: ; %bb.2: ; %exit
1609 ; CHECK-NEXT: ret
1610 ;
1611 ; CHECK-BE-LABEL: zext_v8i8_to_v8i128_in_loop:
1612 ; CHECK-BE: // %bb.0: // %entry
1613 ; CHECK-BE-NEXT: mov x8, xzr
1614 ; CHECK-BE-NEXT: .LBB16_1: // %loop
1615 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
1616 ; CHECK-BE-NEXT: add x9, x0, x8
1617 ; CHECK-BE-NEXT: add x8, x8, #16
1618 ; CHECK-BE-NEXT: ld1 { v0.8b }, [x9]
1619 ; CHECK-BE-NEXT: add x9, x1, #120
1620 ; CHECK-BE-NEXT: str xzr, [x1, #112]
1621 ; CHECK-BE-NEXT: str xzr, [x1, #96]
1622 ; CHECK-BE-NEXT: cmp x8, #128
1623 ; CHECK-BE-NEXT: str xzr, [x1, #80]
1624 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
1625 ; CHECK-BE-NEXT: str xzr, [x1, #64]
1626 ; CHECK-BE-NEXT: str xzr, [x1, #48]
1627 ; CHECK-BE-NEXT: str xzr, [x1, #32]
1628 ; CHECK-BE-NEXT: ushll2 v1.4s, v0.8h, #0
1629 ; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
1630 ; CHECK-BE-NEXT: str xzr, [x1, #16]
1631 ; CHECK-BE-NEXT: str xzr, [x1]
1632 ; CHECK-BE-NEXT: ushll2 v2.2d, v1.4s, #0
1633 ; CHECK-BE-NEXT: ushll v1.2d, v1.2s, #0
1634 ; CHECK-BE-NEXT: ushll2 v3.2d, v0.4s, #0
1635 ; CHECK-BE-NEXT: ushll v0.2d, v0.2s, #0
1636 ; CHECK-BE-NEXT: st1 { v2.d }[1], [x9]
1637 ; CHECK-BE-NEXT: add x9, x1, #88
1638 ; CHECK-BE-NEXT: st1 { v1.d }[1], [x9]
1639 ; CHECK-BE-NEXT: add x9, x1, #56
1640 ; CHECK-BE-NEXT: str d2, [x1, #104]
1641 ; CHECK-BE-NEXT: st1 { v3.d }[1], [x9]
1642 ; CHECK-BE-NEXT: add x9, x1, #24
1643 ; CHECK-BE-NEXT: str d1, [x1, #72]
1644 ; CHECK-BE-NEXT: str d3, [x1, #40]
1645 ; CHECK-BE-NEXT: str d0, [x1, #8]
1646 ; CHECK-BE-NEXT: add x1, x1, #256
1647 ; CHECK-BE-NEXT: st1 { v0.d }[1], [x9]
1648 ; CHECK-BE-NEXT: b.ne .LBB16_1
1649 ; CHECK-BE-NEXT: // %bb.2: // %exit
1650 ; CHECK-BE-NEXT: ret
1653 entry:
1654 br label %loop
1656 loop:
1657 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
1658 %src.gep = getelementptr i8, ptr %src, i64 %iv
1659 %load = load <8 x i8>, ptr %src.gep
1660 %ext = zext <8 x i8> %load to <8 x i128>
1661 %dst.gep = getelementptr i128, ptr %dst, i64 %iv
1662 store <8 x i128> %ext, ptr %dst.gep
1663 %iv.next = add nuw i64 %iv, 16
1664 %ec = icmp eq i64 %iv.next, 128
1665 br i1 %ec, label %exit, label %loop
1667 exit:
1668 ret void
1669 }
1671 ; multiple back-to-back 'zext' of similar type of vectors combined with arithmetic operations
1672 define void @zext_v8i8_to_v8i64_with_add_in_sequence_in_loop(ptr %src, ptr %dst) {
1673 ; CHECK-LABEL: zext_v8i8_to_v8i64_with_add_in_sequence_in_loop:
1674 ; CHECK: ; %bb.0: ; %entry
1675 ; CHECK-NEXT: Lloh18:
1676 ; CHECK-NEXT: adrp x9, lCPI17_0@PAGE
1677 ; CHECK-NEXT: Lloh19:
1678 ; CHECK-NEXT: adrp x10, lCPI17_1@PAGE
1679 ; CHECK-NEXT: mov x8, xzr
1680 ; CHECK-NEXT: Lloh20:
1681 ; CHECK-NEXT: ldr q0, [x9, lCPI17_0@PAGEOFF]
1682 ; CHECK-NEXT: Lloh21:
1683 ; CHECK-NEXT: ldr q1, [x10, lCPI17_1@PAGEOFF]
1684 ; CHECK-NEXT: add x9, x0, #8
1685 ; CHECK-NEXT: LBB17_1: ; %loop
1686 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
1687 ; CHECK-NEXT: ldp d2, d4, [x9, #-8]
1688 ; CHECK-NEXT: add x10, x1, x8
1689 ; CHECK-NEXT: ldp q6, q5, [x10, #32]
1690 ; CHECK-NEXT: add x8, x8, #128
1691 ; CHECK-NEXT: ldp q17, q16, [x10]
1692 ; CHECK-NEXT: cmp x8, #1024
1693 ; CHECK-NEXT: tbl.16b v3, { v2 }, v1
1694 ; CHECK-NEXT: tbl.16b v2, { v2 }, v0
1695 ; CHECK-NEXT: tbl.16b v7, { v4 }, v1
1696 ; CHECK-NEXT: tbl.16b v4, { v4 }, v0
1697 ; CHECK-NEXT: add x9, x9, #16
1698 ; CHECK-NEXT: uaddw2.2d v5, v5, v3
1699 ; CHECK-NEXT: uaddw.2d v3, v6, v3
1700 ; CHECK-NEXT: uaddw2.2d v6, v16, v2
1701 ; CHECK-NEXT: ldp q18, q16, [x10, #96]
1702 ; CHECK-NEXT: uaddw.2d v2, v17, v2
1703 ; CHECK-NEXT: stp q3, q5, [x10, #32]
1704 ; CHECK-NEXT: ldp q17, q5, [x10, #64]
1705 ; CHECK-NEXT: uaddw2.2d v16, v16, v7
1706 ; CHECK-NEXT: uaddw.2d v7, v18, v7
1707 ; CHECK-NEXT: stp q2, q6, [x10]
1708 ; CHECK-NEXT: uaddw2.2d v3, v5, v4
1709 ; CHECK-NEXT: uaddw.2d v4, v17, v4
1710 ; CHECK-NEXT: stp q7, q16, [x10, #96]
1711 ; CHECK-NEXT: stp q4, q3, [x10, #64]
1712 ; CHECK-NEXT: b.ne LBB17_1
1713 ; CHECK-NEXT: ; %bb.2: ; %exit
1714 ; CHECK-NEXT: ret
1715 ; CHECK-NEXT: .loh AdrpLdr Lloh19, Lloh21
1716 ; CHECK-NEXT: .loh AdrpLdr Lloh18, Lloh20
1717 ;
1718 ; CHECK-BE-LABEL: zext_v8i8_to_v8i64_with_add_in_sequence_in_loop:
1719 ; CHECK-BE: // %bb.0: // %entry
1720 ; CHECK-BE-NEXT: adrp x9, .LCPI17_0
1721 ; CHECK-BE-NEXT: add x9, x9, :lo12:.LCPI17_0
1722 ; CHECK-BE-NEXT: mov x8, xzr
1723 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x9]
1724 ; CHECK-BE-NEXT: adrp x9, .LCPI17_1
1725 ; CHECK-BE-NEXT: add x9, x9, :lo12:.LCPI17_1
1726 ; CHECK-BE-NEXT: ld1 { v1.16b }, [x9]
1727 ; CHECK-BE-NEXT: add x9, x0, #8
1728 ; CHECK-BE-NEXT: .LBB17_1: // %loop
1729 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
1730 ; CHECK-BE-NEXT: sub x10, x9, #8
1731 ; CHECK-BE-NEXT: ld1 { v2.8b }, [x9]
1732 ; CHECK-BE-NEXT: add x9, x9, #16
1733 ; CHECK-BE-NEXT: ld1 { v3.8b }, [x10]
1734 ; CHECK-BE-NEXT: add x10, x1, x8
1735 ; CHECK-BE-NEXT: add x8, x8, #128
1736 ; CHECK-BE-NEXT: add x15, x10, #96
1737 ; CHECK-BE-NEXT: add x11, x10, #32
1738 ; CHECK-BE-NEXT: add x14, x10, #64
1739 ; CHECK-BE-NEXT: tbl v4.16b, { v2.16b }, v1.16b
1740 ; CHECK-BE-NEXT: tbl v2.16b, { v2.16b }, v0.16b
1741 ; CHECK-BE-NEXT: ld1 { v16.2d }, [x15]
1742 ; CHECK-BE-NEXT: tbl v5.16b, { v3.16b }, v1.16b
1743 ; CHECK-BE-NEXT: tbl v3.16b, { v3.16b }, v0.16b
1744 ; CHECK-BE-NEXT: ld1 { v6.2d }, [x10]
1745 ; CHECK-BE-NEXT: ld1 { v19.2d }, [x14]
1746 ; CHECK-BE-NEXT: ld1 { v21.2d }, [x11]
1747 ; CHECK-BE-NEXT: add x12, x10, #48
1748 ; CHECK-BE-NEXT: add x13, x10, #16
1749 ; CHECK-BE-NEXT: add x16, x10, #112
1750 ; CHECK-BE-NEXT: add x17, x10, #80
1751 ; CHECK-BE-NEXT: rev32 v7.8b, v4.8b
1752 ; CHECK-BE-NEXT: ext v4.16b, v4.16b, v4.16b, #8
1753 ; CHECK-BE-NEXT: rev32 v17.8b, v2.8b
1754 ; CHECK-BE-NEXT: ext v18.16b, v5.16b, v5.16b, #8
1755 ; CHECK-BE-NEXT: ext v20.16b, v3.16b, v3.16b, #8
1756 ; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
1757 ; CHECK-BE-NEXT: rev32 v5.8b, v5.8b
1758 ; CHECK-BE-NEXT: rev32 v3.8b, v3.8b
1759 ; CHECK-BE-NEXT: cmp x8, #1024
1760 ; CHECK-BE-NEXT: rev32 v4.8b, v4.8b
1761 ; CHECK-BE-NEXT: uaddw v7.2d, v16.2d, v7.2s
1762 ; CHECK-BE-NEXT: ld1 { v16.2d }, [x16]
1763 ; CHECK-BE-NEXT: rev32 v18.8b, v18.8b
1764 ; CHECK-BE-NEXT: rev32 v20.8b, v20.8b
1765 ; CHECK-BE-NEXT: rev32 v2.8b, v2.8b
1766 ; CHECK-BE-NEXT: uaddw v17.2d, v19.2d, v17.2s
1767 ; CHECK-BE-NEXT: ld1 { v19.2d }, [x12]
1768 ; CHECK-BE-NEXT: uaddw v5.2d, v21.2d, v5.2s
1769 ; CHECK-BE-NEXT: ld1 { v21.2d }, [x13]
1770 ; CHECK-BE-NEXT: uaddw v3.2d, v6.2d, v3.2s
1771 ; CHECK-BE-NEXT: ld1 { v6.2d }, [x17]
1772 ; CHECK-BE-NEXT: uaddw v4.2d, v16.2d, v4.2s
1773 ; CHECK-BE-NEXT: st1 { v7.2d }, [x15]
1774 ; CHECK-BE-NEXT: uaddw v7.2d, v19.2d, v18.2s
1775 ; CHECK-BE-NEXT: uaddw v16.2d, v21.2d, v20.2s
1776 ; CHECK-BE-NEXT: uaddw v2.2d, v6.2d, v2.2s
1777 ; CHECK-BE-NEXT: st1 { v17.2d }, [x14]
1778 ; CHECK-BE-NEXT: st1 { v5.2d }, [x11]
1779 ; CHECK-BE-NEXT: st1 { v3.2d }, [x10]
1780 ; CHECK-BE-NEXT: st1 { v4.2d }, [x16]
1781 ; CHECK-BE-NEXT: st1 { v7.2d }, [x12]
1782 ; CHECK-BE-NEXT: st1 { v16.2d }, [x13]
1783 ; CHECK-BE-NEXT: st1 { v2.2d }, [x17]
1784 ; CHECK-BE-NEXT: b.ne .LBB17_1
1785 ; CHECK-BE-NEXT: // %bb.2: // %exit
1786 ; CHECK-BE-NEXT: ret
1789 entry:
1790 br label %loop
1792 loop:
1793 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
1794 %src.gep = getelementptr i8, ptr %src, i64 %iv
1795 %load = load <8 x i8>, ptr %src.gep
1796 %src.gep.2 = getelementptr i8, ptr %src.gep, i64 8
1797 %load.2 = load <8 x i8>, ptr %src.gep.2
1798 %ext = zext <8 x i8> %load to <8 x i64>
1799 %ext.2 = zext <8 x i8> %load.2 to <8 x i64>
1800 %dst.gep = getelementptr i64, ptr %dst, i64 %iv
1801 %load.dst = load <8 x i64>, ptr %dst.gep
1802 %dst.gep.2 = getelementptr i64, ptr %dst.gep, i64 8
1803 %load.dst.2 = load <8 x i64>, ptr %dst.gep.2
1804 %sum = add <8 x i64> %load.dst, %ext
1805 %sum.2 = add <8 x i64> %load.dst.2, %ext.2
1806 store <8 x i64> %sum, ptr %dst.gep
1807 store <8 x i64> %sum.2, ptr %dst.gep.2
1808 %iv.next = add nuw i64 %iv, 16
1809 %ec = icmp eq i64 %iv.next, 128
1810 br i1 %ec, label %exit, label %loop
1812 exit:
1813 ret void
1814 }
1816 ; multiple back-to-back 'zext' of similar type of vectors
1817 define void @zext_v16i8_to_v16i64_in_sequence_in_loop(ptr %src, ptr %dst) {
1818 ; CHECK-LABEL: zext_v16i8_to_v16i64_in_sequence_in_loop:
1819 ; CHECK: ; %bb.0: ; %entry
1820 ; CHECK-NEXT: mov x8, xzr
1821 ; CHECK-NEXT: add x9, x1, #128
1822 ; CHECK-NEXT: LBB18_1: ; %loop
1823 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
1824 ; CHECK-NEXT: add x10, x0, x8
1825 ; CHECK-NEXT: add x8, x8, #16
1826 ; CHECK-NEXT: ldp q0, q1, [x10]
1827 ; CHECK-NEXT: cmp x8, #128
1828 ; CHECK-NEXT: ushll2.8h v2, v0, #0
1829 ; CHECK-NEXT: ushll.8h v0, v0, #0
1830 ; CHECK-NEXT: ushll2.8h v6, v1, #0
1831 ; CHECK-NEXT: ushll.8h v1, v1, #0
1832 ; CHECK-NEXT: ushll2.4s v3, v2, #0
1833 ; CHECK-NEXT: ushll.4s v2, v2, #0
1834 ; CHECK-NEXT: ushll2.4s v5, v0, #0
1835 ; CHECK-NEXT: ushll.4s v0, v0, #0
1836 ; CHECK-NEXT: ushll2.2d v4, v3, #0
1837 ; CHECK-NEXT: ushll.2d v3, v3, #0
1838 ; CHECK-NEXT: ushll2.2d v7, v2, #0
1839 ; CHECK-NEXT: ushll.2d v2, v2, #0
1840 ; CHECK-NEXT: stp q3, q4, [x9, #-32]
1841 ; CHECK-NEXT: ushll2.2d v4, v5, #0
1842 ; CHECK-NEXT: ushll2.4s v3, v6, #0
1843 ; CHECK-NEXT: ushll.2d v5, v5, #0
1844 ; CHECK-NEXT: stp q2, q7, [x9, #-64]
1845 ; CHECK-NEXT: ushll2.2d v7, v0, #0
1846 ; CHECK-NEXT: ushll.2d v0, v0, #0
1847 ; CHECK-NEXT: ushll.4s v2, v6, #0
1848 ; CHECK-NEXT: stp q5, q4, [x9, #-96]
1849 ; CHECK-NEXT: ushll2.2d v4, v3, #0
1850 ; CHECK-NEXT: ushll2.4s v5, v1, #0
1851 ; CHECK-NEXT: ushll.2d v3, v3, #0
1852 ; CHECK-NEXT: stp q0, q7, [x9, #-128]
1853 ; CHECK-NEXT: ushll.4s v0, v1, #0
1854 ; CHECK-NEXT: ushll2.2d v6, v2, #0
1855 ; CHECK-NEXT: ushll.2d v1, v2, #0
1856 ; CHECK-NEXT: ushll2.2d v2, v5, #0
1857 ; CHECK-NEXT: stp q3, q4, [x9, #96]
1858 ; CHECK-NEXT: ushll.2d v3, v5, #0
1859 ; CHECK-NEXT: ushll2.2d v4, v0, #0
1860 ; CHECK-NEXT: ushll.2d v0, v0, #0
1861 ; CHECK-NEXT: stp q1, q6, [x9, #64]
1862 ; CHECK-NEXT: stp q3, q2, [x9, #32]
1863 ; CHECK-NEXT: stp q0, q4, [x9], #128
1864 ; CHECK-NEXT: b.ne LBB18_1
1865 ; CHECK-NEXT: ; %bb.2: ; %exit
1866 ; CHECK-NEXT: ret
1867 ;
1868 ; CHECK-BE-LABEL: zext_v16i8_to_v16i64_in_sequence_in_loop:
1869 ; CHECK-BE: // %bb.0: // %entry
1870 ; CHECK-BE-NEXT: mov x8, xzr
1871 ; CHECK-BE-NEXT: add x9, x1, #128
1872 ; CHECK-BE-NEXT: .LBB18_1: // %loop
1873 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
1874 ; CHECK-BE-NEXT: add x10, x0, x8
1875 ; CHECK-BE-NEXT: sub x11, x9, #32
1876 ; CHECK-BE-NEXT: add x8, x8, #16
1877 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x10]
1878 ; CHECK-BE-NEXT: add x10, x10, #16
1879 ; CHECK-BE-NEXT: cmp x8, #128
1880 ; CHECK-BE-NEXT: ld1 { v5.16b }, [x10]
1881 ; CHECK-BE-NEXT: sub x10, x9, #16
1882 ; CHECK-BE-NEXT: ushll2 v1.8h, v0.16b, #0
1883 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
1884 ; CHECK-BE-NEXT: ushll2 v2.4s, v1.8h, #0
1885 ; CHECK-BE-NEXT: ushll2 v3.4s, v0.8h, #0
1886 ; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #0
1887 ; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
1888 ; CHECK-BE-NEXT: ushll2 v4.2d, v2.4s, #0
1889 ; CHECK-BE-NEXT: ushll v2.2d, v2.2s, #0
1890 ; CHECK-BE-NEXT: ushll2 v6.2d, v1.4s, #0
1891 ; CHECK-BE-NEXT: ushll v1.2d, v1.2s, #0
1892 ; CHECK-BE-NEXT: st1 { v4.2d }, [x10]
1893 ; CHECK-BE-NEXT: ushll2 v4.2d, v3.4s, #0
1894 ; CHECK-BE-NEXT: ushll v3.2d, v3.2s, #0
1895 ; CHECK-BE-NEXT: st1 { v2.2d }, [x11]
1896 ; CHECK-BE-NEXT: ushll2 v2.8h, v5.16b, #0
1897 ; CHECK-BE-NEXT: sub x11, x9, #80
1898 ; CHECK-BE-NEXT: sub x10, x9, #48
1899 ; CHECK-BE-NEXT: st1 { v4.2d }, [x11]
1900 ; CHECK-BE-NEXT: ushll v4.8h, v5.8b, #0
1901 ; CHECK-BE-NEXT: sub x11, x9, #64
1902 ; CHECK-BE-NEXT: ushll2 v5.4s, v2.8h, #0
1903 ; CHECK-BE-NEXT: st1 { v1.2d }, [x11]
1904 ; CHECK-BE-NEXT: sub x11, x9, #96
1905 ; CHECK-BE-NEXT: ushll2 v1.2d, v0.4s, #0
1906 ; CHECK-BE-NEXT: ushll v2.4s, v2.4h, #0
1907 ; CHECK-BE-NEXT: ushll v0.2d, v0.2s, #0
1908 ; CHECK-BE-NEXT: st1 { v6.2d }, [x10]
1909 ; CHECK-BE-NEXT: sub x10, x9, #128
1910 ; CHECK-BE-NEXT: st1 { v3.2d }, [x11]
1911 ; CHECK-BE-NEXT: ushll2 v3.4s, v4.8h, #0
1912 ; CHECK-BE-NEXT: ushll2 v6.2d, v5.4s, #0
1913 ; CHECK-BE-NEXT: sub x11, x9, #112
1914 ; CHECK-BE-NEXT: ushll v5.2d, v5.2s, #0
1915 ; CHECK-BE-NEXT: st1 { v0.2d }, [x10]
1916 ; CHECK-BE-NEXT: st1 { v1.2d }, [x11]
1917 ; CHECK-BE-NEXT: ushll2 v1.2d, v2.4s, #0
1918 ; CHECK-BE-NEXT: add x10, x9, #112
1919 ; CHECK-BE-NEXT: ushll v4.4s, v4.4h, #0
1920 ; CHECK-BE-NEXT: ushll2 v0.2d, v3.4s, #0
1921 ; CHECK-BE-NEXT: st1 { v6.2d }, [x10]
1922 ; CHECK-BE-NEXT: add x10, x9, #96
1923 ; CHECK-BE-NEXT: ushll v2.2d, v2.2s, #0
1924 ; CHECK-BE-NEXT: ushll v3.2d, v3.2s, #0
1925 ; CHECK-BE-NEXT: st1 { v5.2d }, [x10]
1926 ; CHECK-BE-NEXT: add x10, x9, #80
1927 ; CHECK-BE-NEXT: st1 { v1.2d }, [x10]
1928 ; CHECK-BE-NEXT: add x10, x9, #48
1929 ; CHECK-BE-NEXT: ushll2 v1.2d, v4.4s, #0
1930 ; CHECK-BE-NEXT: st1 { v0.2d }, [x10]
1931 ; CHECK-BE-NEXT: ushll v0.2d, v4.2s, #0
1932 ; CHECK-BE-NEXT: add x10, x9, #64
1933 ; CHECK-BE-NEXT: st1 { v2.2d }, [x10]
1934 ; CHECK-BE-NEXT: add x10, x9, #32
1935 ; CHECK-BE-NEXT: st1 { v3.2d }, [x10]
1936 ; CHECK-BE-NEXT: add x10, x9, #16
1937 ; CHECK-BE-NEXT: st1 { v0.2d }, [x9]
1938 ; CHECK-BE-NEXT: add x9, x9, #128
1939 ; CHECK-BE-NEXT: st1 { v1.2d }, [x10]
1940 ; CHECK-BE-NEXT: b.ne .LBB18_1
1941 ; CHECK-BE-NEXT: // %bb.2: // %exit
1942 ; CHECK-BE-NEXT: ret
1945 entry:
1946 br label %loop
1948 loop:
1949 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
1950 %src.gep = getelementptr i8, ptr %src, i64 %iv
1951 %load = load <16 x i8>, ptr %src.gep
1952 %src.gep.2 = getelementptr i8, ptr %src.gep, i64 16
1953 %load.2 = load <16 x i8>, ptr %src.gep.2
1954 %ext = zext <16 x i8> %load to <16 x i64>
1955 %ext.2 = zext <16 x i8> %load.2 to <16 x i64>
1956 %dst.gep = getelementptr i64, ptr %dst, i64 %iv
1957 store <16 x i64> %ext, ptr %dst.gep
1958 %dst.gep.2 = getelementptr i64, ptr %dst.gep, i64 16
1959 store <16 x i64> %ext.2, ptr %dst.gep.2
1960 %iv.next = add nuw i64 %iv, 16
1961 %ec = icmp eq i64 %iv.next, 128
1962 br i1 %ec, label %exit, label %loop
1964 exit:
1965 ret void
1966 }
1968 define void @zext_v16i8_to_v16i32_in_loop_scalable_vectors(ptr %src, ptr %dst) {
1969 ; CHECK-LABEL: zext_v16i8_to_v16i32_in_loop_scalable_vectors:
1970 ; CHECK: ; %bb.0: ; %entry
1971 ; CHECK-NEXT: ptrue p0.s
1972 ; CHECK-NEXT: mov x8, xzr
1973 ; CHECK-NEXT: LBB19_1: ; %loop
1974 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
1975 ; CHECK-NEXT: ld1b { z0.s }, p0/z, [x0, x8]
1976 ; CHECK-NEXT: add x9, x0, x8
1977 ; CHECK-NEXT: ld1b { z1.s }, p0/z, [x9, #2, mul vl]
1978 ; CHECK-NEXT: ld1b { z2.s }, p0/z, [x9, #3, mul vl]
1979 ; CHECK-NEXT: ld1b { z3.s }, p0/z, [x9, #1, mul vl]
1980 ; CHECK-NEXT: add x9, x1, x8, lsl #2
1981 ; CHECK-NEXT: add z0.s, z0.s, z0.s
1982 ; CHECK-NEXT: add z1.s, z1.s, z1.s
1983 ; CHECK-NEXT: add z2.s, z2.s, z2.s
1984 ; CHECK-NEXT: add z3.s, z3.s, z3.s
1985 ; CHECK-NEXT: st1w { z0.s }, p0, [x1, x8, lsl #2]
1986 ; CHECK-NEXT: add x8, x8, #16
1987 ; CHECK-NEXT: cmp x8, #128
1988 ; CHECK-NEXT: st1w { z1.s }, p0, [x9, #2, mul vl]
1989 ; CHECK-NEXT: st1w { z2.s }, p0, [x9, #3, mul vl]
1990 ; CHECK-NEXT: st1w { z3.s }, p0, [x9, #1, mul vl]
1991 ; CHECK-NEXT: b.ne LBB19_1
1992 ; CHECK-NEXT: ; %bb.2: ; %exit
1993 ; CHECK-NEXT: ret
1994 ;
1995 ; CHECK-BE-LABEL: zext_v16i8_to_v16i32_in_loop_scalable_vectors:
1996 ; CHECK-BE: // %bb.0: // %entry
1997 ; CHECK-BE-NEXT: ptrue p0.s
1998 ; CHECK-BE-NEXT: mov x8, xzr
1999 ; CHECK-BE-NEXT: .LBB19_1: // %loop
2000 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
2001 ; CHECK-BE-NEXT: ld1b { z0.s }, p0/z, [x0, x8]
2002 ; CHECK-BE-NEXT: add x9, x0, x8
2003 ; CHECK-BE-NEXT: ld1b { z1.s }, p0/z, [x9, #2, mul vl]
2004 ; CHECK-BE-NEXT: ld1b { z2.s }, p0/z, [x9, #3, mul vl]
2005 ; CHECK-BE-NEXT: ld1b { z3.s }, p0/z, [x9, #1, mul vl]
2006 ; CHECK-BE-NEXT: add x9, x1, x8, lsl #2
2007 ; CHECK-BE-NEXT: add z0.s, z0.s, z0.s
2008 ; CHECK-BE-NEXT: add z1.s, z1.s, z1.s
2009 ; CHECK-BE-NEXT: add z2.s, z2.s, z2.s
2010 ; CHECK-BE-NEXT: add z3.s, z3.s, z3.s
2011 ; CHECK-BE-NEXT: st1w { z0.s }, p0, [x1, x8, lsl #2]
2012 ; CHECK-BE-NEXT: add x8, x8, #16
2013 ; CHECK-BE-NEXT: cmp x8, #128
2014 ; CHECK-BE-NEXT: st1w { z1.s }, p0, [x9, #2, mul vl]
2015 ; CHECK-BE-NEXT: st1w { z2.s }, p0, [x9, #3, mul vl]
2016 ; CHECK-BE-NEXT: st1w { z3.s }, p0, [x9, #1, mul vl]
2017 ; CHECK-BE-NEXT: b.ne .LBB19_1
2018 ; CHECK-BE-NEXT: // %bb.2: // %exit
2019 ; CHECK-BE-NEXT: ret
2020 entry:
2021 br label %loop
2023 loop:
2024 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
2025 %src.gep = getelementptr i8, ptr %src, i64 %iv
2026 %load = load <vscale x 16 x i8>, ptr %src.gep
2027 %ext = zext <vscale x 16 x i8> %load to <vscale x 16 x i32>
2028 %add = add <vscale x 16 x i32> %ext, %ext
2029 %dst.gep = getelementptr i32, ptr %dst, i64 %iv
2030 store <vscale x 16 x i32> %add, ptr %dst.gep
2031 %iv.next = add nuw i64 %iv, 16
2032 %ec = icmp eq i64 %iv.next, 128
2033 br i1 %ec, label %exit, label %loop
2035 exit:
2036 ret void
2037 }
2039 ; CHECK-LABEL: lCPI20_0:
2040 ; CHECK-NEXT: .byte 0 ; 0x0
2041 ; CHECK-NEXT: .byte 255 ; 0xff
2042 ; CHECK-NEXT: .byte 255 ; 0xff
2043 ; CHECK-NEXT: .byte 1 ; 0x1
2044 ; CHECK-NEXT: .byte 255 ; 0xff
2045 ; CHECK-NEXT: .byte 255 ; 0xff
2046 ; CHECK-NEXT: .byte 2 ; 0x2
2047 ; CHECK-NEXT: .byte 255 ; 0xff
2048 ; CHECK-NEXT: .byte 255 ; 0xff
2049 ; CHECK-NEXT: .byte 3 ; 0x3
2050 ; CHECK-NEXT: .byte 255 ; 0xff
2051 ; CHECK-NEXT: .byte 255 ; 0xff
2052 ; CHECK-NEXT: .byte 4 ; 0x4
2053 ; CHECK-NEXT: .byte 255 ; 0xff
2054 ; CHECK-NEXT: .byte 255 ; 0xff
2055 ; CHECK-NEXT: .byte 5 ; 0x5
2056 ; CHECK-NEXT:lCPI20_1:
2057 ; CHECK-NEXT: .byte 255 ; 0xff
2058 ; CHECK-NEXT: .byte 255 ; 0xff
2059 ; CHECK-NEXT: .byte 6 ; 0x6
2060 ; CHECK-NEXT: .byte 255 ; 0xff
2061 ; CHECK-NEXT: .byte 255 ; 0xff
2062 ; CHECK-NEXT: .byte 7 ; 0x7
2063 ; CHECK-NEXT: .byte 255 ; 0xff
2064 ; CHECK-NEXT: .byte 255 ; 0xff
2065 ; CHECK-NEXT: .byte 8 ; 0x8
2066 ; CHECK-NEXT: .byte 255 ; 0xff
2067 ; CHECK-NEXT: .byte 255 ; 0xff
2068 ; CHECK-NEXT: .byte 9 ; 0x9
2069 ; CHECK-NEXT: .byte 255 ; 0xff
2070 ; CHECK-NEXT: .byte 255 ; 0xff
2071 ; CHECK-NEXT: .byte 10 ; 0xa
2072 ; CHECK-NEXT: .byte 255 ; 0xff
2073 ; CHECK-NEXT:lCPI20_2:
2074 ; CHECK-NEXT: .byte 255 ; 0xff
2075 ; CHECK-NEXT: .byte 11 ; 0xb
2076 ; CHECK-NEXT: .byte 255 ; 0xff
2077 ; CHECK-NEXT: .byte 255 ; 0xff
2078 ; CHECK-NEXT: .byte 12 ; 0xc
2079 ; CHECK-NEXT: .byte 255 ; 0xff
2080 ; CHECK-NEXT: .byte 255 ; 0xff
2081 ; CHECK-NEXT: .byte 13 ; 0xd
2082 ; CHECK-NEXT: .byte 255 ; 0xff
2083 ; CHECK-NEXT: .byte 255 ; 0xff
2084 ; CHECK-NEXT: .byte 14 ; 0xe
2085 ; CHECK-NEXT: .byte 255 ; 0xff
2086 ; CHECK-NEXT: .byte 255 ; 0xff
2087 ; CHECK-NEXT: .byte 15 ; 0xf
2088 ; CHECK-NEXT: .byte 255 ; 0xff
2089 ; CHECK-NEXT: .byte 255 ; 0xff
2090 ; CHECK-NEXT:lCPI20_3:
2091 ; CHECK-NEXT: .byte 0 ; 0x0
2092 ; CHECK-NEXT: .byte 255 ; 0xff
2093 ; CHECK-NEXT: .byte 255 ; 0xff
2094 ; CHECK-NEXT: .byte 1 ; 0x1
2095 ; CHECK-NEXT: .byte 255 ; 0xff
2096 ; CHECK-NEXT: .byte 255 ; 0xff
2097 ; CHECK-NEXT: .byte 2 ; 0x2
2098 ; CHECK-NEXT: .byte 255 ; 0xff
2099 ; CHECK-NEXT: .byte 255 ; 0xff
2100 ; CHECK-NEXT: .byte 3 ; 0x3
2101 ; CHECK-NEXT: .byte 255 ; 0xff
2102 ; CHECK-NEXT: .byte 255 ; 0xff
2103 ; CHECK-NEXT: .byte 255 ; 0xff
2104 ; CHECK-NEXT: .byte 255 ; 0xff
2105 ; CHECK-NEXT: .byte 255 ; 0xff
2106 ; CHECK-NEXT: .byte 255 ; 0xff
2108 ; CHECK-BE-LABEL: .LCPI20_0:
2109 ; CHECK-BE-NEXT: .byte 255 // 0xff
2110 ; CHECK-BE-NEXT: .byte 255 // 0xff
2111 ; CHECK-BE-NEXT: .byte 0 // 0x0
2112 ; CHECK-BE-NEXT: .byte 255 // 0xff
2113 ; CHECK-BE-NEXT: .byte 255 // 0xff
2114 ; CHECK-BE-NEXT: .byte 1 // 0x1
2115 ; CHECK-BE-NEXT: .byte 255 // 0xff
2116 ; CHECK-BE-NEXT: .byte 255 // 0xff
2117 ; CHECK-BE-NEXT: .byte 2 // 0x2
2118 ; CHECK-BE-NEXT: .byte 255 // 0xff
2119 ; CHECK-BE-NEXT: .byte 255 // 0xff
2120 ; CHECK-BE-NEXT: .byte 3 // 0x3
2121 ; CHECK-BE-NEXT: .byte 255 // 0xff
2122 ; CHECK-BE-NEXT: .byte 255 // 0xff
2123 ; CHECK-BE-NEXT: .byte 255 // 0xff
2124 ; CHECK-BE-NEXT: .byte 255 // 0xff
2125 ; CHECK-BE-NEXT: .LCPI20_1:
2126 ; CHECK-BE-NEXT: .byte 255 // 0xff
2127 ; CHECK-BE-NEXT: .byte 255 // 0xff
2128 ; CHECK-BE-NEXT: .byte 0 // 0x0
2129 ; CHECK-BE-NEXT: .byte 255 // 0xff
2130 ; CHECK-BE-NEXT: .byte 255 // 0xff
2131 ; CHECK-BE-NEXT: .byte 1 // 0x1
2132 ; CHECK-BE-NEXT: .byte 255 // 0xff
2133 ; CHECK-BE-NEXT: .byte 255 // 0xff
2134 ; CHECK-BE-NEXT: .byte 2 // 0x2
2135 ; CHECK-BE-NEXT: .byte 255 // 0xff
2136 ; CHECK-BE-NEXT: .byte 255 // 0xff
2137 ; CHECK-BE-NEXT: .byte 3 // 0x3
2138 ; CHECK-BE-NEXT: .byte 255 // 0xff
2139 ; CHECK-BE-NEXT: .byte 255 // 0xff
2140 ; CHECK-BE-NEXT: .byte 4 // 0x4
2141 ; CHECK-BE-NEXT: .byte 255 // 0xff
2142 ; CHECK-BE-NEXT: .LCPI20_2:
2143 ; CHECK-BE-NEXT: .byte 255 // 0xff
2144 ; CHECK-BE-NEXT: .byte 5 // 0x5
2145 ; CHECK-BE-NEXT: .byte 255 // 0xff
2146 ; CHECK-BE-NEXT: .byte 255 // 0xff
2147 ; CHECK-BE-NEXT: .byte 6 // 0x6
2148 ; CHECK-BE-NEXT: .byte 255 // 0xff
2149 ; CHECK-BE-NEXT: .byte 255 // 0xff
2150 ; CHECK-BE-NEXT: .byte 7 // 0x7
2151 ; CHECK-BE-NEXT: .byte 255 // 0xff
2152 ; CHECK-BE-NEXT: .byte 255 // 0xff
2153 ; CHECK-BE-NEXT: .byte 8 // 0x8
2154 ; CHECK-BE-NEXT: .byte 255 // 0xff
2155 ; CHECK-BE-NEXT: .byte 255 // 0xff
2156 ; CHECK-BE-NEXT: .byte 9 // 0x9
2157 ; CHECK-BE-NEXT: .byte 255 // 0xff
2158 ; CHECK-BE-NEXT: .byte 255 // 0xff
2159 ; CHECK-BE-NEXT: .LCPI20_3:
2160 ; CHECK-BE-NEXT: .byte 10 // 0xa
2161 ; CHECK-BE-NEXT: .byte 255 // 0xff
2162 ; CHECK-BE-NEXT: .byte 255 // 0xff
2163 ; CHECK-BE-NEXT: .byte 11 // 0xb
2164 ; CHECK-BE-NEXT: .byte 255 // 0xff
2165 ; CHECK-BE-NEXT: .byte 255 // 0xff
2166 ; CHECK-BE-NEXT: .byte 12 // 0xc
2167 ; CHECK-BE-NEXT: .byte 255 // 0xff
2168 ; CHECK-BE-NEXT: .byte 255 // 0xff
2169 ; CHECK-BE-NEXT: .byte 13 // 0xd
2170 ; CHECK-BE-NEXT: .byte 255 // 0xff
2171 ; CHECK-BE-NEXT: .byte 255 // 0xff
2172 ; CHECK-BE-NEXT: .byte 14 // 0xe
2173 ; CHECK-BE-NEXT: .byte 255 // 0xff
2174 ; CHECK-BE-NEXT: .byte 255 // 0xff
2175 ; CHECK-BE-NEXT: .byte 15 // 0xf
2177 define void @zext_v20i8_to_v20i24_in_loop(ptr %src, ptr %dst) {
2178 ; CHECK-LABEL: zext_v20i8_to_v20i24_in_loop:
2179 ; CHECK: ; %bb.0: ; %entry
2180 ; CHECK-NEXT: Lloh22:
2181 ; CHECK-NEXT: adrp x8, lCPI20_0@PAGE
2182 ; CHECK-NEXT: Lloh23:
2183 ; CHECK-NEXT: adrp x9, lCPI20_1@PAGE
2184 ; CHECK-NEXT: Lloh24:
2185 ; CHECK-NEXT: adrp x10, lCPI20_2@PAGE
2186 ; CHECK-NEXT: Lloh25:
2187 ; CHECK-NEXT: ldr q0, [x8, lCPI20_0@PAGEOFF]
2188 ; CHECK-NEXT: Lloh26:
2189 ; CHECK-NEXT: adrp x8, lCPI20_3@PAGE
2190 ; CHECK-NEXT: Lloh27:
2191 ; CHECK-NEXT: ldr q1, [x9, lCPI20_1@PAGEOFF]
2192 ; CHECK-NEXT: Lloh28:
2193 ; CHECK-NEXT: ldr q2, [x10, lCPI20_2@PAGEOFF]
2194 ; CHECK-NEXT: Lloh29:
2195 ; CHECK-NEXT: ldr q3, [x8, lCPI20_3@PAGEOFF]
2196 ; CHECK-NEXT: mov x8, xzr
2197 ; CHECK-NEXT: LBB20_1: ; %loop
2198 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
2199 ; CHECK-NEXT: add x9, x0, x8
2200 ; CHECK-NEXT: add x8, x8, #16
2201 ; CHECK-NEXT: ldp q5, q4, [x9]
2202 ; CHECK-NEXT: add x9, x1, #56
2203 ; CHECK-NEXT: cmp x8, #128
2204 ; CHECK-NEXT: tbl.16b v4, { v4 }, v3
2205 ; CHECK-NEXT: tbl.16b v6, { v5 }, v2
2206 ; CHECK-NEXT: tbl.16b v7, { v5 }, v1
2207 ; CHECK-NEXT: tbl.16b v5, { v5 }, v0
2208 ; CHECK-NEXT: stp q7, q6, [x1, #16]
2209 ; CHECK-NEXT: str q5, [x1]
2210 ; CHECK-NEXT: str d4, [x1, #48]
2211 ; CHECK-NEXT: add x1, x1, #64
2212 ; CHECK-NEXT: st1.s { v4 }[2], [x9]
2213 ; CHECK-NEXT: b.ne LBB20_1
2214 ; CHECK-NEXT: ; %bb.2: ; %exit
2215 ; CHECK-NEXT: ret
2216 ; CHECK-NEXT: .loh AdrpLdr Lloh26, Lloh29
2217 ; CHECK-NEXT: .loh AdrpLdr Lloh24, Lloh28
2218 ; CHECK-NEXT: .loh AdrpLdr Lloh23, Lloh27
2219 ; CHECK-NEXT: .loh AdrpAdrp Lloh22, Lloh26
2220 ; CHECK-NEXT: .loh AdrpLdr Lloh22, Lloh25
2221 ;
2222 ; CHECK-BE-LABEL: zext_v20i8_to_v20i24_in_loop:
2223 ; CHECK-BE: // %bb.0: // %entry
2224 ; CHECK-BE-NEXT: adrp x8, .LCPI20_0
2225 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI20_0
2226 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]
2227 ; CHECK-BE-NEXT: adrp x8, .LCPI20_1
2228 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI20_1
2229 ; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
2230 ; CHECK-BE-NEXT: adrp x8, .LCPI20_2
2231 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI20_2
2232 ; CHECK-BE-NEXT: ld1 { v2.16b }, [x8]
2233 ; CHECK-BE-NEXT: adrp x8, .LCPI20_3
2234 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI20_3
2235 ; CHECK-BE-NEXT: ld1 { v3.16b }, [x8]
2236 ; CHECK-BE-NEXT: mov x8, xzr
2237 ; CHECK-BE-NEXT: .LBB20_1: // %loop
2238 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
2239 ; CHECK-BE-NEXT: add x9, x0, x8
2240 ; CHECK-BE-NEXT: add x8, x8, #16
2241 ; CHECK-BE-NEXT: add x10, x9, #16
2242 ; CHECK-BE-NEXT: ld1 { v5.16b }, [x9]
2243 ; CHECK-BE-NEXT: add x9, x1, #32
2244 ; CHECK-BE-NEXT: ld1 { v4.16b }, [x10]
2245 ; CHECK-BE-NEXT: cmp x8, #128
2246 ; CHECK-BE-NEXT: tbl v6.16b, { v5.16b }, v3.16b
2247 ; CHECK-BE-NEXT: tbl v7.16b, { v5.16b }, v2.16b
2248 ; CHECK-BE-NEXT: tbl v5.16b, { v5.16b }, v1.16b
2249 ; CHECK-BE-NEXT: tbl v4.16b, { v4.16b }, v0.16b
2250 ; CHECK-BE-NEXT: st1 { v6.16b }, [x9]
2251 ; CHECK-BE-NEXT: add x9, x1, #16
2252 ; CHECK-BE-NEXT: rev32 v16.16b, v4.16b
2253 ; CHECK-BE-NEXT: rev64 v4.16b, v4.16b
2254 ; CHECK-BE-NEXT: st1 { v7.16b }, [x9]
2255 ; CHECK-BE-NEXT: add x9, x1, #56
2256 ; CHECK-BE-NEXT: st1 { v5.16b }, [x1]
2257 ; CHECK-BE-NEXT: str d4, [x1, #48]
2258 ; CHECK-BE-NEXT: add x1, x1, #64
2259 ; CHECK-BE-NEXT: st1 { v16.s }[2], [x9]
2260 ; CHECK-BE-NEXT: b.ne .LBB20_1
2261 ; CHECK-BE-NEXT: // %bb.2: // %exit
2262 ; CHECK-BE-NEXT: ret
2264 entry:
2265 br label %loop
2267 loop:
2268 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
2269 %src.gep = getelementptr i8, ptr %src, i64 %iv
2270 %load = load <20 x i8>, ptr %src.gep
2271 %ext = zext <20 x i8> %load to <20 x i24>
2272 %dst.gep = getelementptr i24, ptr %dst, i64 %iv
2273 store <20 x i24> %ext, ptr %dst.gep
2274 %iv.next = add nuw i64 %iv, 16
2275 %ec = icmp eq i64 %iv.next, 128
2276 br i1 %ec, label %exit, label %loop
2278 exit:
2279 ret void
2280 }
2282 ; CHECK-LABEL: lCPI21_0:
2283 ; CHECK-NEXT: .byte 0 ; 0x0
2284 ; CHECK-NEXT: .byte 255 ; 0xff
2285 ; CHECK-NEXT: .byte 255 ; 0xff
2286 ; CHECK-NEXT: .byte 255 ; 0xff
2287 ; CHECK-NEXT: .byte 255 ; 0xff
2288 ; CHECK-NEXT: .byte 255 ; 0xff
2289 ; CHECK-NEXT: .byte 1 ; 0x1
2290 ; CHECK-NEXT: .byte 255 ; 0xff
2291 ; CHECK-NEXT: .byte 255 ; 0xff
2292 ; CHECK-NEXT: .byte 255 ; 0xff
2293 ; CHECK-NEXT: .byte 255 ; 0xff
2294 ; CHECK-NEXT: .byte 255 ; 0xff
2295 ; CHECK-NEXT: .byte 2 ; 0x2
2296 ; CHECK-NEXT: .byte 255 ; 0xff
2297 ; CHECK-NEXT: .byte 255 ; 0xff
2298 ; CHECK-NEXT: .byte 255 ; 0xff
2299 ; CHECK-NEXT: lCPI21_1:
2300 ; CHECK-NEXT: .byte 255 ; 0xff
2301 ; CHECK-NEXT: .byte 255 ; 0xff
2302 ; CHECK-NEXT: .byte 3 ; 0x3
2303 ; CHECK-NEXT: .byte 255 ; 0xff
2304 ; CHECK-NEXT: .byte 255 ; 0xff
2305 ; CHECK-NEXT: .byte 255 ; 0xff
2306 ; CHECK-NEXT: .byte 255 ; 0xff
2307 ; CHECK-NEXT: .byte 255 ; 0xff
2308 ; CHECK-NEXT: .byte 4 ; 0x4
2309 ; CHECK-NEXT: .byte 255 ; 0xff
2310 ; CHECK-NEXT: .byte 255 ; 0xff
2311 ; CHECK-NEXT: .byte 255 ; 0xff
2312 ; CHECK-NEXT: .byte 255 ; 0xff
2313 ; CHECK-NEXT: .byte 255 ; 0xff
2314 ; CHECK-NEXT: .byte 5 ; 0x5
2315 ; CHECK-NEXT: .byte 255 ; 0xff
2316 ; CHECK-NEXT: lCPI21_2:
2317 ; CHECK-NEXT: .byte 255 ; 0xff
2318 ; CHECK-NEXT: .byte 255 ; 0xff
2319 ; CHECK-NEXT: .byte 255 ; 0xff
2320 ; CHECK-NEXT: .byte 255 ; 0xff
2321 ; CHECK-NEXT: .byte 6 ; 0x6
2322 ; CHECK-NEXT: .byte 255 ; 0xff
2323 ; CHECK-NEXT: .byte 255 ; 0xff
2324 ; CHECK-NEXT: .byte 255 ; 0xff
2325 ; CHECK-NEXT: .byte 255 ; 0xff
2326 ; CHECK-NEXT: .byte 255 ; 0xff
2327 ; CHECK-NEXT: .byte 7 ; 0x7
2328 ; CHECK-NEXT: .byte 255 ; 0xff
2329 ; CHECK-NEXT: .byte 255 ; 0xff
2330 ; CHECK-NEXT: .byte 255 ; 0xff
2331 ; CHECK-NEXT: .byte 255 ; 0xff
2332 ; CHECK-NEXT: .byte 255 ; 0xff
2333 ; CHECK-NEXT: lCPI21_3:
2334 ; CHECK-NEXT: .byte 8 ; 0x8
2335 ; CHECK-NEXT: .byte 255 ; 0xff
2336 ; CHECK-NEXT: .byte 255 ; 0xff
2337 ; CHECK-NEXT: .byte 255 ; 0xff
2338 ; CHECK-NEXT: .byte 255 ; 0xff
2339 ; CHECK-NEXT: .byte 255 ; 0xff
2340 ; CHECK-NEXT: .byte 9 ; 0x9
2341 ; CHECK-NEXT: .byte 255 ; 0xff
2342 ; CHECK-NEXT: .byte 255 ; 0xff
2343 ; CHECK-NEXT: .byte 255 ; 0xff
2344 ; CHECK-NEXT: .byte 255 ; 0xff
2345 ; CHECK-NEXT: .byte 255 ; 0xff
2346 ; CHECK-NEXT: .byte 10 ; 0xa
2347 ; CHECK-NEXT: .byte 255 ; 0xff
2348 ; CHECK-NEXT: .byte 255 ; 0xff
2349 ; CHECK-NEXT: .byte 255 ; 0xff
2350 ; CHECK-NEXT: lCPI21_4:
2351 ; CHECK-NEXT: .byte 255 ; 0xff
2352 ; CHECK-NEXT: .byte 255 ; 0xff
2353 ; CHECK-NEXT: .byte 11 ; 0xb
2354 ; CHECK-NEXT: .byte 255 ; 0xff
2355 ; CHECK-NEXT: .byte 255 ; 0xff
2356 ; CHECK-NEXT: .byte 255 ; 0xff
2357 ; CHECK-NEXT: .byte 255 ; 0xff
2358 ; CHECK-NEXT: .byte 255 ; 0xff
2359 ; CHECK-NEXT: .byte 12 ; 0xc
2360 ; CHECK-NEXT: .byte 255 ; 0xff
2361 ; CHECK-NEXT: .byte 255 ; 0xff
2362 ; CHECK-NEXT: .byte 255 ; 0xff
2363 ; CHECK-NEXT: .byte 255 ; 0xff
2364 ; CHECK-NEXT: .byte 255 ; 0xff
2365 ; CHECK-NEXT: .byte 13 ; 0xd
2366 ; CHECK-NEXT: .byte 255 ; 0xff
2367 ; CHECK-NEXT: lCPI21_5:
2368 ; CHECK-NEXT: .byte 255 ; 0xff
2369 ; CHECK-NEXT: .byte 255 ; 0xff
2370 ; CHECK-NEXT: .byte 255 ; 0xff
2371 ; CHECK-NEXT: .byte 255 ; 0xff
2372 ; CHECK-NEXT: .byte 14 ; 0xe
2373 ; CHECK-NEXT: .byte 255 ; 0xff
2374 ; CHECK-NEXT: .byte 255 ; 0xff
2375 ; CHECK-NEXT: .byte 255 ; 0xff
2376 ; CHECK-NEXT: .byte 255 ; 0xff
2377 ; CHECK-NEXT: .byte 255 ; 0xff
2378 ; CHECK-NEXT: .byte 15 ; 0xf
2379 ; CHECK-NEXT: .byte 255 ; 0xff
2380 ; CHECK-NEXT: .byte 255 ; 0xff
2381 ; CHECK-NEXT: .byte 255 ; 0xff
2382 ; CHECK-NEXT: .byte 255 ; 0xff
2383 ; CHECK-NEXT: .byte 255 ; 0xff
2385 ; CHECK-BE-LABEL: .LCPI21_0:
2386 ; CHECK-BE-NEXT: .byte 255 // 0xff
2387 ; CHECK-BE-NEXT: .byte 255 // 0xff
2388 ; CHECK-BE-NEXT: .byte 255 // 0xff
2389 ; CHECK-BE-NEXT: .byte 5 // 0x5
2390 ; CHECK-BE-NEXT: .byte 255 // 0xff
2391 ; CHECK-BE-NEXT: .byte 255 // 0xff
2392 ; CHECK-BE-NEXT: .byte 255 // 0xff
2393 ; CHECK-BE-NEXT: .byte 255 // 0xff
2394 ; CHECK-BE-NEXT: .byte 255 // 0xff
2395 ; CHECK-BE-NEXT: .byte 6 // 0x6
2396 ; CHECK-BE-NEXT: .byte 255 // 0xff
2397 ; CHECK-BE-NEXT: .byte 255 // 0xff
2398 ; CHECK-BE-NEXT: .byte 255 // 0xff
2399 ; CHECK-BE-NEXT: .byte 255 // 0xff
2400 ; CHECK-BE-NEXT: .byte 255 // 0xff
2401 ; CHECK-BE-NEXT: .byte 255 // 0xff
2402 ; CHECK-BE-NEXT: .LCPI21_1:
2403 ; CHECK-BE-NEXT: .byte 255 // 0xff
2404 ; CHECK-BE-NEXT: .byte 255 // 0xff
2405 ; CHECK-BE-NEXT: .byte 255 // 0xff
2406 ; CHECK-BE-NEXT: .byte 255 // 0xff
2407 ; CHECK-BE-NEXT: .byte 255 // 0xff
2408 ; CHECK-BE-NEXT: .byte 0 // 0x0
2409 ; CHECK-BE-NEXT: .byte 255 // 0xff
2410 ; CHECK-BE-NEXT: .byte 255 // 0xff
2411 ; CHECK-BE-NEXT: .byte 255 // 0xff
2412 ; CHECK-BE-NEXT: .byte 255 // 0xff
2413 ; CHECK-BE-NEXT: .byte 255 // 0xff
2414 ; CHECK-BE-NEXT: .byte 1 // 0x1
2415 ; CHECK-BE-NEXT: .byte 255 // 0xff
2416 ; CHECK-BE-NEXT: .byte 255 // 0xff
2417 ; CHECK-BE-NEXT: .byte 255 // 0xff
2418 ; CHECK-BE-NEXT: .byte 255 // 0xff
2419 ; CHECK-BE-NEXT: .LCPI21_2:
2420 ; CHECK-BE-NEXT: .byte 255 // 0xff
2421 ; CHECK-BE-NEXT: .byte 2 // 0x2
2422 ; CHECK-BE-NEXT: .byte 255 // 0xff
2423 ; CHECK-BE-NEXT: .byte 255 // 0xff
2424 ; CHECK-BE-NEXT: .byte 255 // 0xff
2425 ; CHECK-BE-NEXT: .byte 255 // 0xff
2426 ; CHECK-BE-NEXT: .byte 255 // 0xff
2427 ; CHECK-BE-NEXT: .byte 3 // 0x3
2428 ; CHECK-BE-NEXT: .byte 255 // 0xff
2429 ; CHECK-BE-NEXT: .byte 255 // 0xff
2430 ; CHECK-BE-NEXT: .byte 255 // 0xff
2431 ; CHECK-BE-NEXT: .byte 255 // 0xff
2432 ; CHECK-BE-NEXT: .byte 255 // 0xff
2433 ; CHECK-BE-NEXT: .byte 4 // 0x4
2434 ; CHECK-BE-NEXT: .byte 255 // 0xff
2435 ; CHECK-BE-NEXT: .byte 255 // 0xff
2436 ; CHECK-BE-NEXT: .LCPI21_3:
2437 ; CHECK-BE-NEXT: .byte 255 // 0xff
2438 ; CHECK-BE-NEXT: .byte 255 // 0xff
2439 ; CHECK-BE-NEXT: .byte 255 // 0xff
2440 ; CHECK-BE-NEXT: .byte 5 // 0x5
2441 ; CHECK-BE-NEXT: .byte 255 // 0xff
2442 ; CHECK-BE-NEXT: .byte 255 // 0xff
2443 ; CHECK-BE-NEXT: .byte 255 // 0xff
2444 ; CHECK-BE-NEXT: .byte 255 // 0xff
2445 ; CHECK-BE-NEXT: .byte 255 // 0xff
2446 ; CHECK-BE-NEXT: .byte 6 // 0x6
2447 ; CHECK-BE-NEXT: .byte 255 // 0xff
2448 ; CHECK-BE-NEXT: .byte 255 // 0xff
2449 ; CHECK-BE-NEXT: .byte 255 // 0xff
2450 ; CHECK-BE-NEXT: .byte 255 // 0xff
2451 ; CHECK-BE-NEXT: .byte 255 // 0xff
2452 ; CHECK-BE-NEXT: .byte 7 // 0x7
2453 ; CHECK-BE-NEXT: .LCPI21_4:
2454 ; CHECK-BE-NEXT: .byte 255 // 0xff
2455 ; CHECK-BE-NEXT: .byte 255 // 0xff
2456 ; CHECK-BE-NEXT: .byte 255 // 0xff
2457 ; CHECK-BE-NEXT: .byte 255 // 0xff
2458 ; CHECK-BE-NEXT: .byte 255 // 0xff
2459 ; CHECK-BE-NEXT: .byte 8 // 0x8
2460 ; CHECK-BE-NEXT: .byte 255 // 0xff
2461 ; CHECK-BE-NEXT: .byte 255 // 0xff
2462 ; CHECK-BE-NEXT: .byte 255 // 0xff
2463 ; CHECK-BE-NEXT: .byte 255 // 0xff
2464 ; CHECK-BE-NEXT: .byte 255 // 0xff
2465 ; CHECK-BE-NEXT: .byte 9 // 0x9
2466 ; CHECK-BE-NEXT: .byte 255 // 0xff
2467 ; CHECK-BE-NEXT: .byte 255 // 0xff
2468 ; CHECK-BE-NEXT: .byte 255 // 0xff
2469 ; CHECK-BE-NEXT: .byte 255 // 0xff
2470 ; CHECK-BE-NEXT: .LCPI21_5:
2471 ; CHECK-BE-NEXT: .byte 255 // 0xff
2472 ; CHECK-BE-NEXT: .byte 10 // 0xa
2473 ; CHECK-BE-NEXT: .byte 255 // 0xff
2474 ; CHECK-BE-NEXT: .byte 255 // 0xff
2475 ; CHECK-BE-NEXT: .byte 255 // 0xff
2476 ; CHECK-BE-NEXT: .byte 255 // 0xff
2477 ; CHECK-BE-NEXT: .byte 255 // 0xff
2478 ; CHECK-BE-NEXT: .byte 11 // 0xb
2479 ; CHECK-BE-NEXT: .byte 255 // 0xff
2480 ; CHECK-BE-NEXT: .byte 255 // 0xff
2481 ; CHECK-BE-NEXT: .byte 255 // 0xff
2482 ; CHECK-BE-NEXT: .byte 255 // 0xff
2483 ; CHECK-BE-NEXT: .byte 255 // 0xff
2484 ; CHECK-BE-NEXT: .byte 12 // 0xc
2485 ; CHECK-BE-NEXT: .byte 255 // 0xff
2486 ; CHECK-BE-NEXT: .byte 255 // 0xff
2487 ; CHECK-BE-NEXT: .LCPI21_6:
2488 ; CHECK-BE-NEXT: .byte 255 // 0xff
2489 ; CHECK-BE-NEXT: .byte 255 // 0xff
2490 ; CHECK-BE-NEXT: .byte 255 // 0xff
2491 ; CHECK-BE-NEXT: .byte 13 // 0xd
2492 ; CHECK-BE-NEXT: .byte 255 // 0xff
2493 ; CHECK-BE-NEXT: .byte 255 // 0xff
2494 ; CHECK-BE-NEXT: .byte 255 // 0xff
2495 ; CHECK-BE-NEXT: .byte 255 // 0xff
2496 ; CHECK-BE-NEXT: .byte 255 // 0xff
2497 ; CHECK-BE-NEXT: .byte 14 // 0xe
2498 ; CHECK-BE-NEXT: .byte 255 // 0xff
2499 ; CHECK-BE-NEXT: .byte 255 // 0xff
2500 ; CHECK-BE-NEXT: .byte 255 // 0xff
2501 ; CHECK-BE-NEXT: .byte 255 // 0xff
2502 ; CHECK-BE-NEXT: .byte 255 // 0xff
2503 ; CHECK-BE-NEXT: .byte 15 // 0xf
2505 define void @zext_v23i8_to_v23i48_in_loop(ptr %src, ptr %dst) {
2506 ; CHECK-LABEL: zext_v23i8_to_v23i48_in_loop:
2507 ; CHECK: ; %bb.0: ; %entry
2508 ; CHECK-NEXT: Lloh30:
2509 ; CHECK-NEXT: adrp x8, lCPI21_0@PAGE
2510 ; CHECK-NEXT: Lloh31:
2511 ; CHECK-NEXT: adrp x9, lCPI21_1@PAGE
2512 ; CHECK-NEXT: Lloh32:
2513 ; CHECK-NEXT: adrp x10, lCPI21_2@PAGE
2514 ; CHECK-NEXT: Lloh33:
2515 ; CHECK-NEXT: ldr q0, [x8, lCPI21_0@PAGEOFF]
2516 ; CHECK-NEXT: Lloh34:
2517 ; CHECK-NEXT: ldr q1, [x9, lCPI21_1@PAGEOFF]
2518 ; CHECK-NEXT: Lloh35:
2519 ; CHECK-NEXT: ldr q2, [x10, lCPI21_2@PAGEOFF]
2520 ; CHECK-NEXT: Lloh36:
2521 ; CHECK-NEXT: adrp x8, lCPI21_3@PAGE
2522 ; CHECK-NEXT: Lloh37:
2523 ; CHECK-NEXT: adrp x9, lCPI21_4@PAGE
2524 ; CHECK-NEXT: Lloh38:
2525 ; CHECK-NEXT: adrp x10, lCPI21_5@PAGE
2526 ; CHECK-NEXT: Lloh39:
2527 ; CHECK-NEXT: ldr q3, [x8, lCPI21_3@PAGEOFF]
2528 ; CHECK-NEXT: Lloh40:
2529 ; CHECK-NEXT: ldr q4, [x9, lCPI21_4@PAGEOFF]
2530 ; CHECK-NEXT: Lloh41:
2531 ; CHECK-NEXT: ldr q5, [x10, lCPI21_5@PAGEOFF]
2532 ; CHECK-NEXT: mov x8, xzr
2533 ; CHECK-NEXT: LBB21_1: ; %loop
2534 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
2535 ; CHECK-NEXT: add x9, x0, x8
2536 ; CHECK-NEXT: movi.2d v19, #0000000000000000
2537 ; CHECK-NEXT: add x8, x8, #16
2538 ; CHECK-NEXT: ldp q7, q6, [x9]
2539 ; CHECK-NEXT: cmp x8, #128
2540 ; CHECK-NEXT: strh wzr, [x1, #136]
2541 ; CHECK-NEXT: tbl.16b v16, { v6 }, v1
2542 ; CHECK-NEXT: tbl.16b v17, { v6 }, v0
2543 ; CHECK-NEXT: mov.b v19[4], v6[6]
2544 ; CHECK-NEXT: tbl.16b v18, { v7 }, v5
2545 ; CHECK-NEXT: tbl.16b v20, { v7 }, v4
2546 ; CHECK-NEXT: tbl.16b v21, { v7 }, v3
2547 ; CHECK-NEXT: stp q17, q16, [x1, #96]
2548 ; CHECK-NEXT: tbl.16b v16, { v7 }, v2
2549 ; CHECK-NEXT: tbl.16b v17, { v7 }, v1
2550 ; CHECK-NEXT: tbl.16b v7, { v7 }, v0
2551 ; CHECK-NEXT: fmov x9, d19
2552 ; CHECK-NEXT: stp q20, q18, [x1, #64]
2553 ; CHECK-NEXT: stp q16, q21, [x1, #32]
2554 ; CHECK-NEXT: stp q7, q17, [x1]
2555 ; CHECK-NEXT: str x9, [x1, #128]!
2556 ; CHECK-NEXT: b.ne LBB21_1
2557 ; CHECK-NEXT: ; %bb.2: ; %exit
2558 ; CHECK-NEXT: ret
2559 ; CHECK-NEXT: .loh AdrpLdr Lloh38, Lloh41
2560 ; CHECK-NEXT: .loh AdrpLdr Lloh37, Lloh40
2561 ; CHECK-NEXT: .loh AdrpLdr Lloh36, Lloh39
2562 ; CHECK-NEXT: .loh AdrpAdrp Lloh32, Lloh38
2563 ; CHECK-NEXT: .loh AdrpLdr Lloh32, Lloh35
2564 ; CHECK-NEXT: .loh AdrpAdrp Lloh31, Lloh37
2565 ; CHECK-NEXT: .loh AdrpLdr Lloh31, Lloh34
2566 ; CHECK-NEXT: .loh AdrpAdrp Lloh30, Lloh36
2567 ; CHECK-NEXT: .loh AdrpLdr Lloh30, Lloh33
2568 ;
2569 ; CHECK-BE-LABEL: zext_v23i8_to_v23i48_in_loop:
2570 ; CHECK-BE: // %bb.0: // %entry
2571 ; CHECK-BE-NEXT: adrp x8, .LCPI21_0
2572 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI21_0
2573 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]
2574 ; CHECK-BE-NEXT: adrp x8, .LCPI21_1
2575 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI21_1
2576 ; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
2577 ; CHECK-BE-NEXT: adrp x8, .LCPI21_2
2578 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI21_2
2579 ; CHECK-BE-NEXT: ld1 { v2.16b }, [x8]
2580 ; CHECK-BE-NEXT: adrp x8, .LCPI21_3
2581 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI21_3
2582 ; CHECK-BE-NEXT: ld1 { v3.16b }, [x8]
2583 ; CHECK-BE-NEXT: adrp x8, .LCPI21_4
2584 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI21_4
2585 ; CHECK-BE-NEXT: ld1 { v4.16b }, [x8]
2586 ; CHECK-BE-NEXT: adrp x8, .LCPI21_5
2587 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI21_5
2588 ; CHECK-BE-NEXT: ld1 { v5.16b }, [x8]
2589 ; CHECK-BE-NEXT: adrp x8, .LCPI21_6
2590 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI21_6
2591 ; CHECK-BE-NEXT: ld1 { v6.16b }, [x8]
2592 ; CHECK-BE-NEXT: mov x8, xzr
2593 ; CHECK-BE-NEXT: .LBB21_1: // %loop
2594 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
2595 ; CHECK-BE-NEXT: add x9, x0, x8
2596 ; CHECK-BE-NEXT: add x8, x8, #16
2597 ; CHECK-BE-NEXT: ld1 { v7.16b }, [x9]
2598 ; CHECK-BE-NEXT: add x9, x9, #16
2599 ; CHECK-BE-NEXT: cmp x8, #128
2600 ; CHECK-BE-NEXT: ld1 { v17.16b }, [x9]
2601 ; CHECK-BE-NEXT: add x9, x1, #80
2602 ; CHECK-BE-NEXT: tbl v16.16b, { v7.16b }, v6.16b
2603 ; CHECK-BE-NEXT: tbl v18.16b, { v7.16b }, v5.16b
2604 ; CHECK-BE-NEXT: tbl v19.16b, { v7.16b }, v4.16b
2605 ; CHECK-BE-NEXT: tbl v20.16b, { v7.16b }, v3.16b
2606 ; CHECK-BE-NEXT: tbl v21.16b, { v17.16b }, v0.16b
2607 ; CHECK-BE-NEXT: st1 { v16.16b }, [x9]
2608 ; CHECK-BE-NEXT: add x9, x1, #64
2609 ; CHECK-BE-NEXT: tbl v16.16b, { v7.16b }, v2.16b
2610 ; CHECK-BE-NEXT: st1 { v18.16b }, [x9]
2611 ; CHECK-BE-NEXT: add x9, x1, #48
2612 ; CHECK-BE-NEXT: tbl v18.16b, { v17.16b }, v2.16b
2613 ; CHECK-BE-NEXT: st1 { v19.16b }, [x9]
2614 ; CHECK-BE-NEXT: add x9, x1, #32
2615 ; CHECK-BE-NEXT: tbl v17.16b, { v17.16b }, v1.16b
2616 ; CHECK-BE-NEXT: st1 { v20.16b }, [x9]
2617 ; CHECK-BE-NEXT: add x9, x1, #16
2618 ; CHECK-BE-NEXT: rev64 v19.16b, v21.16b
2619 ; CHECK-BE-NEXT: st1 { v16.16b }, [x9]
2620 ; CHECK-BE-NEXT: rev16 v16.16b, v21.16b
2621 ; CHECK-BE-NEXT: add x9, x1, #112
2622 ; CHECK-BE-NEXT: st1 { v18.16b }, [x9]
2623 ; CHECK-BE-NEXT: add x9, x1, #96
2624 ; CHECK-BE-NEXT: tbl v7.16b, { v7.16b }, v1.16b
2625 ; CHECK-BE-NEXT: st1 { v17.16b }, [x9]
2626 ; CHECK-BE-NEXT: add x9, x1, #136
2627 ; CHECK-BE-NEXT: st1 { v16.h }[4], [x9]
2628 ; CHECK-BE-NEXT: fmov x9, d19
2629 ; CHECK-BE-NEXT: st1 { v7.16b }, [x1]
2630 ; CHECK-BE-NEXT: str x9, [x1, #128]!
2631 ; CHECK-BE-NEXT: b.ne .LBB21_1
2632 ; CHECK-BE-NEXT: // %bb.2: // %exit
2633 ; CHECK-BE-NEXT: ret
2634 ; CHECK : Lloh30:
2638 entry:
2639 br label %loop
2641 loop:
2642 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
2643 %src.gep = getelementptr i8, ptr %src, i64 %iv
2644 %load = load <23 x i8>, ptr %src.gep
2645 %ext = zext <23 x i8> %load to <23 x i48>
2646 %dst.gep = getelementptr i48, ptr %dst, i64 %iv
2647 store <23 x i48> %ext, ptr %dst.gep
2648 %iv.next = add nuw i64 %iv, 16
2649 %ec = icmp eq i64 %iv.next, 128
2650 br i1 %ec, label %exit, label %loop
2652 exit:
2653 ret void
2654 }
2656 define void @zext_v8i8_to_v8i33_in_loop(ptr %src, ptr %dst) {
2657 ; CHECK-LABEL: zext_v8i8_to_v8i33_in_loop:
2658 ; CHECK: ; %bb.0: ; %entry
2659 ; CHECK-NEXT: mov x8, xzr
2660 ; CHECK-NEXT: LBB22_1: ; %loop
2661 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
2662 ; CHECK-NEXT: ldr d0, [x0, x8]
2663 ; CHECK-NEXT: add x8, x8, #16
2664 ; CHECK-NEXT: strb wzr, [x1, #32]
2665 ; CHECK-NEXT: cmp x8, #128
2666 ; CHECK-NEXT: ushll.8h v0, v0, #0
2667 ; CHECK-NEXT: ushll2.4s v1, v0, #0
2668 ; CHECK-NEXT: ushll.4s v0, v0, #0
2669 ; CHECK-NEXT: ushll2.2d v2, v1, #0
2670 ; CHECK-NEXT: ushll.2d v1, v1, #0
2671 ; CHECK-NEXT: ushll2.2d v3, v0, #0
2672 ; CHECK-NEXT: ushll.2d v0, v0, #0
2673 ; CHECK-NEXT: mov.d x9, v2[1]
2674 ; CHECK-NEXT: mov.d x10, v1[1]
2675 ; CHECK-NEXT: fmov x12, d2
2676 ; CHECK-NEXT: mov.d x11, v3[1]
2677 ; CHECK-NEXT: mov.d x13, v0[1]
2678 ; CHECK-NEXT: lsl x9, x9, #39
2679 ; CHECK-NEXT: lsl x10, x10, #37
2680 ; CHECK-NEXT: lsl x11, x11, #35
2681 ; CHECK-NEXT: orr x9, x9, x12, lsl #6
2682 ; CHECK-NEXT: fmov x12, d1
2683 ; CHECK-NEXT: orr x10, x10, x12, lsl #4
2684 ; CHECK-NEXT: fmov x12, d3
2685 ; CHECK-NEXT: stp x10, x9, [x1, #16]
2686 ; CHECK-NEXT: orr x11, x11, x12, lsl #2
2687 ; CHECK-NEXT: fmov x12, d0
2688 ; CHECK-NEXT: orr x9, x12, x13, lsl #33
2689 ; CHECK-NEXT: stp x9, x11, [x1], #128
2690 ; CHECK-NEXT: b.ne LBB22_1
2691 ; CHECK-NEXT: ; %bb.2: ; %exit
2692 ; CHECK-NEXT: ret
2693 ;
2694 ; CHECK-BE-LABEL: zext_v8i8_to_v8i33_in_loop:
2695 ; CHECK-BE: // %bb.0: // %entry
2696 ; CHECK-BE-NEXT: mov x8, xzr
2697 ; CHECK-BE-NEXT: .LBB22_1: // %loop
2698 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
2699 ; CHECK-BE-NEXT: add x9, x0, x8
2700 ; CHECK-BE-NEXT: add x8, x8, #16
2701 ; CHECK-BE-NEXT: ld1 { v0.8b }, [x9]
2702 ; CHECK-BE-NEXT: cmp x8, #128
2703 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
2704 ; CHECK-BE-NEXT: ushll2 v1.4s, v0.8h, #0
2705 ; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
2706 ; CHECK-BE-NEXT: ushll v2.2d, v1.2s, #0
2707 ; CHECK-BE-NEXT: ushll2 v1.2d, v1.4s, #0
2708 ; CHECK-BE-NEXT: ushll2 v3.2d, v0.4s, #0
2709 ; CHECK-BE-NEXT: ushll v0.2d, v0.2s, #0
2710 ; CHECK-BE-NEXT: mov x9, v2.d[1]
2711 ; CHECK-BE-NEXT: mov x10, v1.d[1]
2712 ; CHECK-BE-NEXT: fmov x13, d1
2713 ; CHECK-BE-NEXT: mov x11, v3.d[1]
2714 ; CHECK-BE-NEXT: mov x12, v0.d[1]
2715 ; CHECK-BE-NEXT: fmov x14, d2
2716 ; CHECK-BE-NEXT: fmov x15, d3
2717 ; CHECK-BE-NEXT: lsl x9, x9, #2
2718 ; CHECK-BE-NEXT: orr x13, x10, x13, lsl #33
2719 ; CHECK-BE-NEXT: strb w10, [x1, #32]
2720 ; CHECK-BE-NEXT: lsl x11, x11, #4
2721 ; CHECK-BE-NEXT: lsl x12, x12, #6
2722 ; CHECK-BE-NEXT: orr x14, x9, x14, lsl #35
2723 ; CHECK-BE-NEXT: extr x9, x9, x13, #8
2724 ; CHECK-BE-NEXT: fmov x13, d0
2725 ; CHECK-BE-NEXT: orr x15, x11, x15, lsl #37
2726 ; CHECK-BE-NEXT: extr x10, x11, x14, #8
2727 ; CHECK-BE-NEXT: orr x11, x12, x13, lsl #39
2728 ; CHECK-BE-NEXT: extr x12, x12, x15, #8
2729 ; CHECK-BE-NEXT: stp x10, x9, [x1, #16]
2730 ; CHECK-BE-NEXT: lsr x9, x11, #8
2731 ; CHECK-BE-NEXT: stp x9, x12, [x1], #128
2732 ; CHECK-BE-NEXT: b.ne .LBB22_1
2733 ; CHECK-BE-NEXT: // %bb.2: // %exit
2734 ; CHECK-BE-NEXT: ret
2737 entry:
2738 br label %loop
2740 loop:
2741 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
2742 %src.gep = getelementptr i8, ptr %src, i64 %iv
2743 %load = load <8 x i8>, ptr %src.gep
2744 %ext = zext <8 x i8> %load to <8 x i33>
2745 %dst.gep = getelementptr i33, ptr %dst, i64 %iv
2746 store <8 x i33> %ext, ptr %dst.gep
2747 %iv.next = add nuw i64 %iv, 16
2748 %ec = icmp eq i64 %iv.next, 128
2749 br i1 %ec, label %exit, label %loop
2751 exit:
2752 ret void
2753 }
2755 ; FIXME: Widening instructions should be used instead of tbl.
2756 define i32 @test_pr62620_widening_instr(ptr %p1, ptr %p2, i64 %lx, i32 %h) {
2757 ; CHECK-LABEL: test_pr62620_widening_instr:
2758 ; CHECK: ; %bb.0: ; %entry
2759 ; CHECK-NEXT: lsl x9, x2, #4
2760 ; CHECK-NEXT: mov w8, wzr
2761 ; CHECK-NEXT: LBB23_1: ; %loop
2762 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
2763 ; CHECK-NEXT: ldr q0, [x0, x9]
2764 ; CHECK-NEXT: ldr q1, [x1, x9]
2765 ; CHECK-NEXT: subs w3, w3, #1
2766 ; CHECK-NEXT: uabdl.8h v2, v0, v1
2767 ; CHECK-NEXT: uabal2.8h v2, v0, v1
2768 ; CHECK-NEXT: uaddlv.8h s0, v2
2769 ; CHECK-NEXT: fmov w10, s0
2770 ; CHECK-NEXT: add w8, w10, w8
2771 ; CHECK-NEXT: b.ne LBB23_1
2772 ; CHECK-NEXT: ; %bb.2: ; %exit
2773 ; CHECK-NEXT: mov w0, w8
2774 ; CHECK-NEXT: ret
2775 ;
2776 ; CHECK-BE-LABEL: test_pr62620_widening_instr:
2777 ; CHECK-BE: // %bb.0: // %entry
2778 ; CHECK-BE-NEXT: lsl x9, x2, #4
2779 ; CHECK-BE-NEXT: mov x8, x0
2780 ; CHECK-BE-NEXT: mov w0, wzr
2781 ; CHECK-BE-NEXT: add x8, x8, x9
2782 ; CHECK-BE-NEXT: add x9, x1, x9
2783 ; CHECK-BE-NEXT: .LBB23_1: // %loop
2784 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
2785 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]
2786 ; CHECK-BE-NEXT: ld1 { v1.16b }, [x9]
2787 ; CHECK-BE-NEXT: subs w3, w3, #1
2788 ; CHECK-BE-NEXT: uabdl v2.8h, v0.8b, v1.8b
2789 ; CHECK-BE-NEXT: uabal2 v2.8h, v0.16b, v1.16b
2790 ; CHECK-BE-NEXT: uaddlv s0, v2.8h
2791 ; CHECK-BE-NEXT: fmov w10, s0
2792 ; CHECK-BE-NEXT: add w0, w10, w0
2793 ; CHECK-BE-NEXT: b.ne .LBB23_1
2794 ; CHECK-BE-NEXT: // %bb.2: // %exit
2795 ; CHECK-BE-NEXT: ret
2796 entry:
2797 br label %loop
2799 loop:
2800 %s0 = phi i32 [ 0, %entry ], [ %op.rdx, %loop ]
2801 %j.0261 = phi i32 [ 0, %entry ], [ %inc, %loop ]
2802 %gep.1 = getelementptr inbounds <16 x i8>, ptr %p1, i64 %lx
2803 %gep.2 = getelementptr inbounds <16 x i8>, ptr %p2, i64 %lx
2804 %l1 = load <16 x i8>, ptr %gep.1
2805 %z2 = zext <16 x i8> %l1 to <16 x i32>
2806 %l4 = load <16 x i8>, ptr %gep.2
2807 %z5 = zext <16 x i8> %l4 to <16 x i32>
2808 %sub = sub nsw <16 x i32> %z2, %z5
2809 %abs = tail call <16 x i32> @llvm.abs.v16i32(<16 x i32> %sub, i1 true)
2810 %red = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %abs)
2811 %op.rdx = add i32 %red, %s0
2812 %inc = add nuw nsw i32 %j.0261, 1
2813 %exitcond.not = icmp eq i32 %inc, %h
2814 br i1 %exitcond.not, label %exit, label %loop
2816 exit:
2817 %s1 = phi i32 [ %op.rdx, %loop ]
2818 ret i32 %s1
2819 }
2821 declare <16 x i32> @llvm.abs.v16i32(<16 x i32>, i1 immarg)
2823 declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)
2825 define i32 @test_widening_instr_mull(ptr %p1, ptr %p2, i32 %h) {
2826 ; CHECK-LABEL: test_widening_instr_mull:
2827 ; CHECK: ; %bb.0: ; %entry
2828 ; CHECK-NEXT: mov x8, x0
2829 ; CHECK-NEXT: LBB24_1: ; %loop
2830 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
2831 ; CHECK-NEXT: ldr q0, [x1], #16
2832 ; CHECK-NEXT: ldr q3, [x0]
2833 ; CHECK-NEXT: ldr q2, [x8, #16]!
2834 ; CHECK-NEXT: subs w2, w2, #1
2835 ; CHECK-NEXT: ushll2.8h v1, v0, #0
2836 ; CHECK-NEXT: ushll.8h v0, v0, #0
2837 ; CHECK-NEXT: umull2.4s v4, v2, v1
2838 ; CHECK-NEXT: umull.4s v1, v2, v1
2839 ; CHECK-NEXT: umull2.4s v2, v3, v0
2840 ; CHECK-NEXT: umull.4s v0, v3, v0
2841 ; CHECK-NEXT: stp q1, q4, [x0, #32]
2842 ; CHECK-NEXT: str q0, [x0]
2843 ; CHECK-NEXT: mov x0, x8
2844 ; CHECK-NEXT: str q2, [x8]
2845 ; CHECK-NEXT: b.ne LBB24_1
2846 ; CHECK-NEXT: ; %bb.2: ; %exit
2847 ; CHECK-NEXT: mov w0, wzr
2848 ; CHECK-NEXT: ret
2849 ;
2850 ; CHECK-BE-LABEL: test_widening_instr_mull:
2851 ; CHECK-BE: // %bb.0: // %entry
2852 ; CHECK-BE-NEXT: .LBB24_1: // %loop
2853 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
2854 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x1]
2855 ; CHECK-BE-NEXT: ld1 { v1.8h }, [x0]
2856 ; CHECK-BE-NEXT: add x8, x0, #16
2857 ; CHECK-BE-NEXT: ld1 { v3.8h }, [x8]
2858 ; CHECK-BE-NEXT: add x9, x0, #48
2859 ; CHECK-BE-NEXT: add x10, x0, #32
2860 ; CHECK-BE-NEXT: subs w2, w2, #1
2861 ; CHECK-BE-NEXT: add x1, x1, #16
2862 ; CHECK-BE-NEXT: ushll v2.8h, v0.8b, #0
2863 ; CHECK-BE-NEXT: ushll2 v0.8h, v0.16b, #0
2864 ; CHECK-BE-NEXT: umull v4.4s, v1.4h, v2.4h
2865 ; CHECK-BE-NEXT: umull2 v5.4s, v3.8h, v0.8h
2866 ; CHECK-BE-NEXT: umull v0.4s, v3.4h, v0.4h
2867 ; CHECK-BE-NEXT: umull2 v1.4s, v1.8h, v2.8h
2868 ; CHECK-BE-NEXT: st1 { v4.4s }, [x0]
2869 ; CHECK-BE-NEXT: mov x0, x8
2870 ; CHECK-BE-NEXT: st1 { v5.4s }, [x9]
2871 ; CHECK-BE-NEXT: st1 { v0.4s }, [x10]
2872 ; CHECK-BE-NEXT: st1 { v1.4s }, [x8]
2873 ; CHECK-BE-NEXT: b.ne .LBB24_1
2874 ; CHECK-BE-NEXT: // %bb.2: // %exit
2875 ; CHECK-BE-NEXT: mov w0, wzr
2876 ; CHECK-BE-NEXT: ret
2877 entry:
2878 br label %loop
2880 loop:
2881 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
2882 %gep.1 = getelementptr inbounds <16 x i8>, ptr %p1, i32 %iv
2883 %gep.2 = getelementptr inbounds <16 x i8>, ptr %p2, i32 %iv
2884 %l1 = load <16 x i16>, ptr %gep.1
2885 %z2 = zext <16 x i16> %l1 to <16 x i32>
2886 %l4 = load <16 x i8>, ptr %gep.2
2887 %z5 = zext <16 x i8> %l4 to <16 x i32>
2888 %mul = mul <16 x i32> %z2, %z5
2889 store <16 x i32> %mul, ptr %gep.1
2890 %iv.next= add nuw nsw i32 %iv, 1
2891 %exitcond.not = icmp eq i32 %iv.next, %h
2892 br i1 %exitcond.not, label %exit, label %loop
2894 exit:
2895 ret i32 0
2896 }
2898 define i32 @test_widening_instr_mull_64(ptr %p1, ptr %p2, i32 %h) {
2899 ; CHECK-LABEL: test_widening_instr_mull_64:
2900 ; CHECK: ; %bb.0: ; %entry
2901 ; CHECK-NEXT: Lloh42:
2902 ; CHECK-NEXT: adrp x8, lCPI25_0@PAGE
2903 ; CHECK-NEXT: Lloh43:
2904 ; CHECK-NEXT: adrp x9, lCPI25_1@PAGE
2905 ; CHECK-NEXT: Lloh44:
2906 ; CHECK-NEXT: adrp x10, lCPI25_3@PAGE
2907 ; CHECK-NEXT: Lloh45:
2908 ; CHECK-NEXT: ldr q0, [x8, lCPI25_0@PAGEOFF]
2909 ; CHECK-NEXT: Lloh46:
2910 ; CHECK-NEXT: adrp x8, lCPI25_2@PAGE
2911 ; CHECK-NEXT: Lloh47:
2912 ; CHECK-NEXT: ldr q1, [x9, lCPI25_1@PAGEOFF]
2913 ; CHECK-NEXT: Lloh48:
2914 ; CHECK-NEXT: ldr q2, [x8, lCPI25_2@PAGEOFF]
2915 ; CHECK-NEXT: Lloh49:
2916 ; CHECK-NEXT: ldr q3, [x10, lCPI25_3@PAGEOFF]
2917 ; CHECK-NEXT: mov x8, x1
2918 ; CHECK-NEXT: LBB25_1: ; %loop
2919 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
2920 ; CHECK-NEXT: ldr q4, [x0]
2921 ; CHECK-NEXT: ldp q16, q7, [x1, #32]
2922 ; CHECK-NEXT: ldr q18, [x8, #16]!
2923 ; CHECK-NEXT: subs w2, w2, #1
2924 ; CHECK-NEXT: tbl.16b v5, { v4 }, v3
2925 ; CHECK-NEXT: tbl.16b v6, { v4 }, v0
2926 ; CHECK-NEXT: tbl.16b v17, { v4 }, v2
2927 ; CHECK-NEXT: tbl.16b v4, { v4 }, v1
2928 ; CHECK-NEXT: umull2.2d v19, v5, v7
2929 ; CHECK-NEXT: umull.2d v5, v5, v7
2930 ; CHECK-NEXT: ldr q7, [x1]
2931 ; CHECK-NEXT: umull2.2d v20, v6, v16
2932 ; CHECK-NEXT: umull2.2d v21, v17, v18
2933 ; CHECK-NEXT: umull.2d v17, v17, v18
2934 ; CHECK-NEXT: umull2.2d v18, v4, v7
2935 ; CHECK-NEXT: umull.2d v4, v4, v7
2936 ; CHECK-NEXT: mov x1, x8
2937 ; CHECK-NEXT: stp q5, q19, [x0, #96]
2938 ; CHECK-NEXT: umull.2d v5, v6, v16
2939 ; CHECK-NEXT: str q20, [x0, #80]
2940 ; CHECK-NEXT: stp q4, q18, [x0]
2941 ; CHECK-NEXT: stp q17, q21, [x0, #32]
2942 ; CHECK-NEXT: str q5, [x0, #64]!
2943 ; CHECK-NEXT: b.ne LBB25_1
2944 ; CHECK-NEXT: ; %bb.2: ; %exit
2945 ; CHECK-NEXT: mov w0, wzr
2946 ; CHECK-NEXT: ret
2947 ; CHECK-NEXT: .loh AdrpLdr Lloh46, Lloh48
2948 ; CHECK-NEXT: .loh AdrpLdr Lloh44, Lloh49
2949 ; CHECK-NEXT: .loh AdrpLdr Lloh43, Lloh47
2950 ; CHECK-NEXT: .loh AdrpAdrp Lloh42, Lloh46
2951 ; CHECK-NEXT: .loh AdrpLdr Lloh42, Lloh45
2952 ;
2953 ; CHECK-BE-LABEL: test_widening_instr_mull_64:
2954 ; CHECK-BE: // %bb.0: // %entry
2955 ; CHECK-BE-NEXT: adrp x8, .LCPI25_0
2956 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI25_0
2957 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]
2958 ; CHECK-BE-NEXT: adrp x8, .LCPI25_1
2959 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI25_1
2960 ; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
2961 ; CHECK-BE-NEXT: adrp x8, .LCPI25_2
2962 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI25_2
2963 ; CHECK-BE-NEXT: ld1 { v2.16b }, [x8]
2964 ; CHECK-BE-NEXT: adrp x8, .LCPI25_3
2965 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI25_3
2966 ; CHECK-BE-NEXT: ld1 { v3.16b }, [x8]
2967 ; CHECK-BE-NEXT: .LBB25_1: // %loop
2968 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
2969 ; CHECK-BE-NEXT: ld1 { v4.16b }, [x0]
2970 ; CHECK-BE-NEXT: add x9, x1, #48
2971 ; CHECK-BE-NEXT: add x8, x1, #32
2972 ; CHECK-BE-NEXT: ld1 { v18.4s }, [x9]
2973 ; CHECK-BE-NEXT: ld1 { v16.4s }, [x1]
2974 ; CHECK-BE-NEXT: add x1, x1, #16
2975 ; CHECK-BE-NEXT: ld1 { v20.4s }, [x8]
2976 ; CHECK-BE-NEXT: ld1 { v22.4s }, [x1]
2977 ; CHECK-BE-NEXT: add x8, x0, #96
2978 ; CHECK-BE-NEXT: tbl v5.16b, { v4.16b }, v3.16b
2979 ; CHECK-BE-NEXT: tbl v6.16b, { v4.16b }, v2.16b
2980 ; CHECK-BE-NEXT: tbl v7.16b, { v4.16b }, v1.16b
2981 ; CHECK-BE-NEXT: tbl v4.16b, { v4.16b }, v0.16b
2982 ; CHECK-BE-NEXT: ext v24.16b, v18.16b, v18.16b, #8
2983 ; CHECK-BE-NEXT: add x9, x0, #32
2984 ; CHECK-BE-NEXT: ext v25.16b, v20.16b, v20.16b, #8
2985 ; CHECK-BE-NEXT: add x10, x0, #16
2986 ; CHECK-BE-NEXT: subs w2, w2, #1
2987 ; CHECK-BE-NEXT: ext v17.16b, v5.16b, v5.16b, #8
2988 ; CHECK-BE-NEXT: ext v19.16b, v6.16b, v6.16b, #8
2989 ; CHECK-BE-NEXT: rev32 v5.8b, v5.8b
2990 ; CHECK-BE-NEXT: rev32 v21.8b, v7.8b
2991 ; CHECK-BE-NEXT: rev32 v23.8b, v4.8b
2992 ; CHECK-BE-NEXT: ext v7.16b, v7.16b, v7.16b, #8
2993 ; CHECK-BE-NEXT: ext v4.16b, v4.16b, v4.16b, #8
2994 ; CHECK-BE-NEXT: rev32 v6.8b, v6.8b
2995 ; CHECK-BE-NEXT: rev32 v17.8b, v17.8b
2996 ; CHECK-BE-NEXT: rev32 v19.8b, v19.8b
2997 ; CHECK-BE-NEXT: umull v5.2d, v5.2s, v18.2s
2998 ; CHECK-BE-NEXT: umull v18.2d, v21.2s, v22.2s
2999 ; CHECK-BE-NEXT: ext v21.16b, v22.16b, v22.16b, #8
3000 ; CHECK-BE-NEXT: rev32 v7.8b, v7.8b
3001 ; CHECK-BE-NEXT: umull v22.2d, v23.2s, v16.2s
3002 ; CHECK-BE-NEXT: ext v16.16b, v16.16b, v16.16b, #8
3003 ; CHECK-BE-NEXT: rev32 v4.8b, v4.8b
3004 ; CHECK-BE-NEXT: umull v17.2d, v17.2s, v24.2s
3005 ; CHECK-BE-NEXT: umull v19.2d, v19.2s, v25.2s
3006 ; CHECK-BE-NEXT: st1 { v5.2d }, [x8]
3007 ; CHECK-BE-NEXT: umull v5.2d, v6.2s, v20.2s
3008 ; CHECK-BE-NEXT: umull v6.2d, v7.2s, v21.2s
3009 ; CHECK-BE-NEXT: add x8, x0, #112
3010 ; CHECK-BE-NEXT: umull v4.2d, v4.2s, v16.2s
3011 ; CHECK-BE-NEXT: st1 { v18.2d }, [x9]
3012 ; CHECK-BE-NEXT: add x9, x0, #80
3013 ; CHECK-BE-NEXT: st1 { v22.2d }, [x0]
3014 ; CHECK-BE-NEXT: st1 { v17.2d }, [x8]
3015 ; CHECK-BE-NEXT: add x8, x0, #64
3016 ; CHECK-BE-NEXT: st1 { v19.2d }, [x9]
3017 ; CHECK-BE-NEXT: add x9, x0, #48
3018 ; CHECK-BE-NEXT: mov x0, x8
3019 ; CHECK-BE-NEXT: st1 { v5.2d }, [x8]
3020 ; CHECK-BE-NEXT: st1 { v6.2d }, [x9]
3021 ; CHECK-BE-NEXT: st1 { v4.2d }, [x10]
3022 ; CHECK-BE-NEXT: b.ne .LBB25_1
3023 ; CHECK-BE-NEXT: // %bb.2: // %exit
3024 ; CHECK-BE-NEXT: mov w0, wzr
3025 ; CHECK-BE-NEXT: ret
3026 entry:
3027 br label %loop
3029 loop:
3030 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
3031 %gep.1 = getelementptr inbounds <16 x i32>, ptr %p1, i32 %iv
3032 %gep.2 = getelementptr inbounds <16 x i8>, ptr %p2, i32 %iv
3033 %l1 = load <16 x i8>, ptr %gep.1
3034 %z2 = zext <16 x i8> %l1 to <16 x i64>
3035 %l4 = load <16 x i32>, ptr %gep.2
3036 %z5 = zext <16 x i32> %l4 to <16 x i64>
3037 %mul = mul <16 x i64> %z2, %z5
3038 store <16 x i64> %mul, ptr %gep.1
3039 %iv.next= add nuw nsw i32 %iv, 1
3040 %exitcond.not = icmp eq i32 %iv.next, %h
3041 br i1 %exitcond.not, label %exit, label %loop
3043 exit:
3044 ret i32 0
3045 }
3047 define i32 @test_widening_instr_mull_2(ptr %p1, ptr %p2, i32 %h) {
3048 ; CHECK-LABEL: test_widening_instr_mull_2:
3049 ; CHECK: ; %bb.0: ; %entry
3050 ; CHECK-NEXT: Lloh50:
3051 ; CHECK-NEXT: adrp x8, lCPI26_0@PAGE
3052 ; CHECK-NEXT: Lloh51:
3053 ; CHECK-NEXT: adrp x9, lCPI26_1@PAGE
3054 ; CHECK-NEXT: Lloh52:
3055 ; CHECK-NEXT: adrp x10, lCPI26_3@PAGE
3056 ; CHECK-NEXT: Lloh53:
3057 ; CHECK-NEXT: ldr q0, [x8, lCPI26_0@PAGEOFF]
3058 ; CHECK-NEXT: Lloh54:
3059 ; CHECK-NEXT: adrp x8, lCPI26_2@PAGE
3060 ; CHECK-NEXT: Lloh55:
3061 ; CHECK-NEXT: ldr q1, [x9, lCPI26_1@PAGEOFF]
3062 ; CHECK-NEXT: Lloh56:
3063 ; CHECK-NEXT: ldr q2, [x8, lCPI26_2@PAGEOFF]
3064 ; CHECK-NEXT: Lloh57:
3065 ; CHECK-NEXT: ldr q3, [x10, lCPI26_3@PAGEOFF]
3066 ; CHECK-NEXT: mov x8, x0
3067 ; CHECK-NEXT: LBB26_1: ; %loop
3068 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
3069 ; CHECK-NEXT: ldr q4, [x1], #16
3070 ; CHECK-NEXT: ldr q18, [x0]
3071 ; CHECK-NEXT: ldp q16, q17, [x0, #32]
3072 ; CHECK-NEXT: subs w2, w2, #1
3073 ; CHECK-NEXT: tbl.16b v5, { v4 }, v0
3074 ; CHECK-NEXT: tbl.16b v6, { v4 }, v1
3075 ; CHECK-NEXT: tbl.16b v7, { v4 }, v2
3076 ; CHECK-NEXT: tbl.16b v4, { v4 }, v3
3077 ; CHECK-NEXT: mul.4s v5, v16, v5
3078 ; CHECK-NEXT: ldr q16, [x8, #16]!
3079 ; CHECK-NEXT: mul.4s v6, v17, v6
3080 ; CHECK-NEXT: mul.4s v7, v18, v7
3081 ; CHECK-NEXT: mul.4s v4, v16, v4
3082 ; CHECK-NEXT: stp q5, q6, [x0, #32]
3083 ; CHECK-NEXT: str q7, [x0]
3084 ; CHECK-NEXT: mov x0, x8
3085 ; CHECK-NEXT: str q4, [x8]
3086 ; CHECK-NEXT: b.ne LBB26_1
3087 ; CHECK-NEXT: ; %bb.2: ; %exit
3088 ; CHECK-NEXT: mov w0, wzr
3089 ; CHECK-NEXT: ret
3090 ; CHECK-NEXT: .loh AdrpLdr Lloh54, Lloh56
3091 ; CHECK-NEXT: .loh AdrpLdr Lloh52, Lloh57
3092 ; CHECK-NEXT: .loh AdrpLdr Lloh51, Lloh55
3093 ; CHECK-NEXT: .loh AdrpAdrp Lloh50, Lloh54
3094 ; CHECK-NEXT: .loh AdrpLdr Lloh50, Lloh53
3095 ;
3096 ; CHECK-BE-LABEL: test_widening_instr_mull_2:
3097 ; CHECK-BE: // %bb.0: // %entry
3098 ; CHECK-BE-NEXT: adrp x8, .LCPI26_0
3099 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI26_0
3100 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]
3101 ; CHECK-BE-NEXT: adrp x8, .LCPI26_1
3102 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI26_1
3103 ; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
3104 ; CHECK-BE-NEXT: adrp x8, .LCPI26_2
3105 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI26_2
3106 ; CHECK-BE-NEXT: ld1 { v2.16b }, [x8]
3107 ; CHECK-BE-NEXT: adrp x8, .LCPI26_3
3108 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI26_3
3109 ; CHECK-BE-NEXT: ld1 { v3.16b }, [x8]
3110 ; CHECK-BE-NEXT: .LBB26_1: // %loop
3111 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
3112 ; CHECK-BE-NEXT: ld1 { v4.16b }, [x1]
3113 ; CHECK-BE-NEXT: add x8, x0, #32
3114 ; CHECK-BE-NEXT: ld1 { v16.4s }, [x0]
3115 ; CHECK-BE-NEXT: add x9, x0, #48
3116 ; CHECK-BE-NEXT: add x10, x0, #16
3117 ; CHECK-BE-NEXT: ld1 { v17.4s }, [x8]
3118 ; CHECK-BE-NEXT: ld1 { v18.4s }, [x9]
3119 ; CHECK-BE-NEXT: ld1 { v19.4s }, [x10]
3120 ; CHECK-BE-NEXT: subs w2, w2, #1
3121 ; CHECK-BE-NEXT: tbl v5.16b, { v4.16b }, v1.16b
3122 ; CHECK-BE-NEXT: tbl v6.16b, { v4.16b }, v3.16b
3123 ; CHECK-BE-NEXT: tbl v7.16b, { v4.16b }, v2.16b
3124 ; CHECK-BE-NEXT: tbl v4.16b, { v4.16b }, v0.16b
3125 ; CHECK-BE-NEXT: add x1, x1, #16
3126 ; CHECK-BE-NEXT: rev32 v5.16b, v5.16b
3127 ; CHECK-BE-NEXT: rev32 v6.16b, v6.16b
3128 ; CHECK-BE-NEXT: rev32 v7.16b, v7.16b
3129 ; CHECK-BE-NEXT: rev32 v4.16b, v4.16b
3130 ; CHECK-BE-NEXT: mul v5.4s, v16.4s, v5.4s
3131 ; CHECK-BE-NEXT: mul v6.4s, v17.4s, v6.4s
3132 ; CHECK-BE-NEXT: mul v7.4s, v18.4s, v7.4s
3133 ; CHECK-BE-NEXT: mul v4.4s, v19.4s, v4.4s
3134 ; CHECK-BE-NEXT: st1 { v5.4s }, [x0]
3135 ; CHECK-BE-NEXT: mov x0, x10
3136 ; CHECK-BE-NEXT: st1 { v6.4s }, [x8]
3137 ; CHECK-BE-NEXT: st1 { v7.4s }, [x9]
3138 ; CHECK-BE-NEXT: st1 { v4.4s }, [x10]
3139 ; CHECK-BE-NEXT: b.ne .LBB26_1
3140 ; CHECK-BE-NEXT: // %bb.2: // %exit
3141 ; CHECK-BE-NEXT: mov w0, wzr
3142 ; CHECK-BE-NEXT: ret
3143 entry:
3144 br label %loop
3146 loop:
3147 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
3148 %gep.1 = getelementptr inbounds <16 x i8>, ptr %p1, i32 %iv
3149 %gep.2 = getelementptr inbounds <16 x i8>, ptr %p2, i32 %iv
3150 %l1 = load <16 x i32>, ptr %gep.1
3151 %l4 = load <16 x i8>, ptr %gep.2
3152 %z5 = zext <16 x i8> %l4 to <16 x i32>
3153 %mul = mul <16 x i32> %l1, %z5
3154 store <16 x i32> %mul, ptr %gep.1
3155 %iv.next= add nuw nsw i32 %iv, 1
3156 %exitcond.not = icmp eq i32 %iv.next, %h
3157 br i1 %exitcond.not, label %exit, label %loop
3159 exit:
3160 ret i32 0
3161 }