| blob | 66bb131ce724949a40511d7538b73b3529c9d64e |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc -mtriple=arm64-apple-ios -mattr=+sve -o - %s | FileCheck %s
3 ; RUN: llc -mtriple=aarch64_be-unknown-linux -mattr=+sve -o - %s | FileCheck --check-prefix=CHECK-BE %s
5 ; CHECK-LABEL: lCPI0_0:
6 ; CHECK-NEXT: .byte 0 ; 0x0
7 ; CHECK-NEXT: .byte 255 ; 0xff
8 ; CHECK-NEXT: .byte 255 ; 0xff
9 ; CHECK-NEXT: .byte 255 ; 0xff
10 ; CHECK-NEXT: .byte 1 ; 0x1
11 ; CHECK-NEXT: .byte 255 ; 0xff
12 ; CHECK-NEXT: .byte 255 ; 0xff
13 ; CHECK-NEXT: .byte 255 ; 0xff
14 ; CHECK-NEXT: .byte 2 ; 0x2
15 ; CHECK-NEXT: .byte 255 ; 0xff
16 ; CHECK-NEXT: .byte 255 ; 0xff
17 ; CHECK-NEXT: .byte 255 ; 0xff
18 ; CHECK-NEXT: .byte 3 ; 0x3
19 ; CHECK-NEXT: .byte 255 ; 0xff
20 ; CHECK-NEXT: .byte 255 ; 0xff
21 ; CHECK-NEXT: .byte 255 ; 0xff
22 ; CHECK-NEXT:lCPI0_1:
23 ; CHECK-NEXT: .byte 4 ; 0x4
24 ; CHECK-NEXT: .byte 255 ; 0xff
25 ; CHECK-NEXT: .byte 255 ; 0xff
26 ; CHECK-NEXT: .byte 255 ; 0xff
27 ; CHECK-NEXT: .byte 5 ; 0x5
28 ; CHECK-NEXT: .byte 255 ; 0xff
29 ; CHECK-NEXT: .byte 255 ; 0xff
30 ; CHECK-NEXT: .byte 255 ; 0xff
31 ; CHECK-NEXT: .byte 6 ; 0x6
32 ; CHECK-NEXT: .byte 255 ; 0xff
33 ; CHECK-NEXT: .byte 255 ; 0xff
34 ; CHECK-NEXT: .byte 255 ; 0xff
35 ; CHECK-NEXT: .byte 7 ; 0x7
36 ; CHECK-NEXT: .byte 255 ; 0xff
37 ; CHECK-NEXT: .byte 255 ; 0xff
38 ; CHECK-NEXT: .byte 255 ; 0xff
39 ; CHECK-NEXT:lCPI0_2:
40 ; CHECK-NEXT: .byte 8 ; 0x8
41 ; CHECK-NEXT: .byte 255 ; 0xff
42 ; CHECK-NEXT: .byte 255 ; 0xff
43 ; CHECK-NEXT: .byte 255 ; 0xff
44 ; CHECK-NEXT: .byte 9 ; 0x9
45 ; CHECK-NEXT: .byte 255 ; 0xff
46 ; CHECK-NEXT: .byte 255 ; 0xff
47 ; CHECK-NEXT: .byte 255 ; 0xff
48 ; CHECK-NEXT: .byte 10 ; 0xa
49 ; CHECK-NEXT: .byte 255 ; 0xff
50 ; CHECK-NEXT: .byte 255 ; 0xff
51 ; CHECK-NEXT: .byte 255 ; 0xff
52 ; CHECK-NEXT: .byte 11 ; 0xb
53 ; CHECK-NEXT: .byte 255 ; 0xff
54 ; CHECK-NEXT: .byte 255 ; 0xff
55 ; CHECK-NEXT: .byte 255 ; 0xff
56 ; CHECK-NEXT:lCPI0_3:
57 ; CHECK-NEXT: .byte 12 ; 0xc
58 ; CHECK-NEXT: .byte 255 ; 0xff
59 ; CHECK-NEXT: .byte 255 ; 0xff
60 ; CHECK-NEXT: .byte 255 ; 0xff
61 ; CHECK-NEXT: .byte 13 ; 0xd
62 ; CHECK-NEXT: .byte 255 ; 0xff
63 ; CHECK-NEXT: .byte 255 ; 0xff
64 ; CHECK-NEXT: .byte 255 ; 0xff
65 ; CHECK-NEXT: .byte 14 ; 0xe
66 ; CHECK-NEXT: .byte 255 ; 0xff
67 ; CHECK-NEXT: .byte 255 ; 0xff
68 ; CHECK-NEXT: .byte 255 ; 0xff
69 ; CHECK-NEXT: .byte 15 ; 0xf
70 ; CHECK-NEXT: .byte 255 ; 0xff
71 ; CHECK-NEXT: .byte 255 ; 0xff
72 ; CHECK-NEXT: .byte 255 ; 0xff
74 ; CHECK-BE: .LCPI0_0:
75 ; CHECK-BE-NEXT: .byte 255 // 0xff
76 ; CHECK-BE-NEXT: .byte 255 // 0xff
77 ; CHECK-BE-NEXT: .byte 255 // 0xff
78 ; CHECK-BE-NEXT: .byte 0 // 0x0
79 ; CHECK-BE-NEXT: .byte 255 // 0xff
80 ; CHECK-BE-NEXT: .byte 255 // 0xff
81 ; CHECK-BE-NEXT: .byte 255 // 0xff
82 ; CHECK-BE-NEXT: .byte 1 // 0x1
83 ; CHECK-BE-NEXT: .byte 255 // 0xff
84 ; CHECK-BE-NEXT: .byte 255 // 0xff
85 ; CHECK-BE-NEXT: .byte 255 // 0xff
86 ; CHECK-BE-NEXT: .byte 2 // 0x2
87 ; CHECK-BE-NEXT: .byte 255 // 0xff
88 ; CHECK-BE-NEXT: .byte 255 // 0xff
89 ; CHECK-BE-NEXT: .byte 255 // 0xff
90 ; CHECK-BE-NEXT: .byte 3 // 0x3
91 ; CHECK-BE-NEXT: .LCPI0_1:
92 ; CHECK-BE-NEXT: .byte 255 // 0xff
93 ; CHECK-BE-NEXT: .byte 255 // 0xff
94 ; CHECK-BE-NEXT: .byte 255 // 0xff
95 ; CHECK-BE-NEXT: .byte 4 // 0x4
96 ; CHECK-BE-NEXT: .byte 255 // 0xff
97 ; CHECK-BE-NEXT: .byte 255 // 0xff
98 ; CHECK-BE-NEXT: .byte 255 // 0xff
99 ; CHECK-BE-NEXT: .byte 5 // 0x5
100 ; CHECK-BE-NEXT: .byte 255 // 0xff
101 ; CHECK-BE-NEXT: .byte 255 // 0xff
102 ; CHECK-BE-NEXT: .byte 255 // 0xff
103 ; CHECK-BE-NEXT: .byte 6 // 0x6
104 ; CHECK-BE-NEXT: .byte 255 // 0xff
105 ; CHECK-BE-NEXT: .byte 255 // 0xff
106 ; CHECK-BE-NEXT: .byte 255 // 0xff
107 ; CHECK-BE-NEXT: .byte 7 // 0x7
108 ; CHECK-BE-NEXT: .LCPI0_2:
109 ; CHECK-BE-NEXT: .byte 255 // 0xff
110 ; CHECK-BE-NEXT: .byte 255 // 0xff
111 ; CHECK-BE-NEXT: .byte 255 // 0xff
112 ; CHECK-BE-NEXT: .byte 8 // 0x8
113 ; CHECK-BE-NEXT: .byte 255 // 0xff
114 ; CHECK-BE-NEXT: .byte 255 // 0xff
115 ; CHECK-BE-NEXT: .byte 255 // 0xff
116 ; CHECK-BE-NEXT: .byte 9 // 0x9
117 ; CHECK-BE-NEXT: .byte 255 // 0xff
118 ; CHECK-BE-NEXT: .byte 255 // 0xff
119 ; CHECK-BE-NEXT: .byte 255 // 0xff
120 ; CHECK-BE-NEXT: .byte 10 // 0xa
121 ; CHECK-BE-NEXT: .byte 255 // 0xff
122 ; CHECK-BE-NEXT: .byte 255 // 0xff
123 ; CHECK-BE-NEXT: .byte 255 // 0xff
124 ; CHECK-BE-NEXT: .byte 11 // 0xb
125 ; CHECK-BE-NEXT: .LCPI0_3:
126 ; CHECK-BE-NEXT: .byte 255 // 0xff
127 ; CHECK-BE-NEXT: .byte 255 // 0xff
128 ; CHECK-BE-NEXT: .byte 255 // 0xff
129 ; CHECK-BE-NEXT: .byte 12 // 0xc
130 ; CHECK-BE-NEXT: .byte 255 // 0xff
131 ; CHECK-BE-NEXT: .byte 255 // 0xff
132 ; CHECK-BE-NEXT: .byte 255 // 0xff
133 ; CHECK-BE-NEXT: .byte 13 // 0xd
134 ; CHECK-BE-NEXT: .byte 255 // 0xff
135 ; CHECK-BE-NEXT: .byte 255 // 0xff
136 ; CHECK-BE-NEXT: .byte 255 // 0xff
137 ; CHECK-BE-NEXT: .byte 14 // 0xe
138 ; CHECK-BE-NEXT: .byte 255 // 0xff
139 ; CHECK-BE-NEXT: .byte 255 // 0xff
140 ; CHECK-BE-NEXT: .byte 255 // 0xff
141 ; CHECK-BE-NEXT: .byte 15 // 0xf
143 ; It's profitable to convert the zext to a shuffle, which in turn will be
144 ; lowered to 4 tbl instructions. The masks are materialized outside the loop.
145 define void @zext_v16i8_to_v16i32_in_loop(ptr %src, ptr %dst) {
146 ; CHECK-LABEL: zext_v16i8_to_v16i32_in_loop:
147 ; CHECK: ; %bb.0: ; %entry
148 ; CHECK-NEXT: Lloh0:
149 ; CHECK-NEXT: adrp x8, lCPI0_0@PAGE
150 ; CHECK-NEXT: Lloh1:
151 ; CHECK-NEXT: adrp x9, lCPI0_1@PAGE
152 ; CHECK-NEXT: Lloh2:
153 ; CHECK-NEXT: adrp x10, lCPI0_2@PAGE
154 ; CHECK-NEXT: Lloh3:
155 ; CHECK-NEXT: ldr q0, [x8, lCPI0_0@PAGEOFF]
156 ; CHECK-NEXT: Lloh4:
157 ; CHECK-NEXT: adrp x8, lCPI0_3@PAGE
158 ; CHECK-NEXT: Lloh5:
159 ; CHECK-NEXT: ldr q1, [x9, lCPI0_1@PAGEOFF]
160 ; CHECK-NEXT: Lloh6:
161 ; CHECK-NEXT: ldr q2, [x10, lCPI0_2@PAGEOFF]
162 ; CHECK-NEXT: Lloh7:
163 ; CHECK-NEXT: ldr q3, [x8, lCPI0_3@PAGEOFF]
164 ; CHECK-NEXT: mov x8, xzr
165 ; CHECK-NEXT: LBB0_1: ; %loop
166 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
167 ; CHECK-NEXT: ldr q4, [x0, x8]
168 ; CHECK-NEXT: add x8, x8, #16
169 ; CHECK-NEXT: cmp x8, #128
170 ; CHECK-NEXT: tbl.16b v5, { v4 }, v3
171 ; CHECK-NEXT: tbl.16b v6, { v4 }, v2
172 ; CHECK-NEXT: tbl.16b v7, { v4 }, v1
173 ; CHECK-NEXT: tbl.16b v4, { v4 }, v0
174 ; CHECK-NEXT: stp q6, q5, [x1, #32]
175 ; CHECK-NEXT: stp q4, q7, [x1], #64
176 ; CHECK-NEXT: b.ne LBB0_1
177 ; CHECK-NEXT: ; %bb.2: ; %exit
178 ; CHECK-NEXT: ret
179 ; CHECK-NEXT: .loh AdrpLdr Lloh4, Lloh7
180 ; CHECK-NEXT: .loh AdrpLdr Lloh2, Lloh6
181 ; CHECK-NEXT: .loh AdrpLdr Lloh1, Lloh5
182 ; CHECK-NEXT: .loh AdrpAdrp Lloh0, Lloh4
183 ; CHECK-NEXT: .loh AdrpLdr Lloh0, Lloh3
184 ;
185 ; CHECK-BE-LABEL: zext_v16i8_to_v16i32_in_loop:
186 ; CHECK-BE: // %bb.0: // %entry
187 ; CHECK-BE-NEXT: adrp x8, .LCPI0_0
188 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI0_0
189 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]
190 ; CHECK-BE-NEXT: adrp x8, .LCPI0_1
191 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI0_1
192 ; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
193 ; CHECK-BE-NEXT: adrp x8, .LCPI0_2
194 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI0_2
195 ; CHECK-BE-NEXT: ld1 { v2.16b }, [x8]
196 ; CHECK-BE-NEXT: adrp x8, .LCPI0_3
197 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI0_3
198 ; CHECK-BE-NEXT: ld1 { v3.16b }, [x8]
199 ; CHECK-BE-NEXT: mov x8, xzr
200 ; CHECK-BE-NEXT: .LBB0_1: // %loop
201 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
202 ; CHECK-BE-NEXT: add x9, x0, x8
203 ; CHECK-BE-NEXT: add x8, x8, #16
204 ; CHECK-BE-NEXT: ld1 { v4.16b }, [x9]
205 ; CHECK-BE-NEXT: add x9, x1, #48
206 ; CHECK-BE-NEXT: cmp x8, #128
207 ; CHECK-BE-NEXT: tbl v5.16b, { v4.16b }, v3.16b
208 ; CHECK-BE-NEXT: tbl v6.16b, { v4.16b }, v2.16b
209 ; CHECK-BE-NEXT: tbl v7.16b, { v4.16b }, v1.16b
210 ; CHECK-BE-NEXT: tbl v4.16b, { v4.16b }, v0.16b
211 ; CHECK-BE-NEXT: st1 { v5.16b }, [x9]
212 ; CHECK-BE-NEXT: add x9, x1, #32
213 ; CHECK-BE-NEXT: st1 { v6.16b }, [x9]
214 ; CHECK-BE-NEXT: add x9, x1, #16
215 ; CHECK-BE-NEXT: st1 { v4.16b }, [x1]
216 ; CHECK-BE-NEXT: add x1, x1, #64
217 ; CHECK-BE-NEXT: st1 { v7.16b }, [x9]
218 ; CHECK-BE-NEXT: b.ne .LBB0_1
219 ; CHECK-BE-NEXT: // %bb.2: // %exit
220 ; CHECK-BE-NEXT: ret
221 entry:
222 br label %loop
224 loop:
225 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
226 %src.gep = getelementptr i8, ptr %src, i64 %iv
227 %load = load <16 x i8>, ptr %src.gep
228 %ext = zext <16 x i8> %load to <16 x i32>
229 %dst.gep = getelementptr i32, ptr %dst, i64 %iv
230 store <16 x i32> %ext, ptr %dst.gep
231 %iv.next = add nuw i64 %iv, 16
232 %ec = icmp eq i64 %iv.next, 128
233 br i1 %ec, label %exit, label %loop
235 exit:
236 ret void
237 }
239 define void @zext_v16i8_to_v16i32_in_loop_not_header(ptr %src, ptr %dst, i1 %c) {
240 ; CHECK-LABEL: zext_v16i8_to_v16i32_in_loop_not_header:
241 ; CHECK: ; %bb.0: ; %entry
242 ; CHECK-NEXT: mov x8, xzr
243 ; CHECK-NEXT: b LBB1_2
244 ; CHECK-NEXT: LBB1_1: ; %loop.latch
245 ; CHECK-NEXT: ; in Loop: Header=BB1_2 Depth=1
246 ; CHECK-NEXT: add x8, x8, #16
247 ; CHECK-NEXT: add x1, x1, #64
248 ; CHECK-NEXT: cmp x8, #128
249 ; CHECK-NEXT: b.eq LBB1_4
250 ; CHECK-NEXT: LBB1_2: ; %loop
251 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
252 ; CHECK-NEXT: tbz w2, #0, LBB1_1
253 ; CHECK-NEXT: ; %bb.3: ; %then
254 ; CHECK-NEXT: ; in Loop: Header=BB1_2 Depth=1
255 ; CHECK-NEXT: ldr q0, [x0, x8]
256 ; CHECK-NEXT: ushll2.8h v1, v0, #0
257 ; CHECK-NEXT: ushll.8h v0, v0, #0
258 ; CHECK-NEXT: ushll2.4s v2, v1, #0
259 ; CHECK-NEXT: ushll.4s v1, v1, #0
260 ; CHECK-NEXT: ushll2.4s v3, v0, #0
261 ; CHECK-NEXT: ushll.4s v0, v0, #0
262 ; CHECK-NEXT: stp q1, q2, [x1, #32]
263 ; CHECK-NEXT: stp q0, q3, [x1]
264 ; CHECK-NEXT: b LBB1_1
265 ; CHECK-NEXT: LBB1_4: ; %exit
266 ; CHECK-NEXT: ret
267 ;
268 ; CHECK-BE-LABEL: zext_v16i8_to_v16i32_in_loop_not_header:
269 ; CHECK-BE: // %bb.0: // %entry
270 ; CHECK-BE-NEXT: mov x8, xzr
271 ; CHECK-BE-NEXT: b .LBB1_2
272 ; CHECK-BE-NEXT: .LBB1_1: // %loop.latch
273 ; CHECK-BE-NEXT: // in Loop: Header=BB1_2 Depth=1
274 ; CHECK-BE-NEXT: add x8, x8, #16
275 ; CHECK-BE-NEXT: add x1, x1, #64
276 ; CHECK-BE-NEXT: cmp x8, #128
277 ; CHECK-BE-NEXT: b.eq .LBB1_4
278 ; CHECK-BE-NEXT: .LBB1_2: // %loop
279 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
280 ; CHECK-BE-NEXT: tbz w2, #0, .LBB1_1
281 ; CHECK-BE-NEXT: // %bb.3: // %then
282 ; CHECK-BE-NEXT: // in Loop: Header=BB1_2 Depth=1
283 ; CHECK-BE-NEXT: add x9, x0, x8
284 ; CHECK-BE-NEXT: add x10, x1, #32
285 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x9]
286 ; CHECK-BE-NEXT: add x9, x1, #48
287 ; CHECK-BE-NEXT: ushll2 v1.8h, v0.16b, #0
288 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
289 ; CHECK-BE-NEXT: ushll2 v2.4s, v1.8h, #0
290 ; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #0
291 ; CHECK-BE-NEXT: ushll2 v3.4s, v0.8h, #0
292 ; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
293 ; CHECK-BE-NEXT: st1 { v2.4s }, [x9]
294 ; CHECK-BE-NEXT: add x9, x1, #16
295 ; CHECK-BE-NEXT: st1 { v1.4s }, [x10]
296 ; CHECK-BE-NEXT: st1 { v3.4s }, [x9]
297 ; CHECK-BE-NEXT: st1 { v0.4s }, [x1]
298 ; CHECK-BE-NEXT: b .LBB1_1
299 ; CHECK-BE-NEXT: .LBB1_4: // %exit
300 ; CHECK-BE-NEXT: ret
301 entry:
302 br label %loop
304 loop:
305 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
306 br i1 %c, label %then, label %loop.latch
308 then:
309 %src.gep = getelementptr i8, ptr %src, i64 %iv
310 %load = load <16 x i8>, ptr %src.gep
311 %ext = zext <16 x i8> %load to <16 x i32>
312 %dst.gep = getelementptr i32, ptr %dst, i64 %iv
313 store <16 x i32> %ext, ptr %dst.gep
314 br label %loop.latch
316 loop.latch:
317 %iv.next = add nuw i64 %iv, 16
318 %ec = icmp eq i64 %iv.next, 128
319 br i1 %ec, label %exit, label %loop
321 exit:
322 ret void
323 }
325 ; Not profitable to use shuffle/tbl, as 4 tbls + materializing the masks
326 ; require more instructions than lowering zext directly.
327 define void @zext_v16i8_to_v16i32_no_loop(ptr %src, ptr %dst) {
328 ; CHECK-LABEL: zext_v16i8_to_v16i32_no_loop:
329 ; CHECK: ; %bb.0: ; %entry
330 ; CHECK-NEXT: ldr q0, [x0]
331 ; CHECK-NEXT: ushll2.8h v1, v0, #0
332 ; CHECK-NEXT: ushll.8h v0, v0, #0
333 ; CHECK-NEXT: ushll2.4s v2, v1, #0
334 ; CHECK-NEXT: ushll.4s v1, v1, #0
335 ; CHECK-NEXT: ushll2.4s v3, v0, #0
336 ; CHECK-NEXT: ushll.4s v0, v0, #0
337 ; CHECK-NEXT: stp q1, q2, [x1, #32]
338 ; CHECK-NEXT: stp q0, q3, [x1]
339 ; CHECK-NEXT: ret
340 ;
341 ; CHECK-BE-LABEL: zext_v16i8_to_v16i32_no_loop:
342 ; CHECK-BE: // %bb.0: // %entry
343 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x0]
344 ; CHECK-BE-NEXT: add x8, x1, #48
345 ; CHECK-BE-NEXT: ushll2 v1.8h, v0.16b, #0
346 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
347 ; CHECK-BE-NEXT: ushll2 v2.4s, v1.8h, #0
348 ; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #0
349 ; CHECK-BE-NEXT: ushll2 v3.4s, v0.8h, #0
350 ; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
351 ; CHECK-BE-NEXT: st1 { v2.4s }, [x8]
352 ; CHECK-BE-NEXT: add x8, x1, #32
353 ; CHECK-BE-NEXT: st1 { v1.4s }, [x8]
354 ; CHECK-BE-NEXT: add x8, x1, #16
355 ; CHECK-BE-NEXT: st1 { v3.4s }, [x8]
356 ; CHECK-BE-NEXT: st1 { v0.4s }, [x1]
357 ; CHECK-BE-NEXT: ret
358 entry:
359 %load = load <16 x i8>, ptr %src
360 %ext = zext <16 x i8> %load to <16 x i32>
361 store <16 x i32> %ext, ptr %dst
362 ret void
363 }
365 ; Avoid using tbl when optimizing for size.
366 define void @zext_v16i8_to_v16i32_in_loop_optsize(ptr %src, ptr %dst) optsize {
367 ; CHECK-LABEL: zext_v16i8_to_v16i32_in_loop_optsize:
368 ; CHECK: ; %bb.0: ; %entry
369 ; CHECK-NEXT: mov x8, xzr
370 ; CHECK-NEXT: LBB3_1: ; %loop
371 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
372 ; CHECK-NEXT: ldr q0, [x0, x8]
373 ; CHECK-NEXT: add x8, x8, #16
374 ; CHECK-NEXT: cmp x8, #128
375 ; CHECK-NEXT: ushll2.8h v1, v0, #0
376 ; CHECK-NEXT: ushll.8h v0, v0, #0
377 ; CHECK-NEXT: ushll2.4s v2, v1, #0
378 ; CHECK-NEXT: ushll.4s v1, v1, #0
379 ; CHECK-NEXT: ushll2.4s v3, v0, #0
380 ; CHECK-NEXT: ushll.4s v0, v0, #0
381 ; CHECK-NEXT: stp q1, q2, [x1, #32]
382 ; CHECK-NEXT: stp q0, q3, [x1], #64
383 ; CHECK-NEXT: b.ne LBB3_1
384 ; CHECK-NEXT: ; %bb.2: ; %exit
385 ; CHECK-NEXT: ret
386 ;
387 ; CHECK-BE-LABEL: zext_v16i8_to_v16i32_in_loop_optsize:
388 ; CHECK-BE: // %bb.0: // %entry
389 ; CHECK-BE-NEXT: mov x8, xzr
390 ; CHECK-BE-NEXT: .LBB3_1: // %loop
391 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
392 ; CHECK-BE-NEXT: add x9, x0, x8
393 ; CHECK-BE-NEXT: add x8, x8, #16
394 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x9]
395 ; CHECK-BE-NEXT: add x9, x1, #48
396 ; CHECK-BE-NEXT: cmp x8, #128
397 ; CHECK-BE-NEXT: ushll2 v1.8h, v0.16b, #0
398 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
399 ; CHECK-BE-NEXT: ushll2 v2.4s, v1.8h, #0
400 ; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #0
401 ; CHECK-BE-NEXT: ushll2 v3.4s, v0.8h, #0
402 ; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
403 ; CHECK-BE-NEXT: st1 { v2.4s }, [x9]
404 ; CHECK-BE-NEXT: add x9, x1, #32
405 ; CHECK-BE-NEXT: st1 { v1.4s }, [x9]
406 ; CHECK-BE-NEXT: add x9, x1, #16
407 ; CHECK-BE-NEXT: st1 { v0.4s }, [x1]
408 ; CHECK-BE-NEXT: add x1, x1, #64
409 ; CHECK-BE-NEXT: st1 { v3.4s }, [x9]
410 ; CHECK-BE-NEXT: b.ne .LBB3_1
411 ; CHECK-BE-NEXT: // %bb.2: // %exit
412 ; CHECK-BE-NEXT: ret
413 entry:
414 br label %loop
416 loop:
417 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
418 %src.gep = getelementptr i8, ptr %src, i64 %iv
419 %load = load <16 x i8>, ptr %src.gep
420 %ext = zext <16 x i8> %load to <16 x i32>
421 %dst.gep = getelementptr i32, ptr %dst, i64 %iv
422 store <16 x i32> %ext, ptr %dst.gep
423 %iv.next = add nuw i64 %iv, 16
424 %ec = icmp eq i64 %iv.next, 128
425 br i1 %ec, label %exit, label %loop
427 exit:
428 ret void
429 }
431 ; Avoid using tbl when optimizing for size.
432 define void @zext_v16i8_to_v16i32_in_loop_minsize(ptr %src, ptr %dst) minsize {
433 ; CHECK-LABEL: zext_v16i8_to_v16i32_in_loop_minsize:
434 ; CHECK: ; %bb.0: ; %entry
435 ; CHECK-NEXT: mov x8, xzr
436 ; CHECK-NEXT: LBB4_1: ; %loop
437 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
438 ; CHECK-NEXT: ldr q0, [x0, x8]
439 ; CHECK-NEXT: add x8, x8, #16
440 ; CHECK-NEXT: cmp x8, #128
441 ; CHECK-NEXT: ushll2.8h v1, v0, #0
442 ; CHECK-NEXT: ushll.8h v0, v0, #0
443 ; CHECK-NEXT: ushll2.4s v2, v1, #0
444 ; CHECK-NEXT: ushll.4s v1, v1, #0
445 ; CHECK-NEXT: ushll2.4s v3, v0, #0
446 ; CHECK-NEXT: ushll.4s v0, v0, #0
447 ; CHECK-NEXT: stp q1, q2, [x1, #32]
448 ; CHECK-NEXT: stp q0, q3, [x1], #64
449 ; CHECK-NEXT: b.ne LBB4_1
450 ; CHECK-NEXT: ; %bb.2: ; %exit
451 ; CHECK-NEXT: ret
452 ;
453 ; CHECK-BE-LABEL: zext_v16i8_to_v16i32_in_loop_minsize:
454 ; CHECK-BE: // %bb.0: // %entry
455 ; CHECK-BE-NEXT: mov x8, xzr
456 ; CHECK-BE-NEXT: .LBB4_1: // %loop
457 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
458 ; CHECK-BE-NEXT: add x9, x0, x8
459 ; CHECK-BE-NEXT: add x8, x8, #16
460 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x9]
461 ; CHECK-BE-NEXT: add x9, x1, #48
462 ; CHECK-BE-NEXT: cmp x8, #128
463 ; CHECK-BE-NEXT: ushll2 v1.8h, v0.16b, #0
464 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
465 ; CHECK-BE-NEXT: ushll2 v2.4s, v1.8h, #0
466 ; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #0
467 ; CHECK-BE-NEXT: ushll2 v3.4s, v0.8h, #0
468 ; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
469 ; CHECK-BE-NEXT: st1 { v2.4s }, [x9]
470 ; CHECK-BE-NEXT: add x9, x1, #32
471 ; CHECK-BE-NEXT: st1 { v1.4s }, [x9]
472 ; CHECK-BE-NEXT: add x9, x1, #16
473 ; CHECK-BE-NEXT: st1 { v0.4s }, [x1]
474 ; CHECK-BE-NEXT: add x1, x1, #64
475 ; CHECK-BE-NEXT: st1 { v3.4s }, [x9]
476 ; CHECK-BE-NEXT: b.ne .LBB4_1
477 ; CHECK-BE-NEXT: // %bb.2: // %exit
478 ; CHECK-BE-NEXT: ret
479 entry:
480 br label %loop
482 loop:
483 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
484 %src.gep = getelementptr i8, ptr %src, i64 %iv
485 %load = load <16 x i8>, ptr %src.gep
486 %ext = zext <16 x i8> %load to <16 x i32>
487 %dst.gep = getelementptr i32, ptr %dst, i64 %iv
488 store <16 x i32> %ext, ptr %dst.gep
489 %iv.next = add nuw i64 %iv, 16
490 %ec = icmp eq i64 %iv.next, 128
491 br i1 %ec, label %exit, label %loop
493 exit:
494 ret void
495 }
497 define void @zext_v16i8_to_v16i16_in_loop(ptr %src, ptr %dst) {
498 ; CHECK-LABEL: zext_v16i8_to_v16i16_in_loop:
499 ; CHECK: ; %bb.0: ; %entry
500 ; CHECK-NEXT: mov x8, xzr
501 ; CHECK-NEXT: LBB5_1: ; %loop
502 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
503 ; CHECK-NEXT: ldr q0, [x0, x8]
504 ; CHECK-NEXT: add x8, x8, #16
505 ; CHECK-NEXT: cmp x8, #128
506 ; CHECK-NEXT: ushll2.8h v1, v0, #0
507 ; CHECK-NEXT: ushll.8h v0, v0, #0
508 ; CHECK-NEXT: stp q0, q1, [x1], #32
509 ; CHECK-NEXT: b.ne LBB5_1
510 ; CHECK-NEXT: ; %bb.2: ; %exit
511 ; CHECK-NEXT: ret
512 ;
513 ; CHECK-BE-LABEL: zext_v16i8_to_v16i16_in_loop:
514 ; CHECK-BE: // %bb.0: // %entry
515 ; CHECK-BE-NEXT: mov x8, xzr
516 ; CHECK-BE-NEXT: .LBB5_1: // %loop
517 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
518 ; CHECK-BE-NEXT: add x9, x0, x8
519 ; CHECK-BE-NEXT: add x8, x8, #16
520 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x9]
521 ; CHECK-BE-NEXT: add x9, x1, #16
522 ; CHECK-BE-NEXT: cmp x8, #128
523 ; CHECK-BE-NEXT: ushll2 v1.8h, v0.16b, #0
524 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
525 ; CHECK-BE-NEXT: st1 { v0.8h }, [x1]
526 ; CHECK-BE-NEXT: add x1, x1, #32
527 ; CHECK-BE-NEXT: st1 { v1.8h }, [x9]
528 ; CHECK-BE-NEXT: b.ne .LBB5_1
529 ; CHECK-BE-NEXT: // %bb.2: // %exit
530 ; CHECK-BE-NEXT: ret
533 entry:
534 br label %loop
536 loop:
537 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
538 %src.gep = getelementptr i8, ptr %src, i64 %iv
539 %load = load <16 x i8>, ptr %src.gep
540 %ext = zext <16 x i8> %load to <16 x i16>
541 %dst.gep = getelementptr i16, ptr %dst, i64 %iv
542 store <16 x i16> %ext, ptr %dst.gep
543 %iv.next = add nuw i64 %iv, 16
544 %ec = icmp eq i64 %iv.next, 128
545 br i1 %ec, label %exit, label %loop
547 exit:
548 ret void
549 }
551 ; CHECK-LABEL: lCPI6_0:
552 ; CHECK-NEXT: .byte 0 ; 0x0
553 ; CHECK-NEXT: .byte 255 ; 0xff
554 ; CHECK-NEXT: .byte 255 ; 0xff
555 ; CHECK-NEXT: .byte 255 ; 0xff
556 ; CHECK-NEXT: .byte 1 ; 0x1
557 ; CHECK-NEXT: .byte 255 ; 0xff
558 ; CHECK-NEXT: .byte 255 ; 0xff
559 ; CHECK-NEXT: .byte 255 ; 0xff
560 ; CHECK-NEXT: .byte 2 ; 0x2
561 ; CHECK-NEXT: .byte 255 ; 0xff
562 ; CHECK-NEXT: .byte 255 ; 0xff
563 ; CHECK-NEXT: .byte 255 ; 0xff
564 ; CHECK-NEXT: .byte 3 ; 0x3
565 ; CHECK-NEXT: .byte 255 ; 0xff
566 ; CHECK-NEXT: .byte 255 ; 0xff
567 ; CHECK-NEXT: .byte 255 ; 0xff
568 ; CHECK-NEXT: lCPI6_1:
569 ; CHECK-NEXT: .byte 4 ; 0x4
570 ; CHECK-NEXT: .byte 255 ; 0xff
571 ; CHECK-NEXT: .byte 255 ; 0xff
572 ; CHECK-NEXT: .byte 255 ; 0xff
573 ; CHECK-NEXT: .byte 5 ; 0x5
574 ; CHECK-NEXT: .byte 255 ; 0xff
575 ; CHECK-NEXT: .byte 255 ; 0xff
576 ; CHECK-NEXT: .byte 255 ; 0xff
577 ; CHECK-NEXT: .byte 6 ; 0x6
578 ; CHECK-NEXT: .byte 255 ; 0xff
579 ; CHECK-NEXT: .byte 255 ; 0xff
580 ; CHECK-NEXT: .byte 255 ; 0xff
581 ; CHECK-NEXT: .byte 7 ; 0x7
582 ; CHECK-NEXT: .byte 255 ; 0xff
583 ; CHECK-NEXT: .byte 255 ; 0xff
584 ; CHECK-NEXT: .byte 255 ; 0xff
586 ; CHECK-BE: .LCPI6_0:
587 ; CHECK-BE-NEXT: .byte 255 // 0xff
588 ; CHECK-BE-NEXT: .byte 255 // 0xff
589 ; CHECK-BE-NEXT: .byte 255 // 0xff
590 ; CHECK-BE-NEXT: .byte 0 // 0x0
591 ; CHECK-BE-NEXT: .byte 255 // 0xff
592 ; CHECK-BE-NEXT: .byte 255 // 0xff
593 ; CHECK-BE-NEXT: .byte 255 // 0xff
594 ; CHECK-BE-NEXT: .byte 1 // 0x1
595 ; CHECK-BE-NEXT: .byte 255 // 0xff
596 ; CHECK-BE-NEXT: .byte 255 // 0xff
597 ; CHECK-BE-NEXT: .byte 255 // 0xff
598 ; CHECK-BE-NEXT: .byte 2 // 0x2
599 ; CHECK-BE-NEXT: .byte 255 // 0xff
600 ; CHECK-BE-NEXT: .byte 255 // 0xff
601 ; CHECK-BE-NEXT: .byte 255 // 0xff
602 ; CHECK-BE-NEXT: .byte 3 // 0x3
603 ; CHECK-BE-NEXT: .LCPI6_1:
604 ; CHECK-BE-NEXT: .byte 255 // 0xff
605 ; CHECK-BE-NEXT: .byte 255 // 0xff
606 ; CHECK-BE-NEXT: .byte 255 // 0xff
607 ; CHECK-BE-NEXT: .byte 4 // 0x4
608 ; CHECK-BE-NEXT: .byte 255 // 0xff
609 ; CHECK-BE-NEXT: .byte 255 // 0xff
610 ; CHECK-BE-NEXT: .byte 255 // 0xff
611 ; CHECK-BE-NEXT: .byte 5 // 0x5
612 ; CHECK-BE-NEXT: .byte 255 // 0xff
613 ; CHECK-BE-NEXT: .byte 255 // 0xff
614 ; CHECK-BE-NEXT: .byte 255 // 0xff
615 ; CHECK-BE-NEXT: .byte 6 // 0x6
616 ; CHECK-BE-NEXT: .byte 255 // 0xff
617 ; CHECK-BE-NEXT: .byte 255 // 0xff
618 ; CHECK-BE-NEXT: .byte 255 // 0xff
619 ; CHECK-BE-NEXT: .byte 7 // 0x7
621 define void @zext_v8i8_to_v8i32_in_loop(ptr %src, ptr %dst) {
622 ; CHECK-LABEL: zext_v8i8_to_v8i32_in_loop:
623 ; CHECK: ; %bb.0: ; %entry
624 ; CHECK-NEXT: Lloh8:
625 ; CHECK-NEXT: adrp x8, lCPI6_0@PAGE
626 ; CHECK-NEXT: Lloh9:
627 ; CHECK-NEXT: adrp x9, lCPI6_1@PAGE
628 ; CHECK-NEXT: Lloh10:
629 ; CHECK-NEXT: ldr q0, [x8, lCPI6_0@PAGEOFF]
630 ; CHECK-NEXT: Lloh11:
631 ; CHECK-NEXT: ldr q1, [x9, lCPI6_1@PAGEOFF]
632 ; CHECK-NEXT: mov x8, xzr
633 ; CHECK-NEXT: LBB6_1: ; %loop
634 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
635 ; CHECK-NEXT: ldr d2, [x0, x8]
636 ; CHECK-NEXT: add x8, x8, #16
637 ; CHECK-NEXT: cmp x8, #128
638 ; CHECK-NEXT: tbl.16b v3, { v2 }, v1
639 ; CHECK-NEXT: tbl.16b v2, { v2 }, v0
640 ; CHECK-NEXT: stp q2, q3, [x1], #64
641 ; CHECK-NEXT: b.ne LBB6_1
642 ; CHECK-NEXT: ; %bb.2: ; %exit
643 ; CHECK-NEXT: ret
644 ; CHECK-NEXT: .loh AdrpLdr Lloh9, Lloh11
645 ; CHECK-NEXT: .loh AdrpLdr Lloh8, Lloh10
646 ;
647 ; CHECK-BE-LABEL: zext_v8i8_to_v8i32_in_loop:
648 ; CHECK-BE: // %bb.0: // %entry
649 ; CHECK-BE-NEXT: adrp x8, .LCPI6_0
650 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI6_0
651 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]
652 ; CHECK-BE-NEXT: adrp x8, .LCPI6_1
653 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI6_1
654 ; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
655 ; CHECK-BE-NEXT: mov x8, xzr
656 ; CHECK-BE-NEXT: .LBB6_1: // %loop
657 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
658 ; CHECK-BE-NEXT: add x9, x0, x8
659 ; CHECK-BE-NEXT: add x8, x8, #16
660 ; CHECK-BE-NEXT: ld1 { v2.8b }, [x9]
661 ; CHECK-BE-NEXT: add x9, x1, #16
662 ; CHECK-BE-NEXT: cmp x8, #128
663 ; CHECK-BE-NEXT: tbl v3.16b, { v2.16b }, v1.16b
664 ; CHECK-BE-NEXT: tbl v2.16b, { v2.16b }, v0.16b
665 ; CHECK-BE-NEXT: st1 { v2.16b }, [x1]
666 ; CHECK-BE-NEXT: add x1, x1, #64
667 ; CHECK-BE-NEXT: st1 { v3.16b }, [x9]
668 ; CHECK-BE-NEXT: b.ne .LBB6_1
669 ; CHECK-BE-NEXT: // %bb.2: // %exit
670 ; CHECK-BE-NEXT: ret
671 entry:
672 br label %loop
674 loop:
675 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
676 %src.gep = getelementptr i8, ptr %src, i64 %iv
677 %load = load <8 x i8>, ptr %src.gep
678 %ext = zext <8 x i8> %load to <8 x i32>
679 %dst.gep = getelementptr i32, ptr %dst, i64 %iv
680 store <8 x i32> %ext, ptr %dst.gep
681 %iv.next = add nuw i64 %iv, 16
682 %ec = icmp eq i64 %iv.next, 128
683 br i1 %ec, label %exit, label %loop
685 exit:
686 ret void
687 }
689 define void @zext_v16i8_to_v16i64_in_loop(ptr %src, ptr %dst) {
690 ; CHECK-LABEL: zext_v16i8_to_v16i64_in_loop:
691 ; CHECK: ; %bb.0: ; %entry
692 ; CHECK-NEXT: mov x8, xzr
693 ; CHECK-NEXT: LBB7_1: ; %loop
694 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
695 ; CHECK-NEXT: ldr q0, [x0, x8]
696 ; CHECK-NEXT: add x8, x8, #16
697 ; CHECK-NEXT: cmp x8, #128
698 ; CHECK-NEXT: ushll2.8h v1, v0, #0
699 ; CHECK-NEXT: ushll.8h v0, v0, #0
700 ; CHECK-NEXT: ushll2.4s v2, v1, #0
701 ; CHECK-NEXT: ushll.4s v1, v1, #0
702 ; CHECK-NEXT: ushll2.4s v4, v0, #0
703 ; CHECK-NEXT: ushll.4s v0, v0, #0
704 ; CHECK-NEXT: ushll2.2d v3, v2, #0
705 ; CHECK-NEXT: ushll.2d v2, v2, #0
706 ; CHECK-NEXT: ushll2.2d v5, v1, #0
707 ; CHECK-NEXT: ushll.2d v1, v1, #0
708 ; CHECK-NEXT: stp q2, q3, [x1, #96]
709 ; CHECK-NEXT: ushll2.2d v3, v4, #0
710 ; CHECK-NEXT: ushll.2d v2, v4, #0
711 ; CHECK-NEXT: ushll2.2d v4, v0, #0
712 ; CHECK-NEXT: ushll.2d v0, v0, #0
713 ; CHECK-NEXT: stp q1, q5, [x1, #64]
714 ; CHECK-NEXT: stp q2, q3, [x1, #32]
715 ; CHECK-NEXT: stp q0, q4, [x1], #128
716 ; CHECK-NEXT: b.ne LBB7_1
717 ; CHECK-NEXT: ; %bb.2: ; %exit
718 ; CHECK-NEXT: ret
719 ;
720 ; CHECK-BE-LABEL: zext_v16i8_to_v16i64_in_loop:
721 ; CHECK-BE: // %bb.0: // %entry
722 ; CHECK-BE-NEXT: mov x8, xzr
723 ; CHECK-BE-NEXT: .LBB7_1: // %loop
724 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
725 ; CHECK-BE-NEXT: add x9, x0, x8
726 ; CHECK-BE-NEXT: add x8, x8, #16
727 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x9]
728 ; CHECK-BE-NEXT: add x9, x1, #112
729 ; CHECK-BE-NEXT: cmp x8, #128
730 ; CHECK-BE-NEXT: ushll2 v1.8h, v0.16b, #0
731 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
732 ; CHECK-BE-NEXT: ushll2 v2.4s, v1.8h, #0
733 ; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #0
734 ; CHECK-BE-NEXT: ushll2 v4.4s, v0.8h, #0
735 ; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
736 ; CHECK-BE-NEXT: ushll2 v3.2d, v2.4s, #0
737 ; CHECK-BE-NEXT: ushll v2.2d, v2.2s, #0
738 ; CHECK-BE-NEXT: ushll2 v5.2d, v1.4s, #0
739 ; CHECK-BE-NEXT: ushll v1.2d, v1.2s, #0
740 ; CHECK-BE-NEXT: st1 { v3.2d }, [x9]
741 ; CHECK-BE-NEXT: add x9, x1, #96
742 ; CHECK-BE-NEXT: ushll2 v3.2d, v4.4s, #0
743 ; CHECK-BE-NEXT: st1 { v2.2d }, [x9]
744 ; CHECK-BE-NEXT: add x9, x1, #80
745 ; CHECK-BE-NEXT: ushll v2.2d, v4.2s, #0
746 ; CHECK-BE-NEXT: st1 { v5.2d }, [x9]
747 ; CHECK-BE-NEXT: add x9, x1, #48
748 ; CHECK-BE-NEXT: st1 { v3.2d }, [x9]
749 ; CHECK-BE-NEXT: ushll2 v3.2d, v0.4s, #0
750 ; CHECK-BE-NEXT: ushll v0.2d, v0.2s, #0
751 ; CHECK-BE-NEXT: add x9, x1, #64
752 ; CHECK-BE-NEXT: st1 { v1.2d }, [x9]
753 ; CHECK-BE-NEXT: add x9, x1, #32
754 ; CHECK-BE-NEXT: st1 { v2.2d }, [x9]
755 ; CHECK-BE-NEXT: add x9, x1, #16
756 ; CHECK-BE-NEXT: st1 { v0.2d }, [x1]
757 ; CHECK-BE-NEXT: add x1, x1, #128
758 ; CHECK-BE-NEXT: st1 { v3.2d }, [x9]
759 ; CHECK-BE-NEXT: b.ne .LBB7_1
760 ; CHECK-BE-NEXT: // %bb.2: // %exit
761 ; CHECK-BE-NEXT: ret
764 entry:
765 br label %loop
767 loop:
768 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
769 %src.gep = getelementptr i8, ptr %src, i64 %iv
770 %load = load <16 x i8>, ptr %src.gep
771 %ext = zext <16 x i8> %load to <16 x i64>
772 %dst.gep = getelementptr i64, ptr %dst, i64 %iv
773 store <16 x i64> %ext, ptr %dst.gep
774 %iv.next = add nuw i64 %iv, 16
775 %ec = icmp eq i64 %iv.next, 128
776 br i1 %ec, label %exit, label %loop
778 exit:
779 ret void
780 }
782 define void @zext_v8i8_to_v8i64_in_loop(ptr %src, ptr %dst) {
783 ; CHECK-LABEL: zext_v8i8_to_v8i64_in_loop:
784 ; CHECK: ; %bb.0: ; %entry
785 ; CHECK-NEXT: mov x8, xzr
786 ; CHECK-NEXT: LBB8_1: ; %loop
787 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
788 ; CHECK-NEXT: ldr d0, [x0, x8]
789 ; CHECK-NEXT: add x8, x8, #16
790 ; CHECK-NEXT: cmp x8, #128
791 ; CHECK-NEXT: ushll.8h v0, v0, #0
792 ; CHECK-NEXT: ushll2.4s v1, v0, #0
793 ; CHECK-NEXT: ushll.4s v0, v0, #0
794 ; CHECK-NEXT: ushll2.2d v2, v1, #0
795 ; CHECK-NEXT: ushll.2d v1, v1, #0
796 ; CHECK-NEXT: ushll2.2d v3, v0, #0
797 ; CHECK-NEXT: ushll.2d v0, v0, #0
798 ; CHECK-NEXT: stp q1, q2, [x1, #32]
799 ; CHECK-NEXT: stp q0, q3, [x1], #128
800 ; CHECK-NEXT: b.ne LBB8_1
801 ; CHECK-NEXT: ; %bb.2: ; %exit
802 ; CHECK-NEXT: ret
803 ;
804 ; CHECK-BE-LABEL: zext_v8i8_to_v8i64_in_loop:
805 ; CHECK-BE: // %bb.0: // %entry
806 ; CHECK-BE-NEXT: mov x8, xzr
807 ; CHECK-BE-NEXT: .LBB8_1: // %loop
808 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
809 ; CHECK-BE-NEXT: add x9, x0, x8
810 ; CHECK-BE-NEXT: add x8, x8, #16
811 ; CHECK-BE-NEXT: ld1 { v0.8b }, [x9]
812 ; CHECK-BE-NEXT: add x9, x1, #48
813 ; CHECK-BE-NEXT: cmp x8, #128
814 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
815 ; CHECK-BE-NEXT: ushll2 v1.4s, v0.8h, #0
816 ; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
817 ; CHECK-BE-NEXT: ushll2 v2.2d, v1.4s, #0
818 ; CHECK-BE-NEXT: ushll v1.2d, v1.2s, #0
819 ; CHECK-BE-NEXT: ushll2 v3.2d, v0.4s, #0
820 ; CHECK-BE-NEXT: ushll v0.2d, v0.2s, #0
821 ; CHECK-BE-NEXT: st1 { v2.2d }, [x9]
822 ; CHECK-BE-NEXT: add x9, x1, #32
823 ; CHECK-BE-NEXT: st1 { v1.2d }, [x9]
824 ; CHECK-BE-NEXT: add x9, x1, #16
825 ; CHECK-BE-NEXT: st1 { v0.2d }, [x1]
826 ; CHECK-BE-NEXT: add x1, x1, #128
827 ; CHECK-BE-NEXT: st1 { v3.2d }, [x9]
828 ; CHECK-BE-NEXT: b.ne .LBB8_1
829 ; CHECK-BE-NEXT: // %bb.2: // %exit
830 ; CHECK-BE-NEXT: ret
833 entry:
834 br label %loop
836 loop:
837 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
838 %src.gep = getelementptr i8, ptr %src, i64 %iv
839 %load = load <8 x i8>, ptr %src.gep
840 %ext = zext <8 x i8> %load to <8 x i64>
841 %dst.gep = getelementptr i64, ptr %dst, i64 %iv
842 store <8 x i64> %ext, ptr %dst.gep
843 %iv.next = add nuw i64 %iv, 16
844 %ec = icmp eq i64 %iv.next, 128
845 br i1 %ec, label %exit, label %loop
847 exit:
848 ret void
849 }
851 define void @zext_v8i8_to_v8i16_in_loop(ptr %src, ptr %dst) {
852 ; CHECK-LABEL: zext_v8i8_to_v8i16_in_loop:
853 ; CHECK: ; %bb.0: ; %entry
854 ; CHECK-NEXT: mov x8, xzr
855 ; CHECK-NEXT: LBB9_1: ; %loop
856 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
857 ; CHECK-NEXT: ldr d0, [x0, x8]
858 ; CHECK-NEXT: add x8, x8, #16
859 ; CHECK-NEXT: cmp x8, #128
860 ; CHECK-NEXT: ushll.8h v0, v0, #0
861 ; CHECK-NEXT: str q0, [x1], #32
862 ; CHECK-NEXT: b.ne LBB9_1
863 ; CHECK-NEXT: ; %bb.2: ; %exit
864 ; CHECK-NEXT: ret
865 ;
866 ; CHECK-BE-LABEL: zext_v8i8_to_v8i16_in_loop:
867 ; CHECK-BE: // %bb.0: // %entry
868 ; CHECK-BE-NEXT: mov x8, xzr
869 ; CHECK-BE-NEXT: .LBB9_1: // %loop
870 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
871 ; CHECK-BE-NEXT: add x9, x0, x8
872 ; CHECK-BE-NEXT: add x8, x8, #16
873 ; CHECK-BE-NEXT: ld1 { v0.8b }, [x9]
874 ; CHECK-BE-NEXT: cmp x8, #128
875 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
876 ; CHECK-BE-NEXT: st1 { v0.8h }, [x1]
877 ; CHECK-BE-NEXT: add x1, x1, #32
878 ; CHECK-BE-NEXT: b.ne .LBB9_1
879 ; CHECK-BE-NEXT: // %bb.2: // %exit
880 ; CHECK-BE-NEXT: ret
884 entry:
885 br label %loop
887 loop:
888 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
889 %src.gep = getelementptr i8, ptr %src, i64 %iv
890 %load = load <8 x i8>, ptr %src.gep
891 %ext = zext <8 x i8> %load to <8 x i16>
892 %dst.gep = getelementptr i16, ptr %dst, i64 %iv
893 store <8 x i16> %ext, ptr %dst.gep
894 %iv.next = add nuw i64 %iv, 16
895 %ec = icmp eq i64 %iv.next, 128
896 br i1 %ec, label %exit, label %loop
898 exit:
899 ret void
900 }
902 define void @zext_v8i8_to_v8i20_in_loop(ptr %src, ptr %dst) {
903 ; CHECK-LABEL: zext_v8i8_to_v8i20_in_loop:
904 ; CHECK: ; %bb.0: ; %entry
905 ; CHECK-NEXT: mov x8, xzr
906 ; CHECK-NEXT: LBB10_1: ; %loop
907 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
908 ; CHECK-NEXT: ldr d0, [x0, x8]
909 ; CHECK-NEXT: add x8, x8, #16
910 ; CHECK-NEXT: cmp x8, #128
911 ; CHECK-NEXT: ushll.8h v0, v0, #0
912 ; CHECK-NEXT: ushll2.4s v1, v0, #0
913 ; CHECK-NEXT: ushll.4s v0, v0, #0
914 ; CHECK-NEXT: mov.s w9, v1[1]
915 ; CHECK-NEXT: mov.s w11, v0[1]
916 ; CHECK-NEXT: fmov w10, s1
917 ; CHECK-NEXT: fmov w14, s0
918 ; CHECK-NEXT: mov.s w13, v1[2]
919 ; CHECK-NEXT: mov.s w16, v0[2]
920 ; CHECK-NEXT: mov.s w12, v1[3]
921 ; CHECK-NEXT: mov.s w15, v0[3]
922 ; CHECK-NEXT: orr x9, x10, x9, lsl #20
923 ; CHECK-NEXT: orr x10, x14, x11, lsl #20
924 ; CHECK-NEXT: orr x9, x9, x13, lsl #40
925 ; CHECK-NEXT: orr x10, x10, x16, lsl #40
926 ; CHECK-NEXT: lsr w11, w12, #4
927 ; CHECK-NEXT: lsr w13, w15, #4
928 ; CHECK-NEXT: orr x9, x9, x12, lsl #60
929 ; CHECK-NEXT: orr x10, x10, x15, lsl #60
930 ; CHECK-NEXT: strh w11, [x1, #18]
931 ; CHECK-NEXT: strh w13, [x1, #8]
932 ; CHECK-NEXT: stur x9, [x1, #10]
933 ; CHECK-NEXT: str x10, [x1], #64
934 ; CHECK-NEXT: b.ne LBB10_1
935 ; CHECK-NEXT: ; %bb.2: ; %exit
936 ; CHECK-NEXT: ret
937 ;
938 ; CHECK-BE-LABEL: zext_v8i8_to_v8i20_in_loop:
939 ; CHECK-BE: // %bb.0: // %entry
940 ; CHECK-BE-NEXT: mov x8, xzr
941 ; CHECK-BE-NEXT: .LBB10_1: // %loop
942 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
943 ; CHECK-BE-NEXT: add x9, x0, x8
944 ; CHECK-BE-NEXT: add x8, x8, #16
945 ; CHECK-BE-NEXT: ld1 { v0.8b }, [x9]
946 ; CHECK-BE-NEXT: cmp x8, #128
947 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
948 ; CHECK-BE-NEXT: ushll2 v1.4s, v0.8h, #0
949 ; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
950 ; CHECK-BE-NEXT: mov w9, v1.s[1]
951 ; CHECK-BE-NEXT: mov w10, v0.s[1]
952 ; CHECK-BE-NEXT: fmov w12, s1
953 ; CHECK-BE-NEXT: fmov w14, s0
954 ; CHECK-BE-NEXT: mov w11, v1.s[2]
955 ; CHECK-BE-NEXT: mov w13, v0.s[2]
956 ; CHECK-BE-NEXT: mov w15, v1.s[3]
957 ; CHECK-BE-NEXT: lsl x9, x9, #40
958 ; CHECK-BE-NEXT: lsl x10, x10, #40
959 ; CHECK-BE-NEXT: orr x9, x9, x12, lsl #60
960 ; CHECK-BE-NEXT: orr x10, x10, x14, lsl #60
961 ; CHECK-BE-NEXT: lsr x12, x12, #4
962 ; CHECK-BE-NEXT: strh w15, [x1, #18]
963 ; CHECK-BE-NEXT: orr x9, x9, x11, lsl #20
964 ; CHECK-BE-NEXT: orr x10, x10, x13, lsl #20
965 ; CHECK-BE-NEXT: mov w11, v0.s[3]
966 ; CHECK-BE-NEXT: lsr x13, x14, #4
967 ; CHECK-BE-NEXT: lsr x9, x9, #16
968 ; CHECK-BE-NEXT: lsr x10, x10, #16
969 ; CHECK-BE-NEXT: bfi x9, x12, #48, #4
970 ; CHECK-BE-NEXT: bfi x10, x13, #48, #4
971 ; CHECK-BE-NEXT: strh w11, [x1, #8]
972 ; CHECK-BE-NEXT: stur x9, [x1, #10]
973 ; CHECK-BE-NEXT: str x10, [x1], #64
974 ; CHECK-BE-NEXT: b.ne .LBB10_1
975 ; CHECK-BE-NEXT: // %bb.2: // %exit
976 ; CHECK-BE-NEXT: ret
979 entry:
980 br label %loop
982 loop:
983 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
984 %src.gep = getelementptr i8, ptr %src, i64 %iv
985 %load = load <8 x i8>, ptr %src.gep
986 %ext = zext <8 x i8> %load to <8 x i20>
987 %dst.gep = getelementptr i20, ptr %dst, i64 %iv
988 store <8 x i20> %ext, ptr %dst.gep
989 %iv.next = add nuw i64 %iv, 16
990 %ec = icmp eq i64 %iv.next, 128
991 br i1 %ec, label %exit, label %loop
993 exit:
994 ret void
995 }
997 define void @zext_v4i8_to_v4i32_in_loop(ptr %src, ptr %dst) {
998 ; CHECK-LABEL: zext_v4i8_to_v4i32_in_loop:
999 ; CHECK: ; %bb.0: ; %entry
1000 ; CHECK-NEXT: mov x8, xzr
1001 ; CHECK-NEXT: LBB11_1: ; %loop
1002 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
1003 ; CHECK-NEXT: ldr s0, [x0, x8]
1004 ; CHECK-NEXT: add x8, x8, #16
1005 ; CHECK-NEXT: cmp x8, #128
1006 ; CHECK-NEXT: ushll.8h v0, v0, #0
1007 ; CHECK-NEXT: ushll.4s v0, v0, #0
1008 ; CHECK-NEXT: str q0, [x1], #64
1009 ; CHECK-NEXT: b.ne LBB11_1
1010 ; CHECK-NEXT: ; %bb.2: ; %exit
1011 ; CHECK-NEXT: ret
1012 ;
1013 ; CHECK-BE-LABEL: zext_v4i8_to_v4i32_in_loop:
1014 ; CHECK-BE: // %bb.0: // %entry
1015 ; CHECK-BE-NEXT: adrp x8, .LCPI11_0
1016 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI11_0
1017 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]
1018 ; CHECK-BE-NEXT: mov x8, xzr
1019 ; CHECK-BE-NEXT: .LBB11_1: // %loop
1020 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
1021 ; CHECK-BE-NEXT: ldr s1, [x0, x8]
1022 ; CHECK-BE-NEXT: add x8, x8, #16
1023 ; CHECK-BE-NEXT: cmp x8, #128
1024 ; CHECK-BE-NEXT: rev32 v1.16b, v1.16b
1025 ; CHECK-BE-NEXT: tbl v1.16b, { v1.16b }, v0.16b
1026 ; CHECK-BE-NEXT: st1 { v1.16b }, [x1]
1027 ; CHECK-BE-NEXT: add x1, x1, #64
1028 ; CHECK-BE-NEXT: b.ne .LBB11_1
1029 ; CHECK-BE-NEXT: // %bb.2: // %exit
1030 ; CHECK-BE-NEXT: ret
1033 entry:
1034 br label %loop
1036 loop:
1037 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
1038 %src.gep = getelementptr i8, ptr %src, i64 %iv
1039 %load = load <4 x i8>, ptr %src.gep
1040 %ext = zext <4 x i8> %load to <4 x i32>
1041 %dst.gep = getelementptr i32, ptr %dst, i64 %iv
1042 store <4 x i32> %ext, ptr %dst.gep
1043 %iv.next = add nuw i64 %iv, 16
1044 %ec = icmp eq i64 %iv.next, 128
1045 br i1 %ec, label %exit, label %loop
1047 exit:
1048 ret void
1049 }
1051 ; CHECK-LABEL: lCPI12_0:
1052 ; CHECK-NEXT: .byte 0 ; 0x0
1053 ; CHECK-NEXT: .byte 255 ; 0xff
1054 ; CHECK-NEXT: .byte 255 ; 0xff
1055 ; CHECK-NEXT: .byte 255 ; 0xff
1056 ; CHECK-NEXT: .byte 1 ; 0x1
1057 ; CHECK-NEXT: .byte 255 ; 0xff
1058 ; CHECK-NEXT: .byte 255 ; 0xff
1059 ; CHECK-NEXT: .byte 255 ; 0xff
1060 ; CHECK-NEXT: .byte 2 ; 0x2
1061 ; CHECK-NEXT: .byte 255 ; 0xff
1062 ; CHECK-NEXT: .byte 255 ; 0xff
1063 ; CHECK-NEXT: .byte 255 ; 0xff
1064 ; CHECK-NEXT: .byte 3 ; 0x3
1065 ; CHECK-NEXT: .byte 255 ; 0xff
1066 ; CHECK-NEXT: .byte 255 ; 0xff
1067 ; CHECK-NEXT: .byte 255 ; 0xff
1068 ; CHECK-NEXT: lCPI12_1:
1069 ; CHECK-NEXT: .byte 4 ; 0x4
1070 ; CHECK-NEXT: .byte 255 ; 0xff
1071 ; CHECK-NEXT: .byte 255 ; 0xff
1072 ; CHECK-NEXT: .byte 255 ; 0xff
1073 ; CHECK-NEXT: .byte 5 ; 0x5
1074 ; CHECK-NEXT: .byte 255 ; 0xff
1075 ; CHECK-NEXT: .byte 255 ; 0xff
1076 ; CHECK-NEXT: .byte 255 ; 0xff
1077 ; CHECK-NEXT: .byte 6 ; 0x6
1078 ; CHECK-NEXT: .byte 255 ; 0xff
1079 ; CHECK-NEXT: .byte 255 ; 0xff
1080 ; CHECK-NEXT: .byte 255 ; 0xff
1081 ; CHECK-NEXT: .byte 7 ; 0x7
1082 ; CHECK-NEXT: .byte 255 ; 0xff
1083 ; CHECK-NEXT: .byte 255 ; 0xff
1084 ; CHECK-NEXT: .byte 255 ; 0xff
1085 ; CHECK-NEXT: lCPI12_2:
1086 ; CHECK-NEXT: .byte 8 ; 0x8
1087 ; CHECK-NEXT: .byte 255 ; 0xff
1088 ; CHECK-NEXT: .byte 255 ; 0xff
1089 ; CHECK-NEXT: .byte 255 ; 0xff
1090 ; CHECK-NEXT: .byte 9 ; 0x9
1091 ; CHECK-NEXT: .byte 255 ; 0xff
1092 ; CHECK-NEXT: .byte 255 ; 0xff
1093 ; CHECK-NEXT: .byte 255 ; 0xff
1094 ; CHECK-NEXT: .byte 10 ; 0xa
1095 ; CHECK-NEXT: .byte 255 ; 0xff
1096 ; CHECK-NEXT: .byte 255 ; 0xff
1097 ; CHECK-NEXT: .byte 255 ; 0xff
1098 ; CHECK-NEXT: .byte 11 ; 0xb
1099 ; CHECK-NEXT: .byte 255 ; 0xff
1100 ; CHECK-NEXT: .byte 255 ; 0xff
1101 ; CHECK-NEXT: .byte 255 ; 0xff
1103 ; CHECK-BE-LABEL: .LCPI12_0:
1104 ; CHECK-BE-NEXT: .byte 255 // 0xff
1105 ; CHECK-BE-NEXT: .byte 255 // 0xff
1106 ; CHECK-BE-NEXT: .byte 255 // 0xff
1107 ; CHECK-BE-NEXT: .byte 0 // 0x0
1108 ; CHECK-BE-NEXT: .byte 255 // 0xff
1109 ; CHECK-BE-NEXT: .byte 255 // 0xff
1110 ; CHECK-BE-NEXT: .byte 255 // 0xff
1111 ; CHECK-BE-NEXT: .byte 1 // 0x1
1112 ; CHECK-BE-NEXT: .byte 255 // 0xff
1113 ; CHECK-BE-NEXT: .byte 255 // 0xff
1114 ; CHECK-BE-NEXT: .byte 255 // 0xff
1115 ; CHECK-BE-NEXT: .byte 2 // 0x2
1116 ; CHECK-BE-NEXT: .byte 255 // 0xff
1117 ; CHECK-BE-NEXT: .byte 255 // 0xff
1118 ; CHECK-BE-NEXT: .byte 255 // 0xff
1119 ; CHECK-BE-NEXT: .byte 3 // 0x3
1120 ; CHECK-BE-NEXT: .LCPI12_1:
1121 ; CHECK-BE-NEXT: .byte 255 // 0xff
1122 ; CHECK-BE-NEXT: .byte 255 // 0xff
1123 ; CHECK-BE-NEXT: .byte 255 // 0xff
1124 ; CHECK-BE-NEXT: .byte 4 // 0x4
1125 ; CHECK-BE-NEXT: .byte 255 // 0xff
1126 ; CHECK-BE-NEXT: .byte 255 // 0xff
1127 ; CHECK-BE-NEXT: .byte 255 // 0xff
1128 ; CHECK-BE-NEXT: .byte 5 // 0x5
1129 ; CHECK-BE-NEXT: .byte 255 // 0xff
1130 ; CHECK-BE-NEXT: .byte 255 // 0xff
1131 ; CHECK-BE-NEXT: .byte 255 // 0xff
1132 ; CHECK-BE-NEXT: .byte 6 // 0x6
1133 ; CHECK-BE-NEXT: .byte 255 // 0xff
1134 ; CHECK-BE-NEXT: .byte 255 // 0xff
1135 ; CHECK-BE-NEXT: .byte 255 // 0xff
1136 ; CHECK-BE-NEXT: .byte 7 // 0x7
1137 ; CHECK-BE-NEXT: .LCPI12_2:
1138 ; CHECK-BE-NEXT: .byte 255 // 0xff
1139 ; CHECK-BE-NEXT: .byte 255 // 0xff
1140 ; CHECK-BE-NEXT: .byte 255 // 0xff
1141 ; CHECK-BE-NEXT: .byte 8 // 0x8
1142 ; CHECK-BE-NEXT: .byte 255 // 0xff
1143 ; CHECK-BE-NEXT: .byte 255 // 0xff
1144 ; CHECK-BE-NEXT: .byte 255 // 0xff
1145 ; CHECK-BE-NEXT: .byte 9 // 0x9
1146 ; CHECK-BE-NEXT: .byte 255 // 0xff
1147 ; CHECK-BE-NEXT: .byte 255 // 0xff
1148 ; CHECK-BE-NEXT: .byte 255 // 0xff
1149 ; CHECK-BE-NEXT: .byte 10 // 0xa
1150 ; CHECK-BE-NEXT: .byte 255 // 0xff
1151 ; CHECK-BE-NEXT: .byte 255 // 0xff
1152 ; CHECK-BE-NEXT: .byte 255 // 0xff
1153 ; CHECK-BE-NEXT: .byte 11 // 0xb
1155 define void @zext_v12i8_to_v12i32_in_loop(ptr %src, ptr %dst) {
1156 ; CHECK-LABEL: zext_v12i8_to_v12i32_in_loop:
1157 ; CHECK: ; %bb.0: ; %entry
1158 ; CHECK-NEXT: Lloh12:
1159 ; CHECK-NEXT: adrp x8, lCPI12_0@PAGE
1160 ; CHECK-NEXT: Lloh13:
1161 ; CHECK-NEXT: adrp x9, lCPI12_1@PAGE
1162 ; CHECK-NEXT: Lloh14:
1163 ; CHECK-NEXT: adrp x10, lCPI12_2@PAGE
1164 ; CHECK-NEXT: Lloh15:
1165 ; CHECK-NEXT: ldr q0, [x8, lCPI12_0@PAGEOFF]
1166 ; CHECK-NEXT: Lloh16:
1167 ; CHECK-NEXT: ldr q1, [x9, lCPI12_1@PAGEOFF]
1168 ; CHECK-NEXT: Lloh17:
1169 ; CHECK-NEXT: ldr q2, [x10, lCPI12_2@PAGEOFF]
1170 ; CHECK-NEXT: mov x8, xzr
1171 ; CHECK-NEXT: LBB12_1: ; %loop
1172 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
1173 ; CHECK-NEXT: ldr q3, [x0, x8]
1174 ; CHECK-NEXT: add x8, x8, #16
1175 ; CHECK-NEXT: cmp x8, #128
1176 ; CHECK-NEXT: tbl.16b v4, { v3 }, v2
1177 ; CHECK-NEXT: tbl.16b v5, { v3 }, v1
1178 ; CHECK-NEXT: tbl.16b v3, { v3 }, v0
1179 ; CHECK-NEXT: stp q5, q4, [x1, #16]
1180 ; CHECK-NEXT: str q3, [x1], #64
1181 ; CHECK-NEXT: b.ne LBB12_1
1182 ; CHECK-NEXT: ; %bb.2: ; %exit
1183 ; CHECK-NEXT: ret
1184 ; CHECK-NEXT: .loh AdrpLdr Lloh14, Lloh17
1185 ; CHECK-NEXT: .loh AdrpLdr Lloh13, Lloh16
1186 ; CHECK-NEXT: .loh AdrpLdr Lloh12, Lloh15
1187 ;
1188 ; CHECK-BE-LABEL: zext_v12i8_to_v12i32_in_loop:
1189 ; CHECK-BE: // %bb.0: // %entry
1190 ; CHECK-BE-NEXT: adrp x8, .LCPI12_0
1191 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI12_0
1192 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]
1193 ; CHECK-BE-NEXT: adrp x8, .LCPI12_1
1194 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI12_1
1195 ; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
1196 ; CHECK-BE-NEXT: adrp x8, .LCPI12_2
1197 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI12_2
1198 ; CHECK-BE-NEXT: ld1 { v2.16b }, [x8]
1199 ; CHECK-BE-NEXT: mov x8, xzr
1200 ; CHECK-BE-NEXT: .LBB12_1: // %loop
1201 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
1202 ; CHECK-BE-NEXT: add x9, x0, x8
1203 ; CHECK-BE-NEXT: add x8, x8, #16
1204 ; CHECK-BE-NEXT: add x10, x1, #16
1205 ; CHECK-BE-NEXT: ld1 { v3.16b }, [x9]
1206 ; CHECK-BE-NEXT: add x9, x1, #32
1207 ; CHECK-BE-NEXT: cmp x8, #128
1208 ; CHECK-BE-NEXT: tbl v4.16b, { v3.16b }, v2.16b
1209 ; CHECK-BE-NEXT: tbl v5.16b, { v3.16b }, v1.16b
1210 ; CHECK-BE-NEXT: tbl v3.16b, { v3.16b }, v0.16b
1211 ; CHECK-BE-NEXT: st1 { v3.16b }, [x1]
1212 ; CHECK-BE-NEXT: add x1, x1, #64
1213 ; CHECK-BE-NEXT: st1 { v4.16b }, [x9]
1214 ; CHECK-BE-NEXT: st1 { v5.16b }, [x10]
1215 ; CHECK-BE-NEXT: b.ne .LBB12_1
1216 ; CHECK-BE-NEXT: // %bb.2: // %exit
1217 ; CHECK-BE-NEXT: ret
1220 entry:
1221 br label %loop
1223 loop:
1224 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
1225 %src.gep = getelementptr i8, ptr %src, i64 %iv
1226 %load = load <12 x i8>, ptr %src.gep
1227 %ext = zext <12 x i8> %load to <12 x i32>
1228 %dst.gep = getelementptr i32, ptr %dst, i64 %iv
1229 store <12 x i32> %ext, ptr %dst.gep
1230 %iv.next = add nuw i64 %iv, 16
1231 %ec = icmp eq i64 %iv.next, 128
1232 br i1 %ec, label %exit, label %loop
1234 exit:
1235 ret void
1236 }
1238 define void @zext_v16i4_to_v16i32_in_loop(ptr %src, ptr %dst) {
1239 ; CHECK-LABEL: zext_v16i4_to_v16i32_in_loop:
1240 ; CHECK: ; %bb.0: ; %entry
1241 ; CHECK-NEXT: movi.4s v0, #15
1242 ; CHECK-NEXT: mov x8, xzr
1243 ; CHECK-NEXT: LBB13_1: ; %loop
1244 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
1245 ; CHECK-NEXT: ldr x9, [x0, x8]
1246 ; CHECK-NEXT: add x8, x8, #16
1247 ; CHECK-NEXT: cmp x8, #128
1248 ; CHECK-NEXT: and w11, w9, #0xf
1249 ; CHECK-NEXT: ubfx w10, w9, #4, #4
1250 ; CHECK-NEXT: fmov s1, w11
1251 ; CHECK-NEXT: mov.b v1[1], w10
1252 ; CHECK-NEXT: ubfx w10, w9, #8, #4
1253 ; CHECK-NEXT: mov.b v1[2], w10
1254 ; CHECK-NEXT: ubfx w10, w9, #12, #4
1255 ; CHECK-NEXT: mov.b v1[3], w10
1256 ; CHECK-NEXT: ubfx w10, w9, #16, #4
1257 ; CHECK-NEXT: mov.b v1[4], w10
1258 ; CHECK-NEXT: ubfx w10, w9, #20, #4
1259 ; CHECK-NEXT: mov.b v1[5], w10
1260 ; CHECK-NEXT: ubfx w10, w9, #24, #4
1261 ; CHECK-NEXT: mov.b v1[6], w10
1262 ; CHECK-NEXT: ubfx x10, x9, #28, #4
1263 ; CHECK-NEXT: mov.b v1[7], w10
1264 ; CHECK-NEXT: ubfx x10, x9, #32, #4
1265 ; CHECK-NEXT: mov.b v1[8], w10
1266 ; CHECK-NEXT: ubfx x10, x9, #36, #4
1267 ; CHECK-NEXT: mov.b v1[9], w10
1268 ; CHECK-NEXT: ubfx x10, x9, #40, #4
1269 ; CHECK-NEXT: mov.b v1[10], w10
1270 ; CHECK-NEXT: ubfx x10, x9, #44, #4
1271 ; CHECK-NEXT: mov.b v1[11], w10
1272 ; CHECK-NEXT: ubfx x10, x9, #48, #4
1273 ; CHECK-NEXT: mov.b v1[12], w10
1274 ; CHECK-NEXT: ubfx x10, x9, #52, #4
1275 ; CHECK-NEXT: mov.b v1[13], w10
1276 ; CHECK-NEXT: ubfx x10, x9, #56, #4
1277 ; CHECK-NEXT: lsr x9, x9, #60
1278 ; CHECK-NEXT: mov.b v1[14], w10
1279 ; CHECK-NEXT: mov.b v1[15], w9
1280 ; CHECK-NEXT: ext.16b v2, v1, v1, #8
1281 ; CHECK-NEXT: zip2.8b v3, v1, v0
1282 ; CHECK-NEXT: zip1.8b v1, v1, v0
1283 ; CHECK-NEXT: zip2.8b v4, v2, v0
1284 ; CHECK-NEXT: zip1.8b v2, v2, v0
1285 ; CHECK-NEXT: ushll.4s v3, v3, #0
1286 ; CHECK-NEXT: ushll.4s v1, v1, #0
1287 ; CHECK-NEXT: and.16b v3, v3, v0
1288 ; CHECK-NEXT: ushll.4s v4, v4, #0
1289 ; CHECK-NEXT: ushll.4s v2, v2, #0
1290 ; CHECK-NEXT: and.16b v1, v1, v0
1291 ; CHECK-NEXT: and.16b v4, v4, v0
1292 ; CHECK-NEXT: and.16b v2, v2, v0
1293 ; CHECK-NEXT: stp q1, q3, [x1]
1294 ; CHECK-NEXT: stp q2, q4, [x1, #32]
1295 ; CHECK-NEXT: add x1, x1, #64
1296 ; CHECK-NEXT: b.ne LBB13_1
1297 ; CHECK-NEXT: ; %bb.2: ; %exit
1298 ; CHECK-NEXT: ret
1299 ;
1300 ; CHECK-BE-LABEL: zext_v16i4_to_v16i32_in_loop:
1301 ; CHECK-BE: // %bb.0: // %entry
1302 ; CHECK-BE-NEXT: movi v0.4s, #15
1303 ; CHECK-BE-NEXT: mov x8, xzr
1304 ; CHECK-BE-NEXT: .LBB13_1: // %loop
1305 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
1306 ; CHECK-BE-NEXT: ldr x9, [x0, x8]
1307 ; CHECK-BE-NEXT: add x8, x8, #16
1308 ; CHECK-BE-NEXT: cmp x8, #128
1309 ; CHECK-BE-NEXT: lsr x10, x9, #60
1310 ; CHECK-BE-NEXT: ubfx x11, x9, #56, #4
1311 ; CHECK-BE-NEXT: fmov s1, w10
1312 ; CHECK-BE-NEXT: ubfx x10, x9, #52, #4
1313 ; CHECK-BE-NEXT: mov v1.b[1], w11
1314 ; CHECK-BE-NEXT: mov v1.b[2], w10
1315 ; CHECK-BE-NEXT: ubfx x10, x9, #48, #4
1316 ; CHECK-BE-NEXT: mov v1.b[3], w10
1317 ; CHECK-BE-NEXT: ubfx x10, x9, #44, #4
1318 ; CHECK-BE-NEXT: mov v1.b[4], w10
1319 ; CHECK-BE-NEXT: ubfx x10, x9, #40, #4
1320 ; CHECK-BE-NEXT: mov v1.b[5], w10
1321 ; CHECK-BE-NEXT: ubfx x10, x9, #36, #4
1322 ; CHECK-BE-NEXT: mov v1.b[6], w10
1323 ; CHECK-BE-NEXT: ubfx x10, x9, #32, #4
1324 ; CHECK-BE-NEXT: mov v1.b[7], w10
1325 ; CHECK-BE-NEXT: ubfx x10, x9, #28, #4
1326 ; CHECK-BE-NEXT: mov v1.b[8], w10
1327 ; CHECK-BE-NEXT: ubfx w10, w9, #24, #4
1328 ; CHECK-BE-NEXT: mov v1.b[9], w10
1329 ; CHECK-BE-NEXT: ubfx w10, w9, #20, #4
1330 ; CHECK-BE-NEXT: mov v1.b[10], w10
1331 ; CHECK-BE-NEXT: ubfx w10, w9, #16, #4
1332 ; CHECK-BE-NEXT: mov v1.b[11], w10
1333 ; CHECK-BE-NEXT: ubfx w10, w9, #12, #4
1334 ; CHECK-BE-NEXT: mov v1.b[12], w10
1335 ; CHECK-BE-NEXT: ubfx w10, w9, #8, #4
1336 ; CHECK-BE-NEXT: mov v1.b[13], w10
1337 ; CHECK-BE-NEXT: ubfx w10, w9, #4, #4
1338 ; CHECK-BE-NEXT: and w9, w9, #0xf
1339 ; CHECK-BE-NEXT: mov v1.b[14], w10
1340 ; CHECK-BE-NEXT: add x10, x1, #32
1341 ; CHECK-BE-NEXT: mov v1.b[15], w9
1342 ; CHECK-BE-NEXT: add x9, x1, #16
1343 ; CHECK-BE-NEXT: ext v2.16b, v1.16b, v1.16b, #8
1344 ; CHECK-BE-NEXT: zip2 v3.8b, v1.8b, v0.8b
1345 ; CHECK-BE-NEXT: zip1 v1.8b, v1.8b, v0.8b
1346 ; CHECK-BE-NEXT: zip2 v4.8b, v2.8b, v0.8b
1347 ; CHECK-BE-NEXT: zip1 v2.8b, v2.8b, v0.8b
1348 ; CHECK-BE-NEXT: ushll v3.4s, v3.4h, #0
1349 ; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #0
1350 ; CHECK-BE-NEXT: and v3.16b, v3.16b, v0.16b
1351 ; CHECK-BE-NEXT: ushll v4.4s, v4.4h, #0
1352 ; CHECK-BE-NEXT: ushll v2.4s, v2.4h, #0
1353 ; CHECK-BE-NEXT: and v1.16b, v1.16b, v0.16b
1354 ; CHECK-BE-NEXT: st1 { v3.4s }, [x9]
1355 ; CHECK-BE-NEXT: add x9, x1, #48
1356 ; CHECK-BE-NEXT: and v4.16b, v4.16b, v0.16b
1357 ; CHECK-BE-NEXT: and v2.16b, v2.16b, v0.16b
1358 ; CHECK-BE-NEXT: st1 { v1.4s }, [x1]
1359 ; CHECK-BE-NEXT: add x1, x1, #64
1360 ; CHECK-BE-NEXT: st1 { v4.4s }, [x9]
1361 ; CHECK-BE-NEXT: st1 { v2.4s }, [x10]
1362 ; CHECK-BE-NEXT: b.ne .LBB13_1
1363 ; CHECK-BE-NEXT: // %bb.2: // %exit
1364 ; CHECK-BE-NEXT: ret
1367 entry:
1368 br label %loop
1370 loop:
1371 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
1372 %src.gep = getelementptr i4, ptr %src, i64 %iv
1373 %load = load <16 x i4>, ptr %src.gep
1374 %ext = zext <16 x i4> %load to <16 x i32>
1375 %dst.gep = getelementptr i32, ptr %dst, i64 %iv
1376 store <16 x i32> %ext, ptr %dst.gep
1377 %iv.next = add nuw i64 %iv, 16
1378 %ec = icmp eq i64 %iv.next, 128
1379 br i1 %ec, label %exit, label %loop
1381 exit:
1382 ret void
1383 }
1385 define void @zext_v16i16_to_v16i64_in_loop(ptr %src, ptr %dst) {
1386 ; CHECK-LABEL: zext_v16i16_to_v16i64_in_loop:
1387 ; CHECK: ; %bb.0: ; %entry
1388 ; CHECK-NEXT: mov x8, xzr
1389 ; CHECK-NEXT: LBB14_1: ; %loop
1390 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
1391 ; CHECK-NEXT: add x9, x0, x8
1392 ; CHECK-NEXT: add x8, x8, #32
1393 ; CHECK-NEXT: ldp q1, q0, [x9]
1394 ; CHECK-NEXT: cmp x8, #256
1395 ; CHECK-NEXT: ushll2.4s v2, v0, #0
1396 ; CHECK-NEXT: ushll2.4s v3, v1, #0
1397 ; CHECK-NEXT: ushll.4s v0, v0, #0
1398 ; CHECK-NEXT: ushll.4s v1, v1, #0
1399 ; CHECK-NEXT: ushll2.2d v4, v2, #0
1400 ; CHECK-NEXT: ushll.2d v2, v2, #0
1401 ; CHECK-NEXT: ushll2.2d v5, v3, #0
1402 ; CHECK-NEXT: ushll.2d v3, v3, #0
1403 ; CHECK-NEXT: stp q2, q4, [x1, #96]
1404 ; CHECK-NEXT: ushll2.2d v4, v0, #0
1405 ; CHECK-NEXT: ushll.2d v0, v0, #0
1406 ; CHECK-NEXT: ushll2.2d v2, v1, #0
1407 ; CHECK-NEXT: ushll.2d v1, v1, #0
1408 ; CHECK-NEXT: stp q3, q5, [x1, #32]
1409 ; CHECK-NEXT: stp q0, q4, [x1, #64]
1410 ; CHECK-NEXT: stp q1, q2, [x1], #128
1411 ; CHECK-NEXT: b.ne LBB14_1
1412 ; CHECK-NEXT: ; %bb.2: ; %exit
1413 ; CHECK-NEXT: ret
1414 ;
1415 ; CHECK-BE-LABEL: zext_v16i16_to_v16i64_in_loop:
1416 ; CHECK-BE: // %bb.0: // %entry
1417 ; CHECK-BE-NEXT: mov x8, xzr
1418 ; CHECK-BE-NEXT: .LBB14_1: // %loop
1419 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
1420 ; CHECK-BE-NEXT: add x9, x0, x8
1421 ; CHECK-BE-NEXT: add x8, x8, #32
1422 ; CHECK-BE-NEXT: ld1 { v0.8h }, [x9]
1423 ; CHECK-BE-NEXT: add x9, x9, #16
1424 ; CHECK-BE-NEXT: cmp x8, #256
1425 ; CHECK-BE-NEXT: ld1 { v1.8h }, [x9]
1426 ; CHECK-BE-NEXT: add x9, x1, #48
1427 ; CHECK-BE-NEXT: ushll2 v2.4s, v0.8h, #0
1428 ; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
1429 ; CHECK-BE-NEXT: ushll2 v3.4s, v1.8h, #0
1430 ; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #0
1431 ; CHECK-BE-NEXT: ushll2 v4.2d, v2.4s, #0
1432 ; CHECK-BE-NEXT: ushll v2.2d, v2.2s, #0
1433 ; CHECK-BE-NEXT: ushll2 v5.2d, v0.4s, #0
1434 ; CHECK-BE-NEXT: ushll v0.2d, v0.2s, #0
1435 ; CHECK-BE-NEXT: st1 { v4.2d }, [x9]
1436 ; CHECK-BE-NEXT: add x9, x1, #32
1437 ; CHECK-BE-NEXT: ushll2 v4.2d, v3.4s, #0
1438 ; CHECK-BE-NEXT: st1 { v2.2d }, [x9]
1439 ; CHECK-BE-NEXT: add x9, x1, #16
1440 ; CHECK-BE-NEXT: ushll v2.2d, v3.2s, #0
1441 ; CHECK-BE-NEXT: st1 { v5.2d }, [x9]
1442 ; CHECK-BE-NEXT: add x9, x1, #112
1443 ; CHECK-BE-NEXT: ushll2 v3.2d, v1.4s, #0
1444 ; CHECK-BE-NEXT: st1 { v4.2d }, [x9]
1445 ; CHECK-BE-NEXT: add x9, x1, #96
1446 ; CHECK-BE-NEXT: ushll v1.2d, v1.2s, #0
1447 ; CHECK-BE-NEXT: st1 { v2.2d }, [x9]
1448 ; CHECK-BE-NEXT: add x9, x1, #80
1449 ; CHECK-BE-NEXT: st1 { v3.2d }, [x9]
1450 ; CHECK-BE-NEXT: add x9, x1, #64
1451 ; CHECK-BE-NEXT: st1 { v0.2d }, [x1]
1452 ; CHECK-BE-NEXT: add x1, x1, #128
1453 ; CHECK-BE-NEXT: st1 { v1.2d }, [x9]
1454 ; CHECK-BE-NEXT: b.ne .LBB14_1
1455 ; CHECK-BE-NEXT: // %bb.2: // %exit
1456 ; CHECK-BE-NEXT: ret
1459 entry:
1460 br label %loop
1462 loop:
1463 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
1464 %src.gep = getelementptr i16, ptr %src, i64 %iv
1465 %load = load <16 x i16>, ptr %src.gep
1466 %ext = zext <16 x i16> %load to <16 x i64>
1467 %dst.gep = getelementptr i64, ptr %dst, i64 %iv
1468 store <16 x i64> %ext, ptr %dst.gep
1469 %iv.next = add nuw i64 %iv, 16
1470 %ec = icmp eq i64 %iv.next, 128
1471 br i1 %ec, label %exit, label %loop
1473 exit:
1474 ret void
1475 }
1477 define void @zext_v16i32_to_v16i64_in_loop(ptr %src, ptr %dst) {
1478 ; CHECK-LABEL: zext_v16i32_to_v16i64_in_loop:
1479 ; CHECK: ; %bb.0: ; %entry
1480 ; CHECK-NEXT: mov x8, xzr
1481 ; CHECK-NEXT: LBB15_1: ; %loop
1482 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
1483 ; CHECK-NEXT: add x9, x0, x8
1484 ; CHECK-NEXT: add x8, x8, #64
1485 ; CHECK-NEXT: ldp q1, q0, [x9, #32]
1486 ; CHECK-NEXT: cmp x8, #512
1487 ; CHECK-NEXT: ldp q5, q4, [x9]
1488 ; CHECK-NEXT: ushll2.2d v2, v0, #0
1489 ; CHECK-NEXT: ushll.2d v0, v0, #0
1490 ; CHECK-NEXT: ushll2.2d v3, v1, #0
1491 ; CHECK-NEXT: ushll.2d v1, v1, #0
1492 ; CHECK-NEXT: stp q0, q2, [x1, #96]
1493 ; CHECK-NEXT: ushll2.2d v2, v4, #0
1494 ; CHECK-NEXT: ushll.2d v0, v4, #0
1495 ; CHECK-NEXT: stp q1, q3, [x1, #64]
1496 ; CHECK-NEXT: ushll2.2d v3, v5, #0
1497 ; CHECK-NEXT: ushll.2d v1, v5, #0
1498 ; CHECK-NEXT: stp q0, q2, [x1, #32]
1499 ; CHECK-NEXT: stp q1, q3, [x1], #128
1500 ; CHECK-NEXT: b.ne LBB15_1
1501 ; CHECK-NEXT: ; %bb.2: ; %exit
1502 ; CHECK-NEXT: ret
1503 ;
1504 ; CHECK-BE-LABEL: zext_v16i32_to_v16i64_in_loop:
1505 ; CHECK-BE: // %bb.0: // %entry
1506 ; CHECK-BE-NEXT: mov x8, xzr
1507 ; CHECK-BE-NEXT: .LBB15_1: // %loop
1508 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
1509 ; CHECK-BE-NEXT: add x9, x0, x8
1510 ; CHECK-BE-NEXT: add x8, x8, #64
1511 ; CHECK-BE-NEXT: ld1 { v0.4s }, [x9]
1512 ; CHECK-BE-NEXT: add x10, x9, #48
1513 ; CHECK-BE-NEXT: cmp x8, #512
1514 ; CHECK-BE-NEXT: ld1 { v1.4s }, [x10]
1515 ; CHECK-BE-NEXT: add x10, x9, #32
1516 ; CHECK-BE-NEXT: add x9, x9, #16
1517 ; CHECK-BE-NEXT: ld1 { v4.4s }, [x9]
1518 ; CHECK-BE-NEXT: ld1 { v2.4s }, [x10]
1519 ; CHECK-BE-NEXT: add x9, x1, #16
1520 ; CHECK-BE-NEXT: ushll2 v3.2d, v0.4s, #0
1521 ; CHECK-BE-NEXT: ushll v0.2d, v0.2s, #0
1522 ; CHECK-BE-NEXT: add x10, x1, #80
1523 ; CHECK-BE-NEXT: ushll2 v5.2d, v1.4s, #0
1524 ; CHECK-BE-NEXT: ushll2 v6.2d, v2.4s, #0
1525 ; CHECK-BE-NEXT: st1 { v3.2d }, [x9]
1526 ; CHECK-BE-NEXT: ushll2 v3.2d, v4.4s, #0
1527 ; CHECK-BE-NEXT: add x9, x1, #112
1528 ; CHECK-BE-NEXT: st1 { v0.2d }, [x1]
1529 ; CHECK-BE-NEXT: ushll v0.2d, v1.2s, #0
1530 ; CHECK-BE-NEXT: ushll v1.2d, v2.2s, #0
1531 ; CHECK-BE-NEXT: st1 { v5.2d }, [x9]
1532 ; CHECK-BE-NEXT: add x9, x1, #48
1533 ; CHECK-BE-NEXT: ushll v2.2d, v4.2s, #0
1534 ; CHECK-BE-NEXT: st1 { v3.2d }, [x9]
1535 ; CHECK-BE-NEXT: add x9, x1, #64
1536 ; CHECK-BE-NEXT: st1 { v6.2d }, [x10]
1537 ; CHECK-BE-NEXT: add x10, x1, #96
1538 ; CHECK-BE-NEXT: st1 { v1.2d }, [x9]
1539 ; CHECK-BE-NEXT: add x9, x1, #32
1540 ; CHECK-BE-NEXT: add x1, x1, #128
1541 ; CHECK-BE-NEXT: st1 { v0.2d }, [x10]
1542 ; CHECK-BE-NEXT: st1 { v2.2d }, [x9]
1543 ; CHECK-BE-NEXT: b.ne .LBB15_1
1544 ; CHECK-BE-NEXT: // %bb.2: // %exit
1545 ; CHECK-BE-NEXT: ret
1548 entry:
1549 br label %loop
1551 loop:
1552 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
1553 %src.gep = getelementptr i32, ptr %src, i64 %iv
1554 %load = load <16 x i32>, ptr %src.gep
1555 %ext = zext <16 x i32> %load to <16 x i64>
1556 %dst.gep = getelementptr i64, ptr %dst, i64 %iv
1557 store <16 x i64> %ext, ptr %dst.gep
1558 %iv.next = add nuw i64 %iv, 16
1559 %ec = icmp eq i64 %iv.next, 128
1560 br i1 %ec, label %exit, label %loop
1562 exit:
1563 ret void
1564 }
1566 define void @zext_v8i8_to_v8i128_in_loop(ptr %src, ptr %dst) {
1567 ; CHECK-LABEL: zext_v8i8_to_v8i128_in_loop:
1568 ; CHECK: ; %bb.0: ; %entry
1569 ; CHECK-NEXT: mov x8, xzr
1570 ; CHECK-NEXT: LBB16_1: ; %loop
1571 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
1572 ; CHECK-NEXT: ldr d0, [x0, x8]
1573 ; CHECK-NEXT: add x9, x1, #112
1574 ; CHECK-NEXT: add x8, x8, #16
1575 ; CHECK-NEXT: str xzr, [x1, #120]
1576 ; CHECK-NEXT: cmp x8, #128
1577 ; CHECK-NEXT: ushll.8h v0, v0, #0
1578 ; CHECK-NEXT: str xzr, [x1, #104]
1579 ; CHECK-NEXT: str xzr, [x1, #88]
1580 ; CHECK-NEXT: str xzr, [x1, #72]
1581 ; CHECK-NEXT: ushll2.4s v1, v0, #0
1582 ; CHECK-NEXT: ushll.4s v0, v0, #0
1583 ; CHECK-NEXT: str xzr, [x1, #56]
1584 ; CHECK-NEXT: str xzr, [x1, #40]
1585 ; CHECK-NEXT: str xzr, [x1, #24]
1586 ; CHECK-NEXT: ushll2.2d v2, v1, #0
1587 ; CHECK-NEXT: ushll.2d v1, v1, #0
1588 ; CHECK-NEXT: ushll2.2d v3, v0, #0
1589 ; CHECK-NEXT: ushll.2d v0, v0, #0
1590 ; CHECK-NEXT: str xzr, [x1, #8]
1591 ; CHECK-NEXT: st1.d { v2 }[1], [x9]
1592 ; CHECK-NEXT: add x9, x1, #80
1593 ; CHECK-NEXT: st1.d { v1 }[1], [x9]
1594 ; CHECK-NEXT: add x9, x1, #48
1595 ; CHECK-NEXT: str d2, [x1, #96]
1596 ; CHECK-NEXT: st1.d { v3 }[1], [x9]
1597 ; CHECK-NEXT: add x9, x1, #16
1598 ; CHECK-NEXT: str d1, [x1, #64]
1599 ; CHECK-NEXT: str d3, [x1, #32]
1600 ; CHECK-NEXT: str d0, [x1]
1601 ; CHECK-NEXT: add x1, x1, #256
1602 ; CHECK-NEXT: st1.d { v0 }[1], [x9]
1603 ; CHECK-NEXT: b.ne LBB16_1
1604 ; CHECK-NEXT: ; %bb.2: ; %exit
1605 ; CHECK-NEXT: ret
1606 ;
1607 ; CHECK-BE-LABEL: zext_v8i8_to_v8i128_in_loop:
1608 ; CHECK-BE: // %bb.0: // %entry
1609 ; CHECK-BE-NEXT: mov x8, xzr
1610 ; CHECK-BE-NEXT: .LBB16_1: // %loop
1611 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
1612 ; CHECK-BE-NEXT: add x9, x0, x8
1613 ; CHECK-BE-NEXT: add x8, x8, #16
1614 ; CHECK-BE-NEXT: ld1 { v0.8b }, [x9]
1615 ; CHECK-BE-NEXT: add x9, x1, #120
1616 ; CHECK-BE-NEXT: str xzr, [x1, #112]
1617 ; CHECK-BE-NEXT: str xzr, [x1, #96]
1618 ; CHECK-BE-NEXT: cmp x8, #128
1619 ; CHECK-BE-NEXT: str xzr, [x1, #80]
1620 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
1621 ; CHECK-BE-NEXT: str xzr, [x1, #64]
1622 ; CHECK-BE-NEXT: str xzr, [x1, #48]
1623 ; CHECK-BE-NEXT: str xzr, [x1, #32]
1624 ; CHECK-BE-NEXT: ushll2 v1.4s, v0.8h, #0
1625 ; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
1626 ; CHECK-BE-NEXT: str xzr, [x1, #16]
1627 ; CHECK-BE-NEXT: str xzr, [x1]
1628 ; CHECK-BE-NEXT: ushll2 v2.2d, v1.4s, #0
1629 ; CHECK-BE-NEXT: ushll v1.2d, v1.2s, #0
1630 ; CHECK-BE-NEXT: ushll2 v3.2d, v0.4s, #0
1631 ; CHECK-BE-NEXT: ushll v0.2d, v0.2s, #0
1632 ; CHECK-BE-NEXT: st1 { v2.d }[1], [x9]
1633 ; CHECK-BE-NEXT: add x9, x1, #88
1634 ; CHECK-BE-NEXT: st1 { v1.d }[1], [x9]
1635 ; CHECK-BE-NEXT: add x9, x1, #56
1636 ; CHECK-BE-NEXT: str d2, [x1, #104]
1637 ; CHECK-BE-NEXT: st1 { v3.d }[1], [x9]
1638 ; CHECK-BE-NEXT: add x9, x1, #24
1639 ; CHECK-BE-NEXT: str d1, [x1, #72]
1640 ; CHECK-BE-NEXT: str d3, [x1, #40]
1641 ; CHECK-BE-NEXT: str d0, [x1, #8]
1642 ; CHECK-BE-NEXT: add x1, x1, #256
1643 ; CHECK-BE-NEXT: st1 { v0.d }[1], [x9]
1644 ; CHECK-BE-NEXT: b.ne .LBB16_1
1645 ; CHECK-BE-NEXT: // %bb.2: // %exit
1646 ; CHECK-BE-NEXT: ret
1649 entry:
1650 br label %loop
1652 loop:
1653 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
1654 %src.gep = getelementptr i8, ptr %src, i64 %iv
1655 %load = load <8 x i8>, ptr %src.gep
1656 %ext = zext <8 x i8> %load to <8 x i128>
1657 %dst.gep = getelementptr i128, ptr %dst, i64 %iv
1658 store <8 x i128> %ext, ptr %dst.gep
1659 %iv.next = add nuw i64 %iv, 16
1660 %ec = icmp eq i64 %iv.next, 128
1661 br i1 %ec, label %exit, label %loop
1663 exit:
1664 ret void
1665 }
1667 ; multiple back-to-back 'zext' of similar type of vectors combined with arithmetic operations
1668 define void @zext_v8i8_to_v8i64_with_add_in_sequence_in_loop(ptr %src, ptr %dst) {
1669 ; CHECK-LABEL: zext_v8i8_to_v8i64_with_add_in_sequence_in_loop:
1670 ; CHECK: ; %bb.0: ; %entry
1671 ; CHECK-NEXT: Lloh18:
1672 ; CHECK-NEXT: adrp x8, lCPI17_0@PAGE
1673 ; CHECK-NEXT: Lloh19:
1674 ; CHECK-NEXT: adrp x9, lCPI17_1@PAGE
1675 ; CHECK-NEXT: mov w10, #128 ; =0x80
1676 ; CHECK-NEXT: Lloh20:
1677 ; CHECK-NEXT: ldr q0, [x8, lCPI17_0@PAGEOFF]
1678 ; CHECK-NEXT: Lloh21:
1679 ; CHECK-NEXT: ldr q1, [x9, lCPI17_1@PAGEOFF]
1680 ; CHECK-NEXT: add x8, x1, #64
1681 ; CHECK-NEXT: add x9, x0, #8
1682 ; CHECK-NEXT: LBB17_1: ; %loop
1683 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
1684 ; CHECK-NEXT: ldp d2, d3, [x9, #-8]
1685 ; CHECK-NEXT: subs x10, x10, #16
1686 ; CHECK-NEXT: ldp q6, q5, [x8, #-32]
1687 ; CHECK-NEXT: add x9, x9, #16
1688 ; CHECK-NEXT: ldp q17, q16, [x8, #-64]
1689 ; CHECK-NEXT: tbl.16b v4, { v2 }, v1
1690 ; CHECK-NEXT: tbl.16b v2, { v2 }, v0
1691 ; CHECK-NEXT: tbl.16b v7, { v3 }, v1
1692 ; CHECK-NEXT: tbl.16b v3, { v3 }, v0
1693 ; CHECK-NEXT: uaddw2.2d v5, v5, v4
1694 ; CHECK-NEXT: uaddw.2d v4, v6, v4
1695 ; CHECK-NEXT: uaddw2.2d v6, v16, v2
1696 ; CHECK-NEXT: ldp q18, q16, [x8, #32]
1697 ; CHECK-NEXT: uaddw.2d v2, v17, v2
1698 ; CHECK-NEXT: stp q4, q5, [x8, #-32]
1699 ; CHECK-NEXT: uaddw2.2d v5, v16, v7
1700 ; CHECK-NEXT: ldp q16, q4, [x8]
1701 ; CHECK-NEXT: uaddw.2d v7, v18, v7
1702 ; CHECK-NEXT: stp q2, q6, [x8, #-64]
1703 ; CHECK-NEXT: uaddw2.2d v4, v4, v3
1704 ; CHECK-NEXT: uaddw.2d v2, v16, v3
1705 ; CHECK-NEXT: stp q7, q5, [x8, #32]
1706 ; CHECK-NEXT: stp q2, q4, [x8], #128
1707 ; CHECK-NEXT: b.ne LBB17_1
1708 ; CHECK-NEXT: ; %bb.2: ; %exit
1709 ; CHECK-NEXT: ret
1710 ; CHECK-NEXT: .loh AdrpLdr Lloh19, Lloh21
1711 ; CHECK-NEXT: .loh AdrpLdr Lloh18, Lloh20
1712 ;
1713 ; CHECK-BE-LABEL: zext_v8i8_to_v8i64_with_add_in_sequence_in_loop:
1714 ; CHECK-BE: // %bb.0: // %entry
1715 ; CHECK-BE-NEXT: adrp x9, .LCPI17_0
1716 ; CHECK-BE-NEXT: add x9, x9, :lo12:.LCPI17_0
1717 ; CHECK-BE-NEXT: mov w8, #128 // =0x80
1718 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x9]
1719 ; CHECK-BE-NEXT: adrp x9, .LCPI17_1
1720 ; CHECK-BE-NEXT: add x9, x9, :lo12:.LCPI17_1
1721 ; CHECK-BE-NEXT: ld1 { v1.16b }, [x9]
1722 ; CHECK-BE-NEXT: add x9, x1, #64
1723 ; CHECK-BE-NEXT: add x10, x0, #8
1724 ; CHECK-BE-NEXT: .LBB17_1: // %loop
1725 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
1726 ; CHECK-BE-NEXT: ld1 { v2.8b }, [x10]
1727 ; CHECK-BE-NEXT: sub x11, x10, #8
1728 ; CHECK-BE-NEXT: add x15, x9, #32
1729 ; CHECK-BE-NEXT: ld1 { v3.8b }, [x11]
1730 ; CHECK-BE-NEXT: ld1 { v16.2d }, [x15]
1731 ; CHECK-BE-NEXT: sub x11, x9, #64
1732 ; CHECK-BE-NEXT: sub x12, x9, #32
1733 ; CHECK-BE-NEXT: ld1 { v6.2d }, [x9]
1734 ; CHECK-BE-NEXT: ld1 { v21.2d }, [x11]
1735 ; CHECK-BE-NEXT: tbl v4.16b, { v2.16b }, v1.16b
1736 ; CHECK-BE-NEXT: tbl v2.16b, { v2.16b }, v0.16b
1737 ; CHECK-BE-NEXT: ld1 { v19.2d }, [x12]
1738 ; CHECK-BE-NEXT: tbl v5.16b, { v3.16b }, v1.16b
1739 ; CHECK-BE-NEXT: tbl v3.16b, { v3.16b }, v0.16b
1740 ; CHECK-BE-NEXT: sub x13, x9, #16
1741 ; CHECK-BE-NEXT: sub x14, x9, #48
1742 ; CHECK-BE-NEXT: add x16, x9, #48
1743 ; CHECK-BE-NEXT: add x17, x9, #16
1744 ; CHECK-BE-NEXT: ld1 { v22.2d }, [x13]
1745 ; CHECK-BE-NEXT: subs x8, x8, #16
1746 ; CHECK-BE-NEXT: add x10, x10, #16
1747 ; CHECK-BE-NEXT: rev32 v7.8b, v4.8b
1748 ; CHECK-BE-NEXT: ext v4.16b, v4.16b, v4.16b, #8
1749 ; CHECK-BE-NEXT: rev32 v17.8b, v2.8b
1750 ; CHECK-BE-NEXT: ext v18.16b, v5.16b, v5.16b, #8
1751 ; CHECK-BE-NEXT: ext v20.16b, v3.16b, v3.16b, #8
1752 ; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
1753 ; CHECK-BE-NEXT: rev32 v5.8b, v5.8b
1754 ; CHECK-BE-NEXT: rev32 v3.8b, v3.8b
1755 ; CHECK-BE-NEXT: uaddw v7.2d, v16.2d, v7.2s
1756 ; CHECK-BE-NEXT: rev32 v4.8b, v4.8b
1757 ; CHECK-BE-NEXT: uaddw v6.2d, v6.2d, v17.2s
1758 ; CHECK-BE-NEXT: rev32 v17.8b, v18.8b
1759 ; CHECK-BE-NEXT: rev32 v20.8b, v20.8b
1760 ; CHECK-BE-NEXT: rev32 v2.8b, v2.8b
1761 ; CHECK-BE-NEXT: ld1 { v16.2d }, [x16]
1762 ; CHECK-BE-NEXT: ld1 { v18.2d }, [x14]
1763 ; CHECK-BE-NEXT: uaddw v5.2d, v19.2d, v5.2s
1764 ; CHECK-BE-NEXT: uaddw v3.2d, v21.2d, v3.2s
1765 ; CHECK-BE-NEXT: st1 { v7.2d }, [x15]
1766 ; CHECK-BE-NEXT: ld1 { v7.2d }, [x17]
1767 ; CHECK-BE-NEXT: st1 { v6.2d }, [x9]
1768 ; CHECK-BE-NEXT: add x9, x9, #128
1769 ; CHECK-BE-NEXT: uaddw v4.2d, v16.2d, v4.2s
1770 ; CHECK-BE-NEXT: st1 { v5.2d }, [x12]
1771 ; CHECK-BE-NEXT: uaddw v5.2d, v22.2d, v17.2s
1772 ; CHECK-BE-NEXT: st1 { v3.2d }, [x11]
1773 ; CHECK-BE-NEXT: uaddw v3.2d, v18.2d, v20.2s
1774 ; CHECK-BE-NEXT: uaddw v2.2d, v7.2d, v2.2s
1775 ; CHECK-BE-NEXT: st1 { v4.2d }, [x16]
1776 ; CHECK-BE-NEXT: st1 { v5.2d }, [x13]
1777 ; CHECK-BE-NEXT: st1 { v3.2d }, [x14]
1778 ; CHECK-BE-NEXT: st1 { v2.2d }, [x17]
1779 ; CHECK-BE-NEXT: b.ne .LBB17_1
1780 ; CHECK-BE-NEXT: // %bb.2: // %exit
1781 ; CHECK-BE-NEXT: ret
1784 entry:
1785 br label %loop
1787 loop:
1788 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
1789 %src.gep = getelementptr i8, ptr %src, i64 %iv
1790 %load = load <8 x i8>, ptr %src.gep
1791 %src.gep.2 = getelementptr i8, ptr %src.gep, i64 8
1792 %load.2 = load <8 x i8>, ptr %src.gep.2
1793 %ext = zext <8 x i8> %load to <8 x i64>
1794 %ext.2 = zext <8 x i8> %load.2 to <8 x i64>
1795 %dst.gep = getelementptr i64, ptr %dst, i64 %iv
1796 %load.dst = load <8 x i64>, ptr %dst.gep
1797 %dst.gep.2 = getelementptr i64, ptr %dst.gep, i64 8
1798 %load.dst.2 = load <8 x i64>, ptr %dst.gep.2
1799 %sum = add <8 x i64> %load.dst, %ext
1800 %sum.2 = add <8 x i64> %load.dst.2, %ext.2
1801 store <8 x i64> %sum, ptr %dst.gep
1802 store <8 x i64> %sum.2, ptr %dst.gep.2
1803 %iv.next = add nuw i64 %iv, 16
1804 %ec = icmp eq i64 %iv.next, 128
1805 br i1 %ec, label %exit, label %loop
1807 exit:
1808 ret void
1809 }
1811 ; multiple back-to-back 'zext' of similar type of vectors
1812 define void @zext_v16i8_to_v16i64_in_sequence_in_loop(ptr %src, ptr %dst) {
1813 ; CHECK-LABEL: zext_v16i8_to_v16i64_in_sequence_in_loop:
1814 ; CHECK: ; %bb.0: ; %entry
1815 ; CHECK-NEXT: mov w8, #128 ; =0x80
1816 ; CHECK-NEXT: add x9, x1, #128
1817 ; CHECK-NEXT: add x10, x0, #16
1818 ; CHECK-NEXT: LBB18_1: ; %loop
1819 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
1820 ; CHECK-NEXT: ldp q0, q1, [x10, #-16]
1821 ; CHECK-NEXT: subs x8, x8, #16
1822 ; CHECK-NEXT: add x10, x10, #16
1823 ; CHECK-NEXT: ushll2.8h v2, v0, #0
1824 ; CHECK-NEXT: ushll.8h v0, v0, #0
1825 ; CHECK-NEXT: ushll2.8h v6, v1, #0
1826 ; CHECK-NEXT: ushll.8h v1, v1, #0
1827 ; CHECK-NEXT: ushll2.4s v3, v2, #0
1828 ; CHECK-NEXT: ushll.4s v2, v2, #0
1829 ; CHECK-NEXT: ushll2.4s v5, v0, #0
1830 ; CHECK-NEXT: ushll.4s v0, v0, #0
1831 ; CHECK-NEXT: ushll2.2d v4, v3, #0
1832 ; CHECK-NEXT: ushll.2d v3, v3, #0
1833 ; CHECK-NEXT: ushll2.2d v7, v2, #0
1834 ; CHECK-NEXT: ushll.2d v2, v2, #0
1835 ; CHECK-NEXT: stp q3, q4, [x9, #-32]
1836 ; CHECK-NEXT: ushll2.2d v4, v5, #0
1837 ; CHECK-NEXT: ushll2.4s v3, v6, #0
1838 ; CHECK-NEXT: ushll.2d v5, v5, #0
1839 ; CHECK-NEXT: stp q2, q7, [x9, #-64]
1840 ; CHECK-NEXT: ushll2.2d v7, v0, #0
1841 ; CHECK-NEXT: ushll.2d v0, v0, #0
1842 ; CHECK-NEXT: ushll.4s v2, v6, #0
1843 ; CHECK-NEXT: stp q5, q4, [x9, #-96]
1844 ; CHECK-NEXT: ushll2.2d v4, v3, #0
1845 ; CHECK-NEXT: ushll2.4s v5, v1, #0
1846 ; CHECK-NEXT: ushll.2d v3, v3, #0
1847 ; CHECK-NEXT: stp q0, q7, [x9, #-128]
1848 ; CHECK-NEXT: ushll.4s v0, v1, #0
1849 ; CHECK-NEXT: ushll2.2d v6, v2, #0
1850 ; CHECK-NEXT: ushll.2d v1, v2, #0
1851 ; CHECK-NEXT: ushll2.2d v2, v5, #0
1852 ; CHECK-NEXT: stp q3, q4, [x9, #96]
1853 ; CHECK-NEXT: ushll.2d v3, v5, #0
1854 ; CHECK-NEXT: ushll2.2d v4, v0, #0
1855 ; CHECK-NEXT: ushll.2d v0, v0, #0
1856 ; CHECK-NEXT: stp q1, q6, [x9, #64]
1857 ; CHECK-NEXT: stp q3, q2, [x9, #32]
1858 ; CHECK-NEXT: stp q0, q4, [x9], #128
1859 ; CHECK-NEXT: b.ne LBB18_1
1860 ; CHECK-NEXT: ; %bb.2: ; %exit
1861 ; CHECK-NEXT: ret
1862 ;
1863 ; CHECK-BE-LABEL: zext_v16i8_to_v16i64_in_sequence_in_loop:
1864 ; CHECK-BE: // %bb.0: // %entry
1865 ; CHECK-BE-NEXT: mov w8, #128 // =0x80
1866 ; CHECK-BE-NEXT: add x9, x1, #128
1867 ; CHECK-BE-NEXT: add x10, x0, #16
1868 ; CHECK-BE-NEXT: .LBB18_1: // %loop
1869 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
1870 ; CHECK-BE-NEXT: sub x11, x10, #16
1871 ; CHECK-BE-NEXT: ld1 { v5.16b }, [x10]
1872 ; CHECK-BE-NEXT: sub x12, x9, #32
1873 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x11]
1874 ; CHECK-BE-NEXT: sub x11, x9, #16
1875 ; CHECK-BE-NEXT: subs x8, x8, #16
1876 ; CHECK-BE-NEXT: add x10, x10, #16
1877 ; CHECK-BE-NEXT: ushll2 v1.8h, v0.16b, #0
1878 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
1879 ; CHECK-BE-NEXT: ushll2 v2.4s, v1.8h, #0
1880 ; CHECK-BE-NEXT: ushll2 v3.4s, v0.8h, #0
1881 ; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #0
1882 ; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
1883 ; CHECK-BE-NEXT: ushll2 v4.2d, v2.4s, #0
1884 ; CHECK-BE-NEXT: ushll v2.2d, v2.2s, #0
1885 ; CHECK-BE-NEXT: ushll2 v6.2d, v1.4s, #0
1886 ; CHECK-BE-NEXT: ushll v1.2d, v1.2s, #0
1887 ; CHECK-BE-NEXT: st1 { v4.2d }, [x11]
1888 ; CHECK-BE-NEXT: ushll2 v4.2d, v3.4s, #0
1889 ; CHECK-BE-NEXT: ushll v3.2d, v3.2s, #0
1890 ; CHECK-BE-NEXT: st1 { v2.2d }, [x12]
1891 ; CHECK-BE-NEXT: ushll2 v2.8h, v5.16b, #0
1892 ; CHECK-BE-NEXT: sub x12, x9, #80
1893 ; CHECK-BE-NEXT: sub x11, x9, #48
1894 ; CHECK-BE-NEXT: st1 { v4.2d }, [x12]
1895 ; CHECK-BE-NEXT: ushll v4.8h, v5.8b, #0
1896 ; CHECK-BE-NEXT: sub x12, x9, #64
1897 ; CHECK-BE-NEXT: ushll2 v5.4s, v2.8h, #0
1898 ; CHECK-BE-NEXT: st1 { v1.2d }, [x12]
1899 ; CHECK-BE-NEXT: sub x12, x9, #96
1900 ; CHECK-BE-NEXT: ushll2 v1.2d, v0.4s, #0
1901 ; CHECK-BE-NEXT: ushll v2.4s, v2.4h, #0
1902 ; CHECK-BE-NEXT: ushll v0.2d, v0.2s, #0
1903 ; CHECK-BE-NEXT: st1 { v6.2d }, [x11]
1904 ; CHECK-BE-NEXT: sub x11, x9, #128
1905 ; CHECK-BE-NEXT: st1 { v3.2d }, [x12]
1906 ; CHECK-BE-NEXT: ushll2 v3.4s, v4.8h, #0
1907 ; CHECK-BE-NEXT: ushll2 v6.2d, v5.4s, #0
1908 ; CHECK-BE-NEXT: sub x12, x9, #112
1909 ; CHECK-BE-NEXT: ushll v5.2d, v5.2s, #0
1910 ; CHECK-BE-NEXT: st1 { v0.2d }, [x11]
1911 ; CHECK-BE-NEXT: st1 { v1.2d }, [x12]
1912 ; CHECK-BE-NEXT: ushll2 v1.2d, v2.4s, #0
1913 ; CHECK-BE-NEXT: add x11, x9, #112
1914 ; CHECK-BE-NEXT: ushll v4.4s, v4.4h, #0
1915 ; CHECK-BE-NEXT: ushll2 v0.2d, v3.4s, #0
1916 ; CHECK-BE-NEXT: st1 { v6.2d }, [x11]
1917 ; CHECK-BE-NEXT: add x11, x9, #96
1918 ; CHECK-BE-NEXT: ushll v2.2d, v2.2s, #0
1919 ; CHECK-BE-NEXT: ushll v3.2d, v3.2s, #0
1920 ; CHECK-BE-NEXT: st1 { v5.2d }, [x11]
1921 ; CHECK-BE-NEXT: add x11, x9, #80
1922 ; CHECK-BE-NEXT: st1 { v1.2d }, [x11]
1923 ; CHECK-BE-NEXT: add x11, x9, #48
1924 ; CHECK-BE-NEXT: ushll2 v1.2d, v4.4s, #0
1925 ; CHECK-BE-NEXT: st1 { v0.2d }, [x11]
1926 ; CHECK-BE-NEXT: ushll v0.2d, v4.2s, #0
1927 ; CHECK-BE-NEXT: add x11, x9, #64
1928 ; CHECK-BE-NEXT: st1 { v2.2d }, [x11]
1929 ; CHECK-BE-NEXT: add x11, x9, #32
1930 ; CHECK-BE-NEXT: st1 { v3.2d }, [x11]
1931 ; CHECK-BE-NEXT: add x11, x9, #16
1932 ; CHECK-BE-NEXT: st1 { v0.2d }, [x9]
1933 ; CHECK-BE-NEXT: add x9, x9, #128
1934 ; CHECK-BE-NEXT: st1 { v1.2d }, [x11]
1935 ; CHECK-BE-NEXT: b.ne .LBB18_1
1936 ; CHECK-BE-NEXT: // %bb.2: // %exit
1937 ; CHECK-BE-NEXT: ret
1940 entry:
1941 br label %loop
1943 loop:
1944 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
1945 %src.gep = getelementptr i8, ptr %src, i64 %iv
1946 %load = load <16 x i8>, ptr %src.gep
1947 %src.gep.2 = getelementptr i8, ptr %src.gep, i64 16
1948 %load.2 = load <16 x i8>, ptr %src.gep.2
1949 %ext = zext <16 x i8> %load to <16 x i64>
1950 %ext.2 = zext <16 x i8> %load.2 to <16 x i64>
1951 %dst.gep = getelementptr i64, ptr %dst, i64 %iv
1952 store <16 x i64> %ext, ptr %dst.gep
1953 %dst.gep.2 = getelementptr i64, ptr %dst.gep, i64 16
1954 store <16 x i64> %ext.2, ptr %dst.gep.2
1955 %iv.next = add nuw i64 %iv, 16
1956 %ec = icmp eq i64 %iv.next, 128
1957 br i1 %ec, label %exit, label %loop
1959 exit:
1960 ret void
1961 }
1963 define void @zext_v16i8_to_v16i32_in_loop_scalable_vectors(ptr %src, ptr %dst) {
1964 ; CHECK-LABEL: zext_v16i8_to_v16i32_in_loop_scalable_vectors:
1965 ; CHECK: ; %bb.0: ; %entry
1966 ; CHECK-NEXT: ptrue p0.s
1967 ; CHECK-NEXT: mov x8, xzr
1968 ; CHECK-NEXT: LBB19_1: ; %loop
1969 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
1970 ; CHECK-NEXT: ld1b { z0.s }, p0/z, [x0, x8]
1971 ; CHECK-NEXT: add x9, x0, x8
1972 ; CHECK-NEXT: ld1b { z1.s }, p0/z, [x9, #2, mul vl]
1973 ; CHECK-NEXT: ld1b { z2.s }, p0/z, [x9, #3, mul vl]
1974 ; CHECK-NEXT: ld1b { z3.s }, p0/z, [x9, #1, mul vl]
1975 ; CHECK-NEXT: add x9, x1, x8, lsl #2
1976 ; CHECK-NEXT: add z0.s, z0.s, z0.s
1977 ; CHECK-NEXT: add z1.s, z1.s, z1.s
1978 ; CHECK-NEXT: add z2.s, z2.s, z2.s
1979 ; CHECK-NEXT: add z3.s, z3.s, z3.s
1980 ; CHECK-NEXT: st1w { z0.s }, p0, [x1, x8, lsl #2]
1981 ; CHECK-NEXT: add x8, x8, #16
1982 ; CHECK-NEXT: cmp x8, #128
1983 ; CHECK-NEXT: st1w { z1.s }, p0, [x9, #2, mul vl]
1984 ; CHECK-NEXT: st1w { z2.s }, p0, [x9, #3, mul vl]
1985 ; CHECK-NEXT: st1w { z3.s }, p0, [x9, #1, mul vl]
1986 ; CHECK-NEXT: b.ne LBB19_1
1987 ; CHECK-NEXT: ; %bb.2: ; %exit
1988 ; CHECK-NEXT: ret
1989 ;
1990 ; CHECK-BE-LABEL: zext_v16i8_to_v16i32_in_loop_scalable_vectors:
1991 ; CHECK-BE: // %bb.0: // %entry
1992 ; CHECK-BE-NEXT: ptrue p0.s
1993 ; CHECK-BE-NEXT: mov x8, xzr
1994 ; CHECK-BE-NEXT: .LBB19_1: // %loop
1995 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
1996 ; CHECK-BE-NEXT: ld1b { z0.s }, p0/z, [x0, x8]
1997 ; CHECK-BE-NEXT: add x9, x0, x8
1998 ; CHECK-BE-NEXT: ld1b { z1.s }, p0/z, [x9, #2, mul vl]
1999 ; CHECK-BE-NEXT: ld1b { z2.s }, p0/z, [x9, #3, mul vl]
2000 ; CHECK-BE-NEXT: ld1b { z3.s }, p0/z, [x9, #1, mul vl]
2001 ; CHECK-BE-NEXT: add x9, x1, x8, lsl #2
2002 ; CHECK-BE-NEXT: add z0.s, z0.s, z0.s
2003 ; CHECK-BE-NEXT: add z1.s, z1.s, z1.s
2004 ; CHECK-BE-NEXT: add z2.s, z2.s, z2.s
2005 ; CHECK-BE-NEXT: add z3.s, z3.s, z3.s
2006 ; CHECK-BE-NEXT: st1w { z0.s }, p0, [x1, x8, lsl #2]
2007 ; CHECK-BE-NEXT: add x8, x8, #16
2008 ; CHECK-BE-NEXT: cmp x8, #128
2009 ; CHECK-BE-NEXT: st1w { z1.s }, p0, [x9, #2, mul vl]
2010 ; CHECK-BE-NEXT: st1w { z2.s }, p0, [x9, #3, mul vl]
2011 ; CHECK-BE-NEXT: st1w { z3.s }, p0, [x9, #1, mul vl]
2012 ; CHECK-BE-NEXT: b.ne .LBB19_1
2013 ; CHECK-BE-NEXT: // %bb.2: // %exit
2014 ; CHECK-BE-NEXT: ret
2015 entry:
2016 br label %loop
2018 loop:
2019 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
2020 %src.gep = getelementptr i8, ptr %src, i64 %iv
2021 %load = load <vscale x 16 x i8>, ptr %src.gep
2022 %ext = zext <vscale x 16 x i8> %load to <vscale x 16 x i32>
2023 %add = add <vscale x 16 x i32> %ext, %ext
2024 %dst.gep = getelementptr i32, ptr %dst, i64 %iv
2025 store <vscale x 16 x i32> %add, ptr %dst.gep
2026 %iv.next = add nuw i64 %iv, 16
2027 %ec = icmp eq i64 %iv.next, 128
2028 br i1 %ec, label %exit, label %loop
2030 exit:
2031 ret void
2032 }
2034 ; CHECK-LABEL: lCPI20_0:
2035 ; CHECK-NEXT: .byte 0 ; 0x0
2036 ; CHECK-NEXT: .byte 255 ; 0xff
2037 ; CHECK-NEXT: .byte 255 ; 0xff
2038 ; CHECK-NEXT: .byte 1 ; 0x1
2039 ; CHECK-NEXT: .byte 255 ; 0xff
2040 ; CHECK-NEXT: .byte 255 ; 0xff
2041 ; CHECK-NEXT: .byte 2 ; 0x2
2042 ; CHECK-NEXT: .byte 255 ; 0xff
2043 ; CHECK-NEXT: .byte 255 ; 0xff
2044 ; CHECK-NEXT: .byte 3 ; 0x3
2045 ; CHECK-NEXT: .byte 255 ; 0xff
2046 ; CHECK-NEXT: .byte 255 ; 0xff
2047 ; CHECK-NEXT: .byte 4 ; 0x4
2048 ; CHECK-NEXT: .byte 255 ; 0xff
2049 ; CHECK-NEXT: .byte 255 ; 0xff
2050 ; CHECK-NEXT: .byte 5 ; 0x5
2051 ; CHECK-NEXT:lCPI20_1:
2052 ; CHECK-NEXT: .byte 255 ; 0xff
2053 ; CHECK-NEXT: .byte 255 ; 0xff
2054 ; CHECK-NEXT: .byte 6 ; 0x6
2055 ; CHECK-NEXT: .byte 255 ; 0xff
2056 ; CHECK-NEXT: .byte 255 ; 0xff
2057 ; CHECK-NEXT: .byte 7 ; 0x7
2058 ; CHECK-NEXT: .byte 255 ; 0xff
2059 ; CHECK-NEXT: .byte 255 ; 0xff
2060 ; CHECK-NEXT: .byte 8 ; 0x8
2061 ; CHECK-NEXT: .byte 255 ; 0xff
2062 ; CHECK-NEXT: .byte 255 ; 0xff
2063 ; CHECK-NEXT: .byte 9 ; 0x9
2064 ; CHECK-NEXT: .byte 255 ; 0xff
2065 ; CHECK-NEXT: .byte 255 ; 0xff
2066 ; CHECK-NEXT: .byte 10 ; 0xa
2067 ; CHECK-NEXT: .byte 255 ; 0xff
2068 ; CHECK-NEXT:lCPI20_2:
2069 ; CHECK-NEXT: .byte 255 ; 0xff
2070 ; CHECK-NEXT: .byte 11 ; 0xb
2071 ; CHECK-NEXT: .byte 255 ; 0xff
2072 ; CHECK-NEXT: .byte 255 ; 0xff
2073 ; CHECK-NEXT: .byte 12 ; 0xc
2074 ; CHECK-NEXT: .byte 255 ; 0xff
2075 ; CHECK-NEXT: .byte 255 ; 0xff
2076 ; CHECK-NEXT: .byte 13 ; 0xd
2077 ; CHECK-NEXT: .byte 255 ; 0xff
2078 ; CHECK-NEXT: .byte 255 ; 0xff
2079 ; CHECK-NEXT: .byte 14 ; 0xe
2080 ; CHECK-NEXT: .byte 255 ; 0xff
2081 ; CHECK-NEXT: .byte 255 ; 0xff
2082 ; CHECK-NEXT: .byte 15 ; 0xf
2083 ; CHECK-NEXT: .byte 255 ; 0xff
2084 ; CHECK-NEXT: .byte 255 ; 0xff
2085 ; CHECK-NEXT:lCPI20_3:
2086 ; CHECK-NEXT: .byte 0 ; 0x0
2087 ; CHECK-NEXT: .byte 255 ; 0xff
2088 ; CHECK-NEXT: .byte 255 ; 0xff
2089 ; CHECK-NEXT: .byte 1 ; 0x1
2090 ; CHECK-NEXT: .byte 255 ; 0xff
2091 ; CHECK-NEXT: .byte 255 ; 0xff
2092 ; CHECK-NEXT: .byte 2 ; 0x2
2093 ; CHECK-NEXT: .byte 255 ; 0xff
2094 ; CHECK-NEXT: .byte 255 ; 0xff
2095 ; CHECK-NEXT: .byte 3 ; 0x3
2096 ; CHECK-NEXT: .byte 255 ; 0xff
2097 ; CHECK-NEXT: .byte 255 ; 0xff
2098 ; CHECK-NEXT: .byte 255 ; 0xff
2099 ; CHECK-NEXT: .byte 255 ; 0xff
2100 ; CHECK-NEXT: .byte 255 ; 0xff
2101 ; CHECK-NEXT: .byte 255 ; 0xff
2103 ; CHECK-BE-LABEL: .LCPI20_0:
2104 ; CHECK-BE-NEXT: .byte 255 // 0xff
2105 ; CHECK-BE-NEXT: .byte 255 // 0xff
2106 ; CHECK-BE-NEXT: .byte 0 // 0x0
2107 ; CHECK-BE-NEXT: .byte 255 // 0xff
2108 ; CHECK-BE-NEXT: .byte 255 // 0xff
2109 ; CHECK-BE-NEXT: .byte 1 // 0x1
2110 ; CHECK-BE-NEXT: .byte 255 // 0xff
2111 ; CHECK-BE-NEXT: .byte 255 // 0xff
2112 ; CHECK-BE-NEXT: .byte 2 // 0x2
2113 ; CHECK-BE-NEXT: .byte 255 // 0xff
2114 ; CHECK-BE-NEXT: .byte 255 // 0xff
2115 ; CHECK-BE-NEXT: .byte 3 // 0x3
2116 ; CHECK-BE-NEXT: .byte 255 // 0xff
2117 ; CHECK-BE-NEXT: .byte 255 // 0xff
2118 ; CHECK-BE-NEXT: .byte 255 // 0xff
2119 ; CHECK-BE-NEXT: .byte 255 // 0xff
2120 ; CHECK-BE-NEXT: .LCPI20_1:
2121 ; CHECK-BE-NEXT: .byte 255 // 0xff
2122 ; CHECK-BE-NEXT: .byte 255 // 0xff
2123 ; CHECK-BE-NEXT: .byte 0 // 0x0
2124 ; CHECK-BE-NEXT: .byte 255 // 0xff
2125 ; CHECK-BE-NEXT: .byte 255 // 0xff
2126 ; CHECK-BE-NEXT: .byte 1 // 0x1
2127 ; CHECK-BE-NEXT: .byte 255 // 0xff
2128 ; CHECK-BE-NEXT: .byte 255 // 0xff
2129 ; CHECK-BE-NEXT: .byte 2 // 0x2
2130 ; CHECK-BE-NEXT: .byte 255 // 0xff
2131 ; CHECK-BE-NEXT: .byte 255 // 0xff
2132 ; CHECK-BE-NEXT: .byte 3 // 0x3
2133 ; CHECK-BE-NEXT: .byte 255 // 0xff
2134 ; CHECK-BE-NEXT: .byte 255 // 0xff
2135 ; CHECK-BE-NEXT: .byte 4 // 0x4
2136 ; CHECK-BE-NEXT: .byte 255 // 0xff
2137 ; CHECK-BE-NEXT: .LCPI20_2:
2138 ; CHECK-BE-NEXT: .byte 255 // 0xff
2139 ; CHECK-BE-NEXT: .byte 5 // 0x5
2140 ; CHECK-BE-NEXT: .byte 255 // 0xff
2141 ; CHECK-BE-NEXT: .byte 255 // 0xff
2142 ; CHECK-BE-NEXT: .byte 6 // 0x6
2143 ; CHECK-BE-NEXT: .byte 255 // 0xff
2144 ; CHECK-BE-NEXT: .byte 255 // 0xff
2145 ; CHECK-BE-NEXT: .byte 7 // 0x7
2146 ; CHECK-BE-NEXT: .byte 255 // 0xff
2147 ; CHECK-BE-NEXT: .byte 255 // 0xff
2148 ; CHECK-BE-NEXT: .byte 8 // 0x8
2149 ; CHECK-BE-NEXT: .byte 255 // 0xff
2150 ; CHECK-BE-NEXT: .byte 255 // 0xff
2151 ; CHECK-BE-NEXT: .byte 9 // 0x9
2152 ; CHECK-BE-NEXT: .byte 255 // 0xff
2153 ; CHECK-BE-NEXT: .byte 255 // 0xff
2154 ; CHECK-BE-NEXT: .LCPI20_3:
2155 ; CHECK-BE-NEXT: .byte 10 // 0xa
2156 ; CHECK-BE-NEXT: .byte 255 // 0xff
2157 ; CHECK-BE-NEXT: .byte 255 // 0xff
2158 ; CHECK-BE-NEXT: .byte 11 // 0xb
2159 ; CHECK-BE-NEXT: .byte 255 // 0xff
2160 ; CHECK-BE-NEXT: .byte 255 // 0xff
2161 ; CHECK-BE-NEXT: .byte 12 // 0xc
2162 ; CHECK-BE-NEXT: .byte 255 // 0xff
2163 ; CHECK-BE-NEXT: .byte 255 // 0xff
2164 ; CHECK-BE-NEXT: .byte 13 // 0xd
2165 ; CHECK-BE-NEXT: .byte 255 // 0xff
2166 ; CHECK-BE-NEXT: .byte 255 // 0xff
2167 ; CHECK-BE-NEXT: .byte 14 // 0xe
2168 ; CHECK-BE-NEXT: .byte 255 // 0xff
2169 ; CHECK-BE-NEXT: .byte 255 // 0xff
2170 ; CHECK-BE-NEXT: .byte 15 // 0xf
2172 define void @zext_v20i8_to_v20i24_in_loop(ptr %src, ptr %dst) {
2173 ; CHECK-LABEL: zext_v20i8_to_v20i24_in_loop:
2174 ; CHECK: ; %bb.0: ; %entry
2175 ; CHECK-NEXT: Lloh22:
2176 ; CHECK-NEXT: adrp x8, lCPI20_0@PAGE
2177 ; CHECK-NEXT: Lloh23:
2178 ; CHECK-NEXT: adrp x9, lCPI20_1@PAGE
2179 ; CHECK-NEXT: Lloh24:
2180 ; CHECK-NEXT: adrp x10, lCPI20_2@PAGE
2181 ; CHECK-NEXT: Lloh25:
2182 ; CHECK-NEXT: ldr q0, [x8, lCPI20_0@PAGEOFF]
2183 ; CHECK-NEXT: Lloh26:
2184 ; CHECK-NEXT: adrp x8, lCPI20_3@PAGE
2185 ; CHECK-NEXT: Lloh27:
2186 ; CHECK-NEXT: ldr q1, [x9, lCPI20_1@PAGEOFF]
2187 ; CHECK-NEXT: Lloh28:
2188 ; CHECK-NEXT: ldr q2, [x10, lCPI20_2@PAGEOFF]
2189 ; CHECK-NEXT: Lloh29:
2190 ; CHECK-NEXT: ldr q3, [x8, lCPI20_3@PAGEOFF]
2191 ; CHECK-NEXT: mov x8, xzr
2192 ; CHECK-NEXT: LBB20_1: ; %loop
2193 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
2194 ; CHECK-NEXT: add x9, x0, x8
2195 ; CHECK-NEXT: add x8, x8, #16
2196 ; CHECK-NEXT: ldp q5, q4, [x9]
2197 ; CHECK-NEXT: add x9, x1, #56
2198 ; CHECK-NEXT: cmp x8, #128
2199 ; CHECK-NEXT: tbl.16b v4, { v4 }, v3
2200 ; CHECK-NEXT: tbl.16b v6, { v5 }, v2
2201 ; CHECK-NEXT: tbl.16b v7, { v5 }, v1
2202 ; CHECK-NEXT: tbl.16b v5, { v5 }, v0
2203 ; CHECK-NEXT: stp q7, q6, [x1, #16]
2204 ; CHECK-NEXT: str q5, [x1]
2205 ; CHECK-NEXT: str d4, [x1, #48]
2206 ; CHECK-NEXT: add x1, x1, #64
2207 ; CHECK-NEXT: st1.s { v4 }[2], [x9]
2208 ; CHECK-NEXT: b.ne LBB20_1
2209 ; CHECK-NEXT: ; %bb.2: ; %exit
2210 ; CHECK-NEXT: ret
2211 ; CHECK-NEXT: .loh AdrpLdr Lloh26, Lloh29
2212 ; CHECK-NEXT: .loh AdrpLdr Lloh24, Lloh28
2213 ; CHECK-NEXT: .loh AdrpLdr Lloh23, Lloh27
2214 ; CHECK-NEXT: .loh AdrpAdrp Lloh22, Lloh26
2215 ; CHECK-NEXT: .loh AdrpLdr Lloh22, Lloh25
2216 ;
2217 ; CHECK-BE-LABEL: zext_v20i8_to_v20i24_in_loop:
2218 ; CHECK-BE: // %bb.0: // %entry
2219 ; CHECK-BE-NEXT: adrp x8, .LCPI20_0
2220 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI20_0
2221 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]
2222 ; CHECK-BE-NEXT: adrp x8, .LCPI20_1
2223 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI20_1
2224 ; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
2225 ; CHECK-BE-NEXT: adrp x8, .LCPI20_2
2226 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI20_2
2227 ; CHECK-BE-NEXT: ld1 { v2.16b }, [x8]
2228 ; CHECK-BE-NEXT: adrp x8, .LCPI20_3
2229 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI20_3
2230 ; CHECK-BE-NEXT: ld1 { v3.16b }, [x8]
2231 ; CHECK-BE-NEXT: mov x8, xzr
2232 ; CHECK-BE-NEXT: .LBB20_1: // %loop
2233 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
2234 ; CHECK-BE-NEXT: add x9, x0, x8
2235 ; CHECK-BE-NEXT: add x8, x8, #16
2236 ; CHECK-BE-NEXT: add x10, x9, #16
2237 ; CHECK-BE-NEXT: ld1 { v5.16b }, [x9]
2238 ; CHECK-BE-NEXT: add x9, x1, #32
2239 ; CHECK-BE-NEXT: ld1 { v4.16b }, [x10]
2240 ; CHECK-BE-NEXT: cmp x8, #128
2241 ; CHECK-BE-NEXT: tbl v6.16b, { v5.16b }, v3.16b
2242 ; CHECK-BE-NEXT: tbl v7.16b, { v5.16b }, v2.16b
2243 ; CHECK-BE-NEXT: tbl v5.16b, { v5.16b }, v1.16b
2244 ; CHECK-BE-NEXT: tbl v4.16b, { v4.16b }, v0.16b
2245 ; CHECK-BE-NEXT: st1 { v6.16b }, [x9]
2246 ; CHECK-BE-NEXT: add x9, x1, #16
2247 ; CHECK-BE-NEXT: rev32 v16.16b, v4.16b
2248 ; CHECK-BE-NEXT: rev64 v4.16b, v4.16b
2249 ; CHECK-BE-NEXT: st1 { v7.16b }, [x9]
2250 ; CHECK-BE-NEXT: add x9, x1, #56
2251 ; CHECK-BE-NEXT: st1 { v5.16b }, [x1]
2252 ; CHECK-BE-NEXT: str d4, [x1, #48]
2253 ; CHECK-BE-NEXT: add x1, x1, #64
2254 ; CHECK-BE-NEXT: st1 { v16.s }[2], [x9]
2255 ; CHECK-BE-NEXT: b.ne .LBB20_1
2256 ; CHECK-BE-NEXT: // %bb.2: // %exit
2257 ; CHECK-BE-NEXT: ret
2259 entry:
2260 br label %loop
2262 loop:
2263 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
2264 %src.gep = getelementptr i8, ptr %src, i64 %iv
2265 %load = load <20 x i8>, ptr %src.gep
2266 %ext = zext <20 x i8> %load to <20 x i24>
2267 %dst.gep = getelementptr i24, ptr %dst, i64 %iv
2268 store <20 x i24> %ext, ptr %dst.gep
2269 %iv.next = add nuw i64 %iv, 16
2270 %ec = icmp eq i64 %iv.next, 128
2271 br i1 %ec, label %exit, label %loop
2273 exit:
2274 ret void
2275 }
2277 ; CHECK-LABEL: lCPI21_0:
2278 ; CHECK-NEXT: .byte 0 ; 0x0
2279 ; CHECK-NEXT: .byte 255 ; 0xff
2280 ; CHECK-NEXT: .byte 255 ; 0xff
2281 ; CHECK-NEXT: .byte 255 ; 0xff
2282 ; CHECK-NEXT: .byte 255 ; 0xff
2283 ; CHECK-NEXT: .byte 255 ; 0xff
2284 ; CHECK-NEXT: .byte 1 ; 0x1
2285 ; CHECK-NEXT: .byte 255 ; 0xff
2286 ; CHECK-NEXT: .byte 255 ; 0xff
2287 ; CHECK-NEXT: .byte 255 ; 0xff
2288 ; CHECK-NEXT: .byte 255 ; 0xff
2289 ; CHECK-NEXT: .byte 255 ; 0xff
2290 ; CHECK-NEXT: .byte 2 ; 0x2
2291 ; CHECK-NEXT: .byte 255 ; 0xff
2292 ; CHECK-NEXT: .byte 255 ; 0xff
2293 ; CHECK-NEXT: .byte 255 ; 0xff
2294 ; CHECK-NEXT: lCPI21_1:
2295 ; CHECK-NEXT: .byte 255 ; 0xff
2296 ; CHECK-NEXT: .byte 255 ; 0xff
2297 ; CHECK-NEXT: .byte 3 ; 0x3
2298 ; CHECK-NEXT: .byte 255 ; 0xff
2299 ; CHECK-NEXT: .byte 255 ; 0xff
2300 ; CHECK-NEXT: .byte 255 ; 0xff
2301 ; CHECK-NEXT: .byte 255 ; 0xff
2302 ; CHECK-NEXT: .byte 255 ; 0xff
2303 ; CHECK-NEXT: .byte 4 ; 0x4
2304 ; CHECK-NEXT: .byte 255 ; 0xff
2305 ; CHECK-NEXT: .byte 255 ; 0xff
2306 ; CHECK-NEXT: .byte 255 ; 0xff
2307 ; CHECK-NEXT: .byte 255 ; 0xff
2308 ; CHECK-NEXT: .byte 255 ; 0xff
2309 ; CHECK-NEXT: .byte 5 ; 0x5
2310 ; CHECK-NEXT: .byte 255 ; 0xff
2311 ; CHECK-NEXT: lCPI21_2:
2312 ; CHECK-NEXT: .byte 255 ; 0xff
2313 ; CHECK-NEXT: .byte 255 ; 0xff
2314 ; CHECK-NEXT: .byte 255 ; 0xff
2315 ; CHECK-NEXT: .byte 255 ; 0xff
2316 ; CHECK-NEXT: .byte 6 ; 0x6
2317 ; CHECK-NEXT: .byte 255 ; 0xff
2318 ; CHECK-NEXT: .byte 255 ; 0xff
2319 ; CHECK-NEXT: .byte 255 ; 0xff
2320 ; CHECK-NEXT: .byte 255 ; 0xff
2321 ; CHECK-NEXT: .byte 255 ; 0xff
2322 ; CHECK-NEXT: .byte 7 ; 0x7
2323 ; CHECK-NEXT: .byte 255 ; 0xff
2324 ; CHECK-NEXT: .byte 255 ; 0xff
2325 ; CHECK-NEXT: .byte 255 ; 0xff
2326 ; CHECK-NEXT: .byte 255 ; 0xff
2327 ; CHECK-NEXT: .byte 255 ; 0xff
2328 ; CHECK-NEXT: lCPI21_3:
2329 ; CHECK-NEXT: .byte 8 ; 0x8
2330 ; CHECK-NEXT: .byte 255 ; 0xff
2331 ; CHECK-NEXT: .byte 255 ; 0xff
2332 ; CHECK-NEXT: .byte 255 ; 0xff
2333 ; CHECK-NEXT: .byte 255 ; 0xff
2334 ; CHECK-NEXT: .byte 255 ; 0xff
2335 ; CHECK-NEXT: .byte 9 ; 0x9
2336 ; CHECK-NEXT: .byte 255 ; 0xff
2337 ; CHECK-NEXT: .byte 255 ; 0xff
2338 ; CHECK-NEXT: .byte 255 ; 0xff
2339 ; CHECK-NEXT: .byte 255 ; 0xff
2340 ; CHECK-NEXT: .byte 255 ; 0xff
2341 ; CHECK-NEXT: .byte 10 ; 0xa
2342 ; CHECK-NEXT: .byte 255 ; 0xff
2343 ; CHECK-NEXT: .byte 255 ; 0xff
2344 ; CHECK-NEXT: .byte 255 ; 0xff
2345 ; CHECK-NEXT: lCPI21_4:
2346 ; CHECK-NEXT: .byte 255 ; 0xff
2347 ; CHECK-NEXT: .byte 255 ; 0xff
2348 ; CHECK-NEXT: .byte 11 ; 0xb
2349 ; CHECK-NEXT: .byte 255 ; 0xff
2350 ; CHECK-NEXT: .byte 255 ; 0xff
2351 ; CHECK-NEXT: .byte 255 ; 0xff
2352 ; CHECK-NEXT: .byte 255 ; 0xff
2353 ; CHECK-NEXT: .byte 255 ; 0xff
2354 ; CHECK-NEXT: .byte 12 ; 0xc
2355 ; CHECK-NEXT: .byte 255 ; 0xff
2356 ; CHECK-NEXT: .byte 255 ; 0xff
2357 ; CHECK-NEXT: .byte 255 ; 0xff
2358 ; CHECK-NEXT: .byte 255 ; 0xff
2359 ; CHECK-NEXT: .byte 255 ; 0xff
2360 ; CHECK-NEXT: .byte 13 ; 0xd
2361 ; CHECK-NEXT: .byte 255 ; 0xff
2362 ; CHECK-NEXT: lCPI21_5:
2363 ; CHECK-NEXT: .byte 255 ; 0xff
2364 ; CHECK-NEXT: .byte 255 ; 0xff
2365 ; CHECK-NEXT: .byte 255 ; 0xff
2366 ; CHECK-NEXT: .byte 255 ; 0xff
2367 ; CHECK-NEXT: .byte 14 ; 0xe
2368 ; CHECK-NEXT: .byte 255 ; 0xff
2369 ; CHECK-NEXT: .byte 255 ; 0xff
2370 ; CHECK-NEXT: .byte 255 ; 0xff
2371 ; CHECK-NEXT: .byte 255 ; 0xff
2372 ; CHECK-NEXT: .byte 255 ; 0xff
2373 ; CHECK-NEXT: .byte 15 ; 0xf
2374 ; CHECK-NEXT: .byte 255 ; 0xff
2375 ; CHECK-NEXT: .byte 255 ; 0xff
2376 ; CHECK-NEXT: .byte 255 ; 0xff
2377 ; CHECK-NEXT: .byte 255 ; 0xff
2378 ; CHECK-NEXT: .byte 255 ; 0xff
2380 ; CHECK-BE-LABEL: .LCPI21_0:
2381 ; CHECK-BE-NEXT: .byte 255 // 0xff
2382 ; CHECK-BE-NEXT: .byte 255 // 0xff
2383 ; CHECK-BE-NEXT: .byte 255 // 0xff
2384 ; CHECK-BE-NEXT: .byte 5 // 0x5
2385 ; CHECK-BE-NEXT: .byte 255 // 0xff
2386 ; CHECK-BE-NEXT: .byte 255 // 0xff
2387 ; CHECK-BE-NEXT: .byte 255 // 0xff
2388 ; CHECK-BE-NEXT: .byte 255 // 0xff
2389 ; CHECK-BE-NEXT: .byte 255 // 0xff
2390 ; CHECK-BE-NEXT: .byte 6 // 0x6
2391 ; CHECK-BE-NEXT: .byte 255 // 0xff
2392 ; CHECK-BE-NEXT: .byte 255 // 0xff
2393 ; CHECK-BE-NEXT: .byte 255 // 0xff
2394 ; CHECK-BE-NEXT: .byte 255 // 0xff
2395 ; CHECK-BE-NEXT: .byte 255 // 0xff
2396 ; CHECK-BE-NEXT: .byte 255 // 0xff
2397 ; CHECK-BE-NEXT: .LCPI21_1:
2398 ; CHECK-BE-NEXT: .byte 255 // 0xff
2399 ; CHECK-BE-NEXT: .byte 255 // 0xff
2400 ; CHECK-BE-NEXT: .byte 255 // 0xff
2401 ; CHECK-BE-NEXT: .byte 255 // 0xff
2402 ; CHECK-BE-NEXT: .byte 255 // 0xff
2403 ; CHECK-BE-NEXT: .byte 0 // 0x0
2404 ; CHECK-BE-NEXT: .byte 255 // 0xff
2405 ; CHECK-BE-NEXT: .byte 255 // 0xff
2406 ; CHECK-BE-NEXT: .byte 255 // 0xff
2407 ; CHECK-BE-NEXT: .byte 255 // 0xff
2408 ; CHECK-BE-NEXT: .byte 255 // 0xff
2409 ; CHECK-BE-NEXT: .byte 1 // 0x1
2410 ; CHECK-BE-NEXT: .byte 255 // 0xff
2411 ; CHECK-BE-NEXT: .byte 255 // 0xff
2412 ; CHECK-BE-NEXT: .byte 255 // 0xff
2413 ; CHECK-BE-NEXT: .byte 255 // 0xff
2414 ; CHECK-BE-NEXT: .LCPI21_2:
2415 ; CHECK-BE-NEXT: .byte 255 // 0xff
2416 ; CHECK-BE-NEXT: .byte 2 // 0x2
2417 ; CHECK-BE-NEXT: .byte 255 // 0xff
2418 ; CHECK-BE-NEXT: .byte 255 // 0xff
2419 ; CHECK-BE-NEXT: .byte 255 // 0xff
2420 ; CHECK-BE-NEXT: .byte 255 // 0xff
2421 ; CHECK-BE-NEXT: .byte 255 // 0xff
2422 ; CHECK-BE-NEXT: .byte 3 // 0x3
2423 ; CHECK-BE-NEXT: .byte 255 // 0xff
2424 ; CHECK-BE-NEXT: .byte 255 // 0xff
2425 ; CHECK-BE-NEXT: .byte 255 // 0xff
2426 ; CHECK-BE-NEXT: .byte 255 // 0xff
2427 ; CHECK-BE-NEXT: .byte 255 // 0xff
2428 ; CHECK-BE-NEXT: .byte 4 // 0x4
2429 ; CHECK-BE-NEXT: .byte 255 // 0xff
2430 ; CHECK-BE-NEXT: .byte 255 // 0xff
2431 ; CHECK-BE-NEXT: .LCPI21_3:
2432 ; CHECK-BE-NEXT: .byte 255 // 0xff
2433 ; CHECK-BE-NEXT: .byte 255 // 0xff
2434 ; CHECK-BE-NEXT: .byte 255 // 0xff
2435 ; CHECK-BE-NEXT: .byte 5 // 0x5
2436 ; CHECK-BE-NEXT: .byte 255 // 0xff
2437 ; CHECK-BE-NEXT: .byte 255 // 0xff
2438 ; CHECK-BE-NEXT: .byte 255 // 0xff
2439 ; CHECK-BE-NEXT: .byte 255 // 0xff
2440 ; CHECK-BE-NEXT: .byte 255 // 0xff
2441 ; CHECK-BE-NEXT: .byte 6 // 0x6
2442 ; CHECK-BE-NEXT: .byte 255 // 0xff
2443 ; CHECK-BE-NEXT: .byte 255 // 0xff
2444 ; CHECK-BE-NEXT: .byte 255 // 0xff
2445 ; CHECK-BE-NEXT: .byte 255 // 0xff
2446 ; CHECK-BE-NEXT: .byte 255 // 0xff
2447 ; CHECK-BE-NEXT: .byte 7 // 0x7
2448 ; CHECK-BE-NEXT: .LCPI21_4:
2449 ; CHECK-BE-NEXT: .byte 255 // 0xff
2450 ; CHECK-BE-NEXT: .byte 255 // 0xff
2451 ; CHECK-BE-NEXT: .byte 255 // 0xff
2452 ; CHECK-BE-NEXT: .byte 255 // 0xff
2453 ; CHECK-BE-NEXT: .byte 255 // 0xff
2454 ; CHECK-BE-NEXT: .byte 8 // 0x8
2455 ; CHECK-BE-NEXT: .byte 255 // 0xff
2456 ; CHECK-BE-NEXT: .byte 255 // 0xff
2457 ; CHECK-BE-NEXT: .byte 255 // 0xff
2458 ; CHECK-BE-NEXT: .byte 255 // 0xff
2459 ; CHECK-BE-NEXT: .byte 255 // 0xff
2460 ; CHECK-BE-NEXT: .byte 9 // 0x9
2461 ; CHECK-BE-NEXT: .byte 255 // 0xff
2462 ; CHECK-BE-NEXT: .byte 255 // 0xff
2463 ; CHECK-BE-NEXT: .byte 255 // 0xff
2464 ; CHECK-BE-NEXT: .byte 255 // 0xff
2465 ; CHECK-BE-NEXT: .LCPI21_5:
2466 ; CHECK-BE-NEXT: .byte 255 // 0xff
2467 ; CHECK-BE-NEXT: .byte 10 // 0xa
2468 ; CHECK-BE-NEXT: .byte 255 // 0xff
2469 ; CHECK-BE-NEXT: .byte 255 // 0xff
2470 ; CHECK-BE-NEXT: .byte 255 // 0xff
2471 ; CHECK-BE-NEXT: .byte 255 // 0xff
2472 ; CHECK-BE-NEXT: .byte 255 // 0xff
2473 ; CHECK-BE-NEXT: .byte 11 // 0xb
2474 ; CHECK-BE-NEXT: .byte 255 // 0xff
2475 ; CHECK-BE-NEXT: .byte 255 // 0xff
2476 ; CHECK-BE-NEXT: .byte 255 // 0xff
2477 ; CHECK-BE-NEXT: .byte 255 // 0xff
2478 ; CHECK-BE-NEXT: .byte 255 // 0xff
2479 ; CHECK-BE-NEXT: .byte 12 // 0xc
2480 ; CHECK-BE-NEXT: .byte 255 // 0xff
2481 ; CHECK-BE-NEXT: .byte 255 // 0xff
2482 ; CHECK-BE-NEXT: .LCPI21_6:
2483 ; CHECK-BE-NEXT: .byte 255 // 0xff
2484 ; CHECK-BE-NEXT: .byte 255 // 0xff
2485 ; CHECK-BE-NEXT: .byte 255 // 0xff
2486 ; CHECK-BE-NEXT: .byte 13 // 0xd
2487 ; CHECK-BE-NEXT: .byte 255 // 0xff
2488 ; CHECK-BE-NEXT: .byte 255 // 0xff
2489 ; CHECK-BE-NEXT: .byte 255 // 0xff
2490 ; CHECK-BE-NEXT: .byte 255 // 0xff
2491 ; CHECK-BE-NEXT: .byte 255 // 0xff
2492 ; CHECK-BE-NEXT: .byte 14 // 0xe
2493 ; CHECK-BE-NEXT: .byte 255 // 0xff
2494 ; CHECK-BE-NEXT: .byte 255 // 0xff
2495 ; CHECK-BE-NEXT: .byte 255 // 0xff
2496 ; CHECK-BE-NEXT: .byte 255 // 0xff
2497 ; CHECK-BE-NEXT: .byte 255 // 0xff
2498 ; CHECK-BE-NEXT: .byte 15 // 0xf
2500 define void @zext_v23i8_to_v23i48_in_loop(ptr %src, ptr %dst) {
2501 ; CHECK-LABEL: zext_v23i8_to_v23i48_in_loop:
2502 ; CHECK: ; %bb.0: ; %entry
2503 ; CHECK-NEXT: Lloh30:
2504 ; CHECK-NEXT: adrp x8, lCPI21_0@PAGE
2505 ; CHECK-NEXT: Lloh31:
2506 ; CHECK-NEXT: adrp x9, lCPI21_1@PAGE
2507 ; CHECK-NEXT: Lloh32:
2508 ; CHECK-NEXT: adrp x10, lCPI21_2@PAGE
2509 ; CHECK-NEXT: Lloh33:
2510 ; CHECK-NEXT: ldr q0, [x8, lCPI21_0@PAGEOFF]
2511 ; CHECK-NEXT: Lloh34:
2512 ; CHECK-NEXT: ldr q1, [x9, lCPI21_1@PAGEOFF]
2513 ; CHECK-NEXT: Lloh35:
2514 ; CHECK-NEXT: ldr q2, [x10, lCPI21_2@PAGEOFF]
2515 ; CHECK-NEXT: Lloh36:
2516 ; CHECK-NEXT: adrp x8, lCPI21_3@PAGE
2517 ; CHECK-NEXT: Lloh37:
2518 ; CHECK-NEXT: adrp x9, lCPI21_4@PAGE
2519 ; CHECK-NEXT: Lloh38:
2520 ; CHECK-NEXT: adrp x10, lCPI21_5@PAGE
2521 ; CHECK-NEXT: Lloh39:
2522 ; CHECK-NEXT: ldr q3, [x8, lCPI21_3@PAGEOFF]
2523 ; CHECK-NEXT: Lloh40:
2524 ; CHECK-NEXT: ldr q4, [x9, lCPI21_4@PAGEOFF]
2525 ; CHECK-NEXT: Lloh41:
2526 ; CHECK-NEXT: ldr q5, [x10, lCPI21_5@PAGEOFF]
2527 ; CHECK-NEXT: mov x8, xzr
2528 ; CHECK-NEXT: LBB21_1: ; %loop
2529 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
2530 ; CHECK-NEXT: add x9, x0, x8
2531 ; CHECK-NEXT: movi.2d v19, #0000000000000000
2532 ; CHECK-NEXT: add x8, x8, #16
2533 ; CHECK-NEXT: ldp q7, q6, [x9]
2534 ; CHECK-NEXT: cmp x8, #128
2535 ; CHECK-NEXT: strh wzr, [x1, #136]
2536 ; CHECK-NEXT: tbl.16b v16, { v6 }, v1
2537 ; CHECK-NEXT: tbl.16b v17, { v6 }, v0
2538 ; CHECK-NEXT: mov.b v19[4], v6[6]
2539 ; CHECK-NEXT: tbl.16b v18, { v7 }, v5
2540 ; CHECK-NEXT: tbl.16b v20, { v7 }, v4
2541 ; CHECK-NEXT: tbl.16b v21, { v7 }, v3
2542 ; CHECK-NEXT: stp q17, q16, [x1, #96]
2543 ; CHECK-NEXT: tbl.16b v16, { v7 }, v2
2544 ; CHECK-NEXT: tbl.16b v17, { v7 }, v1
2545 ; CHECK-NEXT: tbl.16b v7, { v7 }, v0
2546 ; CHECK-NEXT: fmov x9, d19
2547 ; CHECK-NEXT: stp q20, q18, [x1, #64]
2548 ; CHECK-NEXT: stp q16, q21, [x1, #32]
2549 ; CHECK-NEXT: stp q7, q17, [x1]
2550 ; CHECK-NEXT: str x9, [x1, #128]!
2551 ; CHECK-NEXT: b.ne LBB21_1
2552 ; CHECK-NEXT: ; %bb.2: ; %exit
2553 ; CHECK-NEXT: ret
2554 ; CHECK-NEXT: .loh AdrpLdr Lloh38, Lloh41
2555 ; CHECK-NEXT: .loh AdrpLdr Lloh37, Lloh40
2556 ; CHECK-NEXT: .loh AdrpLdr Lloh36, Lloh39
2557 ; CHECK-NEXT: .loh AdrpAdrp Lloh32, Lloh38
2558 ; CHECK-NEXT: .loh AdrpLdr Lloh32, Lloh35
2559 ; CHECK-NEXT: .loh AdrpAdrp Lloh31, Lloh37
2560 ; CHECK-NEXT: .loh AdrpLdr Lloh31, Lloh34
2561 ; CHECK-NEXT: .loh AdrpAdrp Lloh30, Lloh36
2562 ; CHECK-NEXT: .loh AdrpLdr Lloh30, Lloh33
2563 ;
2564 ; CHECK-BE-LABEL: zext_v23i8_to_v23i48_in_loop:
2565 ; CHECK-BE: // %bb.0: // %entry
2566 ; CHECK-BE-NEXT: adrp x8, .LCPI21_0
2567 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI21_0
2568 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]
2569 ; CHECK-BE-NEXT: adrp x8, .LCPI21_1
2570 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI21_1
2571 ; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
2572 ; CHECK-BE-NEXT: adrp x8, .LCPI21_2
2573 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI21_2
2574 ; CHECK-BE-NEXT: ld1 { v2.16b }, [x8]
2575 ; CHECK-BE-NEXT: adrp x8, .LCPI21_3
2576 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI21_3
2577 ; CHECK-BE-NEXT: ld1 { v3.16b }, [x8]
2578 ; CHECK-BE-NEXT: adrp x8, .LCPI21_4
2579 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI21_4
2580 ; CHECK-BE-NEXT: ld1 { v4.16b }, [x8]
2581 ; CHECK-BE-NEXT: adrp x8, .LCPI21_5
2582 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI21_5
2583 ; CHECK-BE-NEXT: ld1 { v5.16b }, [x8]
2584 ; CHECK-BE-NEXT: adrp x8, .LCPI21_6
2585 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI21_6
2586 ; CHECK-BE-NEXT: ld1 { v6.16b }, [x8]
2587 ; CHECK-BE-NEXT: mov x8, xzr
2588 ; CHECK-BE-NEXT: .LBB21_1: // %loop
2589 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
2590 ; CHECK-BE-NEXT: add x9, x0, x8
2591 ; CHECK-BE-NEXT: add x8, x8, #16
2592 ; CHECK-BE-NEXT: ld1 { v7.16b }, [x9]
2593 ; CHECK-BE-NEXT: add x9, x9, #16
2594 ; CHECK-BE-NEXT: cmp x8, #128
2595 ; CHECK-BE-NEXT: ld1 { v17.16b }, [x9]
2596 ; CHECK-BE-NEXT: add x9, x1, #80
2597 ; CHECK-BE-NEXT: tbl v16.16b, { v7.16b }, v6.16b
2598 ; CHECK-BE-NEXT: tbl v18.16b, { v7.16b }, v5.16b
2599 ; CHECK-BE-NEXT: tbl v19.16b, { v7.16b }, v4.16b
2600 ; CHECK-BE-NEXT: tbl v20.16b, { v7.16b }, v3.16b
2601 ; CHECK-BE-NEXT: tbl v21.16b, { v17.16b }, v0.16b
2602 ; CHECK-BE-NEXT: st1 { v16.16b }, [x9]
2603 ; CHECK-BE-NEXT: add x9, x1, #64
2604 ; CHECK-BE-NEXT: tbl v16.16b, { v7.16b }, v2.16b
2605 ; CHECK-BE-NEXT: st1 { v18.16b }, [x9]
2606 ; CHECK-BE-NEXT: add x9, x1, #48
2607 ; CHECK-BE-NEXT: tbl v18.16b, { v17.16b }, v2.16b
2608 ; CHECK-BE-NEXT: st1 { v19.16b }, [x9]
2609 ; CHECK-BE-NEXT: add x9, x1, #32
2610 ; CHECK-BE-NEXT: tbl v17.16b, { v17.16b }, v1.16b
2611 ; CHECK-BE-NEXT: st1 { v20.16b }, [x9]
2612 ; CHECK-BE-NEXT: add x9, x1, #16
2613 ; CHECK-BE-NEXT: rev64 v19.16b, v21.16b
2614 ; CHECK-BE-NEXT: st1 { v16.16b }, [x9]
2615 ; CHECK-BE-NEXT: rev16 v16.16b, v21.16b
2616 ; CHECK-BE-NEXT: add x9, x1, #112
2617 ; CHECK-BE-NEXT: st1 { v18.16b }, [x9]
2618 ; CHECK-BE-NEXT: add x9, x1, #96
2619 ; CHECK-BE-NEXT: tbl v7.16b, { v7.16b }, v1.16b
2620 ; CHECK-BE-NEXT: st1 { v17.16b }, [x9]
2621 ; CHECK-BE-NEXT: add x9, x1, #136
2622 ; CHECK-BE-NEXT: st1 { v16.h }[4], [x9]
2623 ; CHECK-BE-NEXT: fmov x9, d19
2624 ; CHECK-BE-NEXT: st1 { v7.16b }, [x1]
2625 ; CHECK-BE-NEXT: str x9, [x1, #128]!
2626 ; CHECK-BE-NEXT: b.ne .LBB21_1
2627 ; CHECK-BE-NEXT: // %bb.2: // %exit
2628 ; CHECK-BE-NEXT: ret
2629 ; CHECK : Lloh30:
2633 entry:
2634 br label %loop
2636 loop:
2637 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
2638 %src.gep = getelementptr i8, ptr %src, i64 %iv
2639 %load = load <23 x i8>, ptr %src.gep
2640 %ext = zext <23 x i8> %load to <23 x i48>
2641 %dst.gep = getelementptr i48, ptr %dst, i64 %iv
2642 store <23 x i48> %ext, ptr %dst.gep
2643 %iv.next = add nuw i64 %iv, 16
2644 %ec = icmp eq i64 %iv.next, 128
2645 br i1 %ec, label %exit, label %loop
2647 exit:
2648 ret void
2649 }
2651 define void @zext_v8i8_to_v8i33_in_loop(ptr %src, ptr %dst) {
2652 ; CHECK-LABEL: zext_v8i8_to_v8i33_in_loop:
2653 ; CHECK: ; %bb.0: ; %entry
2654 ; CHECK-NEXT: mov x8, xzr
2655 ; CHECK-NEXT: LBB22_1: ; %loop
2656 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
2657 ; CHECK-NEXT: ldr d0, [x0, x8]
2658 ; CHECK-NEXT: add x8, x8, #16
2659 ; CHECK-NEXT: strb wzr, [x1, #32]
2660 ; CHECK-NEXT: cmp x8, #128
2661 ; CHECK-NEXT: ushll.8h v0, v0, #0
2662 ; CHECK-NEXT: ushll2.4s v1, v0, #0
2663 ; CHECK-NEXT: ushll.4s v0, v0, #0
2664 ; CHECK-NEXT: ushll2.2d v2, v1, #0
2665 ; CHECK-NEXT: ushll.2d v1, v1, #0
2666 ; CHECK-NEXT: ushll2.2d v3, v0, #0
2667 ; CHECK-NEXT: ushll.2d v0, v0, #0
2668 ; CHECK-NEXT: mov.d x9, v2[1]
2669 ; CHECK-NEXT: mov.d x10, v1[1]
2670 ; CHECK-NEXT: fmov x12, d2
2671 ; CHECK-NEXT: mov.d x11, v3[1]
2672 ; CHECK-NEXT: mov.d x13, v0[1]
2673 ; CHECK-NEXT: lsl x9, x9, #39
2674 ; CHECK-NEXT: lsl x10, x10, #37
2675 ; CHECK-NEXT: lsl x11, x11, #35
2676 ; CHECK-NEXT: orr x9, x9, x12, lsl #6
2677 ; CHECK-NEXT: fmov x12, d1
2678 ; CHECK-NEXT: orr x10, x10, x12, lsl #4
2679 ; CHECK-NEXT: fmov x12, d3
2680 ; CHECK-NEXT: stp x10, x9, [x1, #16]
2681 ; CHECK-NEXT: orr x11, x11, x12, lsl #2
2682 ; CHECK-NEXT: fmov x12, d0
2683 ; CHECK-NEXT: orr x9, x12, x13, lsl #33
2684 ; CHECK-NEXT: stp x9, x11, [x1], #128
2685 ; CHECK-NEXT: b.ne LBB22_1
2686 ; CHECK-NEXT: ; %bb.2: ; %exit
2687 ; CHECK-NEXT: ret
2688 ;
2689 ; CHECK-BE-LABEL: zext_v8i8_to_v8i33_in_loop:
2690 ; CHECK-BE: // %bb.0: // %entry
2691 ; CHECK-BE-NEXT: mov x8, xzr
2692 ; CHECK-BE-NEXT: .LBB22_1: // %loop
2693 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
2694 ; CHECK-BE-NEXT: add x9, x0, x8
2695 ; CHECK-BE-NEXT: add x8, x8, #16
2696 ; CHECK-BE-NEXT: ld1 { v0.8b }, [x9]
2697 ; CHECK-BE-NEXT: cmp x8, #128
2698 ; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
2699 ; CHECK-BE-NEXT: ushll2 v1.4s, v0.8h, #0
2700 ; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
2701 ; CHECK-BE-NEXT: ushll v2.2d, v1.2s, #0
2702 ; CHECK-BE-NEXT: ushll2 v1.2d, v1.4s, #0
2703 ; CHECK-BE-NEXT: ushll2 v3.2d, v0.4s, #0
2704 ; CHECK-BE-NEXT: ushll v0.2d, v0.2s, #0
2705 ; CHECK-BE-NEXT: mov x9, v2.d[1]
2706 ; CHECK-BE-NEXT: mov x10, v1.d[1]
2707 ; CHECK-BE-NEXT: fmov x13, d1
2708 ; CHECK-BE-NEXT: mov x11, v3.d[1]
2709 ; CHECK-BE-NEXT: mov x12, v0.d[1]
2710 ; CHECK-BE-NEXT: fmov x14, d2
2711 ; CHECK-BE-NEXT: fmov x15, d3
2712 ; CHECK-BE-NEXT: lsl x9, x9, #2
2713 ; CHECK-BE-NEXT: orr x13, x10, x13, lsl #33
2714 ; CHECK-BE-NEXT: strb w10, [x1, #32]
2715 ; CHECK-BE-NEXT: lsl x11, x11, #4
2716 ; CHECK-BE-NEXT: lsl x12, x12, #6
2717 ; CHECK-BE-NEXT: orr x14, x9, x14, lsl #35
2718 ; CHECK-BE-NEXT: extr x9, x9, x13, #8
2719 ; CHECK-BE-NEXT: fmov x13, d0
2720 ; CHECK-BE-NEXT: orr x15, x11, x15, lsl #37
2721 ; CHECK-BE-NEXT: extr x10, x11, x14, #8
2722 ; CHECK-BE-NEXT: orr x11, x12, x13, lsl #39
2723 ; CHECK-BE-NEXT: extr x12, x12, x15, #8
2724 ; CHECK-BE-NEXT: stp x10, x9, [x1, #16]
2725 ; CHECK-BE-NEXT: lsr x9, x11, #8
2726 ; CHECK-BE-NEXT: stp x9, x12, [x1], #128
2727 ; CHECK-BE-NEXT: b.ne .LBB22_1
2728 ; CHECK-BE-NEXT: // %bb.2: // %exit
2729 ; CHECK-BE-NEXT: ret
2732 entry:
2733 br label %loop
2735 loop:
2736 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
2737 %src.gep = getelementptr i8, ptr %src, i64 %iv
2738 %load = load <8 x i8>, ptr %src.gep
2739 %ext = zext <8 x i8> %load to <8 x i33>
2740 %dst.gep = getelementptr i33, ptr %dst, i64 %iv
2741 store <8 x i33> %ext, ptr %dst.gep
2742 %iv.next = add nuw i64 %iv, 16
2743 %ec = icmp eq i64 %iv.next, 128
2744 br i1 %ec, label %exit, label %loop
2746 exit:
2747 ret void
2748 }
2750 ; FIXME: Widening instructions should be used instead of tbl.
2751 define i32 @test_pr62620_widening_instr(ptr %p1, ptr %p2, i64 %lx, i32 %h) {
2752 ; CHECK-LABEL: test_pr62620_widening_instr:
2753 ; CHECK: ; %bb.0: ; %entry
2754 ; CHECK-NEXT: lsl x9, x2, #4
2755 ; CHECK-NEXT: mov x8, x0
2756 ; CHECK-NEXT: mov w0, wzr
2757 ; CHECK-NEXT: ldr q0, [x8, x9]
2758 ; CHECK-NEXT: ldr q1, [x1, x9]
2759 ; CHECK-NEXT: uabdl.8h v2, v0, v1
2760 ; CHECK-NEXT: uabal2.8h v2, v0, v1
2761 ; CHECK-NEXT: uaddlv.8h s0, v2
2762 ; CHECK-NEXT: fmov w8, s0
2763 ; CHECK-NEXT: LBB23_1: ; %loop
2764 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
2765 ; CHECK-NEXT: subs w3, w3, #1
2766 ; CHECK-NEXT: add w0, w8, w0
2767 ; CHECK-NEXT: b.ne LBB23_1
2768 ; CHECK-NEXT: ; %bb.2: ; %exit
2769 ; CHECK-NEXT: ret
2770 ;
2771 ; CHECK-BE-LABEL: test_pr62620_widening_instr:
2772 ; CHECK-BE: // %bb.0: // %entry
2773 ; CHECK-BE-NEXT: lsl x9, x2, #4
2774 ; CHECK-BE-NEXT: mov x8, x0
2775 ; CHECK-BE-NEXT: mov w0, wzr
2776 ; CHECK-BE-NEXT: add x8, x8, x9
2777 ; CHECK-BE-NEXT: add x9, x1, x9
2778 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]
2779 ; CHECK-BE-NEXT: ld1 { v1.16b }, [x9]
2780 ; CHECK-BE-NEXT: uabdl v2.8h, v0.8b, v1.8b
2781 ; CHECK-BE-NEXT: uabal2 v2.8h, v0.16b, v1.16b
2782 ; CHECK-BE-NEXT: uaddlv s0, v2.8h
2783 ; CHECK-BE-NEXT: fmov w8, s0
2784 ; CHECK-BE-NEXT: .LBB23_1: // %loop
2785 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
2786 ; CHECK-BE-NEXT: subs w3, w3, #1
2787 ; CHECK-BE-NEXT: add w0, w8, w0
2788 ; CHECK-BE-NEXT: b.ne .LBB23_1
2789 ; CHECK-BE-NEXT: // %bb.2: // %exit
2790 ; CHECK-BE-NEXT: ret
2791 entry:
2792 br label %loop
2794 loop:
2795 %s0 = phi i32 [ 0, %entry ], [ %op.rdx, %loop ]
2796 %j.0261 = phi i32 [ 0, %entry ], [ %inc, %loop ]
2797 %gep.1 = getelementptr inbounds <16 x i8>, ptr %p1, i64 %lx
2798 %gep.2 = getelementptr inbounds <16 x i8>, ptr %p2, i64 %lx
2799 %l1 = load <16 x i8>, ptr %gep.1
2800 %z2 = zext <16 x i8> %l1 to <16 x i32>
2801 %l4 = load <16 x i8>, ptr %gep.2
2802 %z5 = zext <16 x i8> %l4 to <16 x i32>
2803 %sub = sub nsw <16 x i32> %z2, %z5
2804 %abs = tail call <16 x i32> @llvm.abs.v16i32(<16 x i32> %sub, i1 true)
2805 %red = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %abs)
2806 %op.rdx = add i32 %red, %s0
2807 %inc = add nuw nsw i32 %j.0261, 1
2808 %exitcond.not = icmp eq i32 %inc, %h
2809 br i1 %exitcond.not, label %exit, label %loop
2811 exit:
2812 %s1 = phi i32 [ %op.rdx, %loop ]
2813 ret i32 %s1
2814 }
2816 declare <16 x i32> @llvm.abs.v16i32(<16 x i32>, i1 immarg)
2818 declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)
2820 define i32 @test_widening_instr_mull(ptr %p1, ptr %p2, i32 %h) {
2821 ; CHECK-LABEL: test_widening_instr_mull:
2822 ; CHECK: ; %bb.0: ; %entry
2823 ; CHECK-NEXT: mov x8, x0
2824 ; CHECK-NEXT: LBB24_1: ; %loop
2825 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
2826 ; CHECK-NEXT: ldr q0, [x1], #16
2827 ; CHECK-NEXT: ldr q3, [x0]
2828 ; CHECK-NEXT: ldr q2, [x8, #16]!
2829 ; CHECK-NEXT: subs w2, w2, #1
2830 ; CHECK-NEXT: ushll2.8h v1, v0, #0
2831 ; CHECK-NEXT: ushll.8h v0, v0, #0
2832 ; CHECK-NEXT: umull2.4s v4, v2, v1
2833 ; CHECK-NEXT: umull.4s v1, v2, v1
2834 ; CHECK-NEXT: umull2.4s v2, v3, v0
2835 ; CHECK-NEXT: umull.4s v0, v3, v0
2836 ; CHECK-NEXT: stp q1, q4, [x0, #32]
2837 ; CHECK-NEXT: str q0, [x0]
2838 ; CHECK-NEXT: mov x0, x8
2839 ; CHECK-NEXT: str q2, [x8]
2840 ; CHECK-NEXT: b.ne LBB24_1
2841 ; CHECK-NEXT: ; %bb.2: ; %exit
2842 ; CHECK-NEXT: mov w0, wzr
2843 ; CHECK-NEXT: ret
2844 ;
2845 ; CHECK-BE-LABEL: test_widening_instr_mull:
2846 ; CHECK-BE: // %bb.0: // %entry
2847 ; CHECK-BE-NEXT: .LBB24_1: // %loop
2848 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
2849 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x1]
2850 ; CHECK-BE-NEXT: ld1 { v1.8h }, [x0]
2851 ; CHECK-BE-NEXT: add x8, x0, #16
2852 ; CHECK-BE-NEXT: ld1 { v3.8h }, [x8]
2853 ; CHECK-BE-NEXT: add x9, x0, #48
2854 ; CHECK-BE-NEXT: add x10, x0, #32
2855 ; CHECK-BE-NEXT: subs w2, w2, #1
2856 ; CHECK-BE-NEXT: add x1, x1, #16
2857 ; CHECK-BE-NEXT: ushll v2.8h, v0.8b, #0
2858 ; CHECK-BE-NEXT: ushll2 v0.8h, v0.16b, #0
2859 ; CHECK-BE-NEXT: umull v4.4s, v1.4h, v2.4h
2860 ; CHECK-BE-NEXT: umull2 v5.4s, v3.8h, v0.8h
2861 ; CHECK-BE-NEXT: umull v0.4s, v3.4h, v0.4h
2862 ; CHECK-BE-NEXT: umull2 v1.4s, v1.8h, v2.8h
2863 ; CHECK-BE-NEXT: st1 { v4.4s }, [x0]
2864 ; CHECK-BE-NEXT: mov x0, x8
2865 ; CHECK-BE-NEXT: st1 { v5.4s }, [x9]
2866 ; CHECK-BE-NEXT: st1 { v0.4s }, [x10]
2867 ; CHECK-BE-NEXT: st1 { v1.4s }, [x8]
2868 ; CHECK-BE-NEXT: b.ne .LBB24_1
2869 ; CHECK-BE-NEXT: // %bb.2: // %exit
2870 ; CHECK-BE-NEXT: mov w0, wzr
2871 ; CHECK-BE-NEXT: ret
2872 entry:
2873 br label %loop
2875 loop:
2876 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
2877 %gep.1 = getelementptr inbounds <16 x i8>, ptr %p1, i32 %iv
2878 %gep.2 = getelementptr inbounds <16 x i8>, ptr %p2, i32 %iv
2879 %l1 = load <16 x i16>, ptr %gep.1
2880 %z2 = zext <16 x i16> %l1 to <16 x i32>
2881 %l4 = load <16 x i8>, ptr %gep.2
2882 %z5 = zext <16 x i8> %l4 to <16 x i32>
2883 %mul = mul <16 x i32> %z2, %z5
2884 store <16 x i32> %mul, ptr %gep.1
2885 %iv.next= add nuw nsw i32 %iv, 1
2886 %exitcond.not = icmp eq i32 %iv.next, %h
2887 br i1 %exitcond.not, label %exit, label %loop
2889 exit:
2890 ret i32 0
2891 }
2893 define i32 @test_widening_instr_mull_64(ptr %p1, ptr %p2, i32 %h) {
2894 ; CHECK-LABEL: test_widening_instr_mull_64:
2895 ; CHECK: ; %bb.0: ; %entry
2896 ; CHECK-NEXT: Lloh42:
2897 ; CHECK-NEXT: adrp x8, lCPI25_0@PAGE
2898 ; CHECK-NEXT: Lloh43:
2899 ; CHECK-NEXT: adrp x9, lCPI25_1@PAGE
2900 ; CHECK-NEXT: Lloh44:
2901 ; CHECK-NEXT: adrp x10, lCPI25_3@PAGE
2902 ; CHECK-NEXT: Lloh45:
2903 ; CHECK-NEXT: ldr q0, [x8, lCPI25_0@PAGEOFF]
2904 ; CHECK-NEXT: Lloh46:
2905 ; CHECK-NEXT: adrp x8, lCPI25_2@PAGE
2906 ; CHECK-NEXT: Lloh47:
2907 ; CHECK-NEXT: ldr q1, [x9, lCPI25_1@PAGEOFF]
2908 ; CHECK-NEXT: Lloh48:
2909 ; CHECK-NEXT: ldr q2, [x8, lCPI25_2@PAGEOFF]
2910 ; CHECK-NEXT: Lloh49:
2911 ; CHECK-NEXT: ldr q3, [x10, lCPI25_3@PAGEOFF]
2912 ; CHECK-NEXT: mov x8, x1
2913 ; CHECK-NEXT: LBB25_1: ; %loop
2914 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
2915 ; CHECK-NEXT: ldr q4, [x0]
2916 ; CHECK-NEXT: ldp q16, q7, [x1, #32]
2917 ; CHECK-NEXT: ldr q18, [x8, #16]!
2918 ; CHECK-NEXT: subs w2, w2, #1
2919 ; CHECK-NEXT: tbl.16b v5, { v4 }, v3
2920 ; CHECK-NEXT: tbl.16b v6, { v4 }, v0
2921 ; CHECK-NEXT: tbl.16b v17, { v4 }, v2
2922 ; CHECK-NEXT: tbl.16b v4, { v4 }, v1
2923 ; CHECK-NEXT: umull2.2d v19, v5, v7
2924 ; CHECK-NEXT: umull.2d v5, v5, v7
2925 ; CHECK-NEXT: ldr q7, [x1]
2926 ; CHECK-NEXT: umull2.2d v20, v6, v16
2927 ; CHECK-NEXT: umull2.2d v21, v17, v18
2928 ; CHECK-NEXT: umull.2d v17, v17, v18
2929 ; CHECK-NEXT: umull2.2d v18, v4, v7
2930 ; CHECK-NEXT: umull.2d v4, v4, v7
2931 ; CHECK-NEXT: mov x1, x8
2932 ; CHECK-NEXT: stp q5, q19, [x0, #96]
2933 ; CHECK-NEXT: umull.2d v5, v6, v16
2934 ; CHECK-NEXT: str q20, [x0, #80]
2935 ; CHECK-NEXT: stp q4, q18, [x0]
2936 ; CHECK-NEXT: stp q17, q21, [x0, #32]
2937 ; CHECK-NEXT: str q5, [x0, #64]!
2938 ; CHECK-NEXT: b.ne LBB25_1
2939 ; CHECK-NEXT: ; %bb.2: ; %exit
2940 ; CHECK-NEXT: mov w0, wzr
2941 ; CHECK-NEXT: ret
2942 ; CHECK-NEXT: .loh AdrpLdr Lloh46, Lloh48
2943 ; CHECK-NEXT: .loh AdrpLdr Lloh44, Lloh49
2944 ; CHECK-NEXT: .loh AdrpLdr Lloh43, Lloh47
2945 ; CHECK-NEXT: .loh AdrpAdrp Lloh42, Lloh46
2946 ; CHECK-NEXT: .loh AdrpLdr Lloh42, Lloh45
2947 ;
2948 ; CHECK-BE-LABEL: test_widening_instr_mull_64:
2949 ; CHECK-BE: // %bb.0: // %entry
2950 ; CHECK-BE-NEXT: adrp x8, .LCPI25_0
2951 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI25_0
2952 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]
2953 ; CHECK-BE-NEXT: adrp x8, .LCPI25_1
2954 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI25_1
2955 ; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
2956 ; CHECK-BE-NEXT: adrp x8, .LCPI25_2
2957 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI25_2
2958 ; CHECK-BE-NEXT: ld1 { v2.16b }, [x8]
2959 ; CHECK-BE-NEXT: adrp x8, .LCPI25_3
2960 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI25_3
2961 ; CHECK-BE-NEXT: ld1 { v3.16b }, [x8]
2962 ; CHECK-BE-NEXT: .LBB25_1: // %loop
2963 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
2964 ; CHECK-BE-NEXT: ld1 { v4.16b }, [x0]
2965 ; CHECK-BE-NEXT: add x9, x1, #48
2966 ; CHECK-BE-NEXT: add x8, x1, #32
2967 ; CHECK-BE-NEXT: ld1 { v18.4s }, [x9]
2968 ; CHECK-BE-NEXT: ld1 { v16.4s }, [x1]
2969 ; CHECK-BE-NEXT: add x1, x1, #16
2970 ; CHECK-BE-NEXT: ld1 { v20.4s }, [x8]
2971 ; CHECK-BE-NEXT: ld1 { v22.4s }, [x1]
2972 ; CHECK-BE-NEXT: add x8, x0, #96
2973 ; CHECK-BE-NEXT: tbl v5.16b, { v4.16b }, v3.16b
2974 ; CHECK-BE-NEXT: tbl v6.16b, { v4.16b }, v2.16b
2975 ; CHECK-BE-NEXT: tbl v7.16b, { v4.16b }, v1.16b
2976 ; CHECK-BE-NEXT: tbl v4.16b, { v4.16b }, v0.16b
2977 ; CHECK-BE-NEXT: ext v24.16b, v18.16b, v18.16b, #8
2978 ; CHECK-BE-NEXT: add x9, x0, #32
2979 ; CHECK-BE-NEXT: ext v25.16b, v20.16b, v20.16b, #8
2980 ; CHECK-BE-NEXT: add x10, x0, #16
2981 ; CHECK-BE-NEXT: subs w2, w2, #1
2982 ; CHECK-BE-NEXT: ext v17.16b, v5.16b, v5.16b, #8
2983 ; CHECK-BE-NEXT: ext v19.16b, v6.16b, v6.16b, #8
2984 ; CHECK-BE-NEXT: rev32 v5.8b, v5.8b
2985 ; CHECK-BE-NEXT: rev32 v21.8b, v7.8b
2986 ; CHECK-BE-NEXT: rev32 v23.8b, v4.8b
2987 ; CHECK-BE-NEXT: ext v7.16b, v7.16b, v7.16b, #8
2988 ; CHECK-BE-NEXT: ext v4.16b, v4.16b, v4.16b, #8
2989 ; CHECK-BE-NEXT: rev32 v6.8b, v6.8b
2990 ; CHECK-BE-NEXT: rev32 v17.8b, v17.8b
2991 ; CHECK-BE-NEXT: rev32 v19.8b, v19.8b
2992 ; CHECK-BE-NEXT: umull v5.2d, v5.2s, v18.2s
2993 ; CHECK-BE-NEXT: umull v18.2d, v21.2s, v22.2s
2994 ; CHECK-BE-NEXT: ext v21.16b, v22.16b, v22.16b, #8
2995 ; CHECK-BE-NEXT: rev32 v7.8b, v7.8b
2996 ; CHECK-BE-NEXT: umull v22.2d, v23.2s, v16.2s
2997 ; CHECK-BE-NEXT: ext v16.16b, v16.16b, v16.16b, #8
2998 ; CHECK-BE-NEXT: rev32 v4.8b, v4.8b
2999 ; CHECK-BE-NEXT: umull v17.2d, v17.2s, v24.2s
3000 ; CHECK-BE-NEXT: umull v19.2d, v19.2s, v25.2s
3001 ; CHECK-BE-NEXT: st1 { v5.2d }, [x8]
3002 ; CHECK-BE-NEXT: umull v5.2d, v6.2s, v20.2s
3003 ; CHECK-BE-NEXT: umull v6.2d, v7.2s, v21.2s
3004 ; CHECK-BE-NEXT: add x8, x0, #112
3005 ; CHECK-BE-NEXT: umull v4.2d, v4.2s, v16.2s
3006 ; CHECK-BE-NEXT: st1 { v18.2d }, [x9]
3007 ; CHECK-BE-NEXT: add x9, x0, #80
3008 ; CHECK-BE-NEXT: st1 { v22.2d }, [x0]
3009 ; CHECK-BE-NEXT: st1 { v17.2d }, [x8]
3010 ; CHECK-BE-NEXT: add x8, x0, #64
3011 ; CHECK-BE-NEXT: st1 { v19.2d }, [x9]
3012 ; CHECK-BE-NEXT: add x9, x0, #48
3013 ; CHECK-BE-NEXT: mov x0, x8
3014 ; CHECK-BE-NEXT: st1 { v5.2d }, [x8]
3015 ; CHECK-BE-NEXT: st1 { v6.2d }, [x9]
3016 ; CHECK-BE-NEXT: st1 { v4.2d }, [x10]
3017 ; CHECK-BE-NEXT: b.ne .LBB25_1
3018 ; CHECK-BE-NEXT: // %bb.2: // %exit
3019 ; CHECK-BE-NEXT: mov w0, wzr
3020 ; CHECK-BE-NEXT: ret
3021 entry:
3022 br label %loop
3024 loop:
3025 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
3026 %gep.1 = getelementptr inbounds <16 x i32>, ptr %p1, i32 %iv
3027 %gep.2 = getelementptr inbounds <16 x i8>, ptr %p2, i32 %iv
3028 %l1 = load <16 x i8>, ptr %gep.1
3029 %z2 = zext <16 x i8> %l1 to <16 x i64>
3030 %l4 = load <16 x i32>, ptr %gep.2
3031 %z5 = zext <16 x i32> %l4 to <16 x i64>
3032 %mul = mul <16 x i64> %z2, %z5
3033 store <16 x i64> %mul, ptr %gep.1
3034 %iv.next= add nuw nsw i32 %iv, 1
3035 %exitcond.not = icmp eq i32 %iv.next, %h
3036 br i1 %exitcond.not, label %exit, label %loop
3038 exit:
3039 ret i32 0
3040 }
3042 define i32 @test_widening_instr_mull_2(ptr %p1, ptr %p2, i32 %h) {
3043 ; CHECK-LABEL: test_widening_instr_mull_2:
3044 ; CHECK: ; %bb.0: ; %entry
3045 ; CHECK-NEXT: Lloh50:
3046 ; CHECK-NEXT: adrp x8, lCPI26_0@PAGE
3047 ; CHECK-NEXT: Lloh51:
3048 ; CHECK-NEXT: adrp x9, lCPI26_1@PAGE
3049 ; CHECK-NEXT: Lloh52:
3050 ; CHECK-NEXT: adrp x10, lCPI26_3@PAGE
3051 ; CHECK-NEXT: Lloh53:
3052 ; CHECK-NEXT: ldr q0, [x8, lCPI26_0@PAGEOFF]
3053 ; CHECK-NEXT: Lloh54:
3054 ; CHECK-NEXT: adrp x8, lCPI26_2@PAGE
3055 ; CHECK-NEXT: Lloh55:
3056 ; CHECK-NEXT: ldr q1, [x9, lCPI26_1@PAGEOFF]
3057 ; CHECK-NEXT: Lloh56:
3058 ; CHECK-NEXT: ldr q2, [x8, lCPI26_2@PAGEOFF]
3059 ; CHECK-NEXT: Lloh57:
3060 ; CHECK-NEXT: ldr q3, [x10, lCPI26_3@PAGEOFF]
3061 ; CHECK-NEXT: mov x8, x0
3062 ; CHECK-NEXT: LBB26_1: ; %loop
3063 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
3064 ; CHECK-NEXT: ldr q4, [x1], #16
3065 ; CHECK-NEXT: ldr q18, [x0]
3066 ; CHECK-NEXT: ldp q16, q17, [x0, #32]
3067 ; CHECK-NEXT: subs w2, w2, #1
3068 ; CHECK-NEXT: tbl.16b v5, { v4 }, v0
3069 ; CHECK-NEXT: tbl.16b v6, { v4 }, v1
3070 ; CHECK-NEXT: tbl.16b v7, { v4 }, v2
3071 ; CHECK-NEXT: tbl.16b v4, { v4 }, v3
3072 ; CHECK-NEXT: mul.4s v5, v16, v5
3073 ; CHECK-NEXT: ldr q16, [x8, #16]!
3074 ; CHECK-NEXT: mul.4s v6, v17, v6
3075 ; CHECK-NEXT: mul.4s v7, v18, v7
3076 ; CHECK-NEXT: mul.4s v4, v16, v4
3077 ; CHECK-NEXT: stp q5, q6, [x0, #32]
3078 ; CHECK-NEXT: str q7, [x0]
3079 ; CHECK-NEXT: mov x0, x8
3080 ; CHECK-NEXT: str q4, [x8]
3081 ; CHECK-NEXT: b.ne LBB26_1
3082 ; CHECK-NEXT: ; %bb.2: ; %exit
3083 ; CHECK-NEXT: mov w0, wzr
3084 ; CHECK-NEXT: ret
3085 ; CHECK-NEXT: .loh AdrpLdr Lloh54, Lloh56
3086 ; CHECK-NEXT: .loh AdrpLdr Lloh52, Lloh57
3087 ; CHECK-NEXT: .loh AdrpLdr Lloh51, Lloh55
3088 ; CHECK-NEXT: .loh AdrpAdrp Lloh50, Lloh54
3089 ; CHECK-NEXT: .loh AdrpLdr Lloh50, Lloh53
3090 ;
3091 ; CHECK-BE-LABEL: test_widening_instr_mull_2:
3092 ; CHECK-BE: // %bb.0: // %entry
3093 ; CHECK-BE-NEXT: adrp x8, .LCPI26_0
3094 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI26_0
3095 ; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]
3096 ; CHECK-BE-NEXT: adrp x8, .LCPI26_1
3097 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI26_1
3098 ; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
3099 ; CHECK-BE-NEXT: adrp x8, .LCPI26_2
3100 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI26_2
3101 ; CHECK-BE-NEXT: ld1 { v2.16b }, [x8]
3102 ; CHECK-BE-NEXT: adrp x8, .LCPI26_3
3103 ; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI26_3
3104 ; CHECK-BE-NEXT: ld1 { v3.16b }, [x8]
3105 ; CHECK-BE-NEXT: .LBB26_1: // %loop
3106 ; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
3107 ; CHECK-BE-NEXT: ld1 { v4.16b }, [x1]
3108 ; CHECK-BE-NEXT: add x8, x0, #32
3109 ; CHECK-BE-NEXT: ld1 { v16.4s }, [x0]
3110 ; CHECK-BE-NEXT: add x9, x0, #48
3111 ; CHECK-BE-NEXT: add x10, x0, #16
3112 ; CHECK-BE-NEXT: ld1 { v17.4s }, [x8]
3113 ; CHECK-BE-NEXT: ld1 { v18.4s }, [x9]
3114 ; CHECK-BE-NEXT: ld1 { v19.4s }, [x10]
3115 ; CHECK-BE-NEXT: subs w2, w2, #1
3116 ; CHECK-BE-NEXT: tbl v5.16b, { v4.16b }, v1.16b
3117 ; CHECK-BE-NEXT: tbl v6.16b, { v4.16b }, v3.16b
3118 ; CHECK-BE-NEXT: tbl v7.16b, { v4.16b }, v2.16b
3119 ; CHECK-BE-NEXT: tbl v4.16b, { v4.16b }, v0.16b
3120 ; CHECK-BE-NEXT: add x1, x1, #16
3121 ; CHECK-BE-NEXT: rev32 v5.16b, v5.16b
3122 ; CHECK-BE-NEXT: rev32 v6.16b, v6.16b
3123 ; CHECK-BE-NEXT: rev32 v7.16b, v7.16b
3124 ; CHECK-BE-NEXT: rev32 v4.16b, v4.16b
3125 ; CHECK-BE-NEXT: mul v5.4s, v16.4s, v5.4s
3126 ; CHECK-BE-NEXT: mul v6.4s, v17.4s, v6.4s
3127 ; CHECK-BE-NEXT: mul v7.4s, v18.4s, v7.4s
3128 ; CHECK-BE-NEXT: mul v4.4s, v19.4s, v4.4s
3129 ; CHECK-BE-NEXT: st1 { v5.4s }, [x0]
3130 ; CHECK-BE-NEXT: mov x0, x10
3131 ; CHECK-BE-NEXT: st1 { v6.4s }, [x8]
3132 ; CHECK-BE-NEXT: st1 { v7.4s }, [x9]
3133 ; CHECK-BE-NEXT: st1 { v4.4s }, [x10]
3134 ; CHECK-BE-NEXT: b.ne .LBB26_1
3135 ; CHECK-BE-NEXT: // %bb.2: // %exit
3136 ; CHECK-BE-NEXT: mov w0, wzr
3137 ; CHECK-BE-NEXT: ret
3138 entry:
3139 br label %loop
3141 loop:
3142 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
3143 %gep.1 = getelementptr inbounds <16 x i8>, ptr %p1, i32 %iv
3144 %gep.2 = getelementptr inbounds <16 x i8>, ptr %p2, i32 %iv
3145 %l1 = load <16 x i32>, ptr %gep.1
3146 %l4 = load <16 x i8>, ptr %gep.2
3147 %z5 = zext <16 x i8> %l4 to <16 x i32>
3148 %mul = mul <16 x i32> %l1, %z5
3149 store <16 x i32> %mul, ptr %gep.1
3150 %iv.next= add nuw nsw i32 %iv, 1
3151 %exitcond.not = icmp eq i32 %iv.next, %h
3152 br i1 %exitcond.not, label %exit, label %loop
3154 exit:
3155 ret i32 0
3156 }