1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc -mtriple=aarch64 -mattr=+v8.6a,+neon < %s | FileCheck %s
3 ; RUN: llc -mtriple=aarch64 -mattr=+v8.6a,+neon,+bf16 < %s | FileCheck %s
4 ; RUN: llc -mtriple=aarch64 -mattr=+v8.6a,+neon,+fullfp16,+bf16 < %s | FileCheck %s
6 %struct.float16x4x2_t = type { [2 x <4 x bfloat>] }
7 %struct.float16x8x2_t = type { [2 x <8 x bfloat>] }
9 define dso_local %struct.float16x4x2_t @test_vzip_bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
10 ; CHECK-LABEL: test_vzip_bf16:
11 ; CHECK: // %bb.0: // %entry
12 ; CHECK-NEXT: zip1 v2.4h, v0.4h, v1.4h
13 ; CHECK-NEXT: zip2 v1.4h, v0.4h, v1.4h
14 ; CHECK-NEXT: fmov d0, d2
17 %vzip.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
18 %vzip1.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
19 %.fca.0.0.insert = insertvalue %struct.float16x4x2_t undef, <4 x bfloat> %vzip.i, 0, 0
20 %.fca.0.1.insert = insertvalue %struct.float16x4x2_t %.fca.0.0.insert, <4 x bfloat> %vzip1.i, 0, 1
21 ret %struct.float16x4x2_t %.fca.0.1.insert
24 define dso_local %struct.float16x8x2_t @test_vzipq_bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
25 ; CHECK-LABEL: test_vzipq_bf16:
26 ; CHECK: // %bb.0: // %entry
27 ; CHECK-NEXT: zip1 v2.8h, v0.8h, v1.8h
28 ; CHECK-NEXT: zip2 v1.8h, v0.8h, v1.8h
29 ; CHECK-NEXT: mov v0.16b, v2.16b
32 %vzip.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
33 %vzip1.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
34 %.fca.0.0.insert = insertvalue %struct.float16x8x2_t undef, <8 x bfloat> %vzip.i, 0, 0
35 %.fca.0.1.insert = insertvalue %struct.float16x8x2_t %.fca.0.0.insert, <8 x bfloat> %vzip1.i, 0, 1
36 ret %struct.float16x8x2_t %.fca.0.1.insert
39 define dso_local %struct.float16x4x2_t @test_vuzp_bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
40 ; CHECK-LABEL: test_vuzp_bf16:
41 ; CHECK: // %bb.0: // %entry
42 ; CHECK-NEXT: uzp1 v2.4h, v0.4h, v1.4h
43 ; CHECK-NEXT: uzp2 v1.4h, v0.4h, v1.4h
44 ; CHECK-NEXT: fmov d0, d2
47 %vuzp.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
48 %vuzp1.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
49 %.fca.0.0.insert = insertvalue %struct.float16x4x2_t undef, <4 x bfloat> %vuzp.i, 0, 0
50 %.fca.0.1.insert = insertvalue %struct.float16x4x2_t %.fca.0.0.insert, <4 x bfloat> %vuzp1.i, 0, 1
51 ret %struct.float16x4x2_t %.fca.0.1.insert
54 define dso_local %struct.float16x8x2_t @test_vuzpq_bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
55 ; CHECK-LABEL: test_vuzpq_bf16:
56 ; CHECK: // %bb.0: // %entry
57 ; CHECK-NEXT: uzp1 v2.8h, v0.8h, v1.8h
58 ; CHECK-NEXT: uzp2 v1.8h, v0.8h, v1.8h
59 ; CHECK-NEXT: mov v0.16b, v2.16b
62 %vuzp.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
63 %vuzp1.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
64 %.fca.0.0.insert = insertvalue %struct.float16x8x2_t undef, <8 x bfloat> %vuzp.i, 0, 0
65 %.fca.0.1.insert = insertvalue %struct.float16x8x2_t %.fca.0.0.insert, <8 x bfloat> %vuzp1.i, 0, 1
66 ret %struct.float16x8x2_t %.fca.0.1.insert
69 define dso_local %struct.float16x4x2_t @test_vtrn_bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
70 ; CHECK-LABEL: test_vtrn_bf16:
71 ; CHECK: // %bb.0: // %entry
72 ; CHECK-NEXT: trn1 v2.4h, v0.4h, v1.4h
73 ; CHECK-NEXT: trn2 v1.4h, v0.4h, v1.4h
74 ; CHECK-NEXT: fmov d0, d2
77 %vtrn.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
78 %vtrn1.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
79 %.fca.0.0.insert = insertvalue %struct.float16x4x2_t undef, <4 x bfloat> %vtrn.i, 0, 0
80 %.fca.0.1.insert = insertvalue %struct.float16x4x2_t %.fca.0.0.insert, <4 x bfloat> %vtrn1.i, 0, 1
81 ret %struct.float16x4x2_t %.fca.0.1.insert
84 define dso_local %struct.float16x8x2_t @test_vtrnq_bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
85 ; CHECK-LABEL: test_vtrnq_bf16:
86 ; CHECK: // %bb.0: // %entry
87 ; CHECK-NEXT: trn1 v2.8h, v0.8h, v1.8h
88 ; CHECK-NEXT: trn2 v1.8h, v0.8h, v1.8h
89 ; CHECK-NEXT: mov v0.16b, v2.16b
92 %vtrn.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
93 %vtrn1.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
94 %.fca.0.0.insert = insertvalue %struct.float16x8x2_t undef, <8 x bfloat> %vtrn.i, 0, 0
95 %.fca.0.1.insert = insertvalue %struct.float16x8x2_t %.fca.0.0.insert, <8 x bfloat> %vtrn1.i, 0, 1
96 ret %struct.float16x8x2_t %.fca.0.1.insert
99 define dso_local <4 x bfloat> @test_vmov_n_bf16(float %a.coerce) {
100 ; CHECK-LABEL: test_vmov_n_bf16:
101 ; CHECK: // %bb.0: // %entry
102 ; CHECK-NEXT: // kill: def $s0 killed $s0 def $q0
103 ; CHECK-NEXT: dup v0.4h, v0.h[0]
106 %0 = bitcast float %a.coerce to i32
107 %tmp.0.extract.trunc = trunc i32 %0 to i16
108 %1 = bitcast i16 %tmp.0.extract.trunc to bfloat
109 %vecinit = insertelement <4 x bfloat> undef, bfloat %1, i32 0
110 %vecinit4 = shufflevector <4 x bfloat> %vecinit, <4 x bfloat> undef, <4 x i32> zeroinitializer
111 ret <4 x bfloat> %vecinit4
114 define dso_local <8 x bfloat> @test_vmovq_n_bf16(float %a.coerce) {
115 ; CHECK-LABEL: test_vmovq_n_bf16:
116 ; CHECK: // %bb.0: // %entry
117 ; CHECK-NEXT: // kill: def $s0 killed $s0 def $q0
118 ; CHECK-NEXT: dup v0.8h, v0.h[0]
121 %0 = bitcast float %a.coerce to i32
122 %tmp.0.extract.trunc = trunc i32 %0 to i16
123 %1 = bitcast i16 %tmp.0.extract.trunc to bfloat
124 %vecinit = insertelement <8 x bfloat> undef, bfloat %1, i32 0
125 %vecinit8 = shufflevector <8 x bfloat> %vecinit, <8 x bfloat> undef, <8 x i32> zeroinitializer
126 ret <8 x bfloat> %vecinit8
129 define dso_local <4 x bfloat> @test_vdup_n_bf16(float %a.coerce) {
130 ; CHECK-LABEL: test_vdup_n_bf16:
131 ; CHECK: // %bb.0: // %entry
132 ; CHECK-NEXT: // kill: def $s0 killed $s0 def $q0
133 ; CHECK-NEXT: dup v0.4h, v0.h[0]
136 %0 = bitcast float %a.coerce to i32
137 %tmp.0.extract.trunc = trunc i32 %0 to i16
138 %1 = bitcast i16 %tmp.0.extract.trunc to bfloat
139 %vecinit = insertelement <4 x bfloat> undef, bfloat %1, i32 0
140 %vecinit4 = shufflevector <4 x bfloat> %vecinit, <4 x bfloat> undef, <4 x i32> zeroinitializer
141 ret <4 x bfloat> %vecinit4
144 define dso_local <8 x bfloat> @test_vdupq_n_bf16(float %a.coerce) {
145 ; CHECK-LABEL: test_vdupq_n_bf16:
146 ; CHECK: // %bb.0: // %entry
147 ; CHECK-NEXT: // kill: def $s0 killed $s0 def $q0
148 ; CHECK-NEXT: dup v0.8h, v0.h[0]
151 %0 = bitcast float %a.coerce to i32
152 %tmp.0.extract.trunc = trunc i32 %0 to i16
153 %1 = bitcast i16 %tmp.0.extract.trunc to bfloat
154 %vecinit = insertelement <8 x bfloat> undef, bfloat %1, i32 0
155 %vecinit8 = shufflevector <8 x bfloat> %vecinit, <8 x bfloat> undef, <8 x i32> zeroinitializer
156 ret <8 x bfloat> %vecinit8
159 define dso_local <4 x bfloat> @test_vdup_lane_bf16(<4 x bfloat> %a) {
160 ; CHECK-LABEL: test_vdup_lane_bf16:
161 ; CHECK: // %bb.0: // %entry
162 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
163 ; CHECK-NEXT: dup v0.4h, v0.h[3]
166 %shuffle = shufflevector <4 x bfloat> %a, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
167 ret <4 x bfloat> %shuffle
170 define dso_local <8 x bfloat> @test_vdupq_lane_bf16(<4 x bfloat> %a) {
171 ; CHECK-LABEL: test_vdupq_lane_bf16:
172 ; CHECK: // %bb.0: // %entry
173 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
174 ; CHECK-NEXT: dup v0.8h, v0.h[3]
177 %shuffle = shufflevector <4 x bfloat> %a, <4 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
178 ret <8 x bfloat> %shuffle
181 define dso_local <4 x bfloat> @test_vext_bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
182 ; CHECK-LABEL: test_vext_bf16:
183 ; CHECK: // %bb.0: // %entry
184 ; CHECK-NEXT: ext v0.8b, v0.8b, v1.8b, #4
187 %vext = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
188 ret <4 x bfloat> %vext
191 define dso_local <8 x bfloat> @test_vextq_bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
192 ; CHECK-LABEL: test_vextq_bf16:
193 ; CHECK: // %bb.0: // %entry
194 ; CHECK-NEXT: ext v0.16b, v0.16b, v1.16b, #10
197 %vext = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12>
198 ret <8 x bfloat> %vext
201 define dso_local <4 x bfloat> @test_vext_aligned_bf16(<8 x bfloat> %a) {
202 ; CHECK-LABEL: test_vext_aligned_bf16:
203 ; CHECK: // %bb.0: // %entry
204 ; CHECK-NEXT: ext v0.16b, v0.16b, v0.16b, #8
205 ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
208 %vext = shufflevector <8 x bfloat> %a, <8 x bfloat> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
209 ret <4 x bfloat> %vext
212 define dso_local <4 x bfloat> @test_vext_unaligned_bf16(<8 x bfloat> %a) {
213 ; CHECK-LABEL: test_vext_unaligned_bf16:
214 ; CHECK: // %bb.0: // %entry
215 ; CHECK-NEXT: ext v0.16b, v0.16b, v0.16b, #6
216 ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
219 %vext = shufflevector <8 x bfloat> %a, <8 x bfloat> undef, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
220 ret <4 x bfloat> %vext
223 define <8 x bfloat> @shuffle3step0_bf16(<32 x bfloat> %src) {
224 ; CHECK-LABEL: shuffle3step0_bf16:
225 ; CHECK: // %bb.0: // %entry
226 ; CHECK-NEXT: adrp x8, .LCPI16_0
227 ; CHECK-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
228 ; CHECK-NEXT: mov v3.16b, v2.16b
229 ; CHECK-NEXT: ldr q4, [x8, :lo12:.LCPI16_0]
230 ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
231 ; CHECK-NEXT: adrp x8, .LCPI16_1
232 ; CHECK-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v4.16b
233 ; CHECK-NEXT: ldr q0, [x8, :lo12:.LCPI16_1]
234 ; CHECK-NEXT: tbl v0.16b, { v2.16b, v3.16b }, v0.16b
237 %s1 = shufflevector <32 x bfloat> %src, <32 x bfloat> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
241 define <8 x bfloat> @shuffle3step1_bf16(<32 x bfloat> %src) {
242 ; CHECK-LABEL: shuffle3step1_bf16:
243 ; CHECK: // %bb.0: // %entry
244 ; CHECK-NEXT: adrp x8, .LCPI17_0
245 ; CHECK-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
246 ; CHECK-NEXT: mov v3.16b, v2.16b
247 ; CHECK-NEXT: ldr q4, [x8, :lo12:.LCPI17_0]
248 ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
249 ; CHECK-NEXT: adrp x8, .LCPI17_1
250 ; CHECK-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v4.16b
251 ; CHECK-NEXT: ldr q0, [x8, :lo12:.LCPI17_1]
252 ; CHECK-NEXT: tbl v0.16b, { v2.16b, v3.16b }, v0.16b
255 %s1 = shufflevector <32 x bfloat> %src, <32 x bfloat> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
259 define <8 x bfloat> @shuffle3step2_bf16(<32 x bfloat> %src) {
260 ; CHECK-LABEL: shuffle3step2_bf16:
261 ; CHECK: // %bb.0: // %entry
262 ; CHECK-NEXT: adrp x8, .LCPI18_0
263 ; CHECK-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
264 ; CHECK-NEXT: mov v3.16b, v2.16b
265 ; CHECK-NEXT: ldr q4, [x8, :lo12:.LCPI18_0]
266 ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
267 ; CHECK-NEXT: adrp x8, .LCPI18_1
268 ; CHECK-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v4.16b
269 ; CHECK-NEXT: ldr q0, [x8, :lo12:.LCPI18_1]
270 ; CHECK-NEXT: tbl v0.16b, { v2.16b, v3.16b }, v0.16b
273 %s1 = shufflevector <32 x bfloat> %src, <32 x bfloat> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
278 define dso_local <4 x bfloat> @test_vrev64_bf16(<4 x bfloat> %a) {
279 ; CHECK-LABEL: test_vrev64_bf16:
280 ; CHECK: // %bb.0: // %entry
281 ; CHECK-NEXT: rev64 v0.4h, v0.4h
284 %shuffle.i = shufflevector <4 x bfloat> %a, <4 x bfloat> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
285 ret <4 x bfloat> %shuffle.i
288 define dso_local <8 x bfloat> @test_vrev64q_bf16(<8 x bfloat> %a) {
289 ; CHECK-LABEL: test_vrev64q_bf16:
290 ; CHECK: // %bb.0: // %entry
291 ; CHECK-NEXT: rev64 v0.8h, v0.8h
294 %shuffle.i = shufflevector <8 x bfloat> %a, <8 x bfloat> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
295 ret <8 x bfloat> %shuffle.i
298 define dso_local <4 x bfloat> @test_vrev32_bf16(<4 x bfloat> %a) {
299 ; CHECK-LABEL: test_vrev32_bf16:
300 ; CHECK: // %bb.0: // %entry
301 ; CHECK-NEXT: rev32 v0.4h, v0.4h
304 %shuffle.i = shufflevector <4 x bfloat> %a, <4 x bfloat> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
305 ret <4 x bfloat> %shuffle.i
308 define dso_local <8 x bfloat> @test_vrev32q_bf16(<8 x bfloat> %a) {
309 ; CHECK-LABEL: test_vrev32q_bf16:
310 ; CHECK: // %bb.0: // %entry
311 ; CHECK-NEXT: rev32 v0.8h, v0.8h
314 %shuffle.i = shufflevector <8 x bfloat> %a, <8 x bfloat> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
315 ret <8 x bfloat> %shuffle.i
318 define <4 x bfloat> @test_vld_dup1_4xbfloat(ptr %b) {
319 ; CHECK-LABEL: test_vld_dup1_4xbfloat:
320 ; CHECK: // %bb.0: // %entry
321 ; CHECK-NEXT: ld1r { v0.4h }, [x0]
324 %b1 = load bfloat, ptr %b, align 2
325 %vecinit = insertelement <4 x bfloat> undef, bfloat %b1, i32 0
326 %vecinit2 = insertelement <4 x bfloat> %vecinit, bfloat %b1, i32 1
327 %vecinit3 = insertelement <4 x bfloat> %vecinit2, bfloat %b1, i32 2
328 %vecinit4 = insertelement <4 x bfloat> %vecinit3, bfloat %b1, i32 3
329 ret <4 x bfloat> %vecinit4
332 define <8 x bfloat> @test_vld_dup1_8xbfloat(ptr %b) local_unnamed_addr {
333 ; CHECK-LABEL: test_vld_dup1_8xbfloat:
334 ; CHECK: // %bb.0: // %entry
335 ; CHECK-NEXT: ld1r { v0.8h }, [x0]
338 %b1 = load bfloat, ptr %b, align 2
339 %vecinit = insertelement <8 x bfloat> undef, bfloat %b1, i32 0
340 %vecinit8 = shufflevector <8 x bfloat> %vecinit, <8 x bfloat> undef, <8 x i32> zeroinitializer
341 ret <8 x bfloat> %vecinit8
344 define <8 x bfloat> @test_shufflevector8xbfloat(<4 x bfloat> %a) {
345 ; CHECK-LABEL: test_shufflevector8xbfloat:
346 ; CHECK: // %bb.0: // %entry
347 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
348 ; CHECK-NEXT: mov v0.d[1], v0.d[0]
351 %r = shufflevector <4 x bfloat> %a, <4 x bfloat> %a, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>