| blob | 3135addec1618353000ab9e747402d3937dea15b |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
2 ; RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=tahiti < %s | FileCheck -check-prefix=SI %s
3 ; RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=tonga < %s | FileCheck -check-prefix=VI %s
4 ; RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX900 %s
5 ; RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 < %s | FileCheck -check-prefix=GFX940 %s
7 define amdgpu_kernel void @s_insertelement_v2bf16_0(ptr addrspace(1) %out, ptr addrspace(4) %vec.ptr) #0 {
8 ; SI-LABEL: s_insertelement_v2bf16_0:
9 ; SI: ; %bb.0:
10 ; SI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
11 ; SI-NEXT: s_waitcnt lgkmcnt(0)
12 ; SI-NEXT: s_load_dword s4, s[2:3], 0x0
13 ; SI-NEXT: s_mov_b32 s3, 0x100f000
14 ; SI-NEXT: s_mov_b32 s2, -1
15 ; SI-NEXT: s_waitcnt lgkmcnt(0)
16 ; SI-NEXT: s_and_b32 s4, s4, 0xffff0000
17 ; SI-NEXT: s_or_b32 s4, s4, 0x40a0
18 ; SI-NEXT: v_mov_b32_e32 v0, s4
19 ; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
20 ; SI-NEXT: s_endpgm
21 ;
22 ; VI-LABEL: s_insertelement_v2bf16_0:
23 ; VI: ; %bb.0:
24 ; VI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
25 ; VI-NEXT: s_waitcnt lgkmcnt(0)
26 ; VI-NEXT: s_load_dword s2, s[2:3], 0x0
27 ; VI-NEXT: v_mov_b32_e32 v0, s0
28 ; VI-NEXT: v_mov_b32_e32 v1, s1
29 ; VI-NEXT: s_waitcnt lgkmcnt(0)
30 ; VI-NEXT: s_and_b32 s0, s2, 0xffff0000
31 ; VI-NEXT: s_or_b32 s0, s0, 0x40a0
32 ; VI-NEXT: v_mov_b32_e32 v2, s0
33 ; VI-NEXT: flat_store_dword v[0:1], v2
34 ; VI-NEXT: s_endpgm
35 ;
36 ; GFX900-LABEL: s_insertelement_v2bf16_0:
37 ; GFX900: ; %bb.0:
38 ; GFX900-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
39 ; GFX900-NEXT: v_mov_b32_e32 v0, 0
40 ; GFX900-NEXT: s_waitcnt lgkmcnt(0)
41 ; GFX900-NEXT: s_load_dword s2, s[2:3], 0x0
42 ; GFX900-NEXT: s_waitcnt lgkmcnt(0)
43 ; GFX900-NEXT: s_lshr_b32 s2, s2, 16
44 ; GFX900-NEXT: s_pack_ll_b32_b16 s2, 0x40a0, s2
45 ; GFX900-NEXT: v_mov_b32_e32 v1, s2
46 ; GFX900-NEXT: global_store_dword v0, v1, s[0:1]
47 ; GFX900-NEXT: s_endpgm
48 ;
49 ; GFX940-LABEL: s_insertelement_v2bf16_0:
50 ; GFX940: ; %bb.0:
51 ; GFX940-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
52 ; GFX940-NEXT: v_mov_b32_e32 v0, 0
53 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
54 ; GFX940-NEXT: s_load_dword s2, s[2:3], 0x0
55 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
56 ; GFX940-NEXT: s_lshr_b32 s2, s2, 16
57 ; GFX940-NEXT: s_pack_ll_b32_b16 s2, 0x40a0, s2
58 ; GFX940-NEXT: v_mov_b32_e32 v1, s2
59 ; GFX940-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
60 ; GFX940-NEXT: s_endpgm
61 %vec = load <2 x bfloat>, ptr addrspace(4) %vec.ptr
62 %vecins = insertelement <2 x bfloat> %vec, bfloat 5.000000e+00, i32 0
63 store <2 x bfloat> %vecins, ptr addrspace(1) %out
64 ret void
65 }
67 define amdgpu_kernel void @s_insertelement_v2bf16_1(ptr addrspace(1) %out, ptr addrspace(4) %vec.ptr) #0 {
68 ; SI-LABEL: s_insertelement_v2bf16_1:
69 ; SI: ; %bb.0:
70 ; SI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
71 ; SI-NEXT: s_waitcnt lgkmcnt(0)
72 ; SI-NEXT: s_load_dword s4, s[2:3], 0x0
73 ; SI-NEXT: s_mov_b32 s3, 0x100f000
74 ; SI-NEXT: s_mov_b32 s2, -1
75 ; SI-NEXT: s_waitcnt lgkmcnt(0)
76 ; SI-NEXT: s_and_b32 s4, s4, 0xffff
77 ; SI-NEXT: s_or_b32 s4, s4, 0x40a00000
78 ; SI-NEXT: v_mov_b32_e32 v0, s4
79 ; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
80 ; SI-NEXT: s_endpgm
81 ;
82 ; VI-LABEL: s_insertelement_v2bf16_1:
83 ; VI: ; %bb.0:
84 ; VI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
85 ; VI-NEXT: s_waitcnt lgkmcnt(0)
86 ; VI-NEXT: s_load_dword s2, s[2:3], 0x0
87 ; VI-NEXT: v_mov_b32_e32 v0, s0
88 ; VI-NEXT: v_mov_b32_e32 v1, s1
89 ; VI-NEXT: s_waitcnt lgkmcnt(0)
90 ; VI-NEXT: s_and_b32 s0, s2, 0xffff
91 ; VI-NEXT: s_or_b32 s0, s0, 0x40a00000
92 ; VI-NEXT: v_mov_b32_e32 v2, s0
93 ; VI-NEXT: flat_store_dword v[0:1], v2
94 ; VI-NEXT: s_endpgm
95 ;
96 ; GFX900-LABEL: s_insertelement_v2bf16_1:
97 ; GFX900: ; %bb.0:
98 ; GFX900-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
99 ; GFX900-NEXT: v_mov_b32_e32 v0, 0
100 ; GFX900-NEXT: s_waitcnt lgkmcnt(0)
101 ; GFX900-NEXT: s_load_dword s2, s[2:3], 0x0
102 ; GFX900-NEXT: s_waitcnt lgkmcnt(0)
103 ; GFX900-NEXT: s_pack_ll_b32_b16 s2, s2, 0x40a0
104 ; GFX900-NEXT: v_mov_b32_e32 v1, s2
105 ; GFX900-NEXT: global_store_dword v0, v1, s[0:1]
106 ; GFX900-NEXT: s_endpgm
107 ;
108 ; GFX940-LABEL: s_insertelement_v2bf16_1:
109 ; GFX940: ; %bb.0:
110 ; GFX940-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
111 ; GFX940-NEXT: v_mov_b32_e32 v0, 0
112 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
113 ; GFX940-NEXT: s_load_dword s2, s[2:3], 0x0
114 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
115 ; GFX940-NEXT: s_pack_ll_b32_b16 s2, s2, 0x40a0
116 ; GFX940-NEXT: v_mov_b32_e32 v1, s2
117 ; GFX940-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
118 ; GFX940-NEXT: s_endpgm
119 %vec = load <2 x bfloat>, ptr addrspace(4) %vec.ptr
120 %vecins = insertelement <2 x bfloat> %vec, bfloat 5.000000e+00, i32 1
121 store <2 x bfloat> %vecins, ptr addrspace(1) %out
122 ret void
123 }
125 define amdgpu_kernel void @v_insertelement_v2bf16_0(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
126 ; SI-LABEL: v_insertelement_v2bf16_0:
127 ; SI: ; %bb.0:
128 ; SI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
129 ; SI-NEXT: s_mov_b32 s7, 0x100f000
130 ; SI-NEXT: s_mov_b32 s6, 0
131 ; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
132 ; SI-NEXT: v_mov_b32_e32 v1, 0
133 ; SI-NEXT: s_waitcnt lgkmcnt(0)
134 ; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
135 ; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
136 ; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
137 ; SI-NEXT: s_waitcnt vmcnt(0)
138 ; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
139 ; SI-NEXT: v_or_b32_e32 v2, 0x40a0, v2
140 ; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
141 ; SI-NEXT: s_endpgm
142 ;
143 ; VI-LABEL: v_insertelement_v2bf16_0:
144 ; VI: ; %bb.0:
145 ; VI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
146 ; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
147 ; VI-NEXT: s_waitcnt lgkmcnt(0)
148 ; VI-NEXT: v_mov_b32_e32 v1, s3
149 ; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
150 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
151 ; VI-NEXT: flat_load_dword v3, v[0:1]
152 ; VI-NEXT: v_mov_b32_e32 v1, s1
153 ; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
154 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
155 ; VI-NEXT: s_waitcnt vmcnt(0)
156 ; VI-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
157 ; VI-NEXT: v_or_b32_e32 v2, 0x40a0, v2
158 ; VI-NEXT: flat_store_dword v[0:1], v2
159 ; VI-NEXT: s_endpgm
160 ;
161 ; GFX900-LABEL: v_insertelement_v2bf16_0:
162 ; GFX900: ; %bb.0:
163 ; GFX900-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
164 ; GFX900-NEXT: v_lshlrev_b32_e32 v0, 2, v0
165 ; GFX900-NEXT: v_mov_b32_e32 v2, 0x40a0
166 ; GFX900-NEXT: s_waitcnt lgkmcnt(0)
167 ; GFX900-NEXT: global_load_dword v1, v0, s[2:3]
168 ; GFX900-NEXT: s_mov_b32 s2, 0xffff
169 ; GFX900-NEXT: s_waitcnt vmcnt(0)
170 ; GFX900-NEXT: v_bfi_b32 v1, s2, v2, v1
171 ; GFX900-NEXT: global_store_dword v0, v1, s[0:1]
172 ; GFX900-NEXT: s_endpgm
173 ;
174 ; GFX940-LABEL: v_insertelement_v2bf16_0:
175 ; GFX940: ; %bb.0:
176 ; GFX940-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
177 ; GFX940-NEXT: v_and_b32_e32 v0, 0x3ff, v0
178 ; GFX940-NEXT: v_lshlrev_b32_e32 v0, 2, v0
179 ; GFX940-NEXT: v_mov_b32_e32 v2, 0x40a0
180 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
181 ; GFX940-NEXT: global_load_dword v1, v0, s[2:3]
182 ; GFX940-NEXT: s_mov_b32 s2, 0xffff
183 ; GFX940-NEXT: s_waitcnt vmcnt(0)
184 ; GFX940-NEXT: v_bfi_b32 v1, s2, v2, v1
185 ; GFX940-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
186 ; GFX940-NEXT: s_endpgm
187 %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
188 %tid.ext = sext i32 %tid to i64
189 %in.gep = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
190 %out.gep = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
191 %vec = load <2 x bfloat>, ptr addrspace(1) %in.gep
192 %vecins = insertelement <2 x bfloat> %vec, bfloat 5.000000e+00, i32 0
193 store <2 x bfloat> %vecins, ptr addrspace(1) %out.gep
194 ret void
195 }
197 define amdgpu_kernel void @v_insertelement_v2bf16_0_inlineimm(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
198 ; SI-LABEL: v_insertelement_v2bf16_0_inlineimm:
199 ; SI: ; %bb.0:
200 ; SI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
201 ; SI-NEXT: s_mov_b32 s7, 0x100f000
202 ; SI-NEXT: s_mov_b32 s6, 0
203 ; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
204 ; SI-NEXT: v_mov_b32_e32 v1, 0
205 ; SI-NEXT: s_waitcnt lgkmcnt(0)
206 ; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
207 ; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
208 ; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
209 ; SI-NEXT: s_waitcnt vmcnt(0)
210 ; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
211 ; SI-NEXT: v_or_b32_e32 v2, 53, v2
212 ; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
213 ; SI-NEXT: s_endpgm
214 ;
215 ; VI-LABEL: v_insertelement_v2bf16_0_inlineimm:
216 ; VI: ; %bb.0:
217 ; VI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
218 ; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
219 ; VI-NEXT: s_waitcnt lgkmcnt(0)
220 ; VI-NEXT: v_mov_b32_e32 v1, s3
221 ; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
222 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
223 ; VI-NEXT: flat_load_dword v3, v[0:1]
224 ; VI-NEXT: v_mov_b32_e32 v1, s1
225 ; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
226 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
227 ; VI-NEXT: s_waitcnt vmcnt(0)
228 ; VI-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
229 ; VI-NEXT: v_or_b32_e32 v2, 53, v2
230 ; VI-NEXT: flat_store_dword v[0:1], v2
231 ; VI-NEXT: s_endpgm
232 ;
233 ; GFX900-LABEL: v_insertelement_v2bf16_0_inlineimm:
234 ; GFX900: ; %bb.0:
235 ; GFX900-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
236 ; GFX900-NEXT: v_lshlrev_b32_e32 v0, 2, v0
237 ; GFX900-NEXT: s_waitcnt lgkmcnt(0)
238 ; GFX900-NEXT: global_load_dword v1, v0, s[2:3]
239 ; GFX900-NEXT: s_mov_b32 s2, 0xffff
240 ; GFX900-NEXT: s_waitcnt vmcnt(0)
241 ; GFX900-NEXT: v_bfi_b32 v1, s2, 53, v1
242 ; GFX900-NEXT: global_store_dword v0, v1, s[0:1]
243 ; GFX900-NEXT: s_endpgm
244 ;
245 ; GFX940-LABEL: v_insertelement_v2bf16_0_inlineimm:
246 ; GFX940: ; %bb.0:
247 ; GFX940-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
248 ; GFX940-NEXT: v_and_b32_e32 v0, 0x3ff, v0
249 ; GFX940-NEXT: v_lshlrev_b32_e32 v0, 2, v0
250 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
251 ; GFX940-NEXT: global_load_dword v1, v0, s[2:3]
252 ; GFX940-NEXT: s_mov_b32 s2, 0xffff
253 ; GFX940-NEXT: s_waitcnt vmcnt(0)
254 ; GFX940-NEXT: v_bfi_b32 v1, s2, 53, v1
255 ; GFX940-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
256 ; GFX940-NEXT: s_endpgm
257 %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
258 %tid.ext = sext i32 %tid to i64
259 %in.gep = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
260 %out.gep = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
261 %vec = load <2 x bfloat>, ptr addrspace(1) %in.gep
262 %vecins = insertelement <2 x bfloat> %vec, bfloat 0xR0035, i32 0
263 store <2 x bfloat> %vecins, ptr addrspace(1) %out.gep
264 ret void
265 }
267 define amdgpu_kernel void @v_insertelement_v2bf16_1(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
268 ; SI-LABEL: v_insertelement_v2bf16_1:
269 ; SI: ; %bb.0:
270 ; SI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
271 ; SI-NEXT: s_mov_b32 s7, 0x100f000
272 ; SI-NEXT: s_mov_b32 s6, 0
273 ; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
274 ; SI-NEXT: v_mov_b32_e32 v1, 0
275 ; SI-NEXT: s_waitcnt lgkmcnt(0)
276 ; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
277 ; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
278 ; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
279 ; SI-NEXT: s_waitcnt vmcnt(0)
280 ; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
281 ; SI-NEXT: v_or_b32_e32 v2, 0x40a00000, v2
282 ; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
283 ; SI-NEXT: s_endpgm
284 ;
285 ; VI-LABEL: v_insertelement_v2bf16_1:
286 ; VI: ; %bb.0:
287 ; VI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
288 ; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
289 ; VI-NEXT: s_waitcnt lgkmcnt(0)
290 ; VI-NEXT: v_mov_b32_e32 v1, s3
291 ; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
292 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
293 ; VI-NEXT: flat_load_dword v3, v[0:1]
294 ; VI-NEXT: v_mov_b32_e32 v1, s1
295 ; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
296 ; VI-NEXT: v_mov_b32_e32 v2, 0x40a00000
297 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
298 ; VI-NEXT: s_waitcnt vmcnt(0)
299 ; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
300 ; VI-NEXT: flat_store_dword v[0:1], v2
301 ; VI-NEXT: s_endpgm
302 ;
303 ; GFX900-LABEL: v_insertelement_v2bf16_1:
304 ; GFX900: ; %bb.0:
305 ; GFX900-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
306 ; GFX900-NEXT: v_lshlrev_b32_e32 v0, 2, v0
307 ; GFX900-NEXT: v_mov_b32_e32 v2, 0x5040100
308 ; GFX900-NEXT: s_waitcnt lgkmcnt(0)
309 ; GFX900-NEXT: global_load_dword v1, v0, s[2:3]
310 ; GFX900-NEXT: s_movk_i32 s2, 0x40a0
311 ; GFX900-NEXT: s_waitcnt vmcnt(0)
312 ; GFX900-NEXT: v_perm_b32 v1, s2, v1, v2
313 ; GFX900-NEXT: global_store_dword v0, v1, s[0:1]
314 ; GFX900-NEXT: s_endpgm
315 ;
316 ; GFX940-LABEL: v_insertelement_v2bf16_1:
317 ; GFX940: ; %bb.0:
318 ; GFX940-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
319 ; GFX940-NEXT: v_and_b32_e32 v0, 0x3ff, v0
320 ; GFX940-NEXT: v_lshlrev_b32_e32 v0, 2, v0
321 ; GFX940-NEXT: v_mov_b32_e32 v2, 0x5040100
322 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
323 ; GFX940-NEXT: global_load_dword v1, v0, s[2:3]
324 ; GFX940-NEXT: s_movk_i32 s2, 0x40a0
325 ; GFX940-NEXT: s_waitcnt vmcnt(0)
326 ; GFX940-NEXT: v_perm_b32 v1, s2, v1, v2
327 ; GFX940-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
328 ; GFX940-NEXT: s_endpgm
329 %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
330 %tid.ext = sext i32 %tid to i64
331 %in.gep = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
332 %out.gep = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
333 %vec = load <2 x bfloat>, ptr addrspace(1) %in.gep
334 %vecins = insertelement <2 x bfloat> %vec, bfloat 5.000000e+00, i32 1
335 store <2 x bfloat> %vecins, ptr addrspace(1) %out.gep
336 ret void
337 }
339 define amdgpu_kernel void @v_insertelement_v2bf16_1_inlineimm(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
340 ; SI-LABEL: v_insertelement_v2bf16_1_inlineimm:
341 ; SI: ; %bb.0:
342 ; SI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
343 ; SI-NEXT: s_mov_b32 s7, 0x100f000
344 ; SI-NEXT: s_mov_b32 s6, 0
345 ; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
346 ; SI-NEXT: v_mov_b32_e32 v1, 0
347 ; SI-NEXT: s_waitcnt lgkmcnt(0)
348 ; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
349 ; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
350 ; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
351 ; SI-NEXT: s_waitcnt vmcnt(0)
352 ; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
353 ; SI-NEXT: v_or_b32_e32 v2, 0x230000, v2
354 ; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
355 ; SI-NEXT: s_endpgm
356 ;
357 ; VI-LABEL: v_insertelement_v2bf16_1_inlineimm:
358 ; VI: ; %bb.0:
359 ; VI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
360 ; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
361 ; VI-NEXT: s_waitcnt lgkmcnt(0)
362 ; VI-NEXT: v_mov_b32_e32 v1, s3
363 ; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
364 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
365 ; VI-NEXT: flat_load_dword v3, v[0:1]
366 ; VI-NEXT: v_mov_b32_e32 v1, s1
367 ; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
368 ; VI-NEXT: v_mov_b32_e32 v2, 0x230000
369 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
370 ; VI-NEXT: s_waitcnt vmcnt(0)
371 ; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
372 ; VI-NEXT: flat_store_dword v[0:1], v2
373 ; VI-NEXT: s_endpgm
374 ;
375 ; GFX900-LABEL: v_insertelement_v2bf16_1_inlineimm:
376 ; GFX900: ; %bb.0:
377 ; GFX900-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
378 ; GFX900-NEXT: v_lshlrev_b32_e32 v0, 2, v0
379 ; GFX900-NEXT: v_mov_b32_e32 v2, 0x5040100
380 ; GFX900-NEXT: s_waitcnt lgkmcnt(0)
381 ; GFX900-NEXT: global_load_dword v1, v0, s[2:3]
382 ; GFX900-NEXT: s_waitcnt vmcnt(0)
383 ; GFX900-NEXT: v_perm_b32 v1, 35, v1, v2
384 ; GFX900-NEXT: global_store_dword v0, v1, s[0:1]
385 ; GFX900-NEXT: s_endpgm
386 ;
387 ; GFX940-LABEL: v_insertelement_v2bf16_1_inlineimm:
388 ; GFX940: ; %bb.0:
389 ; GFX940-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
390 ; GFX940-NEXT: v_and_b32_e32 v0, 0x3ff, v0
391 ; GFX940-NEXT: v_lshlrev_b32_e32 v0, 2, v0
392 ; GFX940-NEXT: v_mov_b32_e32 v2, 0x5040100
393 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
394 ; GFX940-NEXT: global_load_dword v1, v0, s[2:3]
395 ; GFX940-NEXT: s_waitcnt vmcnt(0)
396 ; GFX940-NEXT: v_perm_b32 v1, 35, v1, v2
397 ; GFX940-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
398 ; GFX940-NEXT: s_endpgm
399 %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
400 %tid.ext = sext i32 %tid to i64
401 %in.gep = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
402 %out.gep = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
403 %vec = load <2 x bfloat>, ptr addrspace(1) %in.gep
404 %vecins = insertelement <2 x bfloat> %vec, bfloat 0xR0023, i32 1
405 store <2 x bfloat> %vecins, ptr addrspace(1) %out.gep
406 ret void
407 }
409 define amdgpu_kernel void @v_insertelement_v2bf16_dynamic_vgpr(ptr addrspace(1) %out, ptr addrspace(1) %in, ptr addrspace(1) %idx.ptr) #0 {
410 ; SI-LABEL: v_insertelement_v2bf16_dynamic_vgpr:
411 ; SI: ; %bb.0:
412 ; SI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
413 ; SI-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x4
414 ; SI-NEXT: s_mov_b32 s11, 0x100f000
415 ; SI-NEXT: s_mov_b32 s10, 0
416 ; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
417 ; SI-NEXT: v_mov_b32_e32 v1, 0
418 ; SI-NEXT: s_mov_b64 s[6:7], s[10:11]
419 ; SI-NEXT: s_waitcnt lgkmcnt(0)
420 ; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
421 ; SI-NEXT: s_mov_b64 s[8:9], s[2:3]
422 ; SI-NEXT: buffer_load_dword v3, v[0:1], s[8:11], 0 addr64
423 ; SI-NEXT: s_mov_b32 s4, 0x12341234
424 ; SI-NEXT: s_mov_b64 s[2:3], s[10:11]
425 ; SI-NEXT: s_waitcnt vmcnt(1)
426 ; SI-NEXT: v_lshlrev_b32_e32 v2, 4, v2
427 ; SI-NEXT: v_lshl_b32_e32 v2, 0xffff, v2
428 ; SI-NEXT: s_waitcnt vmcnt(0)
429 ; SI-NEXT: v_bfi_b32 v2, v2, s4, v3
430 ; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
431 ; SI-NEXT: s_endpgm
432 ;
433 ; VI-LABEL: v_insertelement_v2bf16_dynamic_vgpr:
434 ; VI: ; %bb.0:
435 ; VI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
436 ; VI-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x10
437 ; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
438 ; VI-NEXT: s_waitcnt lgkmcnt(0)
439 ; VI-NEXT: v_mov_b32_e32 v3, s3
440 ; VI-NEXT: v_mov_b32_e32 v1, s5
441 ; VI-NEXT: v_add_u32_e32 v0, vcc, s4, v2
442 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
443 ; VI-NEXT: flat_load_dword v4, v[0:1]
444 ; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
445 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
446 ; VI-NEXT: flat_load_dword v3, v[0:1]
447 ; VI-NEXT: s_mov_b32 s2, 0xffff
448 ; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
449 ; VI-NEXT: v_mov_b32_e32 v1, s1
450 ; VI-NEXT: s_mov_b32 s0, 0x12341234
451 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
452 ; VI-NEXT: s_waitcnt vmcnt(1)
453 ; VI-NEXT: v_lshlrev_b32_e32 v2, 4, v4
454 ; VI-NEXT: v_lshlrev_b32_e64 v2, v2, s2
455 ; VI-NEXT: s_waitcnt vmcnt(0)
456 ; VI-NEXT: v_bfi_b32 v2, v2, s0, v3
457 ; VI-NEXT: flat_store_dword v[0:1], v2
458 ; VI-NEXT: s_endpgm
459 ;
460 ; GFX900-LABEL: v_insertelement_v2bf16_dynamic_vgpr:
461 ; GFX900: ; %bb.0:
462 ; GFX900-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x10
463 ; GFX900-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
464 ; GFX900-NEXT: v_lshlrev_b32_e32 v0, 2, v0
465 ; GFX900-NEXT: s_waitcnt lgkmcnt(0)
466 ; GFX900-NEXT: global_load_dword v1, v0, s[4:5]
467 ; GFX900-NEXT: global_load_dword v2, v0, s[2:3]
468 ; GFX900-NEXT: s_mov_b32 s2, 0xffff
469 ; GFX900-NEXT: s_waitcnt vmcnt(1)
470 ; GFX900-NEXT: v_lshlrev_b32_e32 v1, 4, v1
471 ; GFX900-NEXT: v_lshlrev_b32_e64 v1, v1, s2
472 ; GFX900-NEXT: s_mov_b32 s2, 0x12341234
473 ; GFX900-NEXT: s_waitcnt vmcnt(0)
474 ; GFX900-NEXT: v_bfi_b32 v1, v1, s2, v2
475 ; GFX900-NEXT: global_store_dword v0, v1, s[0:1]
476 ; GFX900-NEXT: s_endpgm
477 ;
478 ; GFX940-LABEL: v_insertelement_v2bf16_dynamic_vgpr:
479 ; GFX940: ; %bb.0:
480 ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x10
481 ; GFX940-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
482 ; GFX940-NEXT: v_and_b32_e32 v0, 0x3ff, v0
483 ; GFX940-NEXT: v_lshlrev_b32_e32 v0, 2, v0
484 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
485 ; GFX940-NEXT: global_load_dword v1, v0, s[0:1]
486 ; GFX940-NEXT: global_load_dword v2, v0, s[6:7]
487 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
488 ; GFX940-NEXT: s_waitcnt vmcnt(1)
489 ; GFX940-NEXT: v_lshlrev_b32_e32 v1, 4, v1
490 ; GFX940-NEXT: v_lshlrev_b32_e64 v1, v1, s0
491 ; GFX940-NEXT: s_mov_b32 s0, 0x12341234
492 ; GFX940-NEXT: s_waitcnt vmcnt(0)
493 ; GFX940-NEXT: v_bfi_b32 v1, v1, s0, v2
494 ; GFX940-NEXT: global_store_dword v0, v1, s[4:5] sc0 sc1
495 ; GFX940-NEXT: s_endpgm
496 %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
497 %tid.ext = sext i32 %tid to i64
498 %in.gep = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
499 %idx.gep = getelementptr inbounds i32, ptr addrspace(1) %idx.ptr, i64 %tid.ext
500 %out.gep = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
501 %idx = load i32, ptr addrspace(1) %idx.gep
502 %vec = load <2 x bfloat>, ptr addrspace(1) %in.gep
503 %vecins = insertelement <2 x bfloat> %vec, bfloat 0xR1234, i32 %idx
504 store <2 x bfloat> %vecins, ptr addrspace(1) %out.gep
505 ret void
506 }
508 define amdgpu_kernel void @v_insertelement_v4bf16_0(ptr addrspace(1) %out, ptr addrspace(1) %in, [8 x i32], i32 %val) #0 {
509 ; SI-LABEL: v_insertelement_v4bf16_0:
510 ; SI: ; %bb.0:
511 ; SI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
512 ; SI-NEXT: s_load_dword s4, s[6:7], 0xc
513 ; SI-NEXT: s_mov_b32 s11, 0x100f000
514 ; SI-NEXT: s_mov_b32 s10, 0
515 ; SI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
516 ; SI-NEXT: s_waitcnt lgkmcnt(0)
517 ; SI-NEXT: s_mov_b64 s[8:9], s[2:3]
518 ; SI-NEXT: v_mov_b32_e32 v1, 0
519 ; SI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[8:11], 0 addr64
520 ; SI-NEXT: s_mov_b32 s5, 0xffff
521 ; SI-NEXT: v_mov_b32_e32 v4, s4
522 ; SI-NEXT: s_mov_b64 s[2:3], s[10:11]
523 ; SI-NEXT: s_waitcnt vmcnt(0)
524 ; SI-NEXT: v_bfi_b32 v2, s5, v4, v2
525 ; SI-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
526 ; SI-NEXT: s_endpgm
527 ;
528 ; VI-LABEL: v_insertelement_v4bf16_0:
529 ; VI: ; %bb.0:
530 ; VI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
531 ; VI-NEXT: s_load_dword s4, s[6:7], 0x30
532 ; VI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
533 ; VI-NEXT: v_mov_b32_e32 v4, 0x3020504
534 ; VI-NEXT: s_waitcnt lgkmcnt(0)
535 ; VI-NEXT: v_mov_b32_e32 v1, s3
536 ; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
537 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
538 ; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
539 ; VI-NEXT: v_mov_b32_e32 v3, s1
540 ; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v2
541 ; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
542 ; VI-NEXT: s_waitcnt vmcnt(0)
543 ; VI-NEXT: v_perm_b32 v0, s4, v0, v4
544 ; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
545 ; VI-NEXT: s_endpgm
546 ;
547 ; GFX900-LABEL: v_insertelement_v4bf16_0:
548 ; GFX900: ; %bb.0:
549 ; GFX900-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
550 ; GFX900-NEXT: s_load_dword s4, s[6:7], 0x30
551 ; GFX900-NEXT: v_lshlrev_b32_e32 v2, 3, v0
552 ; GFX900-NEXT: s_waitcnt lgkmcnt(0)
553 ; GFX900-NEXT: global_load_dwordx2 v[0:1], v2, s[2:3]
554 ; GFX900-NEXT: s_mov_b32 s2, 0xffff
555 ; GFX900-NEXT: v_mov_b32_e32 v3, s4
556 ; GFX900-NEXT: s_waitcnt vmcnt(0)
557 ; GFX900-NEXT: v_bfi_b32 v0, s2, v3, v0
558 ; GFX900-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
559 ; GFX900-NEXT: s_endpgm
560 ;
561 ; GFX940-LABEL: v_insertelement_v4bf16_0:
562 ; GFX940: ; %bb.0:
563 ; GFX940-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
564 ; GFX940-NEXT: s_load_dword s0, s[2:3], 0x30
565 ; GFX940-NEXT: v_and_b32_e32 v0, 0x3ff, v0
566 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 3, v0
567 ; GFX940-NEXT: s_mov_b32 s1, 0xffff
568 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
569 ; GFX940-NEXT: global_load_dwordx2 v[0:1], v2, s[6:7]
570 ; GFX940-NEXT: v_mov_b32_e32 v3, s0
571 ; GFX940-NEXT: s_waitcnt vmcnt(0)
572 ; GFX940-NEXT: v_bfi_b32 v0, s1, v3, v0
573 ; GFX940-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5] sc0 sc1
574 ; GFX940-NEXT: s_endpgm
575 %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
576 %tid.ext = sext i32 %tid to i64
577 %in.gep = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
578 %out.gep = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
579 %vec = load <4 x bfloat>, ptr addrspace(1) %in.gep
580 %val.trunc = trunc i32 %val to i16
581 %val.cvt = bitcast i16 %val.trunc to bfloat
582 %vecins = insertelement <4 x bfloat> %vec, bfloat %val.cvt, i32 0
583 store <4 x bfloat> %vecins, ptr addrspace(1) %out.gep
584 ret void
585 }
587 define amdgpu_kernel void @v_insertelement_v4bf16_1(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %val) #0 {
588 ; SI-LABEL: v_insertelement_v4bf16_1:
589 ; SI: ; %bb.0:
590 ; SI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
591 ; SI-NEXT: s_load_dword s4, s[6:7], 0x4
592 ; SI-NEXT: s_mov_b32 s11, 0x100f000
593 ; SI-NEXT: s_mov_b32 s10, 0
594 ; SI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
595 ; SI-NEXT: s_waitcnt lgkmcnt(0)
596 ; SI-NEXT: s_mov_b64 s[8:9], s[2:3]
597 ; SI-NEXT: v_mov_b32_e32 v1, 0
598 ; SI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[8:11], 0 addr64
599 ; SI-NEXT: s_lshl_b32 s4, s4, 16
600 ; SI-NEXT: s_mov_b64 s[2:3], s[10:11]
601 ; SI-NEXT: s_waitcnt vmcnt(0)
602 ; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
603 ; SI-NEXT: v_or_b32_e32 v2, s4, v2
604 ; SI-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
605 ; SI-NEXT: s_endpgm
606 ;
607 ; VI-LABEL: v_insertelement_v4bf16_1:
608 ; VI: ; %bb.0:
609 ; VI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
610 ; VI-NEXT: s_load_dword s4, s[6:7], 0x10
611 ; VI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
612 ; VI-NEXT: v_mov_b32_e32 v4, 0x1000504
613 ; VI-NEXT: s_waitcnt lgkmcnt(0)
614 ; VI-NEXT: v_mov_b32_e32 v1, s3
615 ; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
616 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
617 ; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
618 ; VI-NEXT: v_mov_b32_e32 v3, s1
619 ; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v2
620 ; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
621 ; VI-NEXT: s_waitcnt vmcnt(0)
622 ; VI-NEXT: v_perm_b32 v0, v0, s4, v4
623 ; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
624 ; VI-NEXT: s_endpgm
625 ;
626 ; GFX900-LABEL: v_insertelement_v4bf16_1:
627 ; GFX900: ; %bb.0:
628 ; GFX900-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
629 ; GFX900-NEXT: s_load_dword s4, s[6:7], 0x10
630 ; GFX900-NEXT: v_lshlrev_b32_e32 v2, 3, v0
631 ; GFX900-NEXT: v_mov_b32_e32 v3, 0x5040100
632 ; GFX900-NEXT: s_waitcnt lgkmcnt(0)
633 ; GFX900-NEXT: global_load_dwordx2 v[0:1], v2, s[2:3]
634 ; GFX900-NEXT: s_waitcnt vmcnt(0)
635 ; GFX900-NEXT: v_perm_b32 v0, s4, v0, v3
636 ; GFX900-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
637 ; GFX900-NEXT: s_endpgm
638 ;
639 ; GFX940-LABEL: v_insertelement_v4bf16_1:
640 ; GFX940: ; %bb.0:
641 ; GFX940-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
642 ; GFX940-NEXT: s_load_dword s0, s[2:3], 0x10
643 ; GFX940-NEXT: v_and_b32_e32 v0, 0x3ff, v0
644 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 3, v0
645 ; GFX940-NEXT: v_mov_b32_e32 v3, 0x5040100
646 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
647 ; GFX940-NEXT: global_load_dwordx2 v[0:1], v2, s[6:7]
648 ; GFX940-NEXT: s_waitcnt vmcnt(0)
649 ; GFX940-NEXT: v_perm_b32 v0, s0, v0, v3
650 ; GFX940-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5] sc0 sc1
651 ; GFX940-NEXT: s_endpgm
652 %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
653 %tid.ext = sext i32 %tid to i64
654 %in.gep = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
655 %out.gep = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
656 %vec = load <4 x bfloat>, ptr addrspace(1) %in.gep
657 %val.trunc = trunc i32 %val to i16
658 %val.cvt = bitcast i16 %val.trunc to bfloat
659 %vecins = insertelement <4 x bfloat> %vec, bfloat %val.cvt, i32 1
660 store <4 x bfloat> %vecins, ptr addrspace(1) %out.gep
661 ret void
662 }
664 define amdgpu_kernel void @v_insertelement_v4bf16_2(ptr addrspace(1) %out, ptr addrspace(1) %in, [8 x i32], i32 %val) #0 {
665 ; SI-LABEL: v_insertelement_v4bf16_2:
666 ; SI: ; %bb.0:
667 ; SI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
668 ; SI-NEXT: s_load_dword s4, s[6:7], 0xc
669 ; SI-NEXT: s_mov_b32 s11, 0x100f000
670 ; SI-NEXT: s_mov_b32 s10, 0
671 ; SI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
672 ; SI-NEXT: s_waitcnt lgkmcnt(0)
673 ; SI-NEXT: s_mov_b64 s[8:9], s[2:3]
674 ; SI-NEXT: v_mov_b32_e32 v1, 0
675 ; SI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[8:11], 0 addr64
676 ; SI-NEXT: s_mov_b32 s5, 0xffff
677 ; SI-NEXT: v_mov_b32_e32 v4, s4
678 ; SI-NEXT: s_mov_b64 s[2:3], s[10:11]
679 ; SI-NEXT: s_waitcnt vmcnt(0)
680 ; SI-NEXT: v_bfi_b32 v3, s5, v4, v3
681 ; SI-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
682 ; SI-NEXT: s_endpgm
683 ;
684 ; VI-LABEL: v_insertelement_v4bf16_2:
685 ; VI: ; %bb.0:
686 ; VI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
687 ; VI-NEXT: s_load_dword s4, s[6:7], 0x30
688 ; VI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
689 ; VI-NEXT: v_mov_b32_e32 v4, 0x3020504
690 ; VI-NEXT: s_waitcnt lgkmcnt(0)
691 ; VI-NEXT: v_mov_b32_e32 v1, s3
692 ; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
693 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
694 ; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
695 ; VI-NEXT: v_mov_b32_e32 v3, s1
696 ; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v2
697 ; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
698 ; VI-NEXT: s_waitcnt vmcnt(0)
699 ; VI-NEXT: v_perm_b32 v1, s4, v1, v4
700 ; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
701 ; VI-NEXT: s_endpgm
702 ;
703 ; GFX900-LABEL: v_insertelement_v4bf16_2:
704 ; GFX900: ; %bb.0:
705 ; GFX900-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
706 ; GFX900-NEXT: s_load_dword s4, s[6:7], 0x30
707 ; GFX900-NEXT: v_lshlrev_b32_e32 v2, 3, v0
708 ; GFX900-NEXT: s_waitcnt lgkmcnt(0)
709 ; GFX900-NEXT: global_load_dwordx2 v[0:1], v2, s[2:3]
710 ; GFX900-NEXT: s_mov_b32 s2, 0xffff
711 ; GFX900-NEXT: v_mov_b32_e32 v3, s4
712 ; GFX900-NEXT: s_waitcnt vmcnt(0)
713 ; GFX900-NEXT: v_bfi_b32 v1, s2, v3, v1
714 ; GFX900-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
715 ; GFX900-NEXT: s_endpgm
716 ;
717 ; GFX940-LABEL: v_insertelement_v4bf16_2:
718 ; GFX940: ; %bb.0:
719 ; GFX940-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
720 ; GFX940-NEXT: s_load_dword s0, s[2:3], 0x30
721 ; GFX940-NEXT: v_and_b32_e32 v0, 0x3ff, v0
722 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 3, v0
723 ; GFX940-NEXT: s_mov_b32 s1, 0xffff
724 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
725 ; GFX940-NEXT: global_load_dwordx2 v[0:1], v2, s[6:7]
726 ; GFX940-NEXT: v_mov_b32_e32 v3, s0
727 ; GFX940-NEXT: s_waitcnt vmcnt(0)
728 ; GFX940-NEXT: v_bfi_b32 v1, s1, v3, v1
729 ; GFX940-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5] sc0 sc1
730 ; GFX940-NEXT: s_endpgm
731 %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
732 %tid.ext = sext i32 %tid to i64
733 %in.gep = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
734 %out.gep = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
735 %vec = load <4 x bfloat>, ptr addrspace(1) %in.gep
736 %val.trunc = trunc i32 %val to i16
737 %val.cvt = bitcast i16 %val.trunc to bfloat
738 %vecins = insertelement <4 x bfloat> %vec, bfloat %val.cvt, i32 2
739 store <4 x bfloat> %vecins, ptr addrspace(1) %out.gep
740 ret void
741 }
743 define amdgpu_kernel void @v_insertelement_v4bf16_3(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %val) #0 {
744 ; SI-LABEL: v_insertelement_v4bf16_3:
745 ; SI: ; %bb.0:
746 ; SI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
747 ; SI-NEXT: s_load_dword s4, s[6:7], 0x4
748 ; SI-NEXT: s_mov_b32 s11, 0x100f000
749 ; SI-NEXT: s_mov_b32 s10, 0
750 ; SI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
751 ; SI-NEXT: s_waitcnt lgkmcnt(0)
752 ; SI-NEXT: s_mov_b64 s[8:9], s[2:3]
753 ; SI-NEXT: v_mov_b32_e32 v1, 0
754 ; SI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[8:11], 0 addr64
755 ; SI-NEXT: s_lshl_b32 s4, s4, 16
756 ; SI-NEXT: s_mov_b64 s[2:3], s[10:11]
757 ; SI-NEXT: s_waitcnt vmcnt(0)
758 ; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
759 ; SI-NEXT: v_or_b32_e32 v3, s4, v3
760 ; SI-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
761 ; SI-NEXT: s_endpgm
762 ;
763 ; VI-LABEL: v_insertelement_v4bf16_3:
764 ; VI: ; %bb.0:
765 ; VI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
766 ; VI-NEXT: s_load_dword s4, s[6:7], 0x10
767 ; VI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
768 ; VI-NEXT: v_mov_b32_e32 v4, 0x1000504
769 ; VI-NEXT: s_waitcnt lgkmcnt(0)
770 ; VI-NEXT: v_mov_b32_e32 v1, s3
771 ; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
772 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
773 ; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
774 ; VI-NEXT: v_mov_b32_e32 v3, s1
775 ; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v2
776 ; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
777 ; VI-NEXT: s_waitcnt vmcnt(0)
778 ; VI-NEXT: v_perm_b32 v1, v1, s4, v4
779 ; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
780 ; VI-NEXT: s_endpgm
781 ;
782 ; GFX900-LABEL: v_insertelement_v4bf16_3:
783 ; GFX900: ; %bb.0:
784 ; GFX900-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
785 ; GFX900-NEXT: s_load_dword s4, s[6:7], 0x10
786 ; GFX900-NEXT: v_lshlrev_b32_e32 v2, 3, v0
787 ; GFX900-NEXT: v_mov_b32_e32 v3, 0x5040100
788 ; GFX900-NEXT: s_waitcnt lgkmcnt(0)
789 ; GFX900-NEXT: global_load_dwordx2 v[0:1], v2, s[2:3]
790 ; GFX900-NEXT: s_waitcnt vmcnt(0)
791 ; GFX900-NEXT: v_perm_b32 v1, s4, v1, v3
792 ; GFX900-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
793 ; GFX900-NEXT: s_endpgm
794 ;
795 ; GFX940-LABEL: v_insertelement_v4bf16_3:
796 ; GFX940: ; %bb.0:
797 ; GFX940-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
798 ; GFX940-NEXT: s_load_dword s0, s[2:3], 0x10
799 ; GFX940-NEXT: v_and_b32_e32 v0, 0x3ff, v0
800 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 3, v0
801 ; GFX940-NEXT: v_mov_b32_e32 v3, 0x5040100
802 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
803 ; GFX940-NEXT: global_load_dwordx2 v[0:1], v2, s[6:7]
804 ; GFX940-NEXT: s_waitcnt vmcnt(0)
805 ; GFX940-NEXT: v_perm_b32 v1, s0, v1, v3
806 ; GFX940-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5] sc0 sc1
807 ; GFX940-NEXT: s_endpgm
808 %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
809 %tid.ext = sext i32 %tid to i64
810 %in.gep = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
811 %out.gep = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
812 %vec = load <4 x bfloat>, ptr addrspace(1) %in.gep
813 %val.trunc = trunc i32 %val to i16
814 %val.cvt = bitcast i16 %val.trunc to bfloat
815 %vecins = insertelement <4 x bfloat> %vec, bfloat %val.cvt, i32 3
816 store <4 x bfloat> %vecins, ptr addrspace(1) %out.gep
817 ret void
818 }
820 define amdgpu_kernel void @v_insertelement_v4bf16_dynamic_sgpr(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %val, i32 %idxval) #0 {
821 ; SI-LABEL: v_insertelement_v4bf16_dynamic_sgpr:
822 ; SI: ; %bb.0:
823 ; SI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
824 ; SI-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x4
825 ; SI-NEXT: s_mov_b32 s11, 0x100f000
826 ; SI-NEXT: s_mov_b32 s10, 0
827 ; SI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
828 ; SI-NEXT: s_waitcnt lgkmcnt(0)
829 ; SI-NEXT: s_mov_b64 s[8:9], s[2:3]
830 ; SI-NEXT: v_mov_b32_e32 v1, 0
831 ; SI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[8:11], 0 addr64
832 ; SI-NEXT: s_lshl_b32 s6, s4, 16
833 ; SI-NEXT: s_and_b32 s4, s4, 0xffff
834 ; SI-NEXT: s_lshl_b32 s5, s5, 4
835 ; SI-NEXT: s_or_b32 s6, s4, s6
836 ; SI-NEXT: s_lshl_b64 s[4:5], 0xffff, s5
837 ; SI-NEXT: v_mov_b32_e32 v4, s6
838 ; SI-NEXT: v_mov_b32_e32 v5, s6
839 ; SI-NEXT: s_mov_b64 s[2:3], s[10:11]
840 ; SI-NEXT: s_waitcnt vmcnt(0)
841 ; SI-NEXT: v_bfi_b32 v3, s5, v4, v3
842 ; SI-NEXT: v_bfi_b32 v2, s4, v5, v2
843 ; SI-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
844 ; SI-NEXT: s_endpgm
845 ;
846 ; VI-LABEL: v_insertelement_v4bf16_dynamic_sgpr:
847 ; VI: ; %bb.0:
848 ; VI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
849 ; VI-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x10
850 ; VI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
851 ; VI-NEXT: s_waitcnt lgkmcnt(0)
852 ; VI-NEXT: v_mov_b32_e32 v1, s3
853 ; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
854 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
855 ; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
856 ; VI-NEXT: v_mov_b32_e32 v3, s1
857 ; VI-NEXT: s_lshl_b32 s1, s4, 16
858 ; VI-NEXT: s_and_b32 s2, s4, 0xffff
859 ; VI-NEXT: s_lshl_b32 s3, s5, 4
860 ; VI-NEXT: s_or_b32 s2, s2, s1
861 ; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v2
862 ; VI-NEXT: s_lshl_b64 s[0:1], 0xffff, s3
863 ; VI-NEXT: v_mov_b32_e32 v4, s2
864 ; VI-NEXT: v_mov_b32_e32 v5, s2
865 ; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
866 ; VI-NEXT: s_waitcnt vmcnt(0)
867 ; VI-NEXT: v_bfi_b32 v1, s1, v4, v1
868 ; VI-NEXT: v_bfi_b32 v0, s0, v5, v0
869 ; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
870 ; VI-NEXT: s_endpgm
871 ;
872 ; GFX900-LABEL: v_insertelement_v4bf16_dynamic_sgpr:
873 ; GFX900: ; %bb.0:
874 ; GFX900-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
875 ; GFX900-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x10
876 ; GFX900-NEXT: v_lshlrev_b32_e32 v2, 3, v0
877 ; GFX900-NEXT: s_waitcnt lgkmcnt(0)
878 ; GFX900-NEXT: global_load_dwordx2 v[0:1], v2, s[2:3]
879 ; GFX900-NEXT: s_lshl_b32 s2, s5, 4
880 ; GFX900-NEXT: s_pack_ll_b32_b16 s4, s4, s4
881 ; GFX900-NEXT: s_lshl_b64 s[2:3], 0xffff, s2
882 ; GFX900-NEXT: v_mov_b32_e32 v3, s4
883 ; GFX900-NEXT: v_mov_b32_e32 v4, s4
884 ; GFX900-NEXT: s_waitcnt vmcnt(0)
885 ; GFX900-NEXT: v_bfi_b32 v1, s3, v3, v1
886 ; GFX900-NEXT: v_bfi_b32 v0, s2, v4, v0
887 ; GFX900-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
888 ; GFX900-NEXT: s_endpgm
889 ;
890 ; GFX940-LABEL: v_insertelement_v4bf16_dynamic_sgpr:
891 ; GFX940: ; %bb.0:
892 ; GFX940-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
893 ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x10
894 ; GFX940-NEXT: v_and_b32_e32 v0, 0x3ff, v0
895 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 3, v0
896 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
897 ; GFX940-NEXT: global_load_dwordx2 v[0:1], v2, s[6:7]
898 ; GFX940-NEXT: s_lshl_b32 s1, s1, 4
899 ; GFX940-NEXT: s_pack_ll_b32_b16 s2, s0, s0
900 ; GFX940-NEXT: s_lshl_b64 s[0:1], 0xffff, s1
901 ; GFX940-NEXT: v_mov_b32_e32 v3, s2
902 ; GFX940-NEXT: v_mov_b32_e32 v4, s2
903 ; GFX940-NEXT: s_waitcnt vmcnt(0)
904 ; GFX940-NEXT: v_bfi_b32 v1, s1, v3, v1
905 ; GFX940-NEXT: v_bfi_b32 v0, s0, v4, v0
906 ; GFX940-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5] sc0 sc1
907 ; GFX940-NEXT: s_endpgm
908 %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
909 %tid.ext = sext i32 %tid to i64
910 %in.gep = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
911 %out.gep = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
912 %vec = load <4 x bfloat>, ptr addrspace(1) %in.gep
913 %val.trunc = trunc i32 %val to i16
914 %val.cvt = bitcast i16 %val.trunc to bfloat
915 %vecins = insertelement <4 x bfloat> %vec, bfloat %val.cvt, i32 %idxval
916 store <4 x bfloat> %vecins, ptr addrspace(1) %out.gep
917 ret void
918 }
920 define amdgpu_kernel void @v_insertelement_v8bf16_3(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %val) {
921 ; SI-LABEL: v_insertelement_v8bf16_3:
922 ; SI: ; %bb.0:
923 ; SI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
924 ; SI-NEXT: s_load_dword s4, s[6:7], 0x4
925 ; SI-NEXT: s_mov_b32 s11, 0x100f000
926 ; SI-NEXT: s_mov_b32 s10, 0
927 ; SI-NEXT: v_lshlrev_b32_e32 v4, 4, v0
928 ; SI-NEXT: s_waitcnt lgkmcnt(0)
929 ; SI-NEXT: s_mov_b64 s[8:9], s[2:3]
930 ; SI-NEXT: v_mov_b32_e32 v5, 0
931 ; SI-NEXT: buffer_load_dwordx4 v[0:3], v[4:5], s[8:11], 0 addr64
932 ; SI-NEXT: s_lshl_b32 s4, s4, 16
933 ; SI-NEXT: s_mov_b64 s[2:3], s[10:11]
934 ; SI-NEXT: s_waitcnt vmcnt(0)
935 ; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
936 ; SI-NEXT: v_or_b32_e32 v1, s4, v1
937 ; SI-NEXT: buffer_store_dwordx4 v[0:3], v[4:5], s[0:3], 0 addr64
938 ; SI-NEXT: s_endpgm
939 ;
940 ; VI-LABEL: v_insertelement_v8bf16_3:
941 ; VI: ; %bb.0:
942 ; VI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
943 ; VI-NEXT: s_load_dword s4, s[6:7], 0x10
944 ; VI-NEXT: v_lshlrev_b32_e32 v4, 4, v0
945 ; VI-NEXT: s_waitcnt lgkmcnt(0)
946 ; VI-NEXT: v_mov_b32_e32 v1, s3
947 ; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v4
948 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
949 ; VI-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
950 ; VI-NEXT: v_add_u32_e32 v4, vcc, s0, v4
951 ; VI-NEXT: s_lshl_b32 s0, s4, 16
952 ; VI-NEXT: v_mov_b32_e32 v5, s1
953 ; VI-NEXT: v_mov_b32_e32 v6, s0
954 ; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
955 ; VI-NEXT: s_waitcnt vmcnt(0)
956 ; VI-NEXT: v_or_b32_sdwa v1, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
957 ; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
958 ; VI-NEXT: s_endpgm
959 ;
960 ; GFX900-LABEL: v_insertelement_v8bf16_3:
961 ; GFX900: ; %bb.0:
962 ; GFX900-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
963 ; GFX900-NEXT: s_load_dword s4, s[6:7], 0x10
964 ; GFX900-NEXT: v_lshlrev_b32_e32 v4, 4, v0
965 ; GFX900-NEXT: v_mov_b32_e32 v5, 0x5040100
966 ; GFX900-NEXT: s_waitcnt lgkmcnt(0)
967 ; GFX900-NEXT: global_load_dwordx4 v[0:3], v4, s[2:3]
968 ; GFX900-NEXT: s_mov_b32 s2, 0xffff
969 ; GFX900-NEXT: s_waitcnt vmcnt(0)
970 ; GFX900-NEXT: v_bfi_b32 v3, s2, v3, v3
971 ; GFX900-NEXT: v_bfi_b32 v2, s2, v2, v2
972 ; GFX900-NEXT: v_bfi_b32 v0, s2, v0, v0
973 ; GFX900-NEXT: v_perm_b32 v1, s4, v1, v5
974 ; GFX900-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
975 ; GFX900-NEXT: s_endpgm
976 ;
977 ; GFX940-LABEL: v_insertelement_v8bf16_3:
978 ; GFX940: ; %bb.0:
979 ; GFX940-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
980 ; GFX940-NEXT: s_load_dword s0, s[2:3], 0x10
981 ; GFX940-NEXT: v_and_b32_e32 v0, 0x3ff, v0
982 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 4, v0
983 ; GFX940-NEXT: s_mov_b32 s1, 0xffff
984 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
985 ; GFX940-NEXT: global_load_dwordx4 v[0:3], v4, s[6:7]
986 ; GFX940-NEXT: v_mov_b32_e32 v5, 0x5040100
987 ; GFX940-NEXT: s_waitcnt vmcnt(0)
988 ; GFX940-NEXT: v_bfi_b32 v3, s1, v3, v3
989 ; GFX940-NEXT: v_bfi_b32 v2, s1, v2, v2
990 ; GFX940-NEXT: v_bfi_b32 v0, s1, v0, v0
991 ; GFX940-NEXT: v_perm_b32 v1, s0, v1, v5
992 ; GFX940-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5] sc0 sc1
993 ; GFX940-NEXT: s_endpgm
994 %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
995 %tid.ext = sext i32 %tid to i64
996 %in.gep = getelementptr inbounds <8 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
997 %out.gep = getelementptr inbounds <8 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
998 %vec = load <8 x bfloat>, ptr addrspace(1) %in.gep
999 %val.trunc = trunc i32 %val to i16
1000 %val.cvt = bitcast i16 %val.trunc to bfloat
1001 %vecins = insertelement <8 x bfloat> %vec, bfloat %val.cvt, i32 3
1002 store <8 x bfloat> %vecins, ptr addrspace(1) %out.gep
1003 ret void
1004 }
1006 define amdgpu_kernel void @v_insertelement_v8bf16_dynamic(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %val, i32 %n) {
1007 ; SI-LABEL: v_insertelement_v8bf16_dynamic:
1008 ; SI: ; %bb.0:
1009 ; SI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
1010 ; SI-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x4
1011 ; SI-NEXT: s_mov_b32 s11, 0x100f000
1012 ; SI-NEXT: s_mov_b32 s10, 0
1013 ; SI-NEXT: v_lshlrev_b32_e32 v4, 4, v0
1014 ; SI-NEXT: s_waitcnt lgkmcnt(0)
1015 ; SI-NEXT: s_mov_b64 s[8:9], s[2:3]
1016 ; SI-NEXT: v_mov_b32_e32 v5, 0
1017 ; SI-NEXT: buffer_load_dwordx4 v[0:3], v[4:5], s[8:11], 0 addr64
1018 ; SI-NEXT: s_cmp_eq_u32 s5, 6
1019 ; SI-NEXT: v_mov_b32_e32 v6, s4
1020 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1021 ; SI-NEXT: s_cmp_eq_u32 s5, 7
1022 ; SI-NEXT: s_mov_b64 s[2:3], s[10:11]
1023 ; SI-NEXT: s_waitcnt vmcnt(0)
1024 ; SI-NEXT: v_cndmask_b32_e32 v7, v3, v6, vcc
1025 ; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
1026 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1027 ; SI-NEXT: s_cmp_eq_u32 s5, 4
1028 ; SI-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc
1029 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1030 ; SI-NEXT: s_cmp_eq_u32 s5, 5
1031 ; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v2
1032 ; SI-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc
1033 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1034 ; SI-NEXT: s_cmp_eq_u32 s5, 2
1035 ; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
1036 ; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
1037 ; SI-NEXT: v_cndmask_b32_e32 v8, v8, v6, vcc
1038 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1039 ; SI-NEXT: s_cmp_eq_u32 s5, 3
1040 ; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v1
1041 ; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
1042 ; SI-NEXT: v_or_b32_e32 v3, v7, v3
1043 ; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v8
1044 ; SI-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
1045 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1046 ; SI-NEXT: s_cmp_eq_u32 s5, 0
1047 ; SI-NEXT: v_or_b32_e32 v2, v2, v7
1048 ; SI-NEXT: v_cndmask_b32_e32 v7, v9, v6, vcc
1049 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1050 ; SI-NEXT: s_cmp_eq_u32 s5, 1
1051 ; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v0
1052 ; SI-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc
1053 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1054 ; SI-NEXT: v_cndmask_b32_e32 v6, v10, v6, vcc
1055 ; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
1056 ; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
1057 ; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
1058 ; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
1059 ; SI-NEXT: v_or_b32_e32 v1, v1, v7
1060 ; SI-NEXT: v_or_b32_e32 v0, v0, v6
1061 ; SI-NEXT: buffer_store_dwordx4 v[0:3], v[4:5], s[0:3], 0 addr64
1062 ; SI-NEXT: s_endpgm
1063 ;
1064 ; VI-LABEL: v_insertelement_v8bf16_dynamic:
1065 ; VI: ; %bb.0:
1066 ; VI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
1067 ; VI-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x10
1068 ; VI-NEXT: v_lshlrev_b32_e32 v4, 4, v0
1069 ; VI-NEXT: s_waitcnt lgkmcnt(0)
1070 ; VI-NEXT: v_mov_b32_e32 v1, s3
1071 ; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v4
1072 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1073 ; VI-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
1074 ; VI-NEXT: v_mov_b32_e32 v5, s1
1075 ; VI-NEXT: v_add_u32_e32 v4, vcc, s0, v4
1076 ; VI-NEXT: s_cmp_eq_u32 s5, 6
1077 ; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
1078 ; VI-NEXT: v_mov_b32_e32 v6, s4
1079 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1080 ; VI-NEXT: s_cmp_eq_u32 s5, 7
1081 ; VI-NEXT: s_waitcnt vmcnt(0)
1082 ; VI-NEXT: v_cndmask_b32_e32 v7, v3, v6, vcc
1083 ; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
1084 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1085 ; VI-NEXT: s_cmp_eq_u32 s5, 4
1086 ; VI-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc
1087 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1088 ; VI-NEXT: s_cmp_eq_u32 s5, 5
1089 ; VI-NEXT: v_lshrrev_b32_e32 v8, 16, v2
1090 ; VI-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc
1091 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1092 ; VI-NEXT: s_cmp_eq_u32 s5, 2
1093 ; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
1094 ; VI-NEXT: v_cndmask_b32_e32 v8, v8, v6, vcc
1095 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1096 ; VI-NEXT: s_cmp_eq_u32 s5, 3
1097 ; VI-NEXT: v_lshrrev_b32_e32 v9, 16, v1
1098 ; VI-NEXT: v_or_b32_sdwa v3, v7, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
1099 ; VI-NEXT: v_lshlrev_b32_e32 v7, 16, v8
1100 ; VI-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
1101 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1102 ; VI-NEXT: s_cmp_eq_u32 s5, 0
1103 ; VI-NEXT: v_or_b32_sdwa v2, v2, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
1104 ; VI-NEXT: v_cndmask_b32_e32 v7, v9, v6, vcc
1105 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1106 ; VI-NEXT: s_cmp_eq_u32 s5, 1
1107 ; VI-NEXT: v_lshrrev_b32_e32 v10, 16, v0
1108 ; VI-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc
1109 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1110 ; VI-NEXT: v_cndmask_b32_e32 v6, v10, v6, vcc
1111 ; VI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
1112 ; VI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
1113 ; VI-NEXT: v_or_b32_sdwa v1, v1, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
1114 ; VI-NEXT: v_or_b32_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
1115 ; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
1116 ; VI-NEXT: s_endpgm
1117 ;
1118 ; GFX900-LABEL: v_insertelement_v8bf16_dynamic:
1119 ; GFX900: ; %bb.0:
1120 ; GFX900-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
1121 ; GFX900-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x10
1122 ; GFX900-NEXT: v_lshlrev_b32_e32 v4, 4, v0
1123 ; GFX900-NEXT: s_waitcnt lgkmcnt(0)
1124 ; GFX900-NEXT: global_load_dwordx4 v[0:3], v4, s[2:3]
1125 ; GFX900-NEXT: s_cmp_eq_u32 s5, 6
1126 ; GFX900-NEXT: v_mov_b32_e32 v5, s4
1127 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1128 ; GFX900-NEXT: s_cmp_eq_u32 s5, 7
1129 ; GFX900-NEXT: s_mov_b32 s2, 0x5040100
1130 ; GFX900-NEXT: s_waitcnt vmcnt(0)
1131 ; GFX900-NEXT: v_cndmask_b32_e32 v6, v3, v5, vcc
1132 ; GFX900-NEXT: v_lshrrev_b32_e32 v3, 16, v3
1133 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1134 ; GFX900-NEXT: s_cmp_eq_u32 s5, 4
1135 ; GFX900-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
1136 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1137 ; GFX900-NEXT: s_cmp_eq_u32 s5, 5
1138 ; GFX900-NEXT: v_lshrrev_b32_e32 v7, 16, v2
1139 ; GFX900-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
1140 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1141 ; GFX900-NEXT: s_cmp_eq_u32 s5, 2
1142 ; GFX900-NEXT: v_perm_b32 v3, v3, v6, s2
1143 ; GFX900-NEXT: v_cndmask_b32_e32 v6, v7, v5, vcc
1144 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1145 ; GFX900-NEXT: s_cmp_eq_u32 s5, 3
1146 ; GFX900-NEXT: v_lshrrev_b32_e32 v8, 16, v1
1147 ; GFX900-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
1148 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1149 ; GFX900-NEXT: s_cmp_eq_u32 s5, 0
1150 ; GFX900-NEXT: v_perm_b32 v2, v6, v2, s2
1151 ; GFX900-NEXT: v_cndmask_b32_e32 v6, v8, v5, vcc
1152 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1153 ; GFX900-NEXT: s_cmp_eq_u32 s5, 1
1154 ; GFX900-NEXT: v_lshrrev_b32_e32 v9, 16, v0
1155 ; GFX900-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
1156 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1157 ; GFX900-NEXT: v_cndmask_b32_e32 v5, v9, v5, vcc
1158 ; GFX900-NEXT: v_perm_b32 v1, v6, v1, s2
1159 ; GFX900-NEXT: v_perm_b32 v0, v5, v0, s2
1160 ; GFX900-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
1161 ; GFX900-NEXT: s_endpgm
1162 ;
1163 ; GFX940-LABEL: v_insertelement_v8bf16_dynamic:
1164 ; GFX940: ; %bb.0:
1165 ; GFX940-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
1166 ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x10
1167 ; GFX940-NEXT: v_and_b32_e32 v0, 0x3ff, v0
1168 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 4, v0
1169 ; GFX940-NEXT: s_mov_b32 s2, 0x5040100
1170 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
1171 ; GFX940-NEXT: global_load_dwordx4 v[0:3], v4, s[6:7]
1172 ; GFX940-NEXT: s_cmp_eq_u32 s1, 6
1173 ; GFX940-NEXT: v_mov_b32_e32 v5, s0
1174 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1175 ; GFX940-NEXT: s_cmp_eq_u32 s1, 7
1176 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1177 ; GFX940-NEXT: v_cndmask_b32_e32 v6, v3, v5, vcc
1178 ; GFX940-NEXT: v_lshrrev_b32_e32 v3, 16, v3
1179 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1180 ; GFX940-NEXT: s_cmp_eq_u32 s1, 4
1181 ; GFX940-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
1182 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1183 ; GFX940-NEXT: s_cmp_eq_u32 s1, 5
1184 ; GFX940-NEXT: v_lshrrev_b32_e32 v7, 16, v2
1185 ; GFX940-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
1186 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1187 ; GFX940-NEXT: s_cmp_eq_u32 s1, 2
1188 ; GFX940-NEXT: v_perm_b32 v3, v3, v6, s2
1189 ; GFX940-NEXT: v_cndmask_b32_e32 v6, v7, v5, vcc
1190 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1191 ; GFX940-NEXT: s_cmp_eq_u32 s1, 3
1192 ; GFX940-NEXT: v_lshrrev_b32_e32 v8, 16, v1
1193 ; GFX940-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
1194 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1195 ; GFX940-NEXT: s_cmp_eq_u32 s1, 0
1196 ; GFX940-NEXT: v_perm_b32 v2, v6, v2, s2
1197 ; GFX940-NEXT: v_cndmask_b32_e32 v6, v8, v5, vcc
1198 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1199 ; GFX940-NEXT: s_cmp_eq_u32 s1, 1
1200 ; GFX940-NEXT: v_lshrrev_b32_e32 v9, 16, v0
1201 ; GFX940-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
1202 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1203 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v9, v5, vcc
1204 ; GFX940-NEXT: v_perm_b32 v1, v6, v1, s2
1205 ; GFX940-NEXT: v_perm_b32 v0, v5, v0, s2
1206 ; GFX940-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5] sc0 sc1
1207 ; GFX940-NEXT: s_endpgm
1208 %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
1209 %tid.ext = sext i32 %tid to i64
1210 %in.gep = getelementptr inbounds <8 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
1211 %out.gep = getelementptr inbounds <8 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
1212 %vec = load <8 x bfloat>, ptr addrspace(1) %in.gep
1213 %val.trunc = trunc i32 %val to i16
1214 %val.cvt = bitcast i16 %val.trunc to bfloat
1215 %vecins = insertelement <8 x bfloat> %vec, bfloat %val.cvt, i32 %n
1216 store <8 x bfloat> %vecins, ptr addrspace(1) %out.gep
1217 ret void
1218 }
1220 define amdgpu_kernel void @v_insertelement_v16bf16_3(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %val) {
1221 ; SI-LABEL: v_insertelement_v16bf16_3:
1222 ; SI: ; %bb.0:
1223 ; SI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
1224 ; SI-NEXT: s_load_dword s4, s[6:7], 0x4
1225 ; SI-NEXT: s_mov_b32 s11, 0x100f000
1226 ; SI-NEXT: s_mov_b32 s10, 0
1227 ; SI-NEXT: v_lshlrev_b32_e32 v8, 5, v0
1228 ; SI-NEXT: s_waitcnt lgkmcnt(0)
1229 ; SI-NEXT: s_mov_b64 s[8:9], s[2:3]
1230 ; SI-NEXT: v_mov_b32_e32 v9, 0
1231 ; SI-NEXT: buffer_load_dwordx4 v[0:3], v[8:9], s[8:11], 0 addr64
1232 ; SI-NEXT: buffer_load_dwordx4 v[4:7], v[8:9], s[8:11], 0 addr64 offset:16
1233 ; SI-NEXT: s_mov_b64 s[2:3], s[10:11]
1234 ; SI-NEXT: s_lshl_b32 s4, s4, 16
1235 ; SI-NEXT: s_waitcnt vmcnt(1)
1236 ; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
1237 ; SI-NEXT: v_or_b32_e32 v1, s4, v1
1238 ; SI-NEXT: s_waitcnt vmcnt(0)
1239 ; SI-NEXT: buffer_store_dwordx4 v[4:7], v[8:9], s[0:3], 0 addr64 offset:16
1240 ; SI-NEXT: buffer_store_dwordx4 v[0:3], v[8:9], s[0:3], 0 addr64
1241 ; SI-NEXT: s_endpgm
1242 ;
1243 ; VI-LABEL: v_insertelement_v16bf16_3:
1244 ; VI: ; %bb.0:
1245 ; VI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
1246 ; VI-NEXT: s_load_dword s4, s[6:7], 0x10
1247 ; VI-NEXT: v_lshlrev_b32_e32 v8, 5, v0
1248 ; VI-NEXT: s_waitcnt lgkmcnt(0)
1249 ; VI-NEXT: v_mov_b32_e32 v1, s3
1250 ; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v8
1251 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1252 ; VI-NEXT: v_add_u32_e32 v4, vcc, 16, v0
1253 ; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
1254 ; VI-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
1255 ; VI-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
1256 ; VI-NEXT: v_mov_b32_e32 v9, s1
1257 ; VI-NEXT: v_add_u32_e32 v8, vcc, s0, v8
1258 ; VI-NEXT: v_addc_u32_e32 v9, vcc, 0, v9, vcc
1259 ; VI-NEXT: s_lshl_b32 s1, s4, 16
1260 ; VI-NEXT: v_add_u32_e32 v10, vcc, 16, v8
1261 ; VI-NEXT: v_mov_b32_e32 v12, s1
1262 ; VI-NEXT: v_addc_u32_e32 v11, vcc, 0, v9, vcc
1263 ; VI-NEXT: s_waitcnt vmcnt(1)
1264 ; VI-NEXT: v_or_b32_sdwa v1, v1, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
1265 ; VI-NEXT: s_waitcnt vmcnt(0)
1266 ; VI-NEXT: flat_store_dwordx4 v[10:11], v[4:7]
1267 ; VI-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
1268 ; VI-NEXT: s_endpgm
1269 ;
1270 ; GFX900-LABEL: v_insertelement_v16bf16_3:
1271 ; GFX900: ; %bb.0:
1272 ; GFX900-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
1273 ; GFX900-NEXT: s_load_dword s4, s[6:7], 0x10
1274 ; GFX900-NEXT: v_lshlrev_b32_e32 v8, 5, v0
1275 ; GFX900-NEXT: v_mov_b32_e32 v9, 0x5040100
1276 ; GFX900-NEXT: s_waitcnt lgkmcnt(0)
1277 ; GFX900-NEXT: global_load_dwordx4 v[0:3], v8, s[2:3]
1278 ; GFX900-NEXT: global_load_dwordx4 v[4:7], v8, s[2:3] offset:16
1279 ; GFX900-NEXT: s_waitcnt vmcnt(1)
1280 ; GFX900-NEXT: v_perm_b32 v1, s4, v1, v9
1281 ; GFX900-NEXT: s_waitcnt vmcnt(0)
1282 ; GFX900-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
1283 ; GFX900-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
1284 ; GFX900-NEXT: s_endpgm
1285 ;
1286 ; GFX940-LABEL: v_insertelement_v16bf16_3:
1287 ; GFX940: ; %bb.0:
1288 ; GFX940-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
1289 ; GFX940-NEXT: s_load_dword s0, s[2:3], 0x10
1290 ; GFX940-NEXT: v_and_b32_e32 v0, 0x3ff, v0
1291 ; GFX940-NEXT: v_lshlrev_b32_e32 v8, 5, v0
1292 ; GFX940-NEXT: v_mov_b32_e32 v9, 0x5040100
1293 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
1294 ; GFX940-NEXT: global_load_dwordx4 v[0:3], v8, s[6:7]
1295 ; GFX940-NEXT: global_load_dwordx4 v[4:7], v8, s[6:7] offset:16
1296 ; GFX940-NEXT: s_waitcnt vmcnt(1)
1297 ; GFX940-NEXT: v_perm_b32 v1, s0, v1, v9
1298 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1299 ; GFX940-NEXT: global_store_dwordx4 v8, v[4:7], s[4:5] offset:16 sc0 sc1
1300 ; GFX940-NEXT: global_store_dwordx4 v8, v[0:3], s[4:5] sc0 sc1
1301 ; GFX940-NEXT: s_endpgm
1302 %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
1303 %tid.ext = sext i32 %tid to i64
1304 %in.gep = getelementptr inbounds <16 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
1305 %out.gep = getelementptr inbounds <16 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
1306 %vec = load <16 x bfloat>, ptr addrspace(1) %in.gep
1307 %val.trunc = trunc i32 %val to i16
1308 %val.cvt = bitcast i16 %val.trunc to bfloat
1309 %vecins = insertelement <16 x bfloat> %vec, bfloat %val.cvt, i32 3
1310 store <16 x bfloat> %vecins, ptr addrspace(1) %out.gep
1311 ret void
1312 }
1314 define amdgpu_kernel void @v_insertelement_v16bf16_dynamic(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %val, i32 %n) {
1315 ; SI-LABEL: v_insertelement_v16bf16_dynamic:
1316 ; SI: ; %bb.0:
1317 ; SI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
1318 ; SI-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x4
1319 ; SI-NEXT: s_mov_b32 s11, 0x100f000
1320 ; SI-NEXT: s_mov_b32 s10, 0
1321 ; SI-NEXT: v_lshlrev_b32_e32 v4, 5, v0
1322 ; SI-NEXT: s_waitcnt lgkmcnt(0)
1323 ; SI-NEXT: s_mov_b64 s[8:9], s[2:3]
1324 ; SI-NEXT: v_mov_b32_e32 v5, 0
1325 ; SI-NEXT: buffer_load_dwordx4 v[7:10], v[4:5], s[8:11], 0 addr64
1326 ; SI-NEXT: buffer_load_dwordx4 v[0:3], v[4:5], s[8:11], 0 addr64 offset:16
1327 ; SI-NEXT: s_cmp_eq_u32 s5, 6
1328 ; SI-NEXT: v_mov_b32_e32 v6, s4
1329 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1330 ; SI-NEXT: s_cmp_eq_u32 s5, 7
1331 ; SI-NEXT: s_mov_b64 s[2:3], s[10:11]
1332 ; SI-NEXT: s_waitcnt vmcnt(1)
1333 ; SI-NEXT: v_cndmask_b32_e32 v11, v10, v6, vcc
1334 ; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
1335 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1336 ; SI-NEXT: s_cmp_eq_u32 s5, 4
1337 ; SI-NEXT: v_cndmask_b32_e32 v10, v10, v6, vcc
1338 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1339 ; SI-NEXT: s_cmp_eq_u32 s5, 5
1340 ; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v9
1341 ; SI-NEXT: v_cndmask_b32_e32 v9, v9, v6, vcc
1342 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1343 ; SI-NEXT: s_cmp_eq_u32 s5, 2
1344 ; SI-NEXT: v_and_b32_e32 v11, 0xffff, v11
1345 ; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
1346 ; SI-NEXT: v_cndmask_b32_e32 v12, v12, v6, vcc
1347 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1348 ; SI-NEXT: s_cmp_eq_u32 s5, 3
1349 ; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v8
1350 ; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
1351 ; SI-NEXT: v_or_b32_e32 v10, v11, v10
1352 ; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v12
1353 ; SI-NEXT: v_cndmask_b32_e32 v8, v8, v6, vcc
1354 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1355 ; SI-NEXT: s_cmp_eq_u32 s5, 0
1356 ; SI-NEXT: v_or_b32_e32 v9, v9, v11
1357 ; SI-NEXT: v_cndmask_b32_e32 v11, v13, v6, vcc
1358 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1359 ; SI-NEXT: s_cmp_eq_u32 s5, 1
1360 ; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v7
1361 ; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
1362 ; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
1363 ; SI-NEXT: v_cndmask_b32_e32 v7, v7, v6, vcc
1364 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1365 ; SI-NEXT: s_cmp_eq_u32 s5, 14
1366 ; SI-NEXT: v_or_b32_e32 v8, v8, v11
1367 ; SI-NEXT: v_cndmask_b32_e32 v11, v14, v6, vcc
1368 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1369 ; SI-NEXT: s_cmp_eq_u32 s5, 15
1370 ; SI-NEXT: s_waitcnt vmcnt(0)
1371 ; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v3
1372 ; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
1373 ; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
1374 ; SI-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc
1375 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1376 ; SI-NEXT: s_cmp_eq_u32 s5, 12
1377 ; SI-NEXT: v_or_b32_e32 v7, v7, v11
1378 ; SI-NEXT: v_cndmask_b32_e32 v11, v15, v6, vcc
1379 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1380 ; SI-NEXT: s_cmp_eq_u32 s5, 13
1381 ; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v2
1382 ; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
1383 ; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
1384 ; SI-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc
1385 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1386 ; SI-NEXT: s_cmp_eq_u32 s5, 10
1387 ; SI-NEXT: v_or_b32_e32 v3, v3, v11
1388 ; SI-NEXT: v_cndmask_b32_e32 v11, v16, v6, vcc
1389 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1390 ; SI-NEXT: s_cmp_eq_u32 s5, 11
1391 ; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v1
1392 ; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
1393 ; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
1394 ; SI-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
1395 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1396 ; SI-NEXT: s_cmp_eq_u32 s5, 8
1397 ; SI-NEXT: v_or_b32_e32 v2, v2, v11
1398 ; SI-NEXT: v_cndmask_b32_e32 v11, v17, v6, vcc
1399 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1400 ; SI-NEXT: s_cmp_eq_u32 s5, 9
1401 ; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v0
1402 ; SI-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc
1403 ; SI-NEXT: s_cselect_b64 vcc, -1, 0
1404 ; SI-NEXT: v_cndmask_b32_e32 v6, v18, v6, vcc
1405 ; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
1406 ; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
1407 ; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
1408 ; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
1409 ; SI-NEXT: v_or_b32_e32 v1, v1, v11
1410 ; SI-NEXT: v_or_b32_e32 v0, v0, v6
1411 ; SI-NEXT: buffer_store_dwordx4 v[0:3], v[4:5], s[0:3], 0 addr64 offset:16
1412 ; SI-NEXT: buffer_store_dwordx4 v[7:10], v[4:5], s[0:3], 0 addr64
1413 ; SI-NEXT: s_endpgm
1414 ;
1415 ; VI-LABEL: v_insertelement_v16bf16_dynamic:
1416 ; VI: ; %bb.0:
1417 ; VI-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
1418 ; VI-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x10
1419 ; VI-NEXT: v_lshlrev_b32_e32 v8, 5, v0
1420 ; VI-NEXT: s_waitcnt lgkmcnt(0)
1421 ; VI-NEXT: v_mov_b32_e32 v0, s3
1422 ; VI-NEXT: v_add_u32_e32 v4, vcc, s2, v8
1423 ; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v0, vcc
1424 ; VI-NEXT: v_add_u32_e32 v0, vcc, 16, v4
1425 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
1426 ; VI-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
1427 ; VI-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
1428 ; VI-NEXT: v_mov_b32_e32 v9, s1
1429 ; VI-NEXT: v_add_u32_e32 v8, vcc, s0, v8
1430 ; VI-NEXT: v_addc_u32_e32 v9, vcc, 0, v9, vcc
1431 ; VI-NEXT: v_add_u32_e32 v10, vcc, 16, v8
1432 ; VI-NEXT: s_cmp_eq_u32 s5, 14
1433 ; VI-NEXT: v_addc_u32_e32 v11, vcc, 0, v9, vcc
1434 ; VI-NEXT: v_mov_b32_e32 v12, s4
1435 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1436 ; VI-NEXT: s_cmp_eq_u32 s5, 15
1437 ; VI-NEXT: s_waitcnt vmcnt(1)
1438 ; VI-NEXT: v_cndmask_b32_e32 v13, v3, v12, vcc
1439 ; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
1440 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1441 ; VI-NEXT: s_cmp_eq_u32 s5, 12
1442 ; VI-NEXT: v_cndmask_b32_e32 v3, v3, v12, vcc
1443 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1444 ; VI-NEXT: s_cmp_eq_u32 s5, 13
1445 ; VI-NEXT: v_lshrrev_b32_e32 v14, 16, v2
1446 ; VI-NEXT: v_cndmask_b32_e32 v2, v2, v12, vcc
1447 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1448 ; VI-NEXT: s_cmp_eq_u32 s5, 10
1449 ; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
1450 ; VI-NEXT: v_cndmask_b32_e32 v14, v14, v12, vcc
1451 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1452 ; VI-NEXT: s_cmp_eq_u32 s5, 11
1453 ; VI-NEXT: v_lshrrev_b32_e32 v15, 16, v1
1454 ; VI-NEXT: v_or_b32_sdwa v3, v13, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
1455 ; VI-NEXT: v_lshlrev_b32_e32 v13, 16, v14
1456 ; VI-NEXT: v_cndmask_b32_e32 v1, v1, v12, vcc
1457 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1458 ; VI-NEXT: s_cmp_eq_u32 s5, 8
1459 ; VI-NEXT: v_or_b32_sdwa v2, v2, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
1460 ; VI-NEXT: v_cndmask_b32_e32 v13, v15, v12, vcc
1461 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1462 ; VI-NEXT: s_cmp_eq_u32 s5, 9
1463 ; VI-NEXT: v_lshrrev_b32_e32 v16, 16, v0
1464 ; VI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
1465 ; VI-NEXT: v_cndmask_b32_e32 v0, v0, v12, vcc
1466 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1467 ; VI-NEXT: s_cmp_eq_u32 s5, 6
1468 ; VI-NEXT: v_or_b32_sdwa v1, v1, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
1469 ; VI-NEXT: v_cndmask_b32_e32 v13, v16, v12, vcc
1470 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1471 ; VI-NEXT: s_cmp_eq_u32 s5, 7
1472 ; VI-NEXT: s_waitcnt vmcnt(0)
1473 ; VI-NEXT: v_lshrrev_b32_e32 v17, 16, v7
1474 ; VI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
1475 ; VI-NEXT: v_cndmask_b32_e32 v7, v7, v12, vcc
1476 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1477 ; VI-NEXT: s_cmp_eq_u32 s5, 4
1478 ; VI-NEXT: v_or_b32_sdwa v0, v0, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
1479 ; VI-NEXT: v_cndmask_b32_e32 v13, v17, v12, vcc
1480 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1481 ; VI-NEXT: s_cmp_eq_u32 s5, 5
1482 ; VI-NEXT: v_lshrrev_b32_e32 v18, 16, v6
1483 ; VI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
1484 ; VI-NEXT: v_cndmask_b32_e32 v6, v6, v12, vcc
1485 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1486 ; VI-NEXT: s_cmp_eq_u32 s5, 2
1487 ; VI-NEXT: v_or_b32_sdwa v7, v7, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
1488 ; VI-NEXT: v_cndmask_b32_e32 v13, v18, v12, vcc
1489 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1490 ; VI-NEXT: s_cmp_eq_u32 s5, 3
1491 ; VI-NEXT: v_lshrrev_b32_e32 v19, 16, v5
1492 ; VI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
1493 ; VI-NEXT: v_cndmask_b32_e32 v5, v5, v12, vcc
1494 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1495 ; VI-NEXT: s_cmp_eq_u32 s5, 0
1496 ; VI-NEXT: v_or_b32_sdwa v6, v6, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
1497 ; VI-NEXT: v_cndmask_b32_e32 v13, v19, v12, vcc
1498 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1499 ; VI-NEXT: s_cmp_eq_u32 s5, 1
1500 ; VI-NEXT: v_lshrrev_b32_e32 v20, 16, v4
1501 ; VI-NEXT: v_cndmask_b32_e32 v4, v4, v12, vcc
1502 ; VI-NEXT: s_cselect_b64 vcc, -1, 0
1503 ; VI-NEXT: v_cndmask_b32_e32 v12, v20, v12, vcc
1504 ; VI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
1505 ; VI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
1506 ; VI-NEXT: v_or_b32_sdwa v5, v5, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
1507 ; VI-NEXT: v_or_b32_sdwa v4, v4, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
1508 ; VI-NEXT: flat_store_dwordx4 v[8:9], v[4:7]
1509 ; VI-NEXT: flat_store_dwordx4 v[10:11], v[0:3]
1510 ; VI-NEXT: s_endpgm
1511 ;
1512 ; GFX900-LABEL: v_insertelement_v16bf16_dynamic:
1513 ; GFX900: ; %bb.0:
1514 ; GFX900-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
1515 ; GFX900-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x10
1516 ; GFX900-NEXT: v_lshlrev_b32_e32 v0, 5, v0
1517 ; GFX900-NEXT: s_waitcnt lgkmcnt(0)
1518 ; GFX900-NEXT: global_load_dwordx4 v[1:4], v0, s[2:3]
1519 ; GFX900-NEXT: global_load_dwordx4 v[5:8], v0, s[2:3] offset:16
1520 ; GFX900-NEXT: s_cmp_eq_u32 s5, 6
1521 ; GFX900-NEXT: v_mov_b32_e32 v9, s4
1522 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1523 ; GFX900-NEXT: s_cmp_eq_u32 s5, 7
1524 ; GFX900-NEXT: s_mov_b32 s2, 0x5040100
1525 ; GFX900-NEXT: s_waitcnt vmcnt(1)
1526 ; GFX900-NEXT: v_cndmask_b32_e32 v10, v4, v9, vcc
1527 ; GFX900-NEXT: v_lshrrev_b32_e32 v4, 16, v4
1528 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1529 ; GFX900-NEXT: s_cmp_eq_u32 s5, 4
1530 ; GFX900-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc
1531 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1532 ; GFX900-NEXT: s_cmp_eq_u32 s5, 5
1533 ; GFX900-NEXT: v_lshrrev_b32_e32 v11, 16, v3
1534 ; GFX900-NEXT: v_cndmask_b32_e32 v3, v3, v9, vcc
1535 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1536 ; GFX900-NEXT: s_cmp_eq_u32 s5, 2
1537 ; GFX900-NEXT: v_perm_b32 v4, v4, v10, s2
1538 ; GFX900-NEXT: v_cndmask_b32_e32 v10, v11, v9, vcc
1539 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1540 ; GFX900-NEXT: s_cmp_eq_u32 s5, 3
1541 ; GFX900-NEXT: v_lshrrev_b32_e32 v12, 16, v2
1542 ; GFX900-NEXT: v_cndmask_b32_e32 v2, v2, v9, vcc
1543 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1544 ; GFX900-NEXT: s_cmp_eq_u32 s5, 0
1545 ; GFX900-NEXT: v_perm_b32 v3, v10, v3, s2
1546 ; GFX900-NEXT: v_cndmask_b32_e32 v10, v12, v9, vcc
1547 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1548 ; GFX900-NEXT: s_cmp_eq_u32 s5, 1
1549 ; GFX900-NEXT: v_lshrrev_b32_e32 v13, 16, v1
1550 ; GFX900-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc
1551 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1552 ; GFX900-NEXT: s_cmp_eq_u32 s5, 14
1553 ; GFX900-NEXT: v_perm_b32 v2, v10, v2, s2
1554 ; GFX900-NEXT: v_cndmask_b32_e32 v10, v13, v9, vcc
1555 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1556 ; GFX900-NEXT: s_cmp_eq_u32 s5, 15
1557 ; GFX900-NEXT: s_waitcnt vmcnt(0)
1558 ; GFX900-NEXT: v_lshrrev_b32_e32 v14, 16, v8
1559 ; GFX900-NEXT: v_cndmask_b32_e32 v8, v8, v9, vcc
1560 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1561 ; GFX900-NEXT: s_cmp_eq_u32 s5, 12
1562 ; GFX900-NEXT: v_perm_b32 v1, v10, v1, s2
1563 ; GFX900-NEXT: v_cndmask_b32_e32 v10, v14, v9, vcc
1564 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1565 ; GFX900-NEXT: s_cmp_eq_u32 s5, 13
1566 ; GFX900-NEXT: v_lshrrev_b32_e32 v15, 16, v7
1567 ; GFX900-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc
1568 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1569 ; GFX900-NEXT: s_cmp_eq_u32 s5, 10
1570 ; GFX900-NEXT: v_perm_b32 v8, v10, v8, s2
1571 ; GFX900-NEXT: v_cndmask_b32_e32 v10, v15, v9, vcc
1572 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1573 ; GFX900-NEXT: s_cmp_eq_u32 s5, 11
1574 ; GFX900-NEXT: v_lshrrev_b32_e32 v16, 16, v6
1575 ; GFX900-NEXT: v_cndmask_b32_e32 v6, v6, v9, vcc
1576 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1577 ; GFX900-NEXT: s_cmp_eq_u32 s5, 8
1578 ; GFX900-NEXT: v_perm_b32 v7, v10, v7, s2
1579 ; GFX900-NEXT: v_cndmask_b32_e32 v10, v16, v9, vcc
1580 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1581 ; GFX900-NEXT: s_cmp_eq_u32 s5, 9
1582 ; GFX900-NEXT: v_lshrrev_b32_e32 v17, 16, v5
1583 ; GFX900-NEXT: v_cndmask_b32_e32 v5, v5, v9, vcc
1584 ; GFX900-NEXT: s_cselect_b64 vcc, -1, 0
1585 ; GFX900-NEXT: v_cndmask_b32_e32 v9, v17, v9, vcc
1586 ; GFX900-NEXT: v_perm_b32 v6, v10, v6, s2
1587 ; GFX900-NEXT: v_perm_b32 v5, v9, v5, s2
1588 ; GFX900-NEXT: global_store_dwordx4 v0, v[5:8], s[0:1] offset:16
1589 ; GFX900-NEXT: global_store_dwordx4 v0, v[1:4], s[0:1]
1590 ; GFX900-NEXT: s_endpgm
1591 ;
1592 ; GFX940-LABEL: v_insertelement_v16bf16_dynamic:
1593 ; GFX940: ; %bb.0:
1594 ; GFX940-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
1595 ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x10
1596 ; GFX940-NEXT: v_and_b32_e32 v0, 0x3ff, v0
1597 ; GFX940-NEXT: v_lshlrev_b32_e32 v8, 5, v0
1598 ; GFX940-NEXT: s_mov_b32 s2, 0x5040100
1599 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
1600 ; GFX940-NEXT: global_load_dwordx4 v[0:3], v8, s[6:7]
1601 ; GFX940-NEXT: global_load_dwordx4 v[4:7], v8, s[6:7] offset:16
1602 ; GFX940-NEXT: s_cmp_eq_u32 s1, 6
1603 ; GFX940-NEXT: v_mov_b32_e32 v9, s0
1604 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1605 ; GFX940-NEXT: s_cmp_eq_u32 s1, 7
1606 ; GFX940-NEXT: s_waitcnt vmcnt(1)
1607 ; GFX940-NEXT: v_cndmask_b32_e32 v10, v3, v9, vcc
1608 ; GFX940-NEXT: v_lshrrev_b32_e32 v3, 16, v3
1609 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1610 ; GFX940-NEXT: s_cmp_eq_u32 s1, 4
1611 ; GFX940-NEXT: v_cndmask_b32_e32 v3, v3, v9, vcc
1612 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1613 ; GFX940-NEXT: s_cmp_eq_u32 s1, 5
1614 ; GFX940-NEXT: v_lshrrev_b32_e32 v11, 16, v2
1615 ; GFX940-NEXT: v_cndmask_b32_e32 v2, v2, v9, vcc
1616 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1617 ; GFX940-NEXT: s_cmp_eq_u32 s1, 2
1618 ; GFX940-NEXT: v_perm_b32 v3, v3, v10, s2
1619 ; GFX940-NEXT: v_cndmask_b32_e32 v10, v11, v9, vcc
1620 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1621 ; GFX940-NEXT: s_cmp_eq_u32 s1, 3
1622 ; GFX940-NEXT: v_lshrrev_b32_e32 v12, 16, v1
1623 ; GFX940-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc
1624 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1625 ; GFX940-NEXT: s_cmp_eq_u32 s1, 0
1626 ; GFX940-NEXT: v_perm_b32 v2, v10, v2, s2
1627 ; GFX940-NEXT: v_cndmask_b32_e32 v10, v12, v9, vcc
1628 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1629 ; GFX940-NEXT: s_cmp_eq_u32 s1, 1
1630 ; GFX940-NEXT: v_lshrrev_b32_e32 v13, 16, v0
1631 ; GFX940-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc
1632 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1633 ; GFX940-NEXT: s_cmp_eq_u32 s1, 14
1634 ; GFX940-NEXT: v_perm_b32 v1, v10, v1, s2
1635 ; GFX940-NEXT: v_cndmask_b32_e32 v10, v13, v9, vcc
1636 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1637 ; GFX940-NEXT: s_cmp_eq_u32 s1, 15
1638 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1639 ; GFX940-NEXT: v_lshrrev_b32_e32 v14, 16, v7
1640 ; GFX940-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc
1641 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1642 ; GFX940-NEXT: s_cmp_eq_u32 s1, 12
1643 ; GFX940-NEXT: v_perm_b32 v0, v10, v0, s2
1644 ; GFX940-NEXT: v_cndmask_b32_e32 v10, v14, v9, vcc
1645 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1646 ; GFX940-NEXT: s_cmp_eq_u32 s1, 13
1647 ; GFX940-NEXT: v_lshrrev_b32_e32 v15, 16, v6
1648 ; GFX940-NEXT: v_cndmask_b32_e32 v6, v6, v9, vcc
1649 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1650 ; GFX940-NEXT: s_cmp_eq_u32 s1, 10
1651 ; GFX940-NEXT: v_perm_b32 v7, v10, v7, s2
1652 ; GFX940-NEXT: v_cndmask_b32_e32 v10, v15, v9, vcc
1653 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1654 ; GFX940-NEXT: s_cmp_eq_u32 s1, 11
1655 ; GFX940-NEXT: v_lshrrev_b32_e32 v16, 16, v5
1656 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v5, v9, vcc
1657 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1658 ; GFX940-NEXT: s_cmp_eq_u32 s1, 8
1659 ; GFX940-NEXT: v_perm_b32 v6, v10, v6, s2
1660 ; GFX940-NEXT: v_cndmask_b32_e32 v10, v16, v9, vcc
1661 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1662 ; GFX940-NEXT: s_cmp_eq_u32 s1, 9
1663 ; GFX940-NEXT: v_lshrrev_b32_e32 v17, 16, v4
1664 ; GFX940-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc
1665 ; GFX940-NEXT: s_cselect_b64 vcc, -1, 0
1666 ; GFX940-NEXT: v_cndmask_b32_e32 v9, v17, v9, vcc
1667 ; GFX940-NEXT: v_perm_b32 v5, v10, v5, s2
1668 ; GFX940-NEXT: v_perm_b32 v4, v9, v4, s2
1669 ; GFX940-NEXT: global_store_dwordx4 v8, v[4:7], s[4:5] offset:16 sc0 sc1
1670 ; GFX940-NEXT: global_store_dwordx4 v8, v[0:3], s[4:5] sc0 sc1
1671 ; GFX940-NEXT: s_endpgm
1672 %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
1673 %tid.ext = sext i32 %tid to i64
1674 %in.gep = getelementptr inbounds <16 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
1675 %out.gep = getelementptr inbounds <16 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
1676 %vec = load <16 x bfloat>, ptr addrspace(1) %in.gep
1677 %val.trunc = trunc i32 %val to i16
1678 %val.cvt = bitcast i16 %val.trunc to bfloat
1679 %vecins = insertelement <16 x bfloat> %vec, bfloat %val.cvt, i32 %n
1680 store <16 x bfloat> %vecins, ptr addrspace(1) %out.gep
1681 ret void
1682 }
1684 declare i32 @llvm.amdgcn.workitem.id.x() #1
1686 attributes #0 = { nounwind }
1687 attributes #1 = { nounwind readnone }