| blob | 3dbf6477a7cb898aac656ad5f38f9b46de1a5e32 |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
2 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
3 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx940 < %s | FileCheck -check-prefix=GFX940 %s
4 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 < %s | FileCheck -check-prefix=GFX11 %s
5 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
6 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx90a < %s | FileCheck -check-prefix=GFX90A %s
7 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx908 < %s | FileCheck -check-prefix=GFX908 %s
8 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=tonga < %s | FileCheck -check-prefix=GFX8 %s
9 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=hawaii < %s | FileCheck -check-prefix=GFX7 %s
10 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=tahiti < %s | FileCheck -check-prefix=GFX6 %s
12 ; --------------------------------------------------------------------
13 ; float
14 ; --------------------------------------------------------------------
16 define float @global_agent_atomic_fsub_ret_f32(ptr addrspace(1) %ptr, float %val) #0 {
17 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f32:
18 ; GFX12: ; %bb.0:
19 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
20 ; GFX12-NEXT: s_wait_expcnt 0x0
21 ; GFX12-NEXT: s_wait_samplecnt 0x0
22 ; GFX12-NEXT: s_wait_bvhcnt 0x0
23 ; GFX12-NEXT: s_wait_kmcnt 0x0
24 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
25 ; GFX12-NEXT: s_mov_b32 s0, 0
26 ; GFX12-NEXT: .LBB0_1: ; %atomicrmw.start
27 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
28 ; GFX12-NEXT: s_wait_loadcnt 0x0
29 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
30 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
31 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
32 ; GFX12-NEXT: s_wait_storecnt 0x0
33 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
34 ; GFX12-NEXT: s_wait_loadcnt 0x0
35 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
36 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
37 ; GFX12-NEXT: s_wait_alu 0xfffe
38 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
39 ; GFX12-NEXT: s_wait_alu 0xfffe
40 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
41 ; GFX12-NEXT: s_cbranch_execnz .LBB0_1
42 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
43 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
44 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
45 ; GFX12-NEXT: s_wait_alu 0xfffe
46 ; GFX12-NEXT: s_setpc_b64 s[30:31]
47 ;
48 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f32:
49 ; GFX940: ; %bb.0:
50 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
51 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
52 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
53 ; GFX940-NEXT: .LBB0_1: ; %atomicrmw.start
54 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
55 ; GFX940-NEXT: s_waitcnt vmcnt(0)
56 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
57 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
58 ; GFX940-NEXT: buffer_wbl2 sc1
59 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
60 ; GFX940-NEXT: s_waitcnt vmcnt(0)
61 ; GFX940-NEXT: buffer_inv sc1
62 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
63 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
64 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
65 ; GFX940-NEXT: s_cbranch_execnz .LBB0_1
66 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
67 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
68 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
69 ; GFX940-NEXT: s_setpc_b64 s[30:31]
70 ;
71 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f32:
72 ; GFX11: ; %bb.0:
73 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
74 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
75 ; GFX11-NEXT: s_mov_b32 s0, 0
76 ; GFX11-NEXT: .LBB0_1: ; %atomicrmw.start
77 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
78 ; GFX11-NEXT: s_waitcnt vmcnt(0)
79 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
80 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
81 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
82 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
83 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
84 ; GFX11-NEXT: s_waitcnt vmcnt(0)
85 ; GFX11-NEXT: buffer_gl1_inv
86 ; GFX11-NEXT: buffer_gl0_inv
87 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
88 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
89 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
90 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
91 ; GFX11-NEXT: s_cbranch_execnz .LBB0_1
92 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
93 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
94 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
95 ; GFX11-NEXT: s_setpc_b64 s[30:31]
96 ;
97 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f32:
98 ; GFX10: ; %bb.0:
99 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
100 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
101 ; GFX10-NEXT: s_mov_b32 s4, 0
102 ; GFX10-NEXT: .LBB0_1: ; %atomicrmw.start
103 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
104 ; GFX10-NEXT: s_waitcnt vmcnt(0)
105 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
106 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
107 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
108 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
109 ; GFX10-NEXT: s_waitcnt vmcnt(0)
110 ; GFX10-NEXT: buffer_gl1_inv
111 ; GFX10-NEXT: buffer_gl0_inv
112 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
113 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
114 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
115 ; GFX10-NEXT: s_cbranch_execnz .LBB0_1
116 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
117 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
118 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
119 ; GFX10-NEXT: s_setpc_b64 s[30:31]
120 ;
121 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f32:
122 ; GFX90A: ; %bb.0:
123 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
124 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
125 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
126 ; GFX90A-NEXT: .LBB0_1: ; %atomicrmw.start
127 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
128 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
129 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
130 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
131 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
132 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
133 ; GFX90A-NEXT: buffer_wbinvl1
134 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
135 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
136 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
137 ; GFX90A-NEXT: s_cbranch_execnz .LBB0_1
138 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
139 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
140 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
141 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
142 ;
143 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f32:
144 ; GFX908: ; %bb.0:
145 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
146 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
147 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
148 ; GFX908-NEXT: .LBB0_1: ; %atomicrmw.start
149 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
150 ; GFX908-NEXT: s_waitcnt vmcnt(0)
151 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
152 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
153 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
154 ; GFX908-NEXT: s_waitcnt vmcnt(0)
155 ; GFX908-NEXT: buffer_wbinvl1
156 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
157 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
158 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
159 ; GFX908-NEXT: s_cbranch_execnz .LBB0_1
160 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
161 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
162 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
163 ; GFX908-NEXT: s_setpc_b64 s[30:31]
164 ;
165 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f32:
166 ; GFX8: ; %bb.0:
167 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
168 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
169 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
170 ; GFX8-NEXT: .LBB0_1: ; %atomicrmw.start
171 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
172 ; GFX8-NEXT: s_waitcnt vmcnt(0)
173 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
174 ; GFX8-NEXT: v_sub_f32_e32 v3, v4, v2
175 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
176 ; GFX8-NEXT: s_waitcnt vmcnt(0)
177 ; GFX8-NEXT: buffer_wbinvl1
178 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
179 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
180 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
181 ; GFX8-NEXT: s_cbranch_execnz .LBB0_1
182 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
183 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
184 ; GFX8-NEXT: v_mov_b32_e32 v0, v3
185 ; GFX8-NEXT: s_setpc_b64 s[30:31]
186 ;
187 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f32:
188 ; GFX7: ; %bb.0:
189 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
190 ; GFX7-NEXT: s_mov_b32 s6, 0
191 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
192 ; GFX7-NEXT: s_mov_b32 s4, s6
193 ; GFX7-NEXT: s_mov_b32 s5, s6
194 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
195 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
196 ; GFX7-NEXT: .LBB0_1: ; %atomicrmw.start
197 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
198 ; GFX7-NEXT: s_waitcnt vmcnt(0)
199 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
200 ; GFX7-NEXT: v_sub_f32_e32 v4, v5, v2
201 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
202 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
203 ; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
204 ; GFX7-NEXT: s_waitcnt vmcnt(0)
205 ; GFX7-NEXT: buffer_wbinvl1
206 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
207 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
208 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
209 ; GFX7-NEXT: s_cbranch_execnz .LBB0_1
210 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
211 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
212 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
213 ; GFX7-NEXT: s_setpc_b64 s[30:31]
214 ;
215 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f32:
216 ; GFX6: ; %bb.0:
217 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
218 ; GFX6-NEXT: s_mov_b32 s6, 0
219 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
220 ; GFX6-NEXT: s_mov_b32 s4, s6
221 ; GFX6-NEXT: s_mov_b32 s5, s6
222 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
223 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
224 ; GFX6-NEXT: .LBB0_1: ; %atomicrmw.start
225 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
226 ; GFX6-NEXT: s_waitcnt vmcnt(0)
227 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
228 ; GFX6-NEXT: s_waitcnt expcnt(0)
229 ; GFX6-NEXT: v_sub_f32_e32 v4, v5, v2
230 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
231 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
232 ; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
233 ; GFX6-NEXT: s_waitcnt vmcnt(0)
234 ; GFX6-NEXT: buffer_wbinvl1
235 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
236 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
237 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
238 ; GFX6-NEXT: s_cbranch_execnz .LBB0_1
239 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
240 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
241 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
242 ; GFX6-NEXT: s_waitcnt expcnt(0)
243 ; GFX6-NEXT: s_setpc_b64 s[30:31]
244 %result = atomicrmw fsub ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst
245 ret float %result
246 }
248 define float @global_agent_atomic_fsub_ret_f32__offset12b_pos(ptr addrspace(1) %ptr, float %val) #0 {
249 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos:
250 ; GFX12: ; %bb.0:
251 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
252 ; GFX12-NEXT: s_wait_expcnt 0x0
253 ; GFX12-NEXT: s_wait_samplecnt 0x0
254 ; GFX12-NEXT: s_wait_bvhcnt 0x0
255 ; GFX12-NEXT: s_wait_kmcnt 0x0
256 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
257 ; GFX12-NEXT: s_mov_b32 s0, 0
258 ; GFX12-NEXT: .LBB1_1: ; %atomicrmw.start
259 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
260 ; GFX12-NEXT: s_wait_loadcnt 0x0
261 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
262 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
263 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
264 ; GFX12-NEXT: s_wait_storecnt 0x0
265 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
266 ; GFX12-NEXT: s_wait_loadcnt 0x0
267 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
268 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
269 ; GFX12-NEXT: s_wait_alu 0xfffe
270 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
271 ; GFX12-NEXT: s_wait_alu 0xfffe
272 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
273 ; GFX12-NEXT: s_cbranch_execnz .LBB1_1
274 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
275 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
276 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
277 ; GFX12-NEXT: s_wait_alu 0xfffe
278 ; GFX12-NEXT: s_setpc_b64 s[30:31]
279 ;
280 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos:
281 ; GFX940: ; %bb.0:
282 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
283 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
284 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
285 ; GFX940-NEXT: .LBB1_1: ; %atomicrmw.start
286 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
287 ; GFX940-NEXT: s_waitcnt vmcnt(0)
288 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
289 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
290 ; GFX940-NEXT: buffer_wbl2 sc1
291 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
292 ; GFX940-NEXT: s_waitcnt vmcnt(0)
293 ; GFX940-NEXT: buffer_inv sc1
294 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
295 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
296 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
297 ; GFX940-NEXT: s_cbranch_execnz .LBB1_1
298 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
299 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
300 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
301 ; GFX940-NEXT: s_setpc_b64 s[30:31]
302 ;
303 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos:
304 ; GFX11: ; %bb.0:
305 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
306 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
307 ; GFX11-NEXT: s_mov_b32 s0, 0
308 ; GFX11-NEXT: .LBB1_1: ; %atomicrmw.start
309 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
310 ; GFX11-NEXT: s_waitcnt vmcnt(0)
311 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
312 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
313 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
314 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
315 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
316 ; GFX11-NEXT: s_waitcnt vmcnt(0)
317 ; GFX11-NEXT: buffer_gl1_inv
318 ; GFX11-NEXT: buffer_gl0_inv
319 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
320 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
321 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
322 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
323 ; GFX11-NEXT: s_cbranch_execnz .LBB1_1
324 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
325 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
326 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
327 ; GFX11-NEXT: s_setpc_b64 s[30:31]
328 ;
329 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos:
330 ; GFX10: ; %bb.0:
331 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
332 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
333 ; GFX10-NEXT: s_mov_b32 s4, 0
334 ; GFX10-NEXT: .LBB1_1: ; %atomicrmw.start
335 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
336 ; GFX10-NEXT: s_waitcnt vmcnt(0)
337 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
338 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
339 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
340 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
341 ; GFX10-NEXT: s_waitcnt vmcnt(0)
342 ; GFX10-NEXT: buffer_gl1_inv
343 ; GFX10-NEXT: buffer_gl0_inv
344 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
345 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
346 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
347 ; GFX10-NEXT: s_cbranch_execnz .LBB1_1
348 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
349 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
350 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
351 ; GFX10-NEXT: s_setpc_b64 s[30:31]
352 ;
353 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos:
354 ; GFX90A: ; %bb.0:
355 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
356 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
357 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
358 ; GFX90A-NEXT: .LBB1_1: ; %atomicrmw.start
359 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
360 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
361 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
362 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
363 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
364 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
365 ; GFX90A-NEXT: buffer_wbinvl1
366 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
367 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
368 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
369 ; GFX90A-NEXT: s_cbranch_execnz .LBB1_1
370 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
371 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
372 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
373 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
374 ;
375 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos:
376 ; GFX908: ; %bb.0:
377 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
378 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
379 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
380 ; GFX908-NEXT: .LBB1_1: ; %atomicrmw.start
381 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
382 ; GFX908-NEXT: s_waitcnt vmcnt(0)
383 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
384 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
385 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
386 ; GFX908-NEXT: s_waitcnt vmcnt(0)
387 ; GFX908-NEXT: buffer_wbinvl1
388 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
389 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
390 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
391 ; GFX908-NEXT: s_cbranch_execnz .LBB1_1
392 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
393 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
394 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
395 ; GFX908-NEXT: s_setpc_b64 s[30:31]
396 ;
397 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos:
398 ; GFX8: ; %bb.0:
399 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
400 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
401 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
402 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
403 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
404 ; GFX8-NEXT: .LBB1_1: ; %atomicrmw.start
405 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
406 ; GFX8-NEXT: s_waitcnt vmcnt(0)
407 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
408 ; GFX8-NEXT: v_sub_f32_e32 v0, v1, v2
409 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
410 ; GFX8-NEXT: s_waitcnt vmcnt(0)
411 ; GFX8-NEXT: buffer_wbinvl1
412 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
413 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
414 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
415 ; GFX8-NEXT: s_cbranch_execnz .LBB1_1
416 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
417 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
418 ; GFX8-NEXT: s_setpc_b64 s[30:31]
419 ;
420 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos:
421 ; GFX7: ; %bb.0:
422 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
423 ; GFX7-NEXT: s_mov_b32 s6, 0
424 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
425 ; GFX7-NEXT: s_mov_b32 s4, s6
426 ; GFX7-NEXT: s_mov_b32 s5, s6
427 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
428 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
429 ; GFX7-NEXT: .LBB1_1: ; %atomicrmw.start
430 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
431 ; GFX7-NEXT: s_waitcnt vmcnt(0)
432 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
433 ; GFX7-NEXT: v_sub_f32_e32 v4, v5, v2
434 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
435 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
436 ; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
437 ; GFX7-NEXT: s_waitcnt vmcnt(0)
438 ; GFX7-NEXT: buffer_wbinvl1
439 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
440 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
441 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
442 ; GFX7-NEXT: s_cbranch_execnz .LBB1_1
443 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
444 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
445 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
446 ; GFX7-NEXT: s_setpc_b64 s[30:31]
447 ;
448 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos:
449 ; GFX6: ; %bb.0:
450 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
451 ; GFX6-NEXT: s_mov_b32 s6, 0
452 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
453 ; GFX6-NEXT: s_mov_b32 s4, s6
454 ; GFX6-NEXT: s_mov_b32 s5, s6
455 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
456 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
457 ; GFX6-NEXT: .LBB1_1: ; %atomicrmw.start
458 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
459 ; GFX6-NEXT: s_waitcnt vmcnt(0)
460 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
461 ; GFX6-NEXT: s_waitcnt expcnt(0)
462 ; GFX6-NEXT: v_sub_f32_e32 v4, v5, v2
463 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
464 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
465 ; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
466 ; GFX6-NEXT: s_waitcnt vmcnt(0)
467 ; GFX6-NEXT: buffer_wbinvl1
468 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
469 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
470 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
471 ; GFX6-NEXT: s_cbranch_execnz .LBB1_1
472 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
473 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
474 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
475 ; GFX6-NEXT: s_waitcnt expcnt(0)
476 ; GFX6-NEXT: s_setpc_b64 s[30:31]
477 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
478 %result = atomicrmw fsub ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
479 ret float %result
480 }
482 define float @global_agent_atomic_fsub_ret_f32__offset12b_neg(ptr addrspace(1) %ptr, float %val) #0 {
483 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg:
484 ; GFX12: ; %bb.0:
485 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
486 ; GFX12-NEXT: s_wait_expcnt 0x0
487 ; GFX12-NEXT: s_wait_samplecnt 0x0
488 ; GFX12-NEXT: s_wait_bvhcnt 0x0
489 ; GFX12-NEXT: s_wait_kmcnt 0x0
490 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
491 ; GFX12-NEXT: s_mov_b32 s0, 0
492 ; GFX12-NEXT: .LBB2_1: ; %atomicrmw.start
493 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
494 ; GFX12-NEXT: s_wait_loadcnt 0x0
495 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
496 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
497 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
498 ; GFX12-NEXT: s_wait_storecnt 0x0
499 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
500 ; GFX12-NEXT: s_wait_loadcnt 0x0
501 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
502 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
503 ; GFX12-NEXT: s_wait_alu 0xfffe
504 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
505 ; GFX12-NEXT: s_wait_alu 0xfffe
506 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
507 ; GFX12-NEXT: s_cbranch_execnz .LBB2_1
508 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
509 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
510 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
511 ; GFX12-NEXT: s_wait_alu 0xfffe
512 ; GFX12-NEXT: s_setpc_b64 s[30:31]
513 ;
514 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg:
515 ; GFX940: ; %bb.0:
516 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
517 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
518 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
519 ; GFX940-NEXT: .LBB2_1: ; %atomicrmw.start
520 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
521 ; GFX940-NEXT: s_waitcnt vmcnt(0)
522 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
523 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
524 ; GFX940-NEXT: buffer_wbl2 sc1
525 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
526 ; GFX940-NEXT: s_waitcnt vmcnt(0)
527 ; GFX940-NEXT: buffer_inv sc1
528 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
529 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
530 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
531 ; GFX940-NEXT: s_cbranch_execnz .LBB2_1
532 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
533 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
534 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
535 ; GFX940-NEXT: s_setpc_b64 s[30:31]
536 ;
537 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg:
538 ; GFX11: ; %bb.0:
539 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
540 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
541 ; GFX11-NEXT: s_mov_b32 s0, 0
542 ; GFX11-NEXT: .LBB2_1: ; %atomicrmw.start
543 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
544 ; GFX11-NEXT: s_waitcnt vmcnt(0)
545 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
546 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
547 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
548 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
549 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
550 ; GFX11-NEXT: s_waitcnt vmcnt(0)
551 ; GFX11-NEXT: buffer_gl1_inv
552 ; GFX11-NEXT: buffer_gl0_inv
553 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
554 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
555 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
556 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
557 ; GFX11-NEXT: s_cbranch_execnz .LBB2_1
558 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
559 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
560 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
561 ; GFX11-NEXT: s_setpc_b64 s[30:31]
562 ;
563 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg:
564 ; GFX10: ; %bb.0:
565 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
566 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
567 ; GFX10-NEXT: s_mov_b32 s4, 0
568 ; GFX10-NEXT: .LBB2_1: ; %atomicrmw.start
569 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
570 ; GFX10-NEXT: s_waitcnt vmcnt(0)
571 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
572 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
573 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
574 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
575 ; GFX10-NEXT: s_waitcnt vmcnt(0)
576 ; GFX10-NEXT: buffer_gl1_inv
577 ; GFX10-NEXT: buffer_gl0_inv
578 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
579 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
580 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
581 ; GFX10-NEXT: s_cbranch_execnz .LBB2_1
582 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
583 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
584 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
585 ; GFX10-NEXT: s_setpc_b64 s[30:31]
586 ;
587 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg:
588 ; GFX90A: ; %bb.0:
589 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
590 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
591 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
592 ; GFX90A-NEXT: .LBB2_1: ; %atomicrmw.start
593 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
594 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
595 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
596 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
597 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
598 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
599 ; GFX90A-NEXT: buffer_wbinvl1
600 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
601 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
602 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
603 ; GFX90A-NEXT: s_cbranch_execnz .LBB2_1
604 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
605 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
606 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
607 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
608 ;
609 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg:
610 ; GFX908: ; %bb.0:
611 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
612 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
613 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
614 ; GFX908-NEXT: .LBB2_1: ; %atomicrmw.start
615 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
616 ; GFX908-NEXT: s_waitcnt vmcnt(0)
617 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
618 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
619 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
620 ; GFX908-NEXT: s_waitcnt vmcnt(0)
621 ; GFX908-NEXT: buffer_wbinvl1
622 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
623 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
624 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
625 ; GFX908-NEXT: s_cbranch_execnz .LBB2_1
626 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
627 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
628 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
629 ; GFX908-NEXT: s_setpc_b64 s[30:31]
630 ;
631 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg:
632 ; GFX8: ; %bb.0:
633 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
634 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
635 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
636 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
637 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
638 ; GFX8-NEXT: .LBB2_1: ; %atomicrmw.start
639 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
640 ; GFX8-NEXT: s_waitcnt vmcnt(0)
641 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
642 ; GFX8-NEXT: v_sub_f32_e32 v0, v1, v2
643 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
644 ; GFX8-NEXT: s_waitcnt vmcnt(0)
645 ; GFX8-NEXT: buffer_wbinvl1
646 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
647 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
648 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
649 ; GFX8-NEXT: s_cbranch_execnz .LBB2_1
650 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
651 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
652 ; GFX8-NEXT: s_setpc_b64 s[30:31]
653 ;
654 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg:
655 ; GFX7: ; %bb.0:
656 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
657 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
658 ; GFX7-NEXT: v_mov_b32_e32 v4, v1
659 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
660 ; GFX7-NEXT: s_mov_b32 s5, -1
661 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
662 ; GFX7-NEXT: s_mov_b32 s6, 0
663 ; GFX7-NEXT: buffer_load_dword v0, v[3:4], s[4:7], 0 addr64
664 ; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0xfffff800, v3
665 ; GFX7-NEXT: v_addc_u32_e32 v4, vcc, -1, v4, vcc
666 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
667 ; GFX7-NEXT: s_mov_b32 s4, s6
668 ; GFX7-NEXT: s_mov_b32 s5, s6
669 ; GFX7-NEXT: .LBB2_1: ; %atomicrmw.start
670 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
671 ; GFX7-NEXT: s_waitcnt vmcnt(0)
672 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
673 ; GFX7-NEXT: v_sub_f32_e32 v5, v6, v2
674 ; GFX7-NEXT: v_mov_b32_e32 v0, v5
675 ; GFX7-NEXT: v_mov_b32_e32 v1, v6
676 ; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v[3:4], s[4:7], 0 addr64 glc
677 ; GFX7-NEXT: s_waitcnt vmcnt(0)
678 ; GFX7-NEXT: buffer_wbinvl1
679 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
680 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
681 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
682 ; GFX7-NEXT: s_cbranch_execnz .LBB2_1
683 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
684 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
685 ; GFX7-NEXT: s_setpc_b64 s[30:31]
686 ;
687 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg:
688 ; GFX6: ; %bb.0:
689 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
690 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
691 ; GFX6-NEXT: v_mov_b32_e32 v4, v1
692 ; GFX6-NEXT: v_mov_b32_e32 v3, v0
693 ; GFX6-NEXT: s_mov_b32 s5, -1
694 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
695 ; GFX6-NEXT: s_mov_b32 s6, 0
696 ; GFX6-NEXT: buffer_load_dword v0, v[3:4], s[4:7], 0 addr64
697 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, 0xfffff800, v3
698 ; GFX6-NEXT: v_addc_u32_e32 v4, vcc, -1, v4, vcc
699 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
700 ; GFX6-NEXT: s_mov_b32 s4, s6
701 ; GFX6-NEXT: s_mov_b32 s5, s6
702 ; GFX6-NEXT: .LBB2_1: ; %atomicrmw.start
703 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
704 ; GFX6-NEXT: s_waitcnt vmcnt(0)
705 ; GFX6-NEXT: v_mov_b32_e32 v6, v0
706 ; GFX6-NEXT: v_sub_f32_e32 v5, v6, v2
707 ; GFX6-NEXT: s_waitcnt expcnt(0)
708 ; GFX6-NEXT: v_mov_b32_e32 v0, v5
709 ; GFX6-NEXT: v_mov_b32_e32 v1, v6
710 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v[3:4], s[4:7], 0 addr64 glc
711 ; GFX6-NEXT: s_waitcnt vmcnt(0)
712 ; GFX6-NEXT: buffer_wbinvl1
713 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
714 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
715 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
716 ; GFX6-NEXT: s_cbranch_execnz .LBB2_1
717 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
718 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
719 ; GFX6-NEXT: s_waitcnt expcnt(0)
720 ; GFX6-NEXT: s_setpc_b64 s[30:31]
721 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
722 %result = atomicrmw fsub ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
723 ret float %result
724 }
726 define void @global_agent_atomic_fsub_noret_f32(ptr addrspace(1) %ptr, float %val) #0 {
727 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f32:
728 ; GFX12: ; %bb.0:
729 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
730 ; GFX12-NEXT: s_wait_expcnt 0x0
731 ; GFX12-NEXT: s_wait_samplecnt 0x0
732 ; GFX12-NEXT: s_wait_bvhcnt 0x0
733 ; GFX12-NEXT: s_wait_kmcnt 0x0
734 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off
735 ; GFX12-NEXT: s_mov_b32 s0, 0
736 ; GFX12-NEXT: .LBB3_1: ; %atomicrmw.start
737 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
738 ; GFX12-NEXT: s_wait_loadcnt 0x0
739 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
740 ; GFX12-NEXT: s_wait_storecnt 0x0
741 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
742 ; GFX12-NEXT: s_wait_loadcnt 0x0
743 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
744 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
745 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
746 ; GFX12-NEXT: s_wait_alu 0xfffe
747 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
748 ; GFX12-NEXT: s_wait_alu 0xfffe
749 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
750 ; GFX12-NEXT: s_cbranch_execnz .LBB3_1
751 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
752 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
753 ; GFX12-NEXT: s_wait_alu 0xfffe
754 ; GFX12-NEXT: s_setpc_b64 s[30:31]
755 ;
756 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f32:
757 ; GFX940: ; %bb.0:
758 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
759 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
760 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
761 ; GFX940-NEXT: .LBB3_1: ; %atomicrmw.start
762 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
763 ; GFX940-NEXT: s_waitcnt vmcnt(0)
764 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
765 ; GFX940-NEXT: buffer_wbl2 sc1
766 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
767 ; GFX940-NEXT: s_waitcnt vmcnt(0)
768 ; GFX940-NEXT: buffer_inv sc1
769 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
770 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
771 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
772 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
773 ; GFX940-NEXT: s_cbranch_execnz .LBB3_1
774 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
775 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
776 ; GFX940-NEXT: s_setpc_b64 s[30:31]
777 ;
778 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f32:
779 ; GFX11: ; %bb.0:
780 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
781 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off
782 ; GFX11-NEXT: s_mov_b32 s0, 0
783 ; GFX11-NEXT: .LBB3_1: ; %atomicrmw.start
784 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
785 ; GFX11-NEXT: s_waitcnt vmcnt(0)
786 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
787 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
788 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
789 ; GFX11-NEXT: s_waitcnt vmcnt(0)
790 ; GFX11-NEXT: buffer_gl1_inv
791 ; GFX11-NEXT: buffer_gl0_inv
792 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
793 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
794 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
795 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
796 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
797 ; GFX11-NEXT: s_cbranch_execnz .LBB3_1
798 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
799 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
800 ; GFX11-NEXT: s_setpc_b64 s[30:31]
801 ;
802 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f32:
803 ; GFX10: ; %bb.0:
804 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
805 ; GFX10-NEXT: global_load_dword v4, v[0:1], off
806 ; GFX10-NEXT: s_mov_b32 s4, 0
807 ; GFX10-NEXT: .LBB3_1: ; %atomicrmw.start
808 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
809 ; GFX10-NEXT: s_waitcnt vmcnt(0)
810 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
811 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
812 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
813 ; GFX10-NEXT: s_waitcnt vmcnt(0)
814 ; GFX10-NEXT: buffer_gl1_inv
815 ; GFX10-NEXT: buffer_gl0_inv
816 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
817 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
818 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
819 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
820 ; GFX10-NEXT: s_cbranch_execnz .LBB3_1
821 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
822 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
823 ; GFX10-NEXT: s_setpc_b64 s[30:31]
824 ;
825 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f32:
826 ; GFX90A: ; %bb.0:
827 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
828 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
829 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
830 ; GFX90A-NEXT: .LBB3_1: ; %atomicrmw.start
831 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
832 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
833 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
834 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
835 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
836 ; GFX90A-NEXT: buffer_wbinvl1
837 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
838 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
839 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
840 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
841 ; GFX90A-NEXT: s_cbranch_execnz .LBB3_1
842 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
843 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
844 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
845 ;
846 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f32:
847 ; GFX908: ; %bb.0:
848 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
849 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
850 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
851 ; GFX908-NEXT: .LBB3_1: ; %atomicrmw.start
852 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
853 ; GFX908-NEXT: s_waitcnt vmcnt(0)
854 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
855 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
856 ; GFX908-NEXT: s_waitcnt vmcnt(0)
857 ; GFX908-NEXT: buffer_wbinvl1
858 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
859 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
860 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
861 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
862 ; GFX908-NEXT: s_cbranch_execnz .LBB3_1
863 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
864 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
865 ; GFX908-NEXT: s_setpc_b64 s[30:31]
866 ;
867 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f32:
868 ; GFX8: ; %bb.0:
869 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
870 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
871 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
872 ; GFX8-NEXT: .LBB3_1: ; %atomicrmw.start
873 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
874 ; GFX8-NEXT: s_waitcnt vmcnt(0)
875 ; GFX8-NEXT: v_sub_f32_e32 v3, v4, v2
876 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
877 ; GFX8-NEXT: s_waitcnt vmcnt(0)
878 ; GFX8-NEXT: buffer_wbinvl1
879 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
880 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
881 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
882 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
883 ; GFX8-NEXT: s_cbranch_execnz .LBB3_1
884 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
885 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
886 ; GFX8-NEXT: s_setpc_b64 s[30:31]
887 ;
888 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f32:
889 ; GFX7: ; %bb.0:
890 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
891 ; GFX7-NEXT: s_mov_b32 s6, 0
892 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
893 ; GFX7-NEXT: s_mov_b32 s4, s6
894 ; GFX7-NEXT: s_mov_b32 s5, s6
895 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
896 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
897 ; GFX7-NEXT: .LBB3_1: ; %atomicrmw.start
898 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
899 ; GFX7-NEXT: s_waitcnt vmcnt(0)
900 ; GFX7-NEXT: v_sub_f32_e32 v3, v4, v2
901 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
902 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
903 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc
904 ; GFX7-NEXT: s_waitcnt vmcnt(0)
905 ; GFX7-NEXT: buffer_wbinvl1
906 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
907 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
908 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
909 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
910 ; GFX7-NEXT: s_cbranch_execnz .LBB3_1
911 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
912 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
913 ; GFX7-NEXT: s_setpc_b64 s[30:31]
914 ;
915 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f32:
916 ; GFX6: ; %bb.0:
917 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
918 ; GFX6-NEXT: s_mov_b32 s6, 0
919 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
920 ; GFX6-NEXT: s_mov_b32 s4, s6
921 ; GFX6-NEXT: s_mov_b32 s5, s6
922 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
923 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
924 ; GFX6-NEXT: .LBB3_1: ; %atomicrmw.start
925 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
926 ; GFX6-NEXT: s_waitcnt vmcnt(0)
927 ; GFX6-NEXT: v_sub_f32_e32 v3, v4, v2
928 ; GFX6-NEXT: s_waitcnt expcnt(0)
929 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
930 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
931 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc
932 ; GFX6-NEXT: s_waitcnt vmcnt(0)
933 ; GFX6-NEXT: buffer_wbinvl1
934 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
935 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
936 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
937 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
938 ; GFX6-NEXT: s_cbranch_execnz .LBB3_1
939 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
940 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
941 ; GFX6-NEXT: s_waitcnt expcnt(0)
942 ; GFX6-NEXT: s_setpc_b64 s[30:31]
943 %unused = atomicrmw fsub ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst
944 ret void
945 }
947 define void @global_agent_atomic_fsub_noret_f32__offset12b_pos(ptr addrspace(1) %ptr, float %val) #0 {
948 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos:
949 ; GFX12: ; %bb.0:
950 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
951 ; GFX12-NEXT: s_wait_expcnt 0x0
952 ; GFX12-NEXT: s_wait_samplecnt 0x0
953 ; GFX12-NEXT: s_wait_bvhcnt 0x0
954 ; GFX12-NEXT: s_wait_kmcnt 0x0
955 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:2044
956 ; GFX12-NEXT: s_mov_b32 s0, 0
957 ; GFX12-NEXT: .LBB4_1: ; %atomicrmw.start
958 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
959 ; GFX12-NEXT: s_wait_loadcnt 0x0
960 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
961 ; GFX12-NEXT: s_wait_storecnt 0x0
962 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
963 ; GFX12-NEXT: s_wait_loadcnt 0x0
964 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
965 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
966 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
967 ; GFX12-NEXT: s_wait_alu 0xfffe
968 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
969 ; GFX12-NEXT: s_wait_alu 0xfffe
970 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
971 ; GFX12-NEXT: s_cbranch_execnz .LBB4_1
972 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
973 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
974 ; GFX12-NEXT: s_wait_alu 0xfffe
975 ; GFX12-NEXT: s_setpc_b64 s[30:31]
976 ;
977 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos:
978 ; GFX940: ; %bb.0:
979 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
980 ; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:2044
981 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
982 ; GFX940-NEXT: .LBB4_1: ; %atomicrmw.start
983 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
984 ; GFX940-NEXT: s_waitcnt vmcnt(0)
985 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
986 ; GFX940-NEXT: buffer_wbl2 sc1
987 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
988 ; GFX940-NEXT: s_waitcnt vmcnt(0)
989 ; GFX940-NEXT: buffer_inv sc1
990 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
991 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
992 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
993 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
994 ; GFX940-NEXT: s_cbranch_execnz .LBB4_1
995 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
996 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
997 ; GFX940-NEXT: s_setpc_b64 s[30:31]
998 ;
999 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos:
1000 ; GFX11: ; %bb.0:
1001 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1002 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:2044
1003 ; GFX11-NEXT: s_mov_b32 s0, 0
1004 ; GFX11-NEXT: .LBB4_1: ; %atomicrmw.start
1005 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
1006 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1007 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
1008 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1009 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
1010 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1011 ; GFX11-NEXT: buffer_gl1_inv
1012 ; GFX11-NEXT: buffer_gl0_inv
1013 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1014 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
1015 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
1016 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1017 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1018 ; GFX11-NEXT: s_cbranch_execnz .LBB4_1
1019 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
1020 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
1021 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1022 ;
1023 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos:
1024 ; GFX10: ; %bb.0:
1025 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1026 ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
1027 ; GFX10-NEXT: s_mov_b32 s4, 0
1028 ; GFX10-NEXT: .LBB4_1: ; %atomicrmw.start
1029 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
1030 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1031 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
1032 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1033 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
1034 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1035 ; GFX10-NEXT: buffer_gl1_inv
1036 ; GFX10-NEXT: buffer_gl0_inv
1037 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1038 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
1039 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
1040 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
1041 ; GFX10-NEXT: s_cbranch_execnz .LBB4_1
1042 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
1043 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
1044 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1045 ;
1046 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos:
1047 ; GFX90A: ; %bb.0:
1048 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1049 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
1050 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
1051 ; GFX90A-NEXT: .LBB4_1: ; %atomicrmw.start
1052 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
1053 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1054 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
1055 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
1056 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1057 ; GFX90A-NEXT: buffer_wbinvl1
1058 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1059 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1060 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
1061 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
1062 ; GFX90A-NEXT: s_cbranch_execnz .LBB4_1
1063 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
1064 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
1065 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1066 ;
1067 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos:
1068 ; GFX908: ; %bb.0:
1069 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1070 ; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
1071 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
1072 ; GFX908-NEXT: .LBB4_1: ; %atomicrmw.start
1073 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
1074 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1075 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
1076 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
1077 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1078 ; GFX908-NEXT: buffer_wbinvl1
1079 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1080 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1081 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
1082 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
1083 ; GFX908-NEXT: s_cbranch_execnz .LBB4_1
1084 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
1085 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
1086 ; GFX908-NEXT: s_setpc_b64 s[30:31]
1087 ;
1088 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos:
1089 ; GFX8: ; %bb.0:
1090 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1091 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
1092 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1093 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
1094 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
1095 ; GFX8-NEXT: .LBB4_1: ; %atomicrmw.start
1096 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
1097 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1098 ; GFX8-NEXT: v_sub_f32_e32 v3, v4, v2
1099 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
1100 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1101 ; GFX8-NEXT: buffer_wbinvl1
1102 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1103 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1104 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
1105 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1106 ; GFX8-NEXT: s_cbranch_execnz .LBB4_1
1107 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1108 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1109 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1110 ;
1111 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos:
1112 ; GFX7: ; %bb.0:
1113 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1114 ; GFX7-NEXT: s_mov_b32 s6, 0
1115 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
1116 ; GFX7-NEXT: s_mov_b32 s4, s6
1117 ; GFX7-NEXT: s_mov_b32 s5, s6
1118 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:2044
1119 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
1120 ; GFX7-NEXT: .LBB4_1: ; %atomicrmw.start
1121 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
1122 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1123 ; GFX7-NEXT: v_sub_f32_e32 v3, v4, v2
1124 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
1125 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
1126 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
1127 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1128 ; GFX7-NEXT: buffer_wbinvl1
1129 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
1130 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
1131 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
1132 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
1133 ; GFX7-NEXT: s_cbranch_execnz .LBB4_1
1134 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
1135 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
1136 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1137 ;
1138 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos:
1139 ; GFX6: ; %bb.0:
1140 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1141 ; GFX6-NEXT: s_mov_b32 s6, 0
1142 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
1143 ; GFX6-NEXT: s_mov_b32 s4, s6
1144 ; GFX6-NEXT: s_mov_b32 s5, s6
1145 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:2044
1146 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
1147 ; GFX6-NEXT: .LBB4_1: ; %atomicrmw.start
1148 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
1149 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1150 ; GFX6-NEXT: v_sub_f32_e32 v3, v4, v2
1151 ; GFX6-NEXT: s_waitcnt expcnt(0)
1152 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
1153 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
1154 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
1155 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1156 ; GFX6-NEXT: buffer_wbinvl1
1157 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
1158 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
1159 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
1160 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
1161 ; GFX6-NEXT: s_cbranch_execnz .LBB4_1
1162 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
1163 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
1164 ; GFX6-NEXT: s_waitcnt expcnt(0)
1165 ; GFX6-NEXT: s_setpc_b64 s[30:31]
1166 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
1167 %unused = atomicrmw fsub ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
1168 ret void
1169 }
1171 define void @global_agent_atomic_fsub_noret_f32__offset12b_neg(ptr addrspace(1) %ptr, float %val) #0 {
1172 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg:
1173 ; GFX12: ; %bb.0:
1174 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
1175 ; GFX12-NEXT: s_wait_expcnt 0x0
1176 ; GFX12-NEXT: s_wait_samplecnt 0x0
1177 ; GFX12-NEXT: s_wait_bvhcnt 0x0
1178 ; GFX12-NEXT: s_wait_kmcnt 0x0
1179 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
1180 ; GFX12-NEXT: s_mov_b32 s0, 0
1181 ; GFX12-NEXT: .LBB5_1: ; %atomicrmw.start
1182 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
1183 ; GFX12-NEXT: s_wait_loadcnt 0x0
1184 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
1185 ; GFX12-NEXT: s_wait_storecnt 0x0
1186 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
1187 ; GFX12-NEXT: s_wait_loadcnt 0x0
1188 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
1189 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1190 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
1191 ; GFX12-NEXT: s_wait_alu 0xfffe
1192 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
1193 ; GFX12-NEXT: s_wait_alu 0xfffe
1194 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1195 ; GFX12-NEXT: s_cbranch_execnz .LBB5_1
1196 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
1197 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
1198 ; GFX12-NEXT: s_wait_alu 0xfffe
1199 ; GFX12-NEXT: s_setpc_b64 s[30:31]
1200 ;
1201 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg:
1202 ; GFX940: ; %bb.0:
1203 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1204 ; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:-2048
1205 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
1206 ; GFX940-NEXT: .LBB5_1: ; %atomicrmw.start
1207 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
1208 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1209 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
1210 ; GFX940-NEXT: buffer_wbl2 sc1
1211 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
1212 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1213 ; GFX940-NEXT: buffer_inv sc1
1214 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1215 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
1216 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
1217 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
1218 ; GFX940-NEXT: s_cbranch_execnz .LBB5_1
1219 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
1220 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
1221 ; GFX940-NEXT: s_setpc_b64 s[30:31]
1222 ;
1223 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg:
1224 ; GFX11: ; %bb.0:
1225 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1226 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
1227 ; GFX11-NEXT: s_mov_b32 s0, 0
1228 ; GFX11-NEXT: .LBB5_1: ; %atomicrmw.start
1229 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
1230 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1231 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
1232 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1233 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
1234 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1235 ; GFX11-NEXT: buffer_gl1_inv
1236 ; GFX11-NEXT: buffer_gl0_inv
1237 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1238 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
1239 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
1240 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1241 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1242 ; GFX11-NEXT: s_cbranch_execnz .LBB5_1
1243 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
1244 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
1245 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1246 ;
1247 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg:
1248 ; GFX10: ; %bb.0:
1249 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1250 ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:-2048
1251 ; GFX10-NEXT: s_mov_b32 s4, 0
1252 ; GFX10-NEXT: .LBB5_1: ; %atomicrmw.start
1253 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
1254 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1255 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
1256 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1257 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
1258 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1259 ; GFX10-NEXT: buffer_gl1_inv
1260 ; GFX10-NEXT: buffer_gl0_inv
1261 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1262 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
1263 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
1264 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
1265 ; GFX10-NEXT: s_cbranch_execnz .LBB5_1
1266 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
1267 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
1268 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1269 ;
1270 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg:
1271 ; GFX90A: ; %bb.0:
1272 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1273 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
1274 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
1275 ; GFX90A-NEXT: .LBB5_1: ; %atomicrmw.start
1276 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
1277 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1278 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
1279 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
1280 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1281 ; GFX90A-NEXT: buffer_wbinvl1
1282 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1283 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1284 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
1285 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
1286 ; GFX90A-NEXT: s_cbranch_execnz .LBB5_1
1287 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
1288 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
1289 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1290 ;
1291 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg:
1292 ; GFX908: ; %bb.0:
1293 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1294 ; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:-2048
1295 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
1296 ; GFX908-NEXT: .LBB5_1: ; %atomicrmw.start
1297 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
1298 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1299 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
1300 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
1301 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1302 ; GFX908-NEXT: buffer_wbinvl1
1303 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1304 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1305 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
1306 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
1307 ; GFX908-NEXT: s_cbranch_execnz .LBB5_1
1308 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
1309 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
1310 ; GFX908-NEXT: s_setpc_b64 s[30:31]
1311 ;
1312 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg:
1313 ; GFX8: ; %bb.0:
1314 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1315 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
1316 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
1317 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
1318 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
1319 ; GFX8-NEXT: .LBB5_1: ; %atomicrmw.start
1320 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
1321 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1322 ; GFX8-NEXT: v_sub_f32_e32 v3, v4, v2
1323 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
1324 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1325 ; GFX8-NEXT: buffer_wbinvl1
1326 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1327 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1328 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
1329 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1330 ; GFX8-NEXT: s_cbranch_execnz .LBB5_1
1331 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1332 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1333 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1334 ;
1335 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg:
1336 ; GFX7: ; %bb.0:
1337 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1338 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
1339 ; GFX7-NEXT: s_mov_b32 s5, -1
1340 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
1341 ; GFX7-NEXT: s_mov_b32 s6, 0
1342 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
1343 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
1344 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
1345 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
1346 ; GFX7-NEXT: s_mov_b32 s4, s6
1347 ; GFX7-NEXT: s_mov_b32 s5, s6
1348 ; GFX7-NEXT: .LBB5_1: ; %atomicrmw.start
1349 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
1350 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1351 ; GFX7-NEXT: v_sub_f32_e32 v3, v4, v2
1352 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
1353 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
1354 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc
1355 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1356 ; GFX7-NEXT: buffer_wbinvl1
1357 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
1358 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
1359 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
1360 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
1361 ; GFX7-NEXT: s_cbranch_execnz .LBB5_1
1362 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
1363 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
1364 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1365 ;
1366 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg:
1367 ; GFX6: ; %bb.0:
1368 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1369 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
1370 ; GFX6-NEXT: s_mov_b32 s5, -1
1371 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
1372 ; GFX6-NEXT: s_mov_b32 s6, 0
1373 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
1374 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
1375 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
1376 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
1377 ; GFX6-NEXT: s_mov_b32 s4, s6
1378 ; GFX6-NEXT: s_mov_b32 s5, s6
1379 ; GFX6-NEXT: .LBB5_1: ; %atomicrmw.start
1380 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
1381 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1382 ; GFX6-NEXT: v_sub_f32_e32 v3, v4, v2
1383 ; GFX6-NEXT: s_waitcnt expcnt(0)
1384 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
1385 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
1386 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc
1387 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1388 ; GFX6-NEXT: buffer_wbinvl1
1389 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
1390 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
1391 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
1392 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
1393 ; GFX6-NEXT: s_cbranch_execnz .LBB5_1
1394 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
1395 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
1396 ; GFX6-NEXT: s_waitcnt expcnt(0)
1397 ; GFX6-NEXT: s_setpc_b64 s[30:31]
1398 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
1399 %unused = atomicrmw fsub ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
1400 ret void
1401 }
1403 define float @global_system_atomic_fsub_ret_f32__offset12b_pos(ptr addrspace(1) %ptr, float %val) #0 {
1404 ; GFX12-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos:
1405 ; GFX12: ; %bb.0:
1406 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
1407 ; GFX12-NEXT: s_wait_expcnt 0x0
1408 ; GFX12-NEXT: s_wait_samplecnt 0x0
1409 ; GFX12-NEXT: s_wait_bvhcnt 0x0
1410 ; GFX12-NEXT: s_wait_kmcnt 0x0
1411 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
1412 ; GFX12-NEXT: s_mov_b32 s0, 0
1413 ; GFX12-NEXT: .LBB6_1: ; %atomicrmw.start
1414 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
1415 ; GFX12-NEXT: s_wait_loadcnt 0x0
1416 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
1417 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
1418 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
1419 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
1420 ; GFX12-NEXT: s_wait_storecnt 0x0
1421 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
1422 ; GFX12-NEXT: s_wait_loadcnt 0x0
1423 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
1424 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1425 ; GFX12-NEXT: s_wait_alu 0xfffe
1426 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
1427 ; GFX12-NEXT: s_wait_alu 0xfffe
1428 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1429 ; GFX12-NEXT: s_cbranch_execnz .LBB6_1
1430 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
1431 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
1432 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
1433 ; GFX12-NEXT: s_wait_alu 0xfffe
1434 ; GFX12-NEXT: s_setpc_b64 s[30:31]
1435 ;
1436 ; GFX940-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos:
1437 ; GFX940: ; %bb.0:
1438 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1439 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
1440 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
1441 ; GFX940-NEXT: .LBB6_1: ; %atomicrmw.start
1442 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
1443 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1444 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
1445 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
1446 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
1447 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
1448 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1449 ; GFX940-NEXT: buffer_inv sc0 sc1
1450 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1451 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
1452 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
1453 ; GFX940-NEXT: s_cbranch_execnz .LBB6_1
1454 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
1455 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
1456 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
1457 ; GFX940-NEXT: s_setpc_b64 s[30:31]
1458 ;
1459 ; GFX11-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos:
1460 ; GFX11: ; %bb.0:
1461 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1462 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
1463 ; GFX11-NEXT: s_mov_b32 s0, 0
1464 ; GFX11-NEXT: .LBB6_1: ; %atomicrmw.start
1465 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
1466 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1467 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
1468 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
1469 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
1470 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1471 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
1472 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1473 ; GFX11-NEXT: buffer_gl1_inv
1474 ; GFX11-NEXT: buffer_gl0_inv
1475 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1476 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
1477 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1478 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1479 ; GFX11-NEXT: s_cbranch_execnz .LBB6_1
1480 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
1481 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
1482 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
1483 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1484 ;
1485 ; GFX10-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos:
1486 ; GFX10: ; %bb.0:
1487 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1488 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
1489 ; GFX10-NEXT: s_mov_b32 s4, 0
1490 ; GFX10-NEXT: .LBB6_1: ; %atomicrmw.start
1491 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
1492 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1493 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
1494 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
1495 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1496 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
1497 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1498 ; GFX10-NEXT: buffer_gl1_inv
1499 ; GFX10-NEXT: buffer_gl0_inv
1500 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1501 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
1502 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
1503 ; GFX10-NEXT: s_cbranch_execnz .LBB6_1
1504 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
1505 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
1506 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
1507 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1508 ;
1509 ; GFX90A-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos:
1510 ; GFX90A: ; %bb.0:
1511 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1512 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
1513 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
1514 ; GFX90A-NEXT: .LBB6_1: ; %atomicrmw.start
1515 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
1516 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1517 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
1518 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
1519 ; GFX90A-NEXT: buffer_wbl2
1520 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
1521 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1522 ; GFX90A-NEXT: buffer_invl2
1523 ; GFX90A-NEXT: buffer_wbinvl1
1524 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1525 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1526 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
1527 ; GFX90A-NEXT: s_cbranch_execnz .LBB6_1
1528 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
1529 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
1530 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
1531 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1532 ;
1533 ; GFX908-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos:
1534 ; GFX908: ; %bb.0:
1535 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1536 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
1537 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
1538 ; GFX908-NEXT: .LBB6_1: ; %atomicrmw.start
1539 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
1540 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1541 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
1542 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
1543 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
1544 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1545 ; GFX908-NEXT: buffer_wbinvl1
1546 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1547 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1548 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
1549 ; GFX908-NEXT: s_cbranch_execnz .LBB6_1
1550 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
1551 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
1552 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
1553 ; GFX908-NEXT: s_setpc_b64 s[30:31]
1554 ;
1555 ; GFX8-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos:
1556 ; GFX8: ; %bb.0:
1557 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1558 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
1559 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
1560 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
1561 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
1562 ; GFX8-NEXT: .LBB6_1: ; %atomicrmw.start
1563 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
1564 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1565 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
1566 ; GFX8-NEXT: v_sub_f32_e32 v0, v1, v2
1567 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
1568 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1569 ; GFX8-NEXT: buffer_wbinvl1
1570 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
1571 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1572 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1573 ; GFX8-NEXT: s_cbranch_execnz .LBB6_1
1574 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1575 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1576 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1577 ;
1578 ; GFX7-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos:
1579 ; GFX7: ; %bb.0:
1580 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1581 ; GFX7-NEXT: s_mov_b32 s6, 0
1582 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
1583 ; GFX7-NEXT: s_mov_b32 s4, s6
1584 ; GFX7-NEXT: s_mov_b32 s5, s6
1585 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
1586 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
1587 ; GFX7-NEXT: .LBB6_1: ; %atomicrmw.start
1588 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
1589 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1590 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
1591 ; GFX7-NEXT: v_sub_f32_e32 v4, v5, v2
1592 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
1593 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
1594 ; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
1595 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1596 ; GFX7-NEXT: buffer_wbinvl1
1597 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1598 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
1599 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
1600 ; GFX7-NEXT: s_cbranch_execnz .LBB6_1
1601 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
1602 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
1603 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
1604 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1605 ;
1606 ; GFX6-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos:
1607 ; GFX6: ; %bb.0:
1608 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1609 ; GFX6-NEXT: s_mov_b32 s6, 0
1610 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
1611 ; GFX6-NEXT: s_mov_b32 s4, s6
1612 ; GFX6-NEXT: s_mov_b32 s5, s6
1613 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
1614 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
1615 ; GFX6-NEXT: .LBB6_1: ; %atomicrmw.start
1616 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
1617 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1618 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
1619 ; GFX6-NEXT: s_waitcnt expcnt(0)
1620 ; GFX6-NEXT: v_sub_f32_e32 v4, v5, v2
1621 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
1622 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
1623 ; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
1624 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1625 ; GFX6-NEXT: buffer_wbinvl1
1626 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1627 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
1628 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
1629 ; GFX6-NEXT: s_cbranch_execnz .LBB6_1
1630 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
1631 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
1632 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
1633 ; GFX6-NEXT: s_waitcnt expcnt(0)
1634 ; GFX6-NEXT: s_setpc_b64 s[30:31]
1635 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
1636 %result = atomicrmw fsub ptr addrspace(1) %gep, float %val seq_cst
1637 ret float %result
1638 }
1640 define void @global_system_atomic_fsub_noret_f32__offset12b_pos(ptr addrspace(1) %ptr, float %val) #0 {
1641 ; GFX12-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos:
1642 ; GFX12: ; %bb.0:
1643 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
1644 ; GFX12-NEXT: s_wait_expcnt 0x0
1645 ; GFX12-NEXT: s_wait_samplecnt 0x0
1646 ; GFX12-NEXT: s_wait_bvhcnt 0x0
1647 ; GFX12-NEXT: s_wait_kmcnt 0x0
1648 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:2044
1649 ; GFX12-NEXT: s_mov_b32 s0, 0
1650 ; GFX12-NEXT: .LBB7_1: ; %atomicrmw.start
1651 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
1652 ; GFX12-NEXT: s_wait_loadcnt 0x0
1653 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
1654 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
1655 ; GFX12-NEXT: s_wait_storecnt 0x0
1656 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
1657 ; GFX12-NEXT: s_wait_loadcnt 0x0
1658 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
1659 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1660 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
1661 ; GFX12-NEXT: s_wait_alu 0xfffe
1662 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
1663 ; GFX12-NEXT: s_wait_alu 0xfffe
1664 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1665 ; GFX12-NEXT: s_cbranch_execnz .LBB7_1
1666 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
1667 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
1668 ; GFX12-NEXT: s_wait_alu 0xfffe
1669 ; GFX12-NEXT: s_setpc_b64 s[30:31]
1670 ;
1671 ; GFX940-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos:
1672 ; GFX940: ; %bb.0:
1673 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1674 ; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:2044
1675 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
1676 ; GFX940-NEXT: .LBB7_1: ; %atomicrmw.start
1677 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
1678 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1679 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
1680 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
1681 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
1682 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1683 ; GFX940-NEXT: buffer_inv sc0 sc1
1684 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1685 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
1686 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
1687 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
1688 ; GFX940-NEXT: s_cbranch_execnz .LBB7_1
1689 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
1690 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
1691 ; GFX940-NEXT: s_setpc_b64 s[30:31]
1692 ;
1693 ; GFX11-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos:
1694 ; GFX11: ; %bb.0:
1695 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1696 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:2044
1697 ; GFX11-NEXT: s_mov_b32 s0, 0
1698 ; GFX11-NEXT: .LBB7_1: ; %atomicrmw.start
1699 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
1700 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1701 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
1702 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1703 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
1704 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1705 ; GFX11-NEXT: buffer_gl1_inv
1706 ; GFX11-NEXT: buffer_gl0_inv
1707 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1708 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
1709 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
1710 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1711 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1712 ; GFX11-NEXT: s_cbranch_execnz .LBB7_1
1713 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
1714 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
1715 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1716 ;
1717 ; GFX10-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos:
1718 ; GFX10: ; %bb.0:
1719 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1720 ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
1721 ; GFX10-NEXT: s_mov_b32 s4, 0
1722 ; GFX10-NEXT: .LBB7_1: ; %atomicrmw.start
1723 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
1724 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1725 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
1726 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1727 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
1728 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1729 ; GFX10-NEXT: buffer_gl1_inv
1730 ; GFX10-NEXT: buffer_gl0_inv
1731 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1732 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
1733 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
1734 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
1735 ; GFX10-NEXT: s_cbranch_execnz .LBB7_1
1736 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
1737 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
1738 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1739 ;
1740 ; GFX90A-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos:
1741 ; GFX90A: ; %bb.0:
1742 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1743 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
1744 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
1745 ; GFX90A-NEXT: .LBB7_1: ; %atomicrmw.start
1746 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
1747 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1748 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
1749 ; GFX90A-NEXT: buffer_wbl2
1750 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
1751 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1752 ; GFX90A-NEXT: buffer_invl2
1753 ; GFX90A-NEXT: buffer_wbinvl1
1754 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1755 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1756 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
1757 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
1758 ; GFX90A-NEXT: s_cbranch_execnz .LBB7_1
1759 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
1760 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
1761 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1762 ;
1763 ; GFX908-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos:
1764 ; GFX908: ; %bb.0:
1765 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1766 ; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
1767 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
1768 ; GFX908-NEXT: .LBB7_1: ; %atomicrmw.start
1769 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
1770 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1771 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
1772 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
1773 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1774 ; GFX908-NEXT: buffer_wbinvl1
1775 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1776 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1777 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
1778 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
1779 ; GFX908-NEXT: s_cbranch_execnz .LBB7_1
1780 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
1781 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
1782 ; GFX908-NEXT: s_setpc_b64 s[30:31]
1783 ;
1784 ; GFX8-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos:
1785 ; GFX8: ; %bb.0:
1786 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1787 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
1788 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1789 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
1790 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
1791 ; GFX8-NEXT: .LBB7_1: ; %atomicrmw.start
1792 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
1793 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1794 ; GFX8-NEXT: v_sub_f32_e32 v3, v4, v2
1795 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
1796 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1797 ; GFX8-NEXT: buffer_wbinvl1
1798 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1799 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1800 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
1801 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1802 ; GFX8-NEXT: s_cbranch_execnz .LBB7_1
1803 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1804 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1805 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1806 ;
1807 ; GFX7-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos:
1808 ; GFX7: ; %bb.0:
1809 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1810 ; GFX7-NEXT: s_mov_b32 s6, 0
1811 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
1812 ; GFX7-NEXT: s_mov_b32 s4, s6
1813 ; GFX7-NEXT: s_mov_b32 s5, s6
1814 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:2044
1815 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
1816 ; GFX7-NEXT: .LBB7_1: ; %atomicrmw.start
1817 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
1818 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1819 ; GFX7-NEXT: v_sub_f32_e32 v3, v4, v2
1820 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
1821 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
1822 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
1823 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1824 ; GFX7-NEXT: buffer_wbinvl1
1825 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
1826 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
1827 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
1828 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
1829 ; GFX7-NEXT: s_cbranch_execnz .LBB7_1
1830 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
1831 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
1832 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1833 ;
1834 ; GFX6-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos:
1835 ; GFX6: ; %bb.0:
1836 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1837 ; GFX6-NEXT: s_mov_b32 s6, 0
1838 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
1839 ; GFX6-NEXT: s_mov_b32 s4, s6
1840 ; GFX6-NEXT: s_mov_b32 s5, s6
1841 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:2044
1842 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
1843 ; GFX6-NEXT: .LBB7_1: ; %atomicrmw.start
1844 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
1845 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1846 ; GFX6-NEXT: v_sub_f32_e32 v3, v4, v2
1847 ; GFX6-NEXT: s_waitcnt expcnt(0)
1848 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
1849 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
1850 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
1851 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1852 ; GFX6-NEXT: buffer_wbinvl1
1853 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
1854 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
1855 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
1856 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
1857 ; GFX6-NEXT: s_cbranch_execnz .LBB7_1
1858 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
1859 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
1860 ; GFX6-NEXT: s_waitcnt expcnt(0)
1861 ; GFX6-NEXT: s_setpc_b64 s[30:31]
1862 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
1863 %unused = atomicrmw fsub ptr addrspace(1) %gep, float %val seq_cst
1864 ret void
1865 }
1867 ; --------------------------------------------------------------------
1868 ; float with ftz/daz
1869 ; --------------------------------------------------------------------
1871 define float @global_agent_atomic_fsub_ret_f32__ftz(ptr addrspace(1) %ptr, float %val) #1 {
1872 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f32__ftz:
1873 ; GFX12: ; %bb.0:
1874 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
1875 ; GFX12-NEXT: s_wait_expcnt 0x0
1876 ; GFX12-NEXT: s_wait_samplecnt 0x0
1877 ; GFX12-NEXT: s_wait_bvhcnt 0x0
1878 ; GFX12-NEXT: s_wait_kmcnt 0x0
1879 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
1880 ; GFX12-NEXT: s_mov_b32 s0, 0
1881 ; GFX12-NEXT: .LBB8_1: ; %atomicrmw.start
1882 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
1883 ; GFX12-NEXT: s_wait_loadcnt 0x0
1884 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
1885 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
1886 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
1887 ; GFX12-NEXT: s_wait_storecnt 0x0
1888 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
1889 ; GFX12-NEXT: s_wait_loadcnt 0x0
1890 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
1891 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1892 ; GFX12-NEXT: s_wait_alu 0xfffe
1893 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
1894 ; GFX12-NEXT: s_wait_alu 0xfffe
1895 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1896 ; GFX12-NEXT: s_cbranch_execnz .LBB8_1
1897 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
1898 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
1899 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
1900 ; GFX12-NEXT: s_wait_alu 0xfffe
1901 ; GFX12-NEXT: s_setpc_b64 s[30:31]
1902 ;
1903 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f32__ftz:
1904 ; GFX940: ; %bb.0:
1905 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1906 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
1907 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
1908 ; GFX940-NEXT: .LBB8_1: ; %atomicrmw.start
1909 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
1910 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1911 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
1912 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
1913 ; GFX940-NEXT: buffer_wbl2 sc1
1914 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
1915 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1916 ; GFX940-NEXT: buffer_inv sc1
1917 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1918 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
1919 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
1920 ; GFX940-NEXT: s_cbranch_execnz .LBB8_1
1921 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
1922 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
1923 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
1924 ; GFX940-NEXT: s_setpc_b64 s[30:31]
1925 ;
1926 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f32__ftz:
1927 ; GFX11: ; %bb.0:
1928 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1929 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
1930 ; GFX11-NEXT: s_mov_b32 s0, 0
1931 ; GFX11-NEXT: .LBB8_1: ; %atomicrmw.start
1932 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
1933 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1934 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
1935 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
1936 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
1937 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1938 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
1939 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1940 ; GFX11-NEXT: buffer_gl1_inv
1941 ; GFX11-NEXT: buffer_gl0_inv
1942 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1943 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
1944 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1945 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1946 ; GFX11-NEXT: s_cbranch_execnz .LBB8_1
1947 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
1948 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
1949 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
1950 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1951 ;
1952 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f32__ftz:
1953 ; GFX10: ; %bb.0:
1954 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1955 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
1956 ; GFX10-NEXT: s_mov_b32 s4, 0
1957 ; GFX10-NEXT: .LBB8_1: ; %atomicrmw.start
1958 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
1959 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1960 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
1961 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
1962 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1963 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
1964 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1965 ; GFX10-NEXT: buffer_gl1_inv
1966 ; GFX10-NEXT: buffer_gl0_inv
1967 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1968 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
1969 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
1970 ; GFX10-NEXT: s_cbranch_execnz .LBB8_1
1971 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
1972 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
1973 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
1974 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1975 ;
1976 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f32__ftz:
1977 ; GFX90A: ; %bb.0:
1978 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1979 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
1980 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
1981 ; GFX90A-NEXT: .LBB8_1: ; %atomicrmw.start
1982 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
1983 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1984 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
1985 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
1986 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
1987 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1988 ; GFX90A-NEXT: buffer_wbinvl1
1989 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1990 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1991 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
1992 ; GFX90A-NEXT: s_cbranch_execnz .LBB8_1
1993 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
1994 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
1995 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
1996 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1997 ;
1998 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f32__ftz:
1999 ; GFX908: ; %bb.0:
2000 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2001 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
2002 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
2003 ; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start
2004 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
2005 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2006 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
2007 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
2008 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
2009 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2010 ; GFX908-NEXT: buffer_wbinvl1
2011 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
2012 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2013 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
2014 ; GFX908-NEXT: s_cbranch_execnz .LBB8_1
2015 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
2016 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
2017 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
2018 ; GFX908-NEXT: s_setpc_b64 s[30:31]
2019 ;
2020 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f32__ftz:
2021 ; GFX8: ; %bb.0:
2022 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2023 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
2024 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2025 ; GFX8-NEXT: .LBB8_1: ; %atomicrmw.start
2026 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2027 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2028 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
2029 ; GFX8-NEXT: v_sub_f32_e32 v3, v4, v2
2030 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
2031 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2032 ; GFX8-NEXT: buffer_wbinvl1
2033 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
2034 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2035 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2036 ; GFX8-NEXT: s_cbranch_execnz .LBB8_1
2037 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2038 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2039 ; GFX8-NEXT: v_mov_b32_e32 v0, v3
2040 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2041 ;
2042 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f32__ftz:
2043 ; GFX7: ; %bb.0:
2044 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2045 ; GFX7-NEXT: s_mov_b32 s6, 0
2046 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
2047 ; GFX7-NEXT: s_mov_b32 s4, s6
2048 ; GFX7-NEXT: s_mov_b32 s5, s6
2049 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
2050 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
2051 ; GFX7-NEXT: .LBB8_1: ; %atomicrmw.start
2052 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
2053 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2054 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
2055 ; GFX7-NEXT: v_sub_f32_e32 v4, v5, v2
2056 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
2057 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
2058 ; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
2059 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2060 ; GFX7-NEXT: buffer_wbinvl1
2061 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2062 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
2063 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
2064 ; GFX7-NEXT: s_cbranch_execnz .LBB8_1
2065 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
2066 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
2067 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
2068 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2069 ;
2070 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f32__ftz:
2071 ; GFX6: ; %bb.0:
2072 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2073 ; GFX6-NEXT: s_mov_b32 s6, 0
2074 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
2075 ; GFX6-NEXT: s_mov_b32 s4, s6
2076 ; GFX6-NEXT: s_mov_b32 s5, s6
2077 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
2078 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
2079 ; GFX6-NEXT: .LBB8_1: ; %atomicrmw.start
2080 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
2081 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2082 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
2083 ; GFX6-NEXT: s_waitcnt expcnt(0)
2084 ; GFX6-NEXT: v_sub_f32_e32 v4, v5, v2
2085 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
2086 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
2087 ; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
2088 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2089 ; GFX6-NEXT: buffer_wbinvl1
2090 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2091 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
2092 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
2093 ; GFX6-NEXT: s_cbranch_execnz .LBB8_1
2094 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
2095 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
2096 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
2097 ; GFX6-NEXT: s_waitcnt expcnt(0)
2098 ; GFX6-NEXT: s_setpc_b64 s[30:31]
2099 %result = atomicrmw fsub ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst
2100 ret float %result
2101 }
2103 define float @global_agent_atomic_fsub_ret_f32__offset12b_pos__ftz(ptr addrspace(1) %ptr, float %val) #1 {
2104 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos__ftz:
2105 ; GFX12: ; %bb.0:
2106 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
2107 ; GFX12-NEXT: s_wait_expcnt 0x0
2108 ; GFX12-NEXT: s_wait_samplecnt 0x0
2109 ; GFX12-NEXT: s_wait_bvhcnt 0x0
2110 ; GFX12-NEXT: s_wait_kmcnt 0x0
2111 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
2112 ; GFX12-NEXT: s_mov_b32 s0, 0
2113 ; GFX12-NEXT: .LBB9_1: ; %atomicrmw.start
2114 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
2115 ; GFX12-NEXT: s_wait_loadcnt 0x0
2116 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
2117 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
2118 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
2119 ; GFX12-NEXT: s_wait_storecnt 0x0
2120 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
2121 ; GFX12-NEXT: s_wait_loadcnt 0x0
2122 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
2123 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2124 ; GFX12-NEXT: s_wait_alu 0xfffe
2125 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
2126 ; GFX12-NEXT: s_wait_alu 0xfffe
2127 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2128 ; GFX12-NEXT: s_cbranch_execnz .LBB9_1
2129 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
2130 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
2131 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
2132 ; GFX12-NEXT: s_wait_alu 0xfffe
2133 ; GFX12-NEXT: s_setpc_b64 s[30:31]
2134 ;
2135 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos__ftz:
2136 ; GFX940: ; %bb.0:
2137 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2138 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
2139 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
2140 ; GFX940-NEXT: .LBB9_1: ; %atomicrmw.start
2141 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
2142 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2143 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
2144 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
2145 ; GFX940-NEXT: buffer_wbl2 sc1
2146 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
2147 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2148 ; GFX940-NEXT: buffer_inv sc1
2149 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2150 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
2151 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
2152 ; GFX940-NEXT: s_cbranch_execnz .LBB9_1
2153 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
2154 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
2155 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
2156 ; GFX940-NEXT: s_setpc_b64 s[30:31]
2157 ;
2158 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos__ftz:
2159 ; GFX11: ; %bb.0:
2160 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2161 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
2162 ; GFX11-NEXT: s_mov_b32 s0, 0
2163 ; GFX11-NEXT: .LBB9_1: ; %atomicrmw.start
2164 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
2165 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2166 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
2167 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
2168 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
2169 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2170 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
2171 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2172 ; GFX11-NEXT: buffer_gl1_inv
2173 ; GFX11-NEXT: buffer_gl0_inv
2174 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2175 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
2176 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
2177 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2178 ; GFX11-NEXT: s_cbranch_execnz .LBB9_1
2179 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
2180 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
2181 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
2182 ; GFX11-NEXT: s_setpc_b64 s[30:31]
2183 ;
2184 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos__ftz:
2185 ; GFX10: ; %bb.0:
2186 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2187 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
2188 ; GFX10-NEXT: s_mov_b32 s4, 0
2189 ; GFX10-NEXT: .LBB9_1: ; %atomicrmw.start
2190 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
2191 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2192 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
2193 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
2194 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2195 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
2196 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2197 ; GFX10-NEXT: buffer_gl1_inv
2198 ; GFX10-NEXT: buffer_gl0_inv
2199 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2200 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
2201 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
2202 ; GFX10-NEXT: s_cbranch_execnz .LBB9_1
2203 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
2204 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
2205 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
2206 ; GFX10-NEXT: s_setpc_b64 s[30:31]
2207 ;
2208 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos__ftz:
2209 ; GFX90A: ; %bb.0:
2210 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2211 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
2212 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
2213 ; GFX90A-NEXT: .LBB9_1: ; %atomicrmw.start
2214 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
2215 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2216 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
2217 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
2218 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
2219 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2220 ; GFX90A-NEXT: buffer_wbinvl1
2221 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2222 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2223 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
2224 ; GFX90A-NEXT: s_cbranch_execnz .LBB9_1
2225 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
2226 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
2227 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
2228 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
2229 ;
2230 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos__ftz:
2231 ; GFX908: ; %bb.0:
2232 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2233 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
2234 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
2235 ; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
2236 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
2237 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2238 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
2239 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
2240 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
2241 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2242 ; GFX908-NEXT: buffer_wbinvl1
2243 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
2244 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2245 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
2246 ; GFX908-NEXT: s_cbranch_execnz .LBB9_1
2247 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
2248 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
2249 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
2250 ; GFX908-NEXT: s_setpc_b64 s[30:31]
2251 ;
2252 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos__ftz:
2253 ; GFX8: ; %bb.0:
2254 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2255 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
2256 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
2257 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
2258 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2259 ; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start
2260 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2261 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2262 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
2263 ; GFX8-NEXT: v_sub_f32_e32 v0, v1, v2
2264 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
2265 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2266 ; GFX8-NEXT: buffer_wbinvl1
2267 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
2268 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2269 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2270 ; GFX8-NEXT: s_cbranch_execnz .LBB9_1
2271 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2272 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2273 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2274 ;
2275 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos__ftz:
2276 ; GFX7: ; %bb.0:
2277 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2278 ; GFX7-NEXT: s_mov_b32 s6, 0
2279 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
2280 ; GFX7-NEXT: s_mov_b32 s4, s6
2281 ; GFX7-NEXT: s_mov_b32 s5, s6
2282 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
2283 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
2284 ; GFX7-NEXT: .LBB9_1: ; %atomicrmw.start
2285 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
2286 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2287 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
2288 ; GFX7-NEXT: v_sub_f32_e32 v4, v5, v2
2289 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
2290 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
2291 ; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
2292 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2293 ; GFX7-NEXT: buffer_wbinvl1
2294 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2295 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
2296 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
2297 ; GFX7-NEXT: s_cbranch_execnz .LBB9_1
2298 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
2299 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
2300 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
2301 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2302 ;
2303 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos__ftz:
2304 ; GFX6: ; %bb.0:
2305 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2306 ; GFX6-NEXT: s_mov_b32 s6, 0
2307 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
2308 ; GFX6-NEXT: s_mov_b32 s4, s6
2309 ; GFX6-NEXT: s_mov_b32 s5, s6
2310 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
2311 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
2312 ; GFX6-NEXT: .LBB9_1: ; %atomicrmw.start
2313 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
2314 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2315 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
2316 ; GFX6-NEXT: s_waitcnt expcnt(0)
2317 ; GFX6-NEXT: v_sub_f32_e32 v4, v5, v2
2318 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
2319 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
2320 ; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
2321 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2322 ; GFX6-NEXT: buffer_wbinvl1
2323 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2324 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
2325 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
2326 ; GFX6-NEXT: s_cbranch_execnz .LBB9_1
2327 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
2328 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
2329 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
2330 ; GFX6-NEXT: s_waitcnt expcnt(0)
2331 ; GFX6-NEXT: s_setpc_b64 s[30:31]
2332 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
2333 %result = atomicrmw fsub ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
2334 ret float %result
2335 }
2337 define float @global_agent_atomic_fsub_ret_f32__offset12b_neg__ftz(ptr addrspace(1) %ptr, float %val) #1 {
2338 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg__ftz:
2339 ; GFX12: ; %bb.0:
2340 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
2341 ; GFX12-NEXT: s_wait_expcnt 0x0
2342 ; GFX12-NEXT: s_wait_samplecnt 0x0
2343 ; GFX12-NEXT: s_wait_bvhcnt 0x0
2344 ; GFX12-NEXT: s_wait_kmcnt 0x0
2345 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
2346 ; GFX12-NEXT: s_mov_b32 s0, 0
2347 ; GFX12-NEXT: .LBB10_1: ; %atomicrmw.start
2348 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
2349 ; GFX12-NEXT: s_wait_loadcnt 0x0
2350 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
2351 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
2352 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
2353 ; GFX12-NEXT: s_wait_storecnt 0x0
2354 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
2355 ; GFX12-NEXT: s_wait_loadcnt 0x0
2356 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
2357 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2358 ; GFX12-NEXT: s_wait_alu 0xfffe
2359 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
2360 ; GFX12-NEXT: s_wait_alu 0xfffe
2361 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2362 ; GFX12-NEXT: s_cbranch_execnz .LBB10_1
2363 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
2364 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
2365 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
2366 ; GFX12-NEXT: s_wait_alu 0xfffe
2367 ; GFX12-NEXT: s_setpc_b64 s[30:31]
2368 ;
2369 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg__ftz:
2370 ; GFX940: ; %bb.0:
2371 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2372 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
2373 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
2374 ; GFX940-NEXT: .LBB10_1: ; %atomicrmw.start
2375 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
2376 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2377 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
2378 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
2379 ; GFX940-NEXT: buffer_wbl2 sc1
2380 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
2381 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2382 ; GFX940-NEXT: buffer_inv sc1
2383 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2384 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
2385 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
2386 ; GFX940-NEXT: s_cbranch_execnz .LBB10_1
2387 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
2388 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
2389 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
2390 ; GFX940-NEXT: s_setpc_b64 s[30:31]
2391 ;
2392 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg__ftz:
2393 ; GFX11: ; %bb.0:
2394 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2395 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
2396 ; GFX11-NEXT: s_mov_b32 s0, 0
2397 ; GFX11-NEXT: .LBB10_1: ; %atomicrmw.start
2398 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
2399 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2400 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
2401 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
2402 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
2403 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2404 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
2405 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2406 ; GFX11-NEXT: buffer_gl1_inv
2407 ; GFX11-NEXT: buffer_gl0_inv
2408 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2409 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
2410 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
2411 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2412 ; GFX11-NEXT: s_cbranch_execnz .LBB10_1
2413 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
2414 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
2415 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
2416 ; GFX11-NEXT: s_setpc_b64 s[30:31]
2417 ;
2418 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg__ftz:
2419 ; GFX10: ; %bb.0:
2420 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2421 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
2422 ; GFX10-NEXT: s_mov_b32 s4, 0
2423 ; GFX10-NEXT: .LBB10_1: ; %atomicrmw.start
2424 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
2425 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2426 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
2427 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
2428 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2429 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
2430 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2431 ; GFX10-NEXT: buffer_gl1_inv
2432 ; GFX10-NEXT: buffer_gl0_inv
2433 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2434 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
2435 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
2436 ; GFX10-NEXT: s_cbranch_execnz .LBB10_1
2437 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
2438 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
2439 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
2440 ; GFX10-NEXT: s_setpc_b64 s[30:31]
2441 ;
2442 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg__ftz:
2443 ; GFX90A: ; %bb.0:
2444 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2445 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
2446 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
2447 ; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start
2448 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
2449 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2450 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
2451 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
2452 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
2453 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2454 ; GFX90A-NEXT: buffer_wbinvl1
2455 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2456 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2457 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
2458 ; GFX90A-NEXT: s_cbranch_execnz .LBB10_1
2459 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
2460 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
2461 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
2462 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
2463 ;
2464 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg__ftz:
2465 ; GFX908: ; %bb.0:
2466 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2467 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
2468 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
2469 ; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
2470 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
2471 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2472 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
2473 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
2474 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
2475 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2476 ; GFX908-NEXT: buffer_wbinvl1
2477 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
2478 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2479 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
2480 ; GFX908-NEXT: s_cbranch_execnz .LBB10_1
2481 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
2482 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
2483 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
2484 ; GFX908-NEXT: s_setpc_b64 s[30:31]
2485 ;
2486 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg__ftz:
2487 ; GFX8: ; %bb.0:
2488 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2489 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
2490 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
2491 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
2492 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2493 ; GFX8-NEXT: .LBB10_1: ; %atomicrmw.start
2494 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2495 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2496 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
2497 ; GFX8-NEXT: v_sub_f32_e32 v0, v1, v2
2498 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
2499 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2500 ; GFX8-NEXT: buffer_wbinvl1
2501 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
2502 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2503 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2504 ; GFX8-NEXT: s_cbranch_execnz .LBB10_1
2505 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2506 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2507 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2508 ;
2509 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg__ftz:
2510 ; GFX7: ; %bb.0:
2511 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2512 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
2513 ; GFX7-NEXT: v_mov_b32_e32 v4, v1
2514 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
2515 ; GFX7-NEXT: s_mov_b32 s5, -1
2516 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
2517 ; GFX7-NEXT: s_mov_b32 s6, 0
2518 ; GFX7-NEXT: buffer_load_dword v0, v[3:4], s[4:7], 0 addr64
2519 ; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0xfffff800, v3
2520 ; GFX7-NEXT: v_addc_u32_e32 v4, vcc, -1, v4, vcc
2521 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
2522 ; GFX7-NEXT: s_mov_b32 s4, s6
2523 ; GFX7-NEXT: s_mov_b32 s5, s6
2524 ; GFX7-NEXT: .LBB10_1: ; %atomicrmw.start
2525 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
2526 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2527 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
2528 ; GFX7-NEXT: v_sub_f32_e32 v5, v6, v2
2529 ; GFX7-NEXT: v_mov_b32_e32 v0, v5
2530 ; GFX7-NEXT: v_mov_b32_e32 v1, v6
2531 ; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v[3:4], s[4:7], 0 addr64 glc
2532 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2533 ; GFX7-NEXT: buffer_wbinvl1
2534 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
2535 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
2536 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
2537 ; GFX7-NEXT: s_cbranch_execnz .LBB10_1
2538 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
2539 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
2540 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2541 ;
2542 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg__ftz:
2543 ; GFX6: ; %bb.0:
2544 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2545 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
2546 ; GFX6-NEXT: v_mov_b32_e32 v4, v1
2547 ; GFX6-NEXT: v_mov_b32_e32 v3, v0
2548 ; GFX6-NEXT: s_mov_b32 s5, -1
2549 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
2550 ; GFX6-NEXT: s_mov_b32 s6, 0
2551 ; GFX6-NEXT: buffer_load_dword v0, v[3:4], s[4:7], 0 addr64
2552 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, 0xfffff800, v3
2553 ; GFX6-NEXT: v_addc_u32_e32 v4, vcc, -1, v4, vcc
2554 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
2555 ; GFX6-NEXT: s_mov_b32 s4, s6
2556 ; GFX6-NEXT: s_mov_b32 s5, s6
2557 ; GFX6-NEXT: .LBB10_1: ; %atomicrmw.start
2558 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
2559 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2560 ; GFX6-NEXT: v_mov_b32_e32 v6, v0
2561 ; GFX6-NEXT: v_sub_f32_e32 v5, v6, v2
2562 ; GFX6-NEXT: s_waitcnt expcnt(0)
2563 ; GFX6-NEXT: v_mov_b32_e32 v0, v5
2564 ; GFX6-NEXT: v_mov_b32_e32 v1, v6
2565 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v[3:4], s[4:7], 0 addr64 glc
2566 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2567 ; GFX6-NEXT: buffer_wbinvl1
2568 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
2569 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
2570 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
2571 ; GFX6-NEXT: s_cbranch_execnz .LBB10_1
2572 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
2573 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
2574 ; GFX6-NEXT: s_waitcnt expcnt(0)
2575 ; GFX6-NEXT: s_setpc_b64 s[30:31]
2576 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
2577 %result = atomicrmw fsub ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
2578 ret float %result
2579 }
2581 define void @global_agent_atomic_fsub_noret_f32__ftz(ptr addrspace(1) %ptr, float %val) #1 {
2582 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f32__ftz:
2583 ; GFX12: ; %bb.0:
2584 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
2585 ; GFX12-NEXT: s_wait_expcnt 0x0
2586 ; GFX12-NEXT: s_wait_samplecnt 0x0
2587 ; GFX12-NEXT: s_wait_bvhcnt 0x0
2588 ; GFX12-NEXT: s_wait_kmcnt 0x0
2589 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off
2590 ; GFX12-NEXT: s_mov_b32 s0, 0
2591 ; GFX12-NEXT: .LBB11_1: ; %atomicrmw.start
2592 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
2593 ; GFX12-NEXT: s_wait_loadcnt 0x0
2594 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
2595 ; GFX12-NEXT: s_wait_storecnt 0x0
2596 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
2597 ; GFX12-NEXT: s_wait_loadcnt 0x0
2598 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
2599 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2600 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
2601 ; GFX12-NEXT: s_wait_alu 0xfffe
2602 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
2603 ; GFX12-NEXT: s_wait_alu 0xfffe
2604 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2605 ; GFX12-NEXT: s_cbranch_execnz .LBB11_1
2606 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
2607 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
2608 ; GFX12-NEXT: s_wait_alu 0xfffe
2609 ; GFX12-NEXT: s_setpc_b64 s[30:31]
2610 ;
2611 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f32__ftz:
2612 ; GFX940: ; %bb.0:
2613 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2614 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
2615 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
2616 ; GFX940-NEXT: .LBB11_1: ; %atomicrmw.start
2617 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
2618 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2619 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
2620 ; GFX940-NEXT: buffer_wbl2 sc1
2621 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
2622 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2623 ; GFX940-NEXT: buffer_inv sc1
2624 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2625 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
2626 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
2627 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
2628 ; GFX940-NEXT: s_cbranch_execnz .LBB11_1
2629 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
2630 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
2631 ; GFX940-NEXT: s_setpc_b64 s[30:31]
2632 ;
2633 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f32__ftz:
2634 ; GFX11: ; %bb.0:
2635 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2636 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off
2637 ; GFX11-NEXT: s_mov_b32 s0, 0
2638 ; GFX11-NEXT: .LBB11_1: ; %atomicrmw.start
2639 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
2640 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2641 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
2642 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2643 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
2644 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2645 ; GFX11-NEXT: buffer_gl1_inv
2646 ; GFX11-NEXT: buffer_gl0_inv
2647 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2648 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
2649 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
2650 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
2651 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2652 ; GFX11-NEXT: s_cbranch_execnz .LBB11_1
2653 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
2654 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
2655 ; GFX11-NEXT: s_setpc_b64 s[30:31]
2656 ;
2657 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f32__ftz:
2658 ; GFX10: ; %bb.0:
2659 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2660 ; GFX10-NEXT: global_load_dword v4, v[0:1], off
2661 ; GFX10-NEXT: s_mov_b32 s4, 0
2662 ; GFX10-NEXT: .LBB11_1: ; %atomicrmw.start
2663 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
2664 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2665 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
2666 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2667 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
2668 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2669 ; GFX10-NEXT: buffer_gl1_inv
2670 ; GFX10-NEXT: buffer_gl0_inv
2671 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2672 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
2673 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
2674 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
2675 ; GFX10-NEXT: s_cbranch_execnz .LBB11_1
2676 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
2677 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
2678 ; GFX10-NEXT: s_setpc_b64 s[30:31]
2679 ;
2680 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f32__ftz:
2681 ; GFX90A: ; %bb.0:
2682 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2683 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
2684 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
2685 ; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start
2686 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
2687 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2688 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
2689 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
2690 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2691 ; GFX90A-NEXT: buffer_wbinvl1
2692 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2693 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2694 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
2695 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
2696 ; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
2697 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
2698 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
2699 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
2700 ;
2701 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f32__ftz:
2702 ; GFX908: ; %bb.0:
2703 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2704 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
2705 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
2706 ; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
2707 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
2708 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2709 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
2710 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
2711 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2712 ; GFX908-NEXT: buffer_wbinvl1
2713 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
2714 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2715 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
2716 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
2717 ; GFX908-NEXT: s_cbranch_execnz .LBB11_1
2718 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
2719 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
2720 ; GFX908-NEXT: s_setpc_b64 s[30:31]
2721 ;
2722 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f32__ftz:
2723 ; GFX8: ; %bb.0:
2724 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2725 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
2726 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2727 ; GFX8-NEXT: .LBB11_1: ; %atomicrmw.start
2728 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2729 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2730 ; GFX8-NEXT: v_sub_f32_e32 v3, v4, v2
2731 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
2732 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2733 ; GFX8-NEXT: buffer_wbinvl1
2734 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
2735 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2736 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
2737 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2738 ; GFX8-NEXT: s_cbranch_execnz .LBB11_1
2739 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2740 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2741 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2742 ;
2743 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f32__ftz:
2744 ; GFX7: ; %bb.0:
2745 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2746 ; GFX7-NEXT: s_mov_b32 s6, 0
2747 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
2748 ; GFX7-NEXT: s_mov_b32 s4, s6
2749 ; GFX7-NEXT: s_mov_b32 s5, s6
2750 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
2751 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
2752 ; GFX7-NEXT: .LBB11_1: ; %atomicrmw.start
2753 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
2754 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2755 ; GFX7-NEXT: v_sub_f32_e32 v3, v4, v2
2756 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
2757 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
2758 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc
2759 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2760 ; GFX7-NEXT: buffer_wbinvl1
2761 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
2762 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
2763 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
2764 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
2765 ; GFX7-NEXT: s_cbranch_execnz .LBB11_1
2766 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
2767 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
2768 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2769 ;
2770 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f32__ftz:
2771 ; GFX6: ; %bb.0:
2772 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2773 ; GFX6-NEXT: s_mov_b32 s6, 0
2774 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
2775 ; GFX6-NEXT: s_mov_b32 s4, s6
2776 ; GFX6-NEXT: s_mov_b32 s5, s6
2777 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
2778 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
2779 ; GFX6-NEXT: .LBB11_1: ; %atomicrmw.start
2780 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
2781 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2782 ; GFX6-NEXT: v_sub_f32_e32 v3, v4, v2
2783 ; GFX6-NEXT: s_waitcnt expcnt(0)
2784 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
2785 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
2786 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc
2787 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2788 ; GFX6-NEXT: buffer_wbinvl1
2789 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
2790 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
2791 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
2792 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
2793 ; GFX6-NEXT: s_cbranch_execnz .LBB11_1
2794 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
2795 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
2796 ; GFX6-NEXT: s_waitcnt expcnt(0)
2797 ; GFX6-NEXT: s_setpc_b64 s[30:31]
2798 %unused = atomicrmw fsub ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst
2799 ret void
2800 }
2802 define void @global_agent_atomic_fsub_noret_f32__offset12b_pos__ftz(ptr addrspace(1) %ptr, float %val) #1 {
2803 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos__ftz:
2804 ; GFX12: ; %bb.0:
2805 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
2806 ; GFX12-NEXT: s_wait_expcnt 0x0
2807 ; GFX12-NEXT: s_wait_samplecnt 0x0
2808 ; GFX12-NEXT: s_wait_bvhcnt 0x0
2809 ; GFX12-NEXT: s_wait_kmcnt 0x0
2810 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:2044
2811 ; GFX12-NEXT: s_mov_b32 s0, 0
2812 ; GFX12-NEXT: .LBB12_1: ; %atomicrmw.start
2813 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
2814 ; GFX12-NEXT: s_wait_loadcnt 0x0
2815 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
2816 ; GFX12-NEXT: s_wait_storecnt 0x0
2817 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
2818 ; GFX12-NEXT: s_wait_loadcnt 0x0
2819 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
2820 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2821 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
2822 ; GFX12-NEXT: s_wait_alu 0xfffe
2823 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
2824 ; GFX12-NEXT: s_wait_alu 0xfffe
2825 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2826 ; GFX12-NEXT: s_cbranch_execnz .LBB12_1
2827 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
2828 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
2829 ; GFX12-NEXT: s_wait_alu 0xfffe
2830 ; GFX12-NEXT: s_setpc_b64 s[30:31]
2831 ;
2832 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos__ftz:
2833 ; GFX940: ; %bb.0:
2834 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2835 ; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:2044
2836 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
2837 ; GFX940-NEXT: .LBB12_1: ; %atomicrmw.start
2838 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
2839 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2840 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
2841 ; GFX940-NEXT: buffer_wbl2 sc1
2842 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
2843 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2844 ; GFX940-NEXT: buffer_inv sc1
2845 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2846 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
2847 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
2848 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
2849 ; GFX940-NEXT: s_cbranch_execnz .LBB12_1
2850 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
2851 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
2852 ; GFX940-NEXT: s_setpc_b64 s[30:31]
2853 ;
2854 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos__ftz:
2855 ; GFX11: ; %bb.0:
2856 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2857 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:2044
2858 ; GFX11-NEXT: s_mov_b32 s0, 0
2859 ; GFX11-NEXT: .LBB12_1: ; %atomicrmw.start
2860 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
2861 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2862 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
2863 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2864 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
2865 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2866 ; GFX11-NEXT: buffer_gl1_inv
2867 ; GFX11-NEXT: buffer_gl0_inv
2868 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2869 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
2870 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
2871 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
2872 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2873 ; GFX11-NEXT: s_cbranch_execnz .LBB12_1
2874 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
2875 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
2876 ; GFX11-NEXT: s_setpc_b64 s[30:31]
2877 ;
2878 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos__ftz:
2879 ; GFX10: ; %bb.0:
2880 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2881 ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
2882 ; GFX10-NEXT: s_mov_b32 s4, 0
2883 ; GFX10-NEXT: .LBB12_1: ; %atomicrmw.start
2884 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
2885 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2886 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
2887 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2888 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
2889 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2890 ; GFX10-NEXT: buffer_gl1_inv
2891 ; GFX10-NEXT: buffer_gl0_inv
2892 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2893 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
2894 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
2895 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
2896 ; GFX10-NEXT: s_cbranch_execnz .LBB12_1
2897 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
2898 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
2899 ; GFX10-NEXT: s_setpc_b64 s[30:31]
2900 ;
2901 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos__ftz:
2902 ; GFX90A: ; %bb.0:
2903 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2904 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
2905 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
2906 ; GFX90A-NEXT: .LBB12_1: ; %atomicrmw.start
2907 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
2908 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2909 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
2910 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
2911 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2912 ; GFX90A-NEXT: buffer_wbinvl1
2913 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2914 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2915 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
2916 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
2917 ; GFX90A-NEXT: s_cbranch_execnz .LBB12_1
2918 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
2919 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
2920 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
2921 ;
2922 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos__ftz:
2923 ; GFX908: ; %bb.0:
2924 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2925 ; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
2926 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
2927 ; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start
2928 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
2929 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2930 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
2931 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
2932 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2933 ; GFX908-NEXT: buffer_wbinvl1
2934 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
2935 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2936 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
2937 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
2938 ; GFX908-NEXT: s_cbranch_execnz .LBB12_1
2939 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
2940 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
2941 ; GFX908-NEXT: s_setpc_b64 s[30:31]
2942 ;
2943 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos__ftz:
2944 ; GFX8: ; %bb.0:
2945 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2946 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
2947 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2948 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
2949 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2950 ; GFX8-NEXT: .LBB12_1: ; %atomicrmw.start
2951 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2952 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2953 ; GFX8-NEXT: v_sub_f32_e32 v3, v4, v2
2954 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
2955 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2956 ; GFX8-NEXT: buffer_wbinvl1
2957 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
2958 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2959 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
2960 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2961 ; GFX8-NEXT: s_cbranch_execnz .LBB12_1
2962 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2963 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2964 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2965 ;
2966 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos__ftz:
2967 ; GFX7: ; %bb.0:
2968 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2969 ; GFX7-NEXT: s_mov_b32 s6, 0
2970 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
2971 ; GFX7-NEXT: s_mov_b32 s4, s6
2972 ; GFX7-NEXT: s_mov_b32 s5, s6
2973 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:2044
2974 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
2975 ; GFX7-NEXT: .LBB12_1: ; %atomicrmw.start
2976 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
2977 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2978 ; GFX7-NEXT: v_sub_f32_e32 v3, v4, v2
2979 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
2980 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
2981 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
2982 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2983 ; GFX7-NEXT: buffer_wbinvl1
2984 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
2985 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
2986 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
2987 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
2988 ; GFX7-NEXT: s_cbranch_execnz .LBB12_1
2989 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
2990 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
2991 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2992 ;
2993 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos__ftz:
2994 ; GFX6: ; %bb.0:
2995 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2996 ; GFX6-NEXT: s_mov_b32 s6, 0
2997 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
2998 ; GFX6-NEXT: s_mov_b32 s4, s6
2999 ; GFX6-NEXT: s_mov_b32 s5, s6
3000 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:2044
3001 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
3002 ; GFX6-NEXT: .LBB12_1: ; %atomicrmw.start
3003 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
3004 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3005 ; GFX6-NEXT: v_sub_f32_e32 v3, v4, v2
3006 ; GFX6-NEXT: s_waitcnt expcnt(0)
3007 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
3008 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
3009 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
3010 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3011 ; GFX6-NEXT: buffer_wbinvl1
3012 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
3013 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
3014 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
3015 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
3016 ; GFX6-NEXT: s_cbranch_execnz .LBB12_1
3017 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
3018 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
3019 ; GFX6-NEXT: s_waitcnt expcnt(0)
3020 ; GFX6-NEXT: s_setpc_b64 s[30:31]
3021 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
3022 %unused = atomicrmw fsub ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
3023 ret void
3024 }
3026 define void @global_agent_atomic_fsub_noret_f32__offset12b_neg__ftz(ptr addrspace(1) %ptr, float %val) #1 {
3027 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg__ftz:
3028 ; GFX12: ; %bb.0:
3029 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
3030 ; GFX12-NEXT: s_wait_expcnt 0x0
3031 ; GFX12-NEXT: s_wait_samplecnt 0x0
3032 ; GFX12-NEXT: s_wait_bvhcnt 0x0
3033 ; GFX12-NEXT: s_wait_kmcnt 0x0
3034 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
3035 ; GFX12-NEXT: s_mov_b32 s0, 0
3036 ; GFX12-NEXT: .LBB13_1: ; %atomicrmw.start
3037 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
3038 ; GFX12-NEXT: s_wait_loadcnt 0x0
3039 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
3040 ; GFX12-NEXT: s_wait_storecnt 0x0
3041 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
3042 ; GFX12-NEXT: s_wait_loadcnt 0x0
3043 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
3044 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3045 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
3046 ; GFX12-NEXT: s_wait_alu 0xfffe
3047 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
3048 ; GFX12-NEXT: s_wait_alu 0xfffe
3049 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3050 ; GFX12-NEXT: s_cbranch_execnz .LBB13_1
3051 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
3052 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
3053 ; GFX12-NEXT: s_wait_alu 0xfffe
3054 ; GFX12-NEXT: s_setpc_b64 s[30:31]
3055 ;
3056 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg__ftz:
3057 ; GFX940: ; %bb.0:
3058 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3059 ; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:-2048
3060 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
3061 ; GFX940-NEXT: .LBB13_1: ; %atomicrmw.start
3062 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
3063 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3064 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
3065 ; GFX940-NEXT: buffer_wbl2 sc1
3066 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
3067 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3068 ; GFX940-NEXT: buffer_inv sc1
3069 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
3070 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
3071 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
3072 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
3073 ; GFX940-NEXT: s_cbranch_execnz .LBB13_1
3074 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
3075 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
3076 ; GFX940-NEXT: s_setpc_b64 s[30:31]
3077 ;
3078 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg__ftz:
3079 ; GFX11: ; %bb.0:
3080 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3081 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
3082 ; GFX11-NEXT: s_mov_b32 s0, 0
3083 ; GFX11-NEXT: .LBB13_1: ; %atomicrmw.start
3084 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
3085 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3086 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
3087 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
3088 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
3089 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3090 ; GFX11-NEXT: buffer_gl1_inv
3091 ; GFX11-NEXT: buffer_gl0_inv
3092 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3093 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
3094 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
3095 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3096 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3097 ; GFX11-NEXT: s_cbranch_execnz .LBB13_1
3098 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
3099 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
3100 ; GFX11-NEXT: s_setpc_b64 s[30:31]
3101 ;
3102 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg__ftz:
3103 ; GFX10: ; %bb.0:
3104 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3105 ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:-2048
3106 ; GFX10-NEXT: s_mov_b32 s4, 0
3107 ; GFX10-NEXT: .LBB13_1: ; %atomicrmw.start
3108 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
3109 ; GFX10-NEXT: s_waitcnt vmcnt(0)
3110 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
3111 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
3112 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
3113 ; GFX10-NEXT: s_waitcnt vmcnt(0)
3114 ; GFX10-NEXT: buffer_gl1_inv
3115 ; GFX10-NEXT: buffer_gl0_inv
3116 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3117 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
3118 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
3119 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
3120 ; GFX10-NEXT: s_cbranch_execnz .LBB13_1
3121 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
3122 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
3123 ; GFX10-NEXT: s_setpc_b64 s[30:31]
3124 ;
3125 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg__ftz:
3126 ; GFX90A: ; %bb.0:
3127 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3128 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
3129 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
3130 ; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
3131 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
3132 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3133 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
3134 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
3135 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3136 ; GFX90A-NEXT: buffer_wbinvl1
3137 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
3138 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3139 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
3140 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
3141 ; GFX90A-NEXT: s_cbranch_execnz .LBB13_1
3142 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
3143 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
3144 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
3145 ;
3146 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg__ftz:
3147 ; GFX908: ; %bb.0:
3148 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3149 ; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:-2048
3150 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
3151 ; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
3152 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
3153 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3154 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
3155 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
3156 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3157 ; GFX908-NEXT: buffer_wbinvl1
3158 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
3159 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3160 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
3161 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
3162 ; GFX908-NEXT: s_cbranch_execnz .LBB13_1
3163 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
3164 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
3165 ; GFX908-NEXT: s_setpc_b64 s[30:31]
3166 ;
3167 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg__ftz:
3168 ; GFX8: ; %bb.0:
3169 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3170 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
3171 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
3172 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
3173 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
3174 ; GFX8-NEXT: .LBB13_1: ; %atomicrmw.start
3175 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3176 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3177 ; GFX8-NEXT: v_sub_f32_e32 v3, v4, v2
3178 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
3179 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3180 ; GFX8-NEXT: buffer_wbinvl1
3181 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
3182 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3183 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
3184 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
3185 ; GFX8-NEXT: s_cbranch_execnz .LBB13_1
3186 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
3187 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
3188 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3189 ;
3190 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg__ftz:
3191 ; GFX7: ; %bb.0:
3192 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3193 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
3194 ; GFX7-NEXT: s_mov_b32 s5, -1
3195 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
3196 ; GFX7-NEXT: s_mov_b32 s6, 0
3197 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
3198 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
3199 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
3200 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
3201 ; GFX7-NEXT: s_mov_b32 s4, s6
3202 ; GFX7-NEXT: s_mov_b32 s5, s6
3203 ; GFX7-NEXT: .LBB13_1: ; %atomicrmw.start
3204 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
3205 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3206 ; GFX7-NEXT: v_sub_f32_e32 v3, v4, v2
3207 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
3208 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
3209 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc
3210 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3211 ; GFX7-NEXT: buffer_wbinvl1
3212 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
3213 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
3214 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
3215 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
3216 ; GFX7-NEXT: s_cbranch_execnz .LBB13_1
3217 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
3218 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
3219 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3220 ;
3221 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg__ftz:
3222 ; GFX6: ; %bb.0:
3223 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3224 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
3225 ; GFX6-NEXT: s_mov_b32 s5, -1
3226 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
3227 ; GFX6-NEXT: s_mov_b32 s6, 0
3228 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
3229 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
3230 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
3231 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
3232 ; GFX6-NEXT: s_mov_b32 s4, s6
3233 ; GFX6-NEXT: s_mov_b32 s5, s6
3234 ; GFX6-NEXT: .LBB13_1: ; %atomicrmw.start
3235 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
3236 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3237 ; GFX6-NEXT: v_sub_f32_e32 v3, v4, v2
3238 ; GFX6-NEXT: s_waitcnt expcnt(0)
3239 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
3240 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
3241 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc
3242 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3243 ; GFX6-NEXT: buffer_wbinvl1
3244 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
3245 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
3246 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
3247 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
3248 ; GFX6-NEXT: s_cbranch_execnz .LBB13_1
3249 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
3250 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
3251 ; GFX6-NEXT: s_waitcnt expcnt(0)
3252 ; GFX6-NEXT: s_setpc_b64 s[30:31]
3253 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
3254 %unused = atomicrmw fsub ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
3255 ret void
3256 }
3258 define float @global_system_atomic_fsub_ret_f32__offset12b_pos__ftz(ptr addrspace(1) %ptr, float %val) #1 {
3259 ; GFX12-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos__ftz:
3260 ; GFX12: ; %bb.0:
3261 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
3262 ; GFX12-NEXT: s_wait_expcnt 0x0
3263 ; GFX12-NEXT: s_wait_samplecnt 0x0
3264 ; GFX12-NEXT: s_wait_bvhcnt 0x0
3265 ; GFX12-NEXT: s_wait_kmcnt 0x0
3266 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
3267 ; GFX12-NEXT: s_mov_b32 s0, 0
3268 ; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
3269 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
3270 ; GFX12-NEXT: s_wait_loadcnt 0x0
3271 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
3272 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
3273 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
3274 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
3275 ; GFX12-NEXT: s_wait_storecnt 0x0
3276 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
3277 ; GFX12-NEXT: s_wait_loadcnt 0x0
3278 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
3279 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3280 ; GFX12-NEXT: s_wait_alu 0xfffe
3281 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
3282 ; GFX12-NEXT: s_wait_alu 0xfffe
3283 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3284 ; GFX12-NEXT: s_cbranch_execnz .LBB14_1
3285 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
3286 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
3287 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
3288 ; GFX12-NEXT: s_wait_alu 0xfffe
3289 ; GFX12-NEXT: s_setpc_b64 s[30:31]
3290 ;
3291 ; GFX940-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos__ftz:
3292 ; GFX940: ; %bb.0:
3293 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3294 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
3295 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
3296 ; GFX940-NEXT: .LBB14_1: ; %atomicrmw.start
3297 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
3298 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3299 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
3300 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
3301 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
3302 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
3303 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3304 ; GFX940-NEXT: buffer_inv sc0 sc1
3305 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
3306 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
3307 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
3308 ; GFX940-NEXT: s_cbranch_execnz .LBB14_1
3309 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
3310 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
3311 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
3312 ; GFX940-NEXT: s_setpc_b64 s[30:31]
3313 ;
3314 ; GFX11-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos__ftz:
3315 ; GFX11: ; %bb.0:
3316 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3317 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
3318 ; GFX11-NEXT: s_mov_b32 s0, 0
3319 ; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
3320 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
3321 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3322 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
3323 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
3324 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
3325 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
3326 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
3327 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3328 ; GFX11-NEXT: buffer_gl1_inv
3329 ; GFX11-NEXT: buffer_gl0_inv
3330 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3331 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
3332 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3333 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3334 ; GFX11-NEXT: s_cbranch_execnz .LBB14_1
3335 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
3336 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
3337 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
3338 ; GFX11-NEXT: s_setpc_b64 s[30:31]
3339 ;
3340 ; GFX10-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos__ftz:
3341 ; GFX10: ; %bb.0:
3342 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3343 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
3344 ; GFX10-NEXT: s_mov_b32 s4, 0
3345 ; GFX10-NEXT: .LBB14_1: ; %atomicrmw.start
3346 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
3347 ; GFX10-NEXT: s_waitcnt vmcnt(0)
3348 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
3349 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
3350 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
3351 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
3352 ; GFX10-NEXT: s_waitcnt vmcnt(0)
3353 ; GFX10-NEXT: buffer_gl1_inv
3354 ; GFX10-NEXT: buffer_gl0_inv
3355 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3356 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
3357 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
3358 ; GFX10-NEXT: s_cbranch_execnz .LBB14_1
3359 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
3360 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
3361 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
3362 ; GFX10-NEXT: s_setpc_b64 s[30:31]
3363 ;
3364 ; GFX90A-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos__ftz:
3365 ; GFX90A: ; %bb.0:
3366 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3367 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
3368 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
3369 ; GFX90A-NEXT: .LBB14_1: ; %atomicrmw.start
3370 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
3371 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3372 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
3373 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
3374 ; GFX90A-NEXT: buffer_wbl2
3375 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
3376 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3377 ; GFX90A-NEXT: buffer_invl2
3378 ; GFX90A-NEXT: buffer_wbinvl1
3379 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
3380 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3381 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
3382 ; GFX90A-NEXT: s_cbranch_execnz .LBB14_1
3383 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
3384 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
3385 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
3386 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
3387 ;
3388 ; GFX908-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos__ftz:
3389 ; GFX908: ; %bb.0:
3390 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3391 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
3392 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
3393 ; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
3394 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
3395 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3396 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
3397 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
3398 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
3399 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3400 ; GFX908-NEXT: buffer_wbinvl1
3401 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
3402 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3403 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
3404 ; GFX908-NEXT: s_cbranch_execnz .LBB14_1
3405 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
3406 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
3407 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
3408 ; GFX908-NEXT: s_setpc_b64 s[30:31]
3409 ;
3410 ; GFX8-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos__ftz:
3411 ; GFX8: ; %bb.0:
3412 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3413 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
3414 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
3415 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
3416 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
3417 ; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
3418 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3419 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3420 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
3421 ; GFX8-NEXT: v_sub_f32_e32 v0, v1, v2
3422 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
3423 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3424 ; GFX8-NEXT: buffer_wbinvl1
3425 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
3426 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3427 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
3428 ; GFX8-NEXT: s_cbranch_execnz .LBB14_1
3429 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
3430 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
3431 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3432 ;
3433 ; GFX7-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos__ftz:
3434 ; GFX7: ; %bb.0:
3435 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3436 ; GFX7-NEXT: s_mov_b32 s6, 0
3437 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
3438 ; GFX7-NEXT: s_mov_b32 s4, s6
3439 ; GFX7-NEXT: s_mov_b32 s5, s6
3440 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
3441 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
3442 ; GFX7-NEXT: .LBB14_1: ; %atomicrmw.start
3443 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
3444 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3445 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
3446 ; GFX7-NEXT: v_sub_f32_e32 v4, v5, v2
3447 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
3448 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
3449 ; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
3450 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3451 ; GFX7-NEXT: buffer_wbinvl1
3452 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
3453 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
3454 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
3455 ; GFX7-NEXT: s_cbranch_execnz .LBB14_1
3456 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
3457 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
3458 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
3459 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3460 ;
3461 ; GFX6-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos__ftz:
3462 ; GFX6: ; %bb.0:
3463 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3464 ; GFX6-NEXT: s_mov_b32 s6, 0
3465 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
3466 ; GFX6-NEXT: s_mov_b32 s4, s6
3467 ; GFX6-NEXT: s_mov_b32 s5, s6
3468 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
3469 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
3470 ; GFX6-NEXT: .LBB14_1: ; %atomicrmw.start
3471 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
3472 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3473 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
3474 ; GFX6-NEXT: s_waitcnt expcnt(0)
3475 ; GFX6-NEXT: v_sub_f32_e32 v4, v5, v2
3476 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
3477 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
3478 ; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
3479 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3480 ; GFX6-NEXT: buffer_wbinvl1
3481 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
3482 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
3483 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
3484 ; GFX6-NEXT: s_cbranch_execnz .LBB14_1
3485 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
3486 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
3487 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
3488 ; GFX6-NEXT: s_waitcnt expcnt(0)
3489 ; GFX6-NEXT: s_setpc_b64 s[30:31]
3490 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
3491 %result = atomicrmw fsub ptr addrspace(1) %gep, float %val seq_cst
3492 ret float %result
3493 }
3495 define void @global_system_atomic_fsub_noret_f32__offset12b_pos__ftz(ptr addrspace(1) %ptr, float %val) #1 {
3496 ; GFX12-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos__ftz:
3497 ; GFX12: ; %bb.0:
3498 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
3499 ; GFX12-NEXT: s_wait_expcnt 0x0
3500 ; GFX12-NEXT: s_wait_samplecnt 0x0
3501 ; GFX12-NEXT: s_wait_bvhcnt 0x0
3502 ; GFX12-NEXT: s_wait_kmcnt 0x0
3503 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:2044
3504 ; GFX12-NEXT: s_mov_b32 s0, 0
3505 ; GFX12-NEXT: .LBB15_1: ; %atomicrmw.start
3506 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
3507 ; GFX12-NEXT: s_wait_loadcnt 0x0
3508 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
3509 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
3510 ; GFX12-NEXT: s_wait_storecnt 0x0
3511 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
3512 ; GFX12-NEXT: s_wait_loadcnt 0x0
3513 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
3514 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3515 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
3516 ; GFX12-NEXT: s_wait_alu 0xfffe
3517 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
3518 ; GFX12-NEXT: s_wait_alu 0xfffe
3519 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3520 ; GFX12-NEXT: s_cbranch_execnz .LBB15_1
3521 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
3522 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
3523 ; GFX12-NEXT: s_wait_alu 0xfffe
3524 ; GFX12-NEXT: s_setpc_b64 s[30:31]
3525 ;
3526 ; GFX940-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos__ftz:
3527 ; GFX940: ; %bb.0:
3528 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3529 ; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:2044
3530 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
3531 ; GFX940-NEXT: .LBB15_1: ; %atomicrmw.start
3532 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
3533 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3534 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
3535 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
3536 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
3537 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3538 ; GFX940-NEXT: buffer_inv sc0 sc1
3539 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
3540 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
3541 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
3542 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
3543 ; GFX940-NEXT: s_cbranch_execnz .LBB15_1
3544 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
3545 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
3546 ; GFX940-NEXT: s_setpc_b64 s[30:31]
3547 ;
3548 ; GFX11-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos__ftz:
3549 ; GFX11: ; %bb.0:
3550 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3551 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:2044
3552 ; GFX11-NEXT: s_mov_b32 s0, 0
3553 ; GFX11-NEXT: .LBB15_1: ; %atomicrmw.start
3554 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
3555 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3556 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
3557 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
3558 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
3559 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3560 ; GFX11-NEXT: buffer_gl1_inv
3561 ; GFX11-NEXT: buffer_gl0_inv
3562 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3563 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
3564 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
3565 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3566 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3567 ; GFX11-NEXT: s_cbranch_execnz .LBB15_1
3568 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
3569 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
3570 ; GFX11-NEXT: s_setpc_b64 s[30:31]
3571 ;
3572 ; GFX10-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos__ftz:
3573 ; GFX10: ; %bb.0:
3574 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3575 ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
3576 ; GFX10-NEXT: s_mov_b32 s4, 0
3577 ; GFX10-NEXT: .LBB15_1: ; %atomicrmw.start
3578 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
3579 ; GFX10-NEXT: s_waitcnt vmcnt(0)
3580 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
3581 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
3582 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
3583 ; GFX10-NEXT: s_waitcnt vmcnt(0)
3584 ; GFX10-NEXT: buffer_gl1_inv
3585 ; GFX10-NEXT: buffer_gl0_inv
3586 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3587 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
3588 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
3589 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
3590 ; GFX10-NEXT: s_cbranch_execnz .LBB15_1
3591 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
3592 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
3593 ; GFX10-NEXT: s_setpc_b64 s[30:31]
3594 ;
3595 ; GFX90A-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos__ftz:
3596 ; GFX90A: ; %bb.0:
3597 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3598 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
3599 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
3600 ; GFX90A-NEXT: .LBB15_1: ; %atomicrmw.start
3601 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
3602 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3603 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
3604 ; GFX90A-NEXT: buffer_wbl2
3605 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
3606 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3607 ; GFX90A-NEXT: buffer_invl2
3608 ; GFX90A-NEXT: buffer_wbinvl1
3609 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
3610 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3611 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
3612 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
3613 ; GFX90A-NEXT: s_cbranch_execnz .LBB15_1
3614 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
3615 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
3616 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
3617 ;
3618 ; GFX908-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos__ftz:
3619 ; GFX908: ; %bb.0:
3620 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3621 ; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
3622 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
3623 ; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start
3624 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
3625 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3626 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
3627 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
3628 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3629 ; GFX908-NEXT: buffer_wbinvl1
3630 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
3631 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3632 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
3633 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
3634 ; GFX908-NEXT: s_cbranch_execnz .LBB15_1
3635 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
3636 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
3637 ; GFX908-NEXT: s_setpc_b64 s[30:31]
3638 ;
3639 ; GFX8-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos__ftz:
3640 ; GFX8: ; %bb.0:
3641 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3642 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
3643 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3644 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
3645 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
3646 ; GFX8-NEXT: .LBB15_1: ; %atomicrmw.start
3647 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3648 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3649 ; GFX8-NEXT: v_sub_f32_e32 v3, v4, v2
3650 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
3651 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3652 ; GFX8-NEXT: buffer_wbinvl1
3653 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
3654 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3655 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
3656 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
3657 ; GFX8-NEXT: s_cbranch_execnz .LBB15_1
3658 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
3659 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
3660 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3661 ;
3662 ; GFX7-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos__ftz:
3663 ; GFX7: ; %bb.0:
3664 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3665 ; GFX7-NEXT: s_mov_b32 s6, 0
3666 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
3667 ; GFX7-NEXT: s_mov_b32 s4, s6
3668 ; GFX7-NEXT: s_mov_b32 s5, s6
3669 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:2044
3670 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
3671 ; GFX7-NEXT: .LBB15_1: ; %atomicrmw.start
3672 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
3673 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3674 ; GFX7-NEXT: v_sub_f32_e32 v3, v4, v2
3675 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
3676 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
3677 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
3678 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3679 ; GFX7-NEXT: buffer_wbinvl1
3680 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
3681 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
3682 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
3683 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
3684 ; GFX7-NEXT: s_cbranch_execnz .LBB15_1
3685 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
3686 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
3687 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3688 ;
3689 ; GFX6-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos__ftz:
3690 ; GFX6: ; %bb.0:
3691 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3692 ; GFX6-NEXT: s_mov_b32 s6, 0
3693 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
3694 ; GFX6-NEXT: s_mov_b32 s4, s6
3695 ; GFX6-NEXT: s_mov_b32 s5, s6
3696 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:2044
3697 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
3698 ; GFX6-NEXT: .LBB15_1: ; %atomicrmw.start
3699 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
3700 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3701 ; GFX6-NEXT: v_sub_f32_e32 v3, v4, v2
3702 ; GFX6-NEXT: s_waitcnt expcnt(0)
3703 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
3704 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
3705 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
3706 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3707 ; GFX6-NEXT: buffer_wbinvl1
3708 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
3709 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
3710 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
3711 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
3712 ; GFX6-NEXT: s_cbranch_execnz .LBB15_1
3713 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
3714 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
3715 ; GFX6-NEXT: s_waitcnt expcnt(0)
3716 ; GFX6-NEXT: s_setpc_b64 s[30:31]
3717 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
3718 %unused = atomicrmw fsub ptr addrspace(1) %gep, float %val seq_cst
3719 ret void
3720 }
3722 ; --------------------------------------------------------------------
3723 ; double
3724 ; --------------------------------------------------------------------
3726 define double @global_agent_atomic_fsub_ret_f64(ptr addrspace(1) %ptr, double %val) #0 {
3727 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f64:
3728 ; GFX12: ; %bb.0:
3729 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
3730 ; GFX12-NEXT: s_wait_expcnt 0x0
3731 ; GFX12-NEXT: s_wait_samplecnt 0x0
3732 ; GFX12-NEXT: s_wait_bvhcnt 0x0
3733 ; GFX12-NEXT: s_wait_kmcnt 0x0
3734 ; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
3735 ; GFX12-NEXT: s_mov_b32 s0, 0
3736 ; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start
3737 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
3738 ; GFX12-NEXT: s_wait_loadcnt 0x0
3739 ; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
3740 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
3741 ; GFX12-NEXT: v_add_f64_e64 v[4:5], v[6:7], -v[2:3]
3742 ; GFX12-NEXT: s_wait_storecnt 0x0
3743 ; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
3744 ; GFX12-NEXT: s_wait_loadcnt 0x0
3745 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
3746 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
3747 ; GFX12-NEXT: s_wait_alu 0xfffe
3748 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
3749 ; GFX12-NEXT: s_wait_alu 0xfffe
3750 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3751 ; GFX12-NEXT: s_cbranch_execnz .LBB16_1
3752 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
3753 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
3754 ; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
3755 ; GFX12-NEXT: s_wait_alu 0xfffe
3756 ; GFX12-NEXT: s_setpc_b64 s[30:31]
3757 ;
3758 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f64:
3759 ; GFX940: ; %bb.0:
3760 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3761 ; GFX940-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
3762 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
3763 ; GFX940-NEXT: .LBB16_1: ; %atomicrmw.start
3764 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
3765 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3766 ; GFX940-NEXT: v_mov_b64_e32 v[6:7], v[4:5]
3767 ; GFX940-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
3768 ; GFX940-NEXT: buffer_wbl2 sc1
3769 ; GFX940-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off sc0
3770 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3771 ; GFX940-NEXT: buffer_inv sc1
3772 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
3773 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
3774 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
3775 ; GFX940-NEXT: s_cbranch_execnz .LBB16_1
3776 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
3777 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
3778 ; GFX940-NEXT: v_mov_b32_e32 v0, v4
3779 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
3780 ; GFX940-NEXT: s_setpc_b64 s[30:31]
3781 ;
3782 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f64:
3783 ; GFX11: ; %bb.0:
3784 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3785 ; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
3786 ; GFX11-NEXT: s_mov_b32 s0, 0
3787 ; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start
3788 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
3789 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3790 ; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
3791 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
3792 ; GFX11-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
3793 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
3794 ; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
3795 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3796 ; GFX11-NEXT: buffer_gl1_inv
3797 ; GFX11-NEXT: buffer_gl0_inv
3798 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
3799 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
3800 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3801 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3802 ; GFX11-NEXT: s_cbranch_execnz .LBB16_1
3803 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
3804 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
3805 ; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
3806 ; GFX11-NEXT: s_setpc_b64 s[30:31]
3807 ;
3808 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f64:
3809 ; GFX10: ; %bb.0:
3810 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3811 ; GFX10-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
3812 ; GFX10-NEXT: s_mov_b32 s4, 0
3813 ; GFX10-NEXT: .LBB16_1: ; %atomicrmw.start
3814 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
3815 ; GFX10-NEXT: s_waitcnt vmcnt(0)
3816 ; GFX10-NEXT: v_mov_b32_e32 v7, v5
3817 ; GFX10-NEXT: v_mov_b32_e32 v6, v4
3818 ; GFX10-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
3819 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
3820 ; GFX10-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
3821 ; GFX10-NEXT: s_waitcnt vmcnt(0)
3822 ; GFX10-NEXT: buffer_gl1_inv
3823 ; GFX10-NEXT: buffer_gl0_inv
3824 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
3825 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
3826 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
3827 ; GFX10-NEXT: s_cbranch_execnz .LBB16_1
3828 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
3829 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
3830 ; GFX10-NEXT: v_mov_b32_e32 v0, v4
3831 ; GFX10-NEXT: v_mov_b32_e32 v1, v5
3832 ; GFX10-NEXT: s_setpc_b64 s[30:31]
3833 ;
3834 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f64:
3835 ; GFX90A: ; %bb.0:
3836 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3837 ; GFX90A-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
3838 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
3839 ; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
3840 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
3841 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3842 ; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1]
3843 ; GFX90A-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
3844 ; GFX90A-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
3845 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3846 ; GFX90A-NEXT: buffer_wbinvl1
3847 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
3848 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3849 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
3850 ; GFX90A-NEXT: s_cbranch_execnz .LBB16_1
3851 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
3852 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
3853 ; GFX90A-NEXT: v_mov_b32_e32 v0, v4
3854 ; GFX90A-NEXT: v_mov_b32_e32 v1, v5
3855 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
3856 ;
3857 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f64:
3858 ; GFX908: ; %bb.0:
3859 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3860 ; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
3861 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
3862 ; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
3863 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
3864 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3865 ; GFX908-NEXT: v_mov_b32_e32 v7, v5
3866 ; GFX908-NEXT: v_mov_b32_e32 v6, v4
3867 ; GFX908-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
3868 ; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
3869 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3870 ; GFX908-NEXT: buffer_wbinvl1
3871 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
3872 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3873 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
3874 ; GFX908-NEXT: s_cbranch_execnz .LBB16_1
3875 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
3876 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
3877 ; GFX908-NEXT: v_mov_b32_e32 v0, v4
3878 ; GFX908-NEXT: v_mov_b32_e32 v1, v5
3879 ; GFX908-NEXT: s_setpc_b64 s[30:31]
3880 ;
3881 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f64:
3882 ; GFX8: ; %bb.0:
3883 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3884 ; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
3885 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
3886 ; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start
3887 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3888 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3889 ; GFX8-NEXT: v_mov_b32_e32 v7, v5
3890 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
3891 ; GFX8-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
3892 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
3893 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3894 ; GFX8-NEXT: buffer_wbinvl1
3895 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
3896 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3897 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
3898 ; GFX8-NEXT: s_cbranch_execnz .LBB16_1
3899 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
3900 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
3901 ; GFX8-NEXT: v_mov_b32_e32 v0, v4
3902 ; GFX8-NEXT: v_mov_b32_e32 v1, v5
3903 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3904 ;
3905 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f64:
3906 ; GFX7: ; %bb.0:
3907 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3908 ; GFX7-NEXT: s_mov_b32 s6, 0
3909 ; GFX7-NEXT: v_mov_b32_e32 v7, v1
3910 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
3911 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
3912 ; GFX7-NEXT: s_mov_b32 s4, s6
3913 ; GFX7-NEXT: s_mov_b32 s5, s6
3914 ; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64
3915 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
3916 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
3917 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
3918 ; GFX7-NEXT: .LBB16_1: ; %atomicrmw.start
3919 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
3920 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3921 ; GFX7-NEXT: v_mov_b32_e32 v11, v1
3922 ; GFX7-NEXT: v_mov_b32_e32 v10, v0
3923 ; GFX7-NEXT: v_add_f64 v[8:9], v[10:11], -v[4:5]
3924 ; GFX7-NEXT: v_mov_b32_e32 v0, v8
3925 ; GFX7-NEXT: v_mov_b32_e32 v1, v9
3926 ; GFX7-NEXT: v_mov_b32_e32 v2, v10
3927 ; GFX7-NEXT: v_mov_b32_e32 v3, v11
3928 ; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 glc
3929 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3930 ; GFX7-NEXT: buffer_wbinvl1
3931 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
3932 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
3933 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
3934 ; GFX7-NEXT: s_cbranch_execnz .LBB16_1
3935 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
3936 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
3937 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3938 ;
3939 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f64:
3940 ; GFX6: ; %bb.0:
3941 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3942 ; GFX6-NEXT: s_mov_b32 s6, 0
3943 ; GFX6-NEXT: v_mov_b32_e32 v7, v1
3944 ; GFX6-NEXT: v_mov_b32_e32 v6, v0
3945 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
3946 ; GFX6-NEXT: s_mov_b32 s4, s6
3947 ; GFX6-NEXT: s_mov_b32 s5, s6
3948 ; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64
3949 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
3950 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
3951 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
3952 ; GFX6-NEXT: .LBB16_1: ; %atomicrmw.start
3953 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
3954 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3955 ; GFX6-NEXT: v_mov_b32_e32 v11, v1
3956 ; GFX6-NEXT: v_mov_b32_e32 v10, v0
3957 ; GFX6-NEXT: v_add_f64 v[8:9], v[10:11], -v[4:5]
3958 ; GFX6-NEXT: s_waitcnt expcnt(0)
3959 ; GFX6-NEXT: v_mov_b32_e32 v0, v8
3960 ; GFX6-NEXT: v_mov_b32_e32 v1, v9
3961 ; GFX6-NEXT: v_mov_b32_e32 v2, v10
3962 ; GFX6-NEXT: v_mov_b32_e32 v3, v11
3963 ; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 glc
3964 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3965 ; GFX6-NEXT: buffer_wbinvl1
3966 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
3967 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
3968 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
3969 ; GFX6-NEXT: s_cbranch_execnz .LBB16_1
3970 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
3971 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
3972 ; GFX6-NEXT: s_waitcnt expcnt(0)
3973 ; GFX6-NEXT: s_setpc_b64 s[30:31]
3974 %result = atomicrmw fsub ptr addrspace(1) %ptr, double %val syncscope("agent") seq_cst
3975 ret double %result
3976 }
3978 define double @global_agent_atomic_fsub_ret_f64__offset12b_pos(ptr addrspace(1) %ptr, double %val) #0 {
3979 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_pos:
3980 ; GFX12: ; %bb.0:
3981 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
3982 ; GFX12-NEXT: s_wait_expcnt 0x0
3983 ; GFX12-NEXT: s_wait_samplecnt 0x0
3984 ; GFX12-NEXT: s_wait_bvhcnt 0x0
3985 ; GFX12-NEXT: s_wait_kmcnt 0x0
3986 ; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
3987 ; GFX12-NEXT: s_mov_b32 s0, 0
3988 ; GFX12-NEXT: .LBB17_1: ; %atomicrmw.start
3989 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
3990 ; GFX12-NEXT: s_wait_loadcnt 0x0
3991 ; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
3992 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
3993 ; GFX12-NEXT: v_add_f64_e64 v[4:5], v[6:7], -v[2:3]
3994 ; GFX12-NEXT: s_wait_storecnt 0x0
3995 ; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
3996 ; GFX12-NEXT: s_wait_loadcnt 0x0
3997 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
3998 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
3999 ; GFX12-NEXT: s_wait_alu 0xfffe
4000 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
4001 ; GFX12-NEXT: s_wait_alu 0xfffe
4002 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4003 ; GFX12-NEXT: s_cbranch_execnz .LBB17_1
4004 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
4005 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
4006 ; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
4007 ; GFX12-NEXT: s_wait_alu 0xfffe
4008 ; GFX12-NEXT: s_setpc_b64 s[30:31]
4009 ;
4010 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_pos:
4011 ; GFX940: ; %bb.0:
4012 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4013 ; GFX940-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:2040
4014 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
4015 ; GFX940-NEXT: .LBB17_1: ; %atomicrmw.start
4016 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
4017 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4018 ; GFX940-NEXT: v_mov_b64_e32 v[6:7], v[4:5]
4019 ; GFX940-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4020 ; GFX940-NEXT: buffer_wbl2 sc1
4021 ; GFX940-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 sc0
4022 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4023 ; GFX940-NEXT: buffer_inv sc1
4024 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4025 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4026 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
4027 ; GFX940-NEXT: s_cbranch_execnz .LBB17_1
4028 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
4029 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
4030 ; GFX940-NEXT: v_mov_b32_e32 v0, v4
4031 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
4032 ; GFX940-NEXT: s_setpc_b64 s[30:31]
4033 ;
4034 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_pos:
4035 ; GFX11: ; %bb.0:
4036 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4037 ; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
4038 ; GFX11-NEXT: s_mov_b32 s0, 0
4039 ; GFX11-NEXT: .LBB17_1: ; %atomicrmw.start
4040 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
4041 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4042 ; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
4043 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
4044 ; GFX11-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4045 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
4046 ; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 glc
4047 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4048 ; GFX11-NEXT: buffer_gl1_inv
4049 ; GFX11-NEXT: buffer_gl0_inv
4050 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4051 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
4052 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4053 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4054 ; GFX11-NEXT: s_cbranch_execnz .LBB17_1
4055 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
4056 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
4057 ; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
4058 ; GFX11-NEXT: s_setpc_b64 s[30:31]
4059 ;
4060 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_pos:
4061 ; GFX10: ; %bb.0:
4062 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4063 ; GFX10-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:2040
4064 ; GFX10-NEXT: s_mov_b32 s4, 0
4065 ; GFX10-NEXT: .LBB17_1: ; %atomicrmw.start
4066 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
4067 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4068 ; GFX10-NEXT: v_mov_b32_e32 v7, v5
4069 ; GFX10-NEXT: v_mov_b32_e32 v6, v4
4070 ; GFX10-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4071 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
4072 ; GFX10-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 glc
4073 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4074 ; GFX10-NEXT: buffer_gl1_inv
4075 ; GFX10-NEXT: buffer_gl0_inv
4076 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4077 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
4078 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
4079 ; GFX10-NEXT: s_cbranch_execnz .LBB17_1
4080 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
4081 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
4082 ; GFX10-NEXT: v_mov_b32_e32 v0, v4
4083 ; GFX10-NEXT: v_mov_b32_e32 v1, v5
4084 ; GFX10-NEXT: s_setpc_b64 s[30:31]
4085 ;
4086 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_pos:
4087 ; GFX90A: ; %bb.0:
4088 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4089 ; GFX90A-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:2040
4090 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
4091 ; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
4092 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
4093 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4094 ; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1]
4095 ; GFX90A-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4096 ; GFX90A-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 glc
4097 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4098 ; GFX90A-NEXT: buffer_wbinvl1
4099 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4100 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4101 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
4102 ; GFX90A-NEXT: s_cbranch_execnz .LBB17_1
4103 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
4104 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
4105 ; GFX90A-NEXT: v_mov_b32_e32 v0, v4
4106 ; GFX90A-NEXT: v_mov_b32_e32 v1, v5
4107 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
4108 ;
4109 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_pos:
4110 ; GFX908: ; %bb.0:
4111 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4112 ; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:2040
4113 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
4114 ; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
4115 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
4116 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4117 ; GFX908-NEXT: v_mov_b32_e32 v7, v5
4118 ; GFX908-NEXT: v_mov_b32_e32 v6, v4
4119 ; GFX908-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4120 ; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 glc
4121 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4122 ; GFX908-NEXT: buffer_wbinvl1
4123 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4124 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4125 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
4126 ; GFX908-NEXT: s_cbranch_execnz .LBB17_1
4127 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
4128 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
4129 ; GFX908-NEXT: v_mov_b32_e32 v0, v4
4130 ; GFX908-NEXT: v_mov_b32_e32 v1, v5
4131 ; GFX908-NEXT: s_setpc_b64 s[30:31]
4132 ;
4133 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_pos:
4134 ; GFX8: ; %bb.0:
4135 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4136 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7f8, v0
4137 ; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
4138 ; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
4139 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
4140 ; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
4141 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4142 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4143 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
4144 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
4145 ; GFX8-NEXT: v_add_f64 v[6:7], v[8:9], -v[2:3]
4146 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
4147 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4148 ; GFX8-NEXT: buffer_wbinvl1
4149 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
4150 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4151 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
4152 ; GFX8-NEXT: s_cbranch_execnz .LBB17_1
4153 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4154 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
4155 ; GFX8-NEXT: s_setpc_b64 s[30:31]
4156 ;
4157 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_pos:
4158 ; GFX7: ; %bb.0:
4159 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4160 ; GFX7-NEXT: s_mov_b32 s6, 0
4161 ; GFX7-NEXT: v_mov_b32_e32 v7, v1
4162 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
4163 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
4164 ; GFX7-NEXT: s_mov_b32 s4, s6
4165 ; GFX7-NEXT: s_mov_b32 s5, s6
4166 ; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64 offset:2040
4167 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
4168 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
4169 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
4170 ; GFX7-NEXT: .LBB17_1: ; %atomicrmw.start
4171 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4172 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4173 ; GFX7-NEXT: v_mov_b32_e32 v11, v1
4174 ; GFX7-NEXT: v_mov_b32_e32 v10, v0
4175 ; GFX7-NEXT: v_add_f64 v[8:9], v[10:11], -v[4:5]
4176 ; GFX7-NEXT: v_mov_b32_e32 v0, v8
4177 ; GFX7-NEXT: v_mov_b32_e32 v1, v9
4178 ; GFX7-NEXT: v_mov_b32_e32 v2, v10
4179 ; GFX7-NEXT: v_mov_b32_e32 v3, v11
4180 ; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 offset:2040 glc
4181 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4182 ; GFX7-NEXT: buffer_wbinvl1
4183 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
4184 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
4185 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
4186 ; GFX7-NEXT: s_cbranch_execnz .LBB17_1
4187 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4188 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
4189 ; GFX7-NEXT: s_setpc_b64 s[30:31]
4190 ;
4191 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_pos:
4192 ; GFX6: ; %bb.0:
4193 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4194 ; GFX6-NEXT: s_mov_b32 s6, 0
4195 ; GFX6-NEXT: v_mov_b32_e32 v7, v1
4196 ; GFX6-NEXT: v_mov_b32_e32 v6, v0
4197 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
4198 ; GFX6-NEXT: s_mov_b32 s4, s6
4199 ; GFX6-NEXT: s_mov_b32 s5, s6
4200 ; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64 offset:2040
4201 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
4202 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
4203 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
4204 ; GFX6-NEXT: .LBB17_1: ; %atomicrmw.start
4205 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
4206 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4207 ; GFX6-NEXT: v_mov_b32_e32 v11, v1
4208 ; GFX6-NEXT: v_mov_b32_e32 v10, v0
4209 ; GFX6-NEXT: v_add_f64 v[8:9], v[10:11], -v[4:5]
4210 ; GFX6-NEXT: s_waitcnt expcnt(0)
4211 ; GFX6-NEXT: v_mov_b32_e32 v0, v8
4212 ; GFX6-NEXT: v_mov_b32_e32 v1, v9
4213 ; GFX6-NEXT: v_mov_b32_e32 v2, v10
4214 ; GFX6-NEXT: v_mov_b32_e32 v3, v11
4215 ; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 offset:2040 glc
4216 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4217 ; GFX6-NEXT: buffer_wbinvl1
4218 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
4219 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
4220 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
4221 ; GFX6-NEXT: s_cbranch_execnz .LBB17_1
4222 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
4223 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
4224 ; GFX6-NEXT: s_waitcnt expcnt(0)
4225 ; GFX6-NEXT: s_setpc_b64 s[30:31]
4226 %gep = getelementptr double, ptr addrspace(1) %ptr, i64 255
4227 %result = atomicrmw fsub ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst
4228 ret double %result
4229 }
4231 define double @global_agent_atomic_fsub_ret_f64__offset12b_neg(ptr addrspace(1) %ptr, double %val) #0 {
4232 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_neg:
4233 ; GFX12: ; %bb.0:
4234 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
4235 ; GFX12-NEXT: s_wait_expcnt 0x0
4236 ; GFX12-NEXT: s_wait_samplecnt 0x0
4237 ; GFX12-NEXT: s_wait_bvhcnt 0x0
4238 ; GFX12-NEXT: s_wait_kmcnt 0x0
4239 ; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
4240 ; GFX12-NEXT: s_mov_b32 s0, 0
4241 ; GFX12-NEXT: .LBB18_1: ; %atomicrmw.start
4242 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
4243 ; GFX12-NEXT: s_wait_loadcnt 0x0
4244 ; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
4245 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
4246 ; GFX12-NEXT: v_add_f64_e64 v[4:5], v[6:7], -v[2:3]
4247 ; GFX12-NEXT: s_wait_storecnt 0x0
4248 ; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
4249 ; GFX12-NEXT: s_wait_loadcnt 0x0
4250 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
4251 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4252 ; GFX12-NEXT: s_wait_alu 0xfffe
4253 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
4254 ; GFX12-NEXT: s_wait_alu 0xfffe
4255 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4256 ; GFX12-NEXT: s_cbranch_execnz .LBB18_1
4257 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
4258 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
4259 ; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
4260 ; GFX12-NEXT: s_wait_alu 0xfffe
4261 ; GFX12-NEXT: s_setpc_b64 s[30:31]
4262 ;
4263 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_neg:
4264 ; GFX940: ; %bb.0:
4265 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4266 ; GFX940-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:-2048
4267 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
4268 ; GFX940-NEXT: .LBB18_1: ; %atomicrmw.start
4269 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
4270 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4271 ; GFX940-NEXT: v_mov_b64_e32 v[6:7], v[4:5]
4272 ; GFX940-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4273 ; GFX940-NEXT: buffer_wbl2 sc1
4274 ; GFX940-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 sc0
4275 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4276 ; GFX940-NEXT: buffer_inv sc1
4277 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4278 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4279 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
4280 ; GFX940-NEXT: s_cbranch_execnz .LBB18_1
4281 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
4282 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
4283 ; GFX940-NEXT: v_mov_b32_e32 v0, v4
4284 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
4285 ; GFX940-NEXT: s_setpc_b64 s[30:31]
4286 ;
4287 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_neg:
4288 ; GFX11: ; %bb.0:
4289 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4290 ; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
4291 ; GFX11-NEXT: s_mov_b32 s0, 0
4292 ; GFX11-NEXT: .LBB18_1: ; %atomicrmw.start
4293 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
4294 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4295 ; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
4296 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
4297 ; GFX11-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4298 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
4299 ; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
4300 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4301 ; GFX11-NEXT: buffer_gl1_inv
4302 ; GFX11-NEXT: buffer_gl0_inv
4303 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4304 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
4305 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4306 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4307 ; GFX11-NEXT: s_cbranch_execnz .LBB18_1
4308 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
4309 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
4310 ; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
4311 ; GFX11-NEXT: s_setpc_b64 s[30:31]
4312 ;
4313 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_neg:
4314 ; GFX10: ; %bb.0:
4315 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4316 ; GFX10-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:-2048
4317 ; GFX10-NEXT: s_mov_b32 s4, 0
4318 ; GFX10-NEXT: .LBB18_1: ; %atomicrmw.start
4319 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
4320 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4321 ; GFX10-NEXT: v_mov_b32_e32 v7, v5
4322 ; GFX10-NEXT: v_mov_b32_e32 v6, v4
4323 ; GFX10-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4324 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
4325 ; GFX10-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
4326 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4327 ; GFX10-NEXT: buffer_gl1_inv
4328 ; GFX10-NEXT: buffer_gl0_inv
4329 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4330 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
4331 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
4332 ; GFX10-NEXT: s_cbranch_execnz .LBB18_1
4333 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
4334 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
4335 ; GFX10-NEXT: v_mov_b32_e32 v0, v4
4336 ; GFX10-NEXT: v_mov_b32_e32 v1, v5
4337 ; GFX10-NEXT: s_setpc_b64 s[30:31]
4338 ;
4339 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_neg:
4340 ; GFX90A: ; %bb.0:
4341 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4342 ; GFX90A-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:-2048
4343 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
4344 ; GFX90A-NEXT: .LBB18_1: ; %atomicrmw.start
4345 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
4346 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4347 ; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1]
4348 ; GFX90A-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4349 ; GFX90A-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
4350 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4351 ; GFX90A-NEXT: buffer_wbinvl1
4352 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4353 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4354 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
4355 ; GFX90A-NEXT: s_cbranch_execnz .LBB18_1
4356 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
4357 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
4358 ; GFX90A-NEXT: v_mov_b32_e32 v0, v4
4359 ; GFX90A-NEXT: v_mov_b32_e32 v1, v5
4360 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
4361 ;
4362 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_neg:
4363 ; GFX908: ; %bb.0:
4364 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4365 ; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:-2048
4366 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
4367 ; GFX908-NEXT: .LBB18_1: ; %atomicrmw.start
4368 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
4369 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4370 ; GFX908-NEXT: v_mov_b32_e32 v7, v5
4371 ; GFX908-NEXT: v_mov_b32_e32 v6, v4
4372 ; GFX908-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4373 ; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
4374 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4375 ; GFX908-NEXT: buffer_wbinvl1
4376 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4377 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4378 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
4379 ; GFX908-NEXT: s_cbranch_execnz .LBB18_1
4380 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
4381 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
4382 ; GFX908-NEXT: v_mov_b32_e32 v0, v4
4383 ; GFX908-NEXT: v_mov_b32_e32 v1, v5
4384 ; GFX908-NEXT: s_setpc_b64 s[30:31]
4385 ;
4386 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_neg:
4387 ; GFX8: ; %bb.0:
4388 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4389 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
4390 ; GFX8-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
4391 ; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
4392 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
4393 ; GFX8-NEXT: .LBB18_1: ; %atomicrmw.start
4394 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4395 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4396 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
4397 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
4398 ; GFX8-NEXT: v_add_f64 v[6:7], v[8:9], -v[2:3]
4399 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
4400 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4401 ; GFX8-NEXT: buffer_wbinvl1
4402 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
4403 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4404 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
4405 ; GFX8-NEXT: s_cbranch_execnz .LBB18_1
4406 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4407 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
4408 ; GFX8-NEXT: s_setpc_b64 s[30:31]
4409 ;
4410 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_neg:
4411 ; GFX7: ; %bb.0:
4412 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4413 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
4414 ; GFX7-NEXT: v_mov_b32_e32 v7, v1
4415 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
4416 ; GFX7-NEXT: s_mov_b32 s5, -1
4417 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
4418 ; GFX7-NEXT: s_mov_b32 s6, 0
4419 ; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64
4420 ; GFX7-NEXT: v_add_i32_e32 v6, vcc, 0xfffff800, v6
4421 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
4422 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
4423 ; GFX7-NEXT: v_addc_u32_e32 v7, vcc, -1, v7, vcc
4424 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
4425 ; GFX7-NEXT: s_mov_b32 s4, s6
4426 ; GFX7-NEXT: s_mov_b32 s5, s6
4427 ; GFX7-NEXT: .LBB18_1: ; %atomicrmw.start
4428 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4429 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4430 ; GFX7-NEXT: v_mov_b32_e32 v11, v1
4431 ; GFX7-NEXT: v_mov_b32_e32 v10, v0
4432 ; GFX7-NEXT: v_add_f64 v[8:9], v[10:11], -v[4:5]
4433 ; GFX7-NEXT: v_mov_b32_e32 v0, v8
4434 ; GFX7-NEXT: v_mov_b32_e32 v1, v9
4435 ; GFX7-NEXT: v_mov_b32_e32 v2, v10
4436 ; GFX7-NEXT: v_mov_b32_e32 v3, v11
4437 ; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 glc
4438 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4439 ; GFX7-NEXT: buffer_wbinvl1
4440 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
4441 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
4442 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
4443 ; GFX7-NEXT: s_cbranch_execnz .LBB18_1
4444 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4445 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
4446 ; GFX7-NEXT: s_setpc_b64 s[30:31]
4447 ;
4448 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_neg:
4449 ; GFX6: ; %bb.0:
4450 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4451 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
4452 ; GFX6-NEXT: v_mov_b32_e32 v7, v1
4453 ; GFX6-NEXT: v_mov_b32_e32 v6, v0
4454 ; GFX6-NEXT: s_mov_b32 s5, -1
4455 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
4456 ; GFX6-NEXT: s_mov_b32 s6, 0
4457 ; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64
4458 ; GFX6-NEXT: v_add_i32_e32 v6, vcc, 0xfffff800, v6
4459 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
4460 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
4461 ; GFX6-NEXT: v_addc_u32_e32 v7, vcc, -1, v7, vcc
4462 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
4463 ; GFX6-NEXT: s_mov_b32 s4, s6
4464 ; GFX6-NEXT: s_mov_b32 s5, s6
4465 ; GFX6-NEXT: .LBB18_1: ; %atomicrmw.start
4466 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
4467 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4468 ; GFX6-NEXT: v_mov_b32_e32 v11, v1
4469 ; GFX6-NEXT: v_mov_b32_e32 v10, v0
4470 ; GFX6-NEXT: v_add_f64 v[8:9], v[10:11], -v[4:5]
4471 ; GFX6-NEXT: s_waitcnt expcnt(0)
4472 ; GFX6-NEXT: v_mov_b32_e32 v0, v8
4473 ; GFX6-NEXT: v_mov_b32_e32 v1, v9
4474 ; GFX6-NEXT: v_mov_b32_e32 v2, v10
4475 ; GFX6-NEXT: v_mov_b32_e32 v3, v11
4476 ; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 glc
4477 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4478 ; GFX6-NEXT: buffer_wbinvl1
4479 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
4480 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
4481 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
4482 ; GFX6-NEXT: s_cbranch_execnz .LBB18_1
4483 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
4484 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
4485 ; GFX6-NEXT: s_waitcnt expcnt(0)
4486 ; GFX6-NEXT: s_setpc_b64 s[30:31]
4487 %gep = getelementptr double, ptr addrspace(1) %ptr, i64 -256
4488 %result = atomicrmw fsub ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst
4489 ret double %result
4490 }
4492 define void @global_agent_atomic_fsub_noret_f64(ptr addrspace(1) %ptr, double %val) #0 {
4493 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f64:
4494 ; GFX12: ; %bb.0:
4495 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
4496 ; GFX12-NEXT: s_wait_expcnt 0x0
4497 ; GFX12-NEXT: s_wait_samplecnt 0x0
4498 ; GFX12-NEXT: s_wait_bvhcnt 0x0
4499 ; GFX12-NEXT: s_wait_kmcnt 0x0
4500 ; GFX12-NEXT: global_load_b64 v[6:7], v[0:1], off
4501 ; GFX12-NEXT: s_mov_b32 s0, 0
4502 ; GFX12-NEXT: .LBB19_1: ; %atomicrmw.start
4503 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
4504 ; GFX12-NEXT: s_wait_loadcnt 0x0
4505 ; GFX12-NEXT: v_add_f64_e64 v[4:5], v[6:7], -v[2:3]
4506 ; GFX12-NEXT: s_wait_storecnt 0x0
4507 ; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
4508 ; GFX12-NEXT: s_wait_loadcnt 0x0
4509 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
4510 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4511 ; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
4512 ; GFX12-NEXT: s_wait_alu 0xfffe
4513 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
4514 ; GFX12-NEXT: s_wait_alu 0xfffe
4515 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4516 ; GFX12-NEXT: s_cbranch_execnz .LBB19_1
4517 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
4518 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
4519 ; GFX12-NEXT: s_wait_alu 0xfffe
4520 ; GFX12-NEXT: s_setpc_b64 s[30:31]
4521 ;
4522 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f64:
4523 ; GFX940: ; %bb.0:
4524 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4525 ; GFX940-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
4526 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
4527 ; GFX940-NEXT: .LBB19_1: ; %atomicrmw.start
4528 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
4529 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4530 ; GFX940-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4531 ; GFX940-NEXT: buffer_wbl2 sc1
4532 ; GFX940-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off sc0
4533 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4534 ; GFX940-NEXT: buffer_inv sc1
4535 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4536 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4537 ; GFX940-NEXT: v_mov_b64_e32 v[6:7], v[4:5]
4538 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
4539 ; GFX940-NEXT: s_cbranch_execnz .LBB19_1
4540 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
4541 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
4542 ; GFX940-NEXT: s_setpc_b64 s[30:31]
4543 ;
4544 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f64:
4545 ; GFX11: ; %bb.0:
4546 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4547 ; GFX11-NEXT: global_load_b64 v[6:7], v[0:1], off
4548 ; GFX11-NEXT: s_mov_b32 s0, 0
4549 ; GFX11-NEXT: .LBB19_1: ; %atomicrmw.start
4550 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
4551 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4552 ; GFX11-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4553 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
4554 ; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
4555 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4556 ; GFX11-NEXT: buffer_gl1_inv
4557 ; GFX11-NEXT: buffer_gl0_inv
4558 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4559 ; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
4560 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
4561 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4562 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4563 ; GFX11-NEXT: s_cbranch_execnz .LBB19_1
4564 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
4565 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
4566 ; GFX11-NEXT: s_setpc_b64 s[30:31]
4567 ;
4568 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f64:
4569 ; GFX10: ; %bb.0:
4570 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4571 ; GFX10-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
4572 ; GFX10-NEXT: s_mov_b32 s4, 0
4573 ; GFX10-NEXT: .LBB19_1: ; %atomicrmw.start
4574 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
4575 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4576 ; GFX10-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4577 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
4578 ; GFX10-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
4579 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4580 ; GFX10-NEXT: buffer_gl1_inv
4581 ; GFX10-NEXT: buffer_gl0_inv
4582 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4583 ; GFX10-NEXT: v_mov_b32_e32 v7, v5
4584 ; GFX10-NEXT: v_mov_b32_e32 v6, v4
4585 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
4586 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
4587 ; GFX10-NEXT: s_cbranch_execnz .LBB19_1
4588 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
4589 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
4590 ; GFX10-NEXT: s_setpc_b64 s[30:31]
4591 ;
4592 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f64:
4593 ; GFX90A: ; %bb.0:
4594 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4595 ; GFX90A-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
4596 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
4597 ; GFX90A-NEXT: .LBB19_1: ; %atomicrmw.start
4598 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
4599 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4600 ; GFX90A-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4601 ; GFX90A-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
4602 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4603 ; GFX90A-NEXT: buffer_wbinvl1
4604 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4605 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4606 ; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1]
4607 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
4608 ; GFX90A-NEXT: s_cbranch_execnz .LBB19_1
4609 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
4610 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
4611 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
4612 ;
4613 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f64:
4614 ; GFX908: ; %bb.0:
4615 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4616 ; GFX908-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
4617 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
4618 ; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
4619 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
4620 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4621 ; GFX908-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4622 ; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
4623 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4624 ; GFX908-NEXT: buffer_wbinvl1
4625 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4626 ; GFX908-NEXT: v_mov_b32_e32 v7, v5
4627 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4628 ; GFX908-NEXT: v_mov_b32_e32 v6, v4
4629 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
4630 ; GFX908-NEXT: s_cbranch_execnz .LBB19_1
4631 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
4632 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
4633 ; GFX908-NEXT: s_setpc_b64 s[30:31]
4634 ;
4635 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f64:
4636 ; GFX8: ; %bb.0:
4637 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4638 ; GFX8-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
4639 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
4640 ; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
4641 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4642 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4643 ; GFX8-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4644 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
4645 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4646 ; GFX8-NEXT: buffer_wbinvl1
4647 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4648 ; GFX8-NEXT: v_mov_b32_e32 v7, v5
4649 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4650 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
4651 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
4652 ; GFX8-NEXT: s_cbranch_execnz .LBB19_1
4653 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4654 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
4655 ; GFX8-NEXT: s_setpc_b64 s[30:31]
4656 ;
4657 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f64:
4658 ; GFX7: ; %bb.0:
4659 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4660 ; GFX7-NEXT: s_mov_b32 s6, 0
4661 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
4662 ; GFX7-NEXT: s_mov_b32 s4, s6
4663 ; GFX7-NEXT: s_mov_b32 s5, s6
4664 ; GFX7-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64
4665 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
4666 ; GFX7-NEXT: .LBB19_1: ; %atomicrmw.start
4667 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4668 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4669 ; GFX7-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4670 ; GFX7-NEXT: v_mov_b32_e32 v11, v7
4671 ; GFX7-NEXT: v_mov_b32_e32 v10, v6
4672 ; GFX7-NEXT: v_mov_b32_e32 v9, v5
4673 ; GFX7-NEXT: v_mov_b32_e32 v8, v4
4674 ; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 glc
4675 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4676 ; GFX7-NEXT: buffer_wbinvl1
4677 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
4678 ; GFX7-NEXT: v_mov_b32_e32 v6, v8
4679 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
4680 ; GFX7-NEXT: v_mov_b32_e32 v7, v9
4681 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
4682 ; GFX7-NEXT: s_cbranch_execnz .LBB19_1
4683 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4684 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
4685 ; GFX7-NEXT: s_setpc_b64 s[30:31]
4686 ;
4687 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f64:
4688 ; GFX6: ; %bb.0:
4689 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4690 ; GFX6-NEXT: s_mov_b32 s6, 0
4691 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
4692 ; GFX6-NEXT: s_mov_b32 s4, s6
4693 ; GFX6-NEXT: s_mov_b32 s5, s6
4694 ; GFX6-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64
4695 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
4696 ; GFX6-NEXT: .LBB19_1: ; %atomicrmw.start
4697 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
4698 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4699 ; GFX6-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4700 ; GFX6-NEXT: s_waitcnt expcnt(0)
4701 ; GFX6-NEXT: v_mov_b32_e32 v11, v7
4702 ; GFX6-NEXT: v_mov_b32_e32 v10, v6
4703 ; GFX6-NEXT: v_mov_b32_e32 v9, v5
4704 ; GFX6-NEXT: v_mov_b32_e32 v8, v4
4705 ; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 glc
4706 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4707 ; GFX6-NEXT: buffer_wbinvl1
4708 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
4709 ; GFX6-NEXT: v_mov_b32_e32 v6, v8
4710 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
4711 ; GFX6-NEXT: v_mov_b32_e32 v7, v9
4712 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
4713 ; GFX6-NEXT: s_cbranch_execnz .LBB19_1
4714 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
4715 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
4716 ; GFX6-NEXT: s_waitcnt expcnt(0)
4717 ; GFX6-NEXT: s_setpc_b64 s[30:31]
4718 %unused = atomicrmw fsub ptr addrspace(1) %ptr, double %val syncscope("agent") seq_cst
4719 ret void
4720 }
4722 define void @global_agent_atomic_fsub_noret_f64__offset12b_pos(ptr addrspace(1) %ptr, double %val) #0 {
4723 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_pos:
4724 ; GFX12: ; %bb.0:
4725 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
4726 ; GFX12-NEXT: s_wait_expcnt 0x0
4727 ; GFX12-NEXT: s_wait_samplecnt 0x0
4728 ; GFX12-NEXT: s_wait_bvhcnt 0x0
4729 ; GFX12-NEXT: s_wait_kmcnt 0x0
4730 ; GFX12-NEXT: global_load_b64 v[6:7], v[0:1], off offset:2040
4731 ; GFX12-NEXT: s_mov_b32 s0, 0
4732 ; GFX12-NEXT: .LBB20_1: ; %atomicrmw.start
4733 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
4734 ; GFX12-NEXT: s_wait_loadcnt 0x0
4735 ; GFX12-NEXT: v_add_f64_e64 v[4:5], v[6:7], -v[2:3]
4736 ; GFX12-NEXT: s_wait_storecnt 0x0
4737 ; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
4738 ; GFX12-NEXT: s_wait_loadcnt 0x0
4739 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
4740 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4741 ; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
4742 ; GFX12-NEXT: s_wait_alu 0xfffe
4743 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
4744 ; GFX12-NEXT: s_wait_alu 0xfffe
4745 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4746 ; GFX12-NEXT: s_cbranch_execnz .LBB20_1
4747 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
4748 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
4749 ; GFX12-NEXT: s_wait_alu 0xfffe
4750 ; GFX12-NEXT: s_setpc_b64 s[30:31]
4751 ;
4752 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_pos:
4753 ; GFX940: ; %bb.0:
4754 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4755 ; GFX940-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:2040
4756 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
4757 ; GFX940-NEXT: .LBB20_1: ; %atomicrmw.start
4758 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
4759 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4760 ; GFX940-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4761 ; GFX940-NEXT: buffer_wbl2 sc1
4762 ; GFX940-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 sc0
4763 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4764 ; GFX940-NEXT: buffer_inv sc1
4765 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4766 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4767 ; GFX940-NEXT: v_mov_b64_e32 v[6:7], v[4:5]
4768 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
4769 ; GFX940-NEXT: s_cbranch_execnz .LBB20_1
4770 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
4771 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
4772 ; GFX940-NEXT: s_setpc_b64 s[30:31]
4773 ;
4774 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_pos:
4775 ; GFX11: ; %bb.0:
4776 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4777 ; GFX11-NEXT: global_load_b64 v[6:7], v[0:1], off offset:2040
4778 ; GFX11-NEXT: s_mov_b32 s0, 0
4779 ; GFX11-NEXT: .LBB20_1: ; %atomicrmw.start
4780 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
4781 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4782 ; GFX11-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4783 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
4784 ; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 glc
4785 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4786 ; GFX11-NEXT: buffer_gl1_inv
4787 ; GFX11-NEXT: buffer_gl0_inv
4788 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4789 ; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
4790 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
4791 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4792 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4793 ; GFX11-NEXT: s_cbranch_execnz .LBB20_1
4794 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
4795 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
4796 ; GFX11-NEXT: s_setpc_b64 s[30:31]
4797 ;
4798 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_pos:
4799 ; GFX10: ; %bb.0:
4800 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4801 ; GFX10-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:2040
4802 ; GFX10-NEXT: s_mov_b32 s4, 0
4803 ; GFX10-NEXT: .LBB20_1: ; %atomicrmw.start
4804 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
4805 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4806 ; GFX10-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4807 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
4808 ; GFX10-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 glc
4809 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4810 ; GFX10-NEXT: buffer_gl1_inv
4811 ; GFX10-NEXT: buffer_gl0_inv
4812 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4813 ; GFX10-NEXT: v_mov_b32_e32 v7, v5
4814 ; GFX10-NEXT: v_mov_b32_e32 v6, v4
4815 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
4816 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
4817 ; GFX10-NEXT: s_cbranch_execnz .LBB20_1
4818 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
4819 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
4820 ; GFX10-NEXT: s_setpc_b64 s[30:31]
4821 ;
4822 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_pos:
4823 ; GFX90A: ; %bb.0:
4824 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4825 ; GFX90A-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:2040
4826 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
4827 ; GFX90A-NEXT: .LBB20_1: ; %atomicrmw.start
4828 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
4829 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4830 ; GFX90A-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4831 ; GFX90A-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 glc
4832 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4833 ; GFX90A-NEXT: buffer_wbinvl1
4834 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4835 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4836 ; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1]
4837 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
4838 ; GFX90A-NEXT: s_cbranch_execnz .LBB20_1
4839 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
4840 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
4841 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
4842 ;
4843 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_pos:
4844 ; GFX908: ; %bb.0:
4845 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4846 ; GFX908-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:2040
4847 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
4848 ; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
4849 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
4850 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4851 ; GFX908-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4852 ; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 glc
4853 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4854 ; GFX908-NEXT: buffer_wbinvl1
4855 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4856 ; GFX908-NEXT: v_mov_b32_e32 v7, v5
4857 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4858 ; GFX908-NEXT: v_mov_b32_e32 v6, v4
4859 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
4860 ; GFX908-NEXT: s_cbranch_execnz .LBB20_1
4861 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
4862 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
4863 ; GFX908-NEXT: s_setpc_b64 s[30:31]
4864 ;
4865 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_pos:
4866 ; GFX8: ; %bb.0:
4867 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4868 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7f8, v0
4869 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
4870 ; GFX8-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
4871 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
4872 ; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
4873 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4874 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4875 ; GFX8-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4876 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
4877 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4878 ; GFX8-NEXT: buffer_wbinvl1
4879 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4880 ; GFX8-NEXT: v_mov_b32_e32 v7, v5
4881 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4882 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
4883 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
4884 ; GFX8-NEXT: s_cbranch_execnz .LBB20_1
4885 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4886 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
4887 ; GFX8-NEXT: s_setpc_b64 s[30:31]
4888 ;
4889 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_pos:
4890 ; GFX7: ; %bb.0:
4891 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4892 ; GFX7-NEXT: s_mov_b32 s6, 0
4893 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
4894 ; GFX7-NEXT: s_mov_b32 s4, s6
4895 ; GFX7-NEXT: s_mov_b32 s5, s6
4896 ; GFX7-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:2040
4897 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
4898 ; GFX7-NEXT: .LBB20_1: ; %atomicrmw.start
4899 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4900 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4901 ; GFX7-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4902 ; GFX7-NEXT: v_mov_b32_e32 v11, v7
4903 ; GFX7-NEXT: v_mov_b32_e32 v10, v6
4904 ; GFX7-NEXT: v_mov_b32_e32 v9, v5
4905 ; GFX7-NEXT: v_mov_b32_e32 v8, v4
4906 ; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:2040 glc
4907 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4908 ; GFX7-NEXT: buffer_wbinvl1
4909 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
4910 ; GFX7-NEXT: v_mov_b32_e32 v6, v8
4911 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
4912 ; GFX7-NEXT: v_mov_b32_e32 v7, v9
4913 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
4914 ; GFX7-NEXT: s_cbranch_execnz .LBB20_1
4915 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4916 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
4917 ; GFX7-NEXT: s_setpc_b64 s[30:31]
4918 ;
4919 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_pos:
4920 ; GFX6: ; %bb.0:
4921 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4922 ; GFX6-NEXT: s_mov_b32 s6, 0
4923 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
4924 ; GFX6-NEXT: s_mov_b32 s4, s6
4925 ; GFX6-NEXT: s_mov_b32 s5, s6
4926 ; GFX6-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:2040
4927 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
4928 ; GFX6-NEXT: .LBB20_1: ; %atomicrmw.start
4929 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
4930 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4931 ; GFX6-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4932 ; GFX6-NEXT: s_waitcnt expcnt(0)
4933 ; GFX6-NEXT: v_mov_b32_e32 v11, v7
4934 ; GFX6-NEXT: v_mov_b32_e32 v10, v6
4935 ; GFX6-NEXT: v_mov_b32_e32 v9, v5
4936 ; GFX6-NEXT: v_mov_b32_e32 v8, v4
4937 ; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:2040 glc
4938 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4939 ; GFX6-NEXT: buffer_wbinvl1
4940 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
4941 ; GFX6-NEXT: v_mov_b32_e32 v6, v8
4942 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
4943 ; GFX6-NEXT: v_mov_b32_e32 v7, v9
4944 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
4945 ; GFX6-NEXT: s_cbranch_execnz .LBB20_1
4946 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
4947 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
4948 ; GFX6-NEXT: s_waitcnt expcnt(0)
4949 ; GFX6-NEXT: s_setpc_b64 s[30:31]
4950 %gep = getelementptr double, ptr addrspace(1) %ptr, i64 255
4951 %unused = atomicrmw fsub ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst
4952 ret void
4953 }
4955 define void @global_agent_atomic_fsub_noret_f64__offset12b_neg(ptr addrspace(1) %ptr, double %val) #0 {
4956 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_neg:
4957 ; GFX12: ; %bb.0:
4958 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
4959 ; GFX12-NEXT: s_wait_expcnt 0x0
4960 ; GFX12-NEXT: s_wait_samplecnt 0x0
4961 ; GFX12-NEXT: s_wait_bvhcnt 0x0
4962 ; GFX12-NEXT: s_wait_kmcnt 0x0
4963 ; GFX12-NEXT: global_load_b64 v[6:7], v[0:1], off offset:-2048
4964 ; GFX12-NEXT: s_mov_b32 s0, 0
4965 ; GFX12-NEXT: .LBB21_1: ; %atomicrmw.start
4966 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
4967 ; GFX12-NEXT: s_wait_loadcnt 0x0
4968 ; GFX12-NEXT: v_add_f64_e64 v[4:5], v[6:7], -v[2:3]
4969 ; GFX12-NEXT: s_wait_storecnt 0x0
4970 ; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
4971 ; GFX12-NEXT: s_wait_loadcnt 0x0
4972 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
4973 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4974 ; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
4975 ; GFX12-NEXT: s_wait_alu 0xfffe
4976 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
4977 ; GFX12-NEXT: s_wait_alu 0xfffe
4978 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4979 ; GFX12-NEXT: s_cbranch_execnz .LBB21_1
4980 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
4981 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
4982 ; GFX12-NEXT: s_wait_alu 0xfffe
4983 ; GFX12-NEXT: s_setpc_b64 s[30:31]
4984 ;
4985 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_neg:
4986 ; GFX940: ; %bb.0:
4987 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4988 ; GFX940-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:-2048
4989 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
4990 ; GFX940-NEXT: .LBB21_1: ; %atomicrmw.start
4991 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
4992 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4993 ; GFX940-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4994 ; GFX940-NEXT: buffer_wbl2 sc1
4995 ; GFX940-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 sc0
4996 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4997 ; GFX940-NEXT: buffer_inv sc1
4998 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4999 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
5000 ; GFX940-NEXT: v_mov_b64_e32 v[6:7], v[4:5]
5001 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
5002 ; GFX940-NEXT: s_cbranch_execnz .LBB21_1
5003 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
5004 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
5005 ; GFX940-NEXT: s_setpc_b64 s[30:31]
5006 ;
5007 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_neg:
5008 ; GFX11: ; %bb.0:
5009 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5010 ; GFX11-NEXT: global_load_b64 v[6:7], v[0:1], off offset:-2048
5011 ; GFX11-NEXT: s_mov_b32 s0, 0
5012 ; GFX11-NEXT: .LBB21_1: ; %atomicrmw.start
5013 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
5014 ; GFX11-NEXT: s_waitcnt vmcnt(0)
5015 ; GFX11-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
5016 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
5017 ; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
5018 ; GFX11-NEXT: s_waitcnt vmcnt(0)
5019 ; GFX11-NEXT: buffer_gl1_inv
5020 ; GFX11-NEXT: buffer_gl0_inv
5021 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
5022 ; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
5023 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
5024 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
5025 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5026 ; GFX11-NEXT: s_cbranch_execnz .LBB21_1
5027 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
5028 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
5029 ; GFX11-NEXT: s_setpc_b64 s[30:31]
5030 ;
5031 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_neg:
5032 ; GFX10: ; %bb.0:
5033 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5034 ; GFX10-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:-2048
5035 ; GFX10-NEXT: s_mov_b32 s4, 0
5036 ; GFX10-NEXT: .LBB21_1: ; %atomicrmw.start
5037 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
5038 ; GFX10-NEXT: s_waitcnt vmcnt(0)
5039 ; GFX10-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
5040 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
5041 ; GFX10-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
5042 ; GFX10-NEXT: s_waitcnt vmcnt(0)
5043 ; GFX10-NEXT: buffer_gl1_inv
5044 ; GFX10-NEXT: buffer_gl0_inv
5045 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
5046 ; GFX10-NEXT: v_mov_b32_e32 v7, v5
5047 ; GFX10-NEXT: v_mov_b32_e32 v6, v4
5048 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
5049 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
5050 ; GFX10-NEXT: s_cbranch_execnz .LBB21_1
5051 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
5052 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
5053 ; GFX10-NEXT: s_setpc_b64 s[30:31]
5054 ;
5055 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_neg:
5056 ; GFX90A: ; %bb.0:
5057 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5058 ; GFX90A-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:-2048
5059 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
5060 ; GFX90A-NEXT: .LBB21_1: ; %atomicrmw.start
5061 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
5062 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
5063 ; GFX90A-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
5064 ; GFX90A-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
5065 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
5066 ; GFX90A-NEXT: buffer_wbinvl1
5067 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
5068 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5069 ; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1]
5070 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
5071 ; GFX90A-NEXT: s_cbranch_execnz .LBB21_1
5072 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
5073 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
5074 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
5075 ;
5076 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_neg:
5077 ; GFX908: ; %bb.0:
5078 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5079 ; GFX908-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:-2048
5080 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
5081 ; GFX908-NEXT: .LBB21_1: ; %atomicrmw.start
5082 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
5083 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5084 ; GFX908-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
5085 ; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
5086 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5087 ; GFX908-NEXT: buffer_wbinvl1
5088 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
5089 ; GFX908-NEXT: v_mov_b32_e32 v7, v5
5090 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5091 ; GFX908-NEXT: v_mov_b32_e32 v6, v4
5092 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
5093 ; GFX908-NEXT: s_cbranch_execnz .LBB21_1
5094 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
5095 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
5096 ; GFX908-NEXT: s_setpc_b64 s[30:31]
5097 ;
5098 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_neg:
5099 ; GFX8: ; %bb.0:
5100 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5101 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
5102 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
5103 ; GFX8-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
5104 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
5105 ; GFX8-NEXT: .LBB21_1: ; %atomicrmw.start
5106 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5107 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5108 ; GFX8-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
5109 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
5110 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5111 ; GFX8-NEXT: buffer_wbinvl1
5112 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
5113 ; GFX8-NEXT: v_mov_b32_e32 v7, v5
5114 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5115 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
5116 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
5117 ; GFX8-NEXT: s_cbranch_execnz .LBB21_1
5118 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5119 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
5120 ; GFX8-NEXT: s_setpc_b64 s[30:31]
5121 ;
5122 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_neg:
5123 ; GFX7: ; %bb.0:
5124 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5125 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
5126 ; GFX7-NEXT: s_mov_b32 s5, -1
5127 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
5128 ; GFX7-NEXT: s_mov_b32 s6, 0
5129 ; GFX7-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64
5130 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
5131 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
5132 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
5133 ; GFX7-NEXT: s_mov_b32 s4, s6
5134 ; GFX7-NEXT: s_mov_b32 s5, s6
5135 ; GFX7-NEXT: .LBB21_1: ; %atomicrmw.start
5136 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5137 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5138 ; GFX7-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
5139 ; GFX7-NEXT: v_mov_b32_e32 v11, v7
5140 ; GFX7-NEXT: v_mov_b32_e32 v10, v6
5141 ; GFX7-NEXT: v_mov_b32_e32 v9, v5
5142 ; GFX7-NEXT: v_mov_b32_e32 v8, v4
5143 ; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 glc
5144 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5145 ; GFX7-NEXT: buffer_wbinvl1
5146 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
5147 ; GFX7-NEXT: v_mov_b32_e32 v6, v8
5148 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
5149 ; GFX7-NEXT: v_mov_b32_e32 v7, v9
5150 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
5151 ; GFX7-NEXT: s_cbranch_execnz .LBB21_1
5152 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5153 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
5154 ; GFX7-NEXT: s_setpc_b64 s[30:31]
5155 ;
5156 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_neg:
5157 ; GFX6: ; %bb.0:
5158 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5159 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
5160 ; GFX6-NEXT: s_mov_b32 s5, -1
5161 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
5162 ; GFX6-NEXT: s_mov_b32 s6, 0
5163 ; GFX6-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64
5164 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
5165 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
5166 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
5167 ; GFX6-NEXT: s_mov_b32 s4, s6
5168 ; GFX6-NEXT: s_mov_b32 s5, s6
5169 ; GFX6-NEXT: .LBB21_1: ; %atomicrmw.start
5170 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
5171 ; GFX6-NEXT: s_waitcnt vmcnt(0)
5172 ; GFX6-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
5173 ; GFX6-NEXT: s_waitcnt expcnt(0)
5174 ; GFX6-NEXT: v_mov_b32_e32 v11, v7
5175 ; GFX6-NEXT: v_mov_b32_e32 v10, v6
5176 ; GFX6-NEXT: v_mov_b32_e32 v9, v5
5177 ; GFX6-NEXT: v_mov_b32_e32 v8, v4
5178 ; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 glc
5179 ; GFX6-NEXT: s_waitcnt vmcnt(0)
5180 ; GFX6-NEXT: buffer_wbinvl1
5181 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
5182 ; GFX6-NEXT: v_mov_b32_e32 v6, v8
5183 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
5184 ; GFX6-NEXT: v_mov_b32_e32 v7, v9
5185 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
5186 ; GFX6-NEXT: s_cbranch_execnz .LBB21_1
5187 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
5188 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
5189 ; GFX6-NEXT: s_waitcnt expcnt(0)
5190 ; GFX6-NEXT: s_setpc_b64 s[30:31]
5191 %gep = getelementptr double, ptr addrspace(1) %ptr, i64 -256
5192 %unused = atomicrmw fsub ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst
5193 ret void
5194 }
5196 ; --------------------------------------------------------------------
5197 ; half
5198 ; --------------------------------------------------------------------
5200 define half @global_agent_atomic_fsub_ret_f16(ptr addrspace(1) %ptr, half %val) #0 {
5201 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f16:
5202 ; GFX12: ; %bb.0:
5203 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
5204 ; GFX12-NEXT: s_wait_expcnt 0x0
5205 ; GFX12-NEXT: s_wait_samplecnt 0x0
5206 ; GFX12-NEXT: s_wait_bvhcnt 0x0
5207 ; GFX12-NEXT: s_wait_kmcnt 0x0
5208 ; GFX12-NEXT: v_mov_b32_e32 v3, v0
5209 ; GFX12-NEXT: s_mov_b32 s0, 0
5210 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
5211 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
5212 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
5213 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
5214 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5215 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
5216 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
5217 ; GFX12-NEXT: v_not_b32_e32 v4, v4
5218 ; GFX12-NEXT: .LBB22_1: ; %atomicrmw.start
5219 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
5220 ; GFX12-NEXT: s_wait_loadcnt 0x0
5221 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
5222 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5223 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5224 ; GFX12-NEXT: v_sub_f16_e32 v5, v5, v2
5225 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5226 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff, v5
5227 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5228 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
5229 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
5230 ; GFX12-NEXT: s_wait_storecnt 0x0
5231 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
5232 ; GFX12-NEXT: s_wait_loadcnt 0x0
5233 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
5234 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
5235 ; GFX12-NEXT: s_wait_alu 0xfffe
5236 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
5237 ; GFX12-NEXT: s_wait_alu 0xfffe
5238 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5239 ; GFX12-NEXT: s_cbranch_execnz .LBB22_1
5240 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
5241 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
5242 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5243 ; GFX12-NEXT: s_wait_alu 0xfffe
5244 ; GFX12-NEXT: s_setpc_b64 s[30:31]
5245 ;
5246 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f16:
5247 ; GFX940: ; %bb.0:
5248 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5249 ; GFX940-NEXT: v_mov_b32_e32 v3, v0
5250 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v3
5251 ; GFX940-NEXT: global_load_dword v4, v[0:1], off
5252 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v3
5253 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5254 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
5255 ; GFX940-NEXT: v_lshlrev_b32_e64 v5, v3, s0
5256 ; GFX940-NEXT: v_not_b32_e32 v5, v5
5257 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
5258 ; GFX940-NEXT: .LBB22_1: ; %atomicrmw.start
5259 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
5260 ; GFX940-NEXT: s_waitcnt vmcnt(0)
5261 ; GFX940-NEXT: v_mov_b32_e32 v7, v4
5262 ; GFX940-NEXT: v_lshrrev_b32_e32 v4, v3, v7
5263 ; GFX940-NEXT: v_sub_f16_e32 v4, v4, v2
5264 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, v3, v4
5265 ; GFX940-NEXT: v_and_or_b32 v6, v7, v5, v4
5266 ; GFX940-NEXT: buffer_wbl2 sc1
5267 ; GFX940-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off sc0
5268 ; GFX940-NEXT: s_waitcnt vmcnt(0)
5269 ; GFX940-NEXT: buffer_inv sc1
5270 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
5271 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
5272 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
5273 ; GFX940-NEXT: s_cbranch_execnz .LBB22_1
5274 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
5275 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
5276 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v4
5277 ; GFX940-NEXT: s_setpc_b64 s[30:31]
5278 ;
5279 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f16:
5280 ; GFX11: ; %bb.0:
5281 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5282 ; GFX11-NEXT: v_mov_b32_e32 v3, v0
5283 ; GFX11-NEXT: s_mov_b32 s0, 0
5284 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
5285 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
5286 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
5287 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
5288 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5289 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
5290 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
5291 ; GFX11-NEXT: v_not_b32_e32 v4, v4
5292 ; GFX11-NEXT: .LBB22_1: ; %atomicrmw.start
5293 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
5294 ; GFX11-NEXT: s_waitcnt vmcnt(0)
5295 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
5296 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5297 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5298 ; GFX11-NEXT: v_sub_f16_e32 v5, v5, v2
5299 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5300 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
5301 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5302 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
5303 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
5304 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
5305 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
5306 ; GFX11-NEXT: s_waitcnt vmcnt(0)
5307 ; GFX11-NEXT: buffer_gl1_inv
5308 ; GFX11-NEXT: buffer_gl0_inv
5309 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
5310 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
5311 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
5312 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5313 ; GFX11-NEXT: s_cbranch_execnz .LBB22_1
5314 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
5315 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
5316 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5317 ; GFX11-NEXT: s_setpc_b64 s[30:31]
5318 ;
5319 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f16:
5320 ; GFX10: ; %bb.0:
5321 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5322 ; GFX10-NEXT: v_mov_b32_e32 v3, v0
5323 ; GFX10-NEXT: s_mov_b32 s4, 0
5324 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
5325 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
5326 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
5327 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5328 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
5329 ; GFX10-NEXT: v_not_b32_e32 v4, v4
5330 ; GFX10-NEXT: .LBB22_1: ; %atomicrmw.start
5331 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
5332 ; GFX10-NEXT: s_waitcnt vmcnt(0)
5333 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
5334 ; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5335 ; GFX10-NEXT: v_sub_f16_e32 v5, v5, v2
5336 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
5337 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
5338 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
5339 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
5340 ; GFX10-NEXT: s_waitcnt vmcnt(0)
5341 ; GFX10-NEXT: buffer_gl1_inv
5342 ; GFX10-NEXT: buffer_gl0_inv
5343 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
5344 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
5345 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
5346 ; GFX10-NEXT: s_cbranch_execnz .LBB22_1
5347 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
5348 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
5349 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5350 ; GFX10-NEXT: s_setpc_b64 s[30:31]
5351 ;
5352 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f16:
5353 ; GFX90A: ; %bb.0:
5354 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5355 ; GFX90A-NEXT: v_mov_b32_e32 v3, v0
5356 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
5357 ; GFX90A-NEXT: global_load_dword v4, v[0:1], off
5358 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
5359 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5360 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
5361 ; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v3, s4
5362 ; GFX90A-NEXT: v_not_b32_e32 v5, v5
5363 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
5364 ; GFX90A-NEXT: .LBB22_1: ; %atomicrmw.start
5365 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
5366 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
5367 ; GFX90A-NEXT: v_mov_b32_e32 v7, v4
5368 ; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v7
5369 ; GFX90A-NEXT: v_sub_f16_e32 v4, v4, v2
5370 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
5371 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v5, v4
5372 ; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off glc
5373 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
5374 ; GFX90A-NEXT: buffer_wbinvl1
5375 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
5376 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5377 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
5378 ; GFX90A-NEXT: s_cbranch_execnz .LBB22_1
5379 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
5380 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
5381 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v4
5382 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
5383 ;
5384 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f16:
5385 ; GFX908: ; %bb.0:
5386 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5387 ; GFX908-NEXT: v_mov_b32_e32 v3, v0
5388 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
5389 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
5390 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
5391 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5392 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
5393 ; GFX908-NEXT: v_lshlrev_b32_e64 v5, v3, s4
5394 ; GFX908-NEXT: v_not_b32_e32 v5, v5
5395 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
5396 ; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
5397 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
5398 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5399 ; GFX908-NEXT: v_mov_b32_e32 v7, v4
5400 ; GFX908-NEXT: v_lshrrev_b32_e32 v4, v3, v7
5401 ; GFX908-NEXT: v_sub_f16_e32 v4, v4, v2
5402 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, v3, v4
5403 ; GFX908-NEXT: v_and_or_b32 v6, v7, v5, v4
5404 ; GFX908-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off glc
5405 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5406 ; GFX908-NEXT: buffer_wbinvl1
5407 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
5408 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5409 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
5410 ; GFX908-NEXT: s_cbranch_execnz .LBB22_1
5411 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
5412 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
5413 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v4
5414 ; GFX908-NEXT: s_setpc_b64 s[30:31]
5415 ;
5416 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f16:
5417 ; GFX8: ; %bb.0:
5418 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5419 ; GFX8-NEXT: v_mov_b32_e32 v3, v0
5420 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
5421 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
5422 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
5423 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5424 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
5425 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
5426 ; GFX8-NEXT: v_not_b32_e32 v4, v4
5427 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
5428 ; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
5429 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5430 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5431 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
5432 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5433 ; GFX8-NEXT: v_sub_f16_e32 v5, v5, v2
5434 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
5435 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5436 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
5437 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
5438 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5439 ; GFX8-NEXT: buffer_wbinvl1
5440 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
5441 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5442 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
5443 ; GFX8-NEXT: s_cbranch_execnz .LBB22_1
5444 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5445 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
5446 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5447 ; GFX8-NEXT: s_setpc_b64 s[30:31]
5448 ;
5449 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f16:
5450 ; GFX7: ; %bb.0:
5451 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5452 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
5453 ; GFX7-NEXT: s_mov_b32 s6, 0
5454 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
5455 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
5456 ; GFX7-NEXT: s_mov_b32 s4, s6
5457 ; GFX7-NEXT: s_mov_b32 s5, s6
5458 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
5459 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
5460 ; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
5461 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 3, v3
5462 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
5463 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v2
5464 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v5
5465 ; GFX7-NEXT: v_not_b32_e32 v7, v2
5466 ; GFX7-NEXT: .LBB22_1: ; %atomicrmw.start
5467 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5468 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5469 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v5, v4
5470 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
5471 ; GFX7-NEXT: v_and_b32_e32 v3, v4, v7
5472 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v6
5473 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
5474 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v5, v2
5475 ; GFX7-NEXT: v_or_b32_e32 v3, v3, v2
5476 ; GFX7-NEXT: v_mov_b32_e32 v2, v3
5477 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
5478 ; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v[0:1], s[4:7], 0 addr64 glc
5479 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5480 ; GFX7-NEXT: buffer_wbinvl1
5481 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
5482 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
5483 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
5484 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
5485 ; GFX7-NEXT: s_cbranch_execnz .LBB22_1
5486 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5487 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
5488 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v5, v2
5489 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
5490 ; GFX7-NEXT: s_setpc_b64 s[30:31]
5491 ;
5492 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f16:
5493 ; GFX6: ; %bb.0:
5494 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5495 ; GFX6-NEXT: v_mov_b32_e32 v3, v0
5496 ; GFX6-NEXT: s_mov_b32 s6, 0
5497 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v3
5498 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
5499 ; GFX6-NEXT: s_mov_b32 s4, s6
5500 ; GFX6-NEXT: s_mov_b32 s5, s6
5501 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
5502 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
5503 ; GFX6-NEXT: v_and_b32_e32 v3, 3, v3
5504 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 3, v3
5505 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
5506 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v2
5507 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v5
5508 ; GFX6-NEXT: v_not_b32_e32 v7, v2
5509 ; GFX6-NEXT: .LBB22_1: ; %atomicrmw.start
5510 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
5511 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
5512 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v5, v4
5513 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
5514 ; GFX6-NEXT: v_and_b32_e32 v3, v4, v7
5515 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v6
5516 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
5517 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v5, v2
5518 ; GFX6-NEXT: v_or_b32_e32 v3, v3, v2
5519 ; GFX6-NEXT: v_mov_b32_e32 v2, v3
5520 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
5521 ; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v[0:1], s[4:7], 0 addr64 glc
5522 ; GFX6-NEXT: s_waitcnt vmcnt(0)
5523 ; GFX6-NEXT: buffer_wbinvl1
5524 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
5525 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
5526 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
5527 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
5528 ; GFX6-NEXT: s_cbranch_execnz .LBB22_1
5529 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
5530 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
5531 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v5, v2
5532 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
5533 ; GFX6-NEXT: s_waitcnt expcnt(0)
5534 ; GFX6-NEXT: s_setpc_b64 s[30:31]
5535 %result = atomicrmw fsub ptr addrspace(1) %ptr, half %val syncscope("agent") seq_cst
5536 ret half %result
5537 }
5539 define half @global_agent_atomic_fsub_ret_f16__offset12b_pos(ptr addrspace(1) %ptr, half %val) #0 {
5540 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos:
5541 ; GFX12: ; %bb.0:
5542 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
5543 ; GFX12-NEXT: s_wait_expcnt 0x0
5544 ; GFX12-NEXT: s_wait_samplecnt 0x0
5545 ; GFX12-NEXT: s_wait_bvhcnt 0x0
5546 ; GFX12-NEXT: s_wait_kmcnt 0x0
5547 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
5548 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
5549 ; GFX12-NEXT: s_mov_b32 s0, 0
5550 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
5551 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
5552 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
5553 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
5554 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5555 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
5556 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
5557 ; GFX12-NEXT: v_not_b32_e32 v4, v4
5558 ; GFX12-NEXT: .LBB23_1: ; %atomicrmw.start
5559 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
5560 ; GFX12-NEXT: s_wait_loadcnt 0x0
5561 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
5562 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5563 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5564 ; GFX12-NEXT: v_sub_f16_e32 v5, v5, v2
5565 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5566 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff, v5
5567 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5568 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
5569 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
5570 ; GFX12-NEXT: s_wait_storecnt 0x0
5571 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
5572 ; GFX12-NEXT: s_wait_loadcnt 0x0
5573 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
5574 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
5575 ; GFX12-NEXT: s_wait_alu 0xfffe
5576 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
5577 ; GFX12-NEXT: s_wait_alu 0xfffe
5578 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5579 ; GFX12-NEXT: s_cbranch_execnz .LBB23_1
5580 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
5581 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
5582 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5583 ; GFX12-NEXT: s_wait_alu 0xfffe
5584 ; GFX12-NEXT: s_setpc_b64 s[30:31]
5585 ;
5586 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos:
5587 ; GFX940: ; %bb.0:
5588 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5589 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
5590 ; GFX940-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, s[0:1]
5591 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v6
5592 ; GFX940-NEXT: v_mov_b32_e32 v1, v7
5593 ; GFX940-NEXT: global_load_dword v4, v[0:1], off
5594 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v6
5595 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5596 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
5597 ; GFX940-NEXT: v_lshlrev_b32_e64 v5, v3, s0
5598 ; GFX940-NEXT: v_not_b32_e32 v5, v5
5599 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
5600 ; GFX940-NEXT: .LBB23_1: ; %atomicrmw.start
5601 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
5602 ; GFX940-NEXT: s_waitcnt vmcnt(0)
5603 ; GFX940-NEXT: v_mov_b32_e32 v7, v4
5604 ; GFX940-NEXT: v_lshrrev_b32_e32 v4, v3, v7
5605 ; GFX940-NEXT: v_sub_f16_e32 v4, v4, v2
5606 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, v3, v4
5607 ; GFX940-NEXT: v_and_or_b32 v6, v7, v5, v4
5608 ; GFX940-NEXT: buffer_wbl2 sc1
5609 ; GFX940-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off sc0
5610 ; GFX940-NEXT: s_waitcnt vmcnt(0)
5611 ; GFX940-NEXT: buffer_inv sc1
5612 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
5613 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
5614 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
5615 ; GFX940-NEXT: s_cbranch_execnz .LBB23_1
5616 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
5617 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
5618 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v4
5619 ; GFX940-NEXT: s_setpc_b64 s[30:31]
5620 ;
5621 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos:
5622 ; GFX11: ; %bb.0:
5623 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5624 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
5625 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
5626 ; GFX11-NEXT: s_mov_b32 s0, 0
5627 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
5628 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
5629 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
5630 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
5631 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5632 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
5633 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
5634 ; GFX11-NEXT: v_not_b32_e32 v4, v4
5635 ; GFX11-NEXT: .LBB23_1: ; %atomicrmw.start
5636 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
5637 ; GFX11-NEXT: s_waitcnt vmcnt(0)
5638 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
5639 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5640 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5641 ; GFX11-NEXT: v_sub_f16_e32 v5, v5, v2
5642 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5643 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
5644 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5645 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
5646 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
5647 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
5648 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
5649 ; GFX11-NEXT: s_waitcnt vmcnt(0)
5650 ; GFX11-NEXT: buffer_gl1_inv
5651 ; GFX11-NEXT: buffer_gl0_inv
5652 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
5653 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
5654 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
5655 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5656 ; GFX11-NEXT: s_cbranch_execnz .LBB23_1
5657 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
5658 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
5659 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5660 ; GFX11-NEXT: s_setpc_b64 s[30:31]
5661 ;
5662 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos:
5663 ; GFX10: ; %bb.0:
5664 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5665 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
5666 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
5667 ; GFX10-NEXT: s_mov_b32 s4, 0
5668 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
5669 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
5670 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
5671 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5672 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
5673 ; GFX10-NEXT: v_not_b32_e32 v4, v4
5674 ; GFX10-NEXT: .LBB23_1: ; %atomicrmw.start
5675 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
5676 ; GFX10-NEXT: s_waitcnt vmcnt(0)
5677 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
5678 ; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5679 ; GFX10-NEXT: v_sub_f16_e32 v5, v5, v2
5680 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
5681 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
5682 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
5683 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
5684 ; GFX10-NEXT: s_waitcnt vmcnt(0)
5685 ; GFX10-NEXT: buffer_gl1_inv
5686 ; GFX10-NEXT: buffer_gl0_inv
5687 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
5688 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
5689 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
5690 ; GFX10-NEXT: s_cbranch_execnz .LBB23_1
5691 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
5692 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
5693 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5694 ; GFX10-NEXT: s_setpc_b64 s[30:31]
5695 ;
5696 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos:
5697 ; GFX90A: ; %bb.0:
5698 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5699 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
5700 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
5701 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
5702 ; GFX90A-NEXT: global_load_dword v4, v[0:1], off
5703 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
5704 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5705 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
5706 ; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v3, s4
5707 ; GFX90A-NEXT: v_not_b32_e32 v5, v5
5708 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
5709 ; GFX90A-NEXT: .LBB23_1: ; %atomicrmw.start
5710 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
5711 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
5712 ; GFX90A-NEXT: v_mov_b32_e32 v7, v4
5713 ; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v7
5714 ; GFX90A-NEXT: v_sub_f16_e32 v4, v4, v2
5715 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
5716 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v5, v4
5717 ; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off glc
5718 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
5719 ; GFX90A-NEXT: buffer_wbinvl1
5720 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
5721 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5722 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
5723 ; GFX90A-NEXT: s_cbranch_execnz .LBB23_1
5724 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
5725 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
5726 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v4
5727 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
5728 ;
5729 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos:
5730 ; GFX908: ; %bb.0:
5731 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5732 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
5733 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
5734 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
5735 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
5736 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
5737 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5738 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
5739 ; GFX908-NEXT: v_lshlrev_b32_e64 v5, v3, s4
5740 ; GFX908-NEXT: v_not_b32_e32 v5, v5
5741 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
5742 ; GFX908-NEXT: .LBB23_1: ; %atomicrmw.start
5743 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
5744 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5745 ; GFX908-NEXT: v_mov_b32_e32 v7, v4
5746 ; GFX908-NEXT: v_lshrrev_b32_e32 v4, v3, v7
5747 ; GFX908-NEXT: v_sub_f16_e32 v4, v4, v2
5748 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, v3, v4
5749 ; GFX908-NEXT: v_and_or_b32 v6, v7, v5, v4
5750 ; GFX908-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off glc
5751 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5752 ; GFX908-NEXT: buffer_wbinvl1
5753 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
5754 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5755 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
5756 ; GFX908-NEXT: s_cbranch_execnz .LBB23_1
5757 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
5758 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
5759 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v4
5760 ; GFX908-NEXT: s_setpc_b64 s[30:31]
5761 ;
5762 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos:
5763 ; GFX8: ; %bb.0:
5764 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5765 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
5766 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
5767 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
5768 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
5769 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
5770 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5771 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
5772 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
5773 ; GFX8-NEXT: v_not_b32_e32 v4, v4
5774 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
5775 ; GFX8-NEXT: .LBB23_1: ; %atomicrmw.start
5776 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5777 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5778 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
5779 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5780 ; GFX8-NEXT: v_sub_f16_e32 v5, v5, v2
5781 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
5782 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5783 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
5784 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
5785 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5786 ; GFX8-NEXT: buffer_wbinvl1
5787 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
5788 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5789 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
5790 ; GFX8-NEXT: s_cbranch_execnz .LBB23_1
5791 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5792 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
5793 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5794 ; GFX8-NEXT: s_setpc_b64 s[30:31]
5795 ;
5796 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos:
5797 ; GFX7: ; %bb.0:
5798 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5799 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
5800 ; GFX7-NEXT: s_mov_b32 s6, 0
5801 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
5802 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
5803 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
5804 ; GFX7-NEXT: s_mov_b32 s4, s6
5805 ; GFX7-NEXT: s_mov_b32 s5, s6
5806 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
5807 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
5808 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
5809 ; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
5810 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
5811 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
5812 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
5813 ; GFX7-NEXT: v_not_b32_e32 v8, v2
5814 ; GFX7-NEXT: .LBB23_1: ; %atomicrmw.start
5815 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5816 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5817 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
5818 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
5819 ; GFX7-NEXT: v_and_b32_e32 v4, v3, v8
5820 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v7
5821 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
5822 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v6, v2
5823 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
5824 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
5825 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
5826 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
5827 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5828 ; GFX7-NEXT: buffer_wbinvl1
5829 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
5830 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
5831 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
5832 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
5833 ; GFX7-NEXT: s_cbranch_execnz .LBB23_1
5834 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5835 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
5836 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
5837 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
5838 ; GFX7-NEXT: s_setpc_b64 s[30:31]
5839 ;
5840 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos:
5841 ; GFX6: ; %bb.0:
5842 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5843 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
5844 ; GFX6-NEXT: s_mov_b32 s6, 0
5845 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
5846 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
5847 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
5848 ; GFX6-NEXT: s_mov_b32 s4, s6
5849 ; GFX6-NEXT: s_mov_b32 s5, s6
5850 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
5851 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
5852 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
5853 ; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
5854 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
5855 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
5856 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
5857 ; GFX6-NEXT: v_not_b32_e32 v8, v2
5858 ; GFX6-NEXT: .LBB23_1: ; %atomicrmw.start
5859 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
5860 ; GFX6-NEXT: s_waitcnt vmcnt(0)
5861 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
5862 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
5863 ; GFX6-NEXT: s_waitcnt expcnt(0)
5864 ; GFX6-NEXT: v_and_b32_e32 v4, v3, v8
5865 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v7
5866 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
5867 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v6, v2
5868 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
5869 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
5870 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
5871 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
5872 ; GFX6-NEXT: s_waitcnt vmcnt(0)
5873 ; GFX6-NEXT: buffer_wbinvl1
5874 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
5875 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
5876 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
5877 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
5878 ; GFX6-NEXT: s_cbranch_execnz .LBB23_1
5879 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
5880 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
5881 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
5882 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
5883 ; GFX6-NEXT: s_waitcnt expcnt(0)
5884 ; GFX6-NEXT: s_setpc_b64 s[30:31]
5885 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
5886 %result = atomicrmw fsub ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst
5887 ret half %result
5888 }
5890 define half @global_agent_atomic_fsub_ret_f16__offset12b_neg(ptr addrspace(1) %ptr, half %val) #0 {
5891 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_neg:
5892 ; GFX12: ; %bb.0:
5893 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
5894 ; GFX12-NEXT: s_wait_expcnt 0x0
5895 ; GFX12-NEXT: s_wait_samplecnt 0x0
5896 ; GFX12-NEXT: s_wait_bvhcnt 0x0
5897 ; GFX12-NEXT: s_wait_kmcnt 0x0
5898 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
5899 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
5900 ; GFX12-NEXT: s_mov_b32 s0, 0
5901 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
5902 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
5903 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
5904 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
5905 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5906 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
5907 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
5908 ; GFX12-NEXT: v_not_b32_e32 v4, v4
5909 ; GFX12-NEXT: .LBB24_1: ; %atomicrmw.start
5910 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
5911 ; GFX12-NEXT: s_wait_loadcnt 0x0
5912 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
5913 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5914 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5915 ; GFX12-NEXT: v_sub_f16_e32 v5, v5, v2
5916 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5917 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff, v5
5918 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5919 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
5920 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
5921 ; GFX12-NEXT: s_wait_storecnt 0x0
5922 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
5923 ; GFX12-NEXT: s_wait_loadcnt 0x0
5924 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
5925 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
5926 ; GFX12-NEXT: s_wait_alu 0xfffe
5927 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
5928 ; GFX12-NEXT: s_wait_alu 0xfffe
5929 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5930 ; GFX12-NEXT: s_cbranch_execnz .LBB24_1
5931 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
5932 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
5933 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5934 ; GFX12-NEXT: s_wait_alu 0xfffe
5935 ; GFX12-NEXT: s_setpc_b64 s[30:31]
5936 ;
5937 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_neg:
5938 ; GFX940: ; %bb.0:
5939 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5940 ; GFX940-NEXT: s_movk_i32 s0, 0xf800
5941 ; GFX940-NEXT: s_mov_b32 s1, -1
5942 ; GFX940-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, s[0:1]
5943 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v6
5944 ; GFX940-NEXT: v_mov_b32_e32 v1, v7
5945 ; GFX940-NEXT: global_load_dword v4, v[0:1], off
5946 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v6
5947 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5948 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
5949 ; GFX940-NEXT: v_lshlrev_b32_e64 v5, v3, s0
5950 ; GFX940-NEXT: v_not_b32_e32 v5, v5
5951 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
5952 ; GFX940-NEXT: .LBB24_1: ; %atomicrmw.start
5953 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
5954 ; GFX940-NEXT: s_waitcnt vmcnt(0)
5955 ; GFX940-NEXT: v_mov_b32_e32 v7, v4
5956 ; GFX940-NEXT: v_lshrrev_b32_e32 v4, v3, v7
5957 ; GFX940-NEXT: v_sub_f16_e32 v4, v4, v2
5958 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, v3, v4
5959 ; GFX940-NEXT: v_and_or_b32 v6, v7, v5, v4
5960 ; GFX940-NEXT: buffer_wbl2 sc1
5961 ; GFX940-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off sc0
5962 ; GFX940-NEXT: s_waitcnt vmcnt(0)
5963 ; GFX940-NEXT: buffer_inv sc1
5964 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
5965 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
5966 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
5967 ; GFX940-NEXT: s_cbranch_execnz .LBB24_1
5968 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
5969 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
5970 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v4
5971 ; GFX940-NEXT: s_setpc_b64 s[30:31]
5972 ;
5973 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_neg:
5974 ; GFX11: ; %bb.0:
5975 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5976 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
5977 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
5978 ; GFX11-NEXT: s_mov_b32 s0, 0
5979 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
5980 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
5981 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
5982 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
5983 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5984 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
5985 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
5986 ; GFX11-NEXT: v_not_b32_e32 v4, v4
5987 ; GFX11-NEXT: .LBB24_1: ; %atomicrmw.start
5988 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
5989 ; GFX11-NEXT: s_waitcnt vmcnt(0)
5990 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
5991 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5992 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5993 ; GFX11-NEXT: v_sub_f16_e32 v5, v5, v2
5994 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5995 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
5996 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5997 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
5998 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
5999 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
6000 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
6001 ; GFX11-NEXT: s_waitcnt vmcnt(0)
6002 ; GFX11-NEXT: buffer_gl1_inv
6003 ; GFX11-NEXT: buffer_gl0_inv
6004 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
6005 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
6006 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
6007 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
6008 ; GFX11-NEXT: s_cbranch_execnz .LBB24_1
6009 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
6010 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
6011 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
6012 ; GFX11-NEXT: s_setpc_b64 s[30:31]
6013 ;
6014 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_neg:
6015 ; GFX10: ; %bb.0:
6016 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6017 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
6018 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
6019 ; GFX10-NEXT: s_mov_b32 s4, 0
6020 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
6021 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
6022 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
6023 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
6024 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
6025 ; GFX10-NEXT: v_not_b32_e32 v4, v4
6026 ; GFX10-NEXT: .LBB24_1: ; %atomicrmw.start
6027 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
6028 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6029 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
6030 ; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
6031 ; GFX10-NEXT: v_sub_f16_e32 v5, v5, v2
6032 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
6033 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
6034 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
6035 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
6036 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6037 ; GFX10-NEXT: buffer_gl1_inv
6038 ; GFX10-NEXT: buffer_gl0_inv
6039 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
6040 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
6041 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
6042 ; GFX10-NEXT: s_cbranch_execnz .LBB24_1
6043 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
6044 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
6045 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
6046 ; GFX10-NEXT: s_setpc_b64 s[30:31]
6047 ;
6048 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_neg:
6049 ; GFX90A: ; %bb.0:
6050 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6051 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
6052 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
6053 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
6054 ; GFX90A-NEXT: global_load_dword v4, v[0:1], off
6055 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
6056 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
6057 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
6058 ; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v3, s4
6059 ; GFX90A-NEXT: v_not_b32_e32 v5, v5
6060 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
6061 ; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
6062 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
6063 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6064 ; GFX90A-NEXT: v_mov_b32_e32 v7, v4
6065 ; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v7
6066 ; GFX90A-NEXT: v_sub_f16_e32 v4, v4, v2
6067 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
6068 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v5, v4
6069 ; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off glc
6070 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6071 ; GFX90A-NEXT: buffer_wbinvl1
6072 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
6073 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6074 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
6075 ; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
6076 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
6077 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
6078 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v4
6079 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
6080 ;
6081 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_neg:
6082 ; GFX908: ; %bb.0:
6083 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6084 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
6085 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
6086 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
6087 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
6088 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
6089 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
6090 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
6091 ; GFX908-NEXT: v_lshlrev_b32_e64 v5, v3, s4
6092 ; GFX908-NEXT: v_not_b32_e32 v5, v5
6093 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
6094 ; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start
6095 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
6096 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6097 ; GFX908-NEXT: v_mov_b32_e32 v7, v4
6098 ; GFX908-NEXT: v_lshrrev_b32_e32 v4, v3, v7
6099 ; GFX908-NEXT: v_sub_f16_e32 v4, v4, v2
6100 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, v3, v4
6101 ; GFX908-NEXT: v_and_or_b32 v6, v7, v5, v4
6102 ; GFX908-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off glc
6103 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6104 ; GFX908-NEXT: buffer_wbinvl1
6105 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
6106 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6107 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
6108 ; GFX908-NEXT: s_cbranch_execnz .LBB24_1
6109 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
6110 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
6111 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v4
6112 ; GFX908-NEXT: s_setpc_b64 s[30:31]
6113 ;
6114 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_neg:
6115 ; GFX8: ; %bb.0:
6116 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6117 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
6118 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
6119 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
6120 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
6121 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
6122 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
6123 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
6124 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
6125 ; GFX8-NEXT: v_not_b32_e32 v4, v4
6126 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
6127 ; GFX8-NEXT: .LBB24_1: ; %atomicrmw.start
6128 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6129 ; GFX8-NEXT: s_waitcnt vmcnt(0)
6130 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
6131 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
6132 ; GFX8-NEXT: v_sub_f16_e32 v5, v5, v2
6133 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
6134 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
6135 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
6136 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
6137 ; GFX8-NEXT: s_waitcnt vmcnt(0)
6138 ; GFX8-NEXT: buffer_wbinvl1
6139 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
6140 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6141 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
6142 ; GFX8-NEXT: s_cbranch_execnz .LBB24_1
6143 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6144 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
6145 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
6146 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6147 ;
6148 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_neg:
6149 ; GFX7: ; %bb.0:
6150 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6151 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
6152 ; GFX7-NEXT: s_mov_b32 s6, 0
6153 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
6154 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
6155 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
6156 ; GFX7-NEXT: s_mov_b32 s4, s6
6157 ; GFX7-NEXT: s_mov_b32 s5, s6
6158 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
6159 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
6160 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
6161 ; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
6162 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
6163 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
6164 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
6165 ; GFX7-NEXT: v_not_b32_e32 v8, v2
6166 ; GFX7-NEXT: .LBB24_1: ; %atomicrmw.start
6167 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
6168 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6169 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
6170 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
6171 ; GFX7-NEXT: v_and_b32_e32 v4, v3, v8
6172 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v7
6173 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
6174 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v6, v2
6175 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
6176 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
6177 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
6178 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
6179 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6180 ; GFX7-NEXT: buffer_wbinvl1
6181 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
6182 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
6183 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
6184 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
6185 ; GFX7-NEXT: s_cbranch_execnz .LBB24_1
6186 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6187 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
6188 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
6189 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
6190 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6191 ;
6192 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_neg:
6193 ; GFX6: ; %bb.0:
6194 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6195 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
6196 ; GFX6-NEXT: s_mov_b32 s6, 0
6197 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
6198 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
6199 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
6200 ; GFX6-NEXT: s_mov_b32 s4, s6
6201 ; GFX6-NEXT: s_mov_b32 s5, s6
6202 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
6203 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
6204 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
6205 ; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
6206 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
6207 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
6208 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
6209 ; GFX6-NEXT: v_not_b32_e32 v8, v2
6210 ; GFX6-NEXT: .LBB24_1: ; %atomicrmw.start
6211 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
6212 ; GFX6-NEXT: s_waitcnt vmcnt(0)
6213 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
6214 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
6215 ; GFX6-NEXT: s_waitcnt expcnt(0)
6216 ; GFX6-NEXT: v_and_b32_e32 v4, v3, v8
6217 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v7
6218 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
6219 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v6, v2
6220 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
6221 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
6222 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
6223 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
6224 ; GFX6-NEXT: s_waitcnt vmcnt(0)
6225 ; GFX6-NEXT: buffer_wbinvl1
6226 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
6227 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
6228 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
6229 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
6230 ; GFX6-NEXT: s_cbranch_execnz .LBB24_1
6231 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
6232 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
6233 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
6234 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
6235 ; GFX6-NEXT: s_waitcnt expcnt(0)
6236 ; GFX6-NEXT: s_setpc_b64 s[30:31]
6237 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 -1024
6238 %result = atomicrmw fsub ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst
6239 ret half %result
6240 }
6242 define void @global_agent_atomic_fsub_noret_f16(ptr addrspace(1) %ptr, half %val) #0 {
6243 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f16:
6244 ; GFX12: ; %bb.0:
6245 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
6246 ; GFX12-NEXT: s_wait_expcnt 0x0
6247 ; GFX12-NEXT: s_wait_samplecnt 0x0
6248 ; GFX12-NEXT: s_wait_bvhcnt 0x0
6249 ; GFX12-NEXT: s_wait_kmcnt 0x0
6250 ; GFX12-NEXT: v_mov_b32_e32 v3, v0
6251 ; GFX12-NEXT: s_mov_b32 s0, 0
6252 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
6253 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
6254 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
6255 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off
6256 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6257 ; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
6258 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
6259 ; GFX12-NEXT: v_not_b32_e32 v6, v3
6260 ; GFX12-NEXT: .LBB25_1: ; %atomicrmw.start
6261 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
6262 ; GFX12-NEXT: s_wait_loadcnt 0x0
6263 ; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
6264 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6265 ; GFX12-NEXT: v_sub_f16_e32 v3, v3, v2
6266 ; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
6267 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6268 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
6269 ; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
6270 ; GFX12-NEXT: s_wait_storecnt 0x0
6271 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
6272 ; GFX12-NEXT: s_wait_loadcnt 0x0
6273 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
6274 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
6275 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
6276 ; GFX12-NEXT: s_wait_alu 0xfffe
6277 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
6278 ; GFX12-NEXT: s_wait_alu 0xfffe
6279 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
6280 ; GFX12-NEXT: s_cbranch_execnz .LBB25_1
6281 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
6282 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
6283 ; GFX12-NEXT: s_wait_alu 0xfffe
6284 ; GFX12-NEXT: s_setpc_b64 s[30:31]
6285 ;
6286 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f16:
6287 ; GFX940: ; %bb.0:
6288 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6289 ; GFX940-NEXT: v_mov_b32_e32 v3, v0
6290 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v3
6291 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
6292 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v3
6293 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
6294 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
6295 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
6296 ; GFX940-NEXT: v_not_b32_e32 v6, v4
6297 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
6298 ; GFX940-NEXT: .LBB25_1: ; %atomicrmw.start
6299 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
6300 ; GFX940-NEXT: s_waitcnt vmcnt(0)
6301 ; GFX940-NEXT: v_lshrrev_b32_e32 v4, v3, v5
6302 ; GFX940-NEXT: v_sub_f16_e32 v4, v4, v2
6303 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, v3, v4
6304 ; GFX940-NEXT: v_and_or_b32 v4, v5, v6, v4
6305 ; GFX940-NEXT: buffer_wbl2 sc1
6306 ; GFX940-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0
6307 ; GFX940-NEXT: s_waitcnt vmcnt(0)
6308 ; GFX940-NEXT: buffer_inv sc1
6309 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
6310 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
6311 ; GFX940-NEXT: v_mov_b32_e32 v5, v4
6312 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
6313 ; GFX940-NEXT: s_cbranch_execnz .LBB25_1
6314 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
6315 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
6316 ; GFX940-NEXT: s_setpc_b64 s[30:31]
6317 ;
6318 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f16:
6319 ; GFX11: ; %bb.0:
6320 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6321 ; GFX11-NEXT: v_mov_b32_e32 v3, v0
6322 ; GFX11-NEXT: s_mov_b32 s0, 0
6323 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
6324 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
6325 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
6326 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off
6327 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6328 ; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
6329 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
6330 ; GFX11-NEXT: v_not_b32_e32 v6, v3
6331 ; GFX11-NEXT: .LBB25_1: ; %atomicrmw.start
6332 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
6333 ; GFX11-NEXT: s_waitcnt vmcnt(0)
6334 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
6335 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6336 ; GFX11-NEXT: v_sub_f16_e32 v3, v3, v2
6337 ; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
6338 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6339 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
6340 ; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
6341 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
6342 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
6343 ; GFX11-NEXT: s_waitcnt vmcnt(0)
6344 ; GFX11-NEXT: buffer_gl1_inv
6345 ; GFX11-NEXT: buffer_gl0_inv
6346 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
6347 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
6348 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
6349 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
6350 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
6351 ; GFX11-NEXT: s_cbranch_execnz .LBB25_1
6352 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
6353 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
6354 ; GFX11-NEXT: s_setpc_b64 s[30:31]
6355 ;
6356 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f16:
6357 ; GFX10: ; %bb.0:
6358 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6359 ; GFX10-NEXT: v_mov_b32_e32 v3, v0
6360 ; GFX10-NEXT: s_mov_b32 s4, 0
6361 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
6362 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
6363 ; GFX10-NEXT: global_load_dword v4, v[0:1], off
6364 ; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6365 ; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
6366 ; GFX10-NEXT: v_not_b32_e32 v6, v3
6367 ; GFX10-NEXT: .LBB25_1: ; %atomicrmw.start
6368 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
6369 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6370 ; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
6371 ; GFX10-NEXT: v_sub_f16_e32 v3, v3, v2
6372 ; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
6373 ; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
6374 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
6375 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
6376 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6377 ; GFX10-NEXT: buffer_gl1_inv
6378 ; GFX10-NEXT: buffer_gl0_inv
6379 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
6380 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
6381 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
6382 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
6383 ; GFX10-NEXT: s_cbranch_execnz .LBB25_1
6384 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
6385 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
6386 ; GFX10-NEXT: s_setpc_b64 s[30:31]
6387 ;
6388 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f16:
6389 ; GFX90A: ; %bb.0:
6390 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6391 ; GFX90A-NEXT: v_mov_b32_e32 v3, v0
6392 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
6393 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
6394 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
6395 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
6396 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
6397 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
6398 ; GFX90A-NEXT: v_not_b32_e32 v6, v4
6399 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
6400 ; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
6401 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
6402 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6403 ; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
6404 ; GFX90A-NEXT: v_sub_f16_e32 v4, v4, v2
6405 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
6406 ; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
6407 ; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
6408 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6409 ; GFX90A-NEXT: buffer_wbinvl1
6410 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
6411 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6412 ; GFX90A-NEXT: v_mov_b32_e32 v5, v4
6413 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
6414 ; GFX90A-NEXT: s_cbranch_execnz .LBB25_1
6415 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
6416 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
6417 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
6418 ;
6419 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f16:
6420 ; GFX908: ; %bb.0:
6421 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6422 ; GFX908-NEXT: v_mov_b32_e32 v3, v0
6423 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
6424 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
6425 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
6426 ; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6427 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
6428 ; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
6429 ; GFX908-NEXT: v_not_b32_e32 v6, v3
6430 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
6431 ; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start
6432 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
6433 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6434 ; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
6435 ; GFX908-NEXT: v_sub_f16_e32 v3, v3, v2
6436 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
6437 ; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
6438 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
6439 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6440 ; GFX908-NEXT: buffer_wbinvl1
6441 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
6442 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6443 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
6444 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
6445 ; GFX908-NEXT: s_cbranch_execnz .LBB25_1
6446 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
6447 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
6448 ; GFX908-NEXT: s_setpc_b64 s[30:31]
6449 ;
6450 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f16:
6451 ; GFX8: ; %bb.0:
6452 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6453 ; GFX8-NEXT: v_mov_b32_e32 v3, v0
6454 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
6455 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
6456 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
6457 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6458 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
6459 ; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
6460 ; GFX8-NEXT: v_not_b32_e32 v6, v3
6461 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
6462 ; GFX8-NEXT: .LBB25_1: ; %atomicrmw.start
6463 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6464 ; GFX8-NEXT: s_waitcnt vmcnt(0)
6465 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
6466 ; GFX8-NEXT: v_sub_f16_e32 v3, v3, v2
6467 ; GFX8-NEXT: v_and_b32_e32 v7, v4, v6
6468 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
6469 ; GFX8-NEXT: v_or_b32_e32 v3, v7, v3
6470 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
6471 ; GFX8-NEXT: s_waitcnt vmcnt(0)
6472 ; GFX8-NEXT: buffer_wbinvl1
6473 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
6474 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6475 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
6476 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
6477 ; GFX8-NEXT: s_cbranch_execnz .LBB25_1
6478 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6479 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
6480 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6481 ;
6482 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f16:
6483 ; GFX7: ; %bb.0:
6484 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6485 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
6486 ; GFX7-NEXT: s_mov_b32 s6, 0
6487 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
6488 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
6489 ; GFX7-NEXT: s_mov_b32 s4, s6
6490 ; GFX7-NEXT: s_mov_b32 s5, s6
6491 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
6492 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v2
6493 ; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
6494 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
6495 ; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
6496 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
6497 ; GFX7-NEXT: v_not_b32_e32 v6, v3
6498 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
6499 ; GFX7-NEXT: .LBB25_1: ; %atomicrmw.start
6500 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
6501 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6502 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, v2, v4
6503 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
6504 ; GFX7-NEXT: v_and_b32_e32 v7, v4, v6
6505 ; GFX7-NEXT: v_sub_f32_e32 v3, v3, v5
6506 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
6507 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, v2, v3
6508 ; GFX7-NEXT: v_or_b32_e32 v3, v7, v3
6509 ; GFX7-NEXT: v_mov_b32_e32 v8, v4
6510 ; GFX7-NEXT: v_mov_b32_e32 v7, v3
6511 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
6512 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6513 ; GFX7-NEXT: buffer_wbinvl1
6514 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v4
6515 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
6516 ; GFX7-NEXT: v_mov_b32_e32 v4, v7
6517 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
6518 ; GFX7-NEXT: s_cbranch_execnz .LBB25_1
6519 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6520 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
6521 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6522 ;
6523 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f16:
6524 ; GFX6: ; %bb.0:
6525 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6526 ; GFX6-NEXT: v_mov_b32_e32 v3, v0
6527 ; GFX6-NEXT: s_mov_b32 s6, 0
6528 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v3
6529 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
6530 ; GFX6-NEXT: s_mov_b32 s4, s6
6531 ; GFX6-NEXT: s_mov_b32 s5, s6
6532 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
6533 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v2
6534 ; GFX6-NEXT: v_and_b32_e32 v2, 3, v3
6535 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 3, v2
6536 ; GFX6-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
6537 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
6538 ; GFX6-NEXT: v_not_b32_e32 v6, v3
6539 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
6540 ; GFX6-NEXT: .LBB25_1: ; %atomicrmw.start
6541 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
6542 ; GFX6-NEXT: s_waitcnt vmcnt(0)
6543 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, v2, v4
6544 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
6545 ; GFX6-NEXT: s_waitcnt expcnt(0)
6546 ; GFX6-NEXT: v_and_b32_e32 v7, v4, v6
6547 ; GFX6-NEXT: v_sub_f32_e32 v3, v3, v5
6548 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
6549 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, v2, v3
6550 ; GFX6-NEXT: v_or_b32_e32 v3, v7, v3
6551 ; GFX6-NEXT: v_mov_b32_e32 v8, v4
6552 ; GFX6-NEXT: v_mov_b32_e32 v7, v3
6553 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
6554 ; GFX6-NEXT: s_waitcnt vmcnt(0)
6555 ; GFX6-NEXT: buffer_wbinvl1
6556 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v4
6557 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
6558 ; GFX6-NEXT: v_mov_b32_e32 v4, v7
6559 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
6560 ; GFX6-NEXT: s_cbranch_execnz .LBB25_1
6561 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
6562 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
6563 ; GFX6-NEXT: s_waitcnt expcnt(0)
6564 ; GFX6-NEXT: s_setpc_b64 s[30:31]
6565 %unused = atomicrmw fsub ptr addrspace(1) %ptr, half %val syncscope("agent") seq_cst
6566 ret void
6567 }
6569 define void @global_agent_atomic_fsub_noret_f16__offset12b_pos(ptr addrspace(1) %ptr, half %val) #0 {
6570 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_pos:
6571 ; GFX12: ; %bb.0:
6572 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
6573 ; GFX12-NEXT: s_wait_expcnt 0x0
6574 ; GFX12-NEXT: s_wait_samplecnt 0x0
6575 ; GFX12-NEXT: s_wait_bvhcnt 0x0
6576 ; GFX12-NEXT: s_wait_kmcnt 0x0
6577 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
6578 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
6579 ; GFX12-NEXT: s_mov_b32 s0, 0
6580 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
6581 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
6582 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
6583 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off
6584 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6585 ; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
6586 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
6587 ; GFX12-NEXT: v_not_b32_e32 v6, v3
6588 ; GFX12-NEXT: .LBB26_1: ; %atomicrmw.start
6589 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
6590 ; GFX12-NEXT: s_wait_loadcnt 0x0
6591 ; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
6592 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6593 ; GFX12-NEXT: v_sub_f16_e32 v3, v3, v2
6594 ; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
6595 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6596 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
6597 ; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
6598 ; GFX12-NEXT: s_wait_storecnt 0x0
6599 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
6600 ; GFX12-NEXT: s_wait_loadcnt 0x0
6601 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
6602 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
6603 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
6604 ; GFX12-NEXT: s_wait_alu 0xfffe
6605 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
6606 ; GFX12-NEXT: s_wait_alu 0xfffe
6607 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
6608 ; GFX12-NEXT: s_cbranch_execnz .LBB26_1
6609 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
6610 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
6611 ; GFX12-NEXT: s_wait_alu 0xfffe
6612 ; GFX12-NEXT: s_setpc_b64 s[30:31]
6613 ;
6614 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_pos:
6615 ; GFX940: ; %bb.0:
6616 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6617 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
6618 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
6619 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
6620 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
6621 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
6622 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v4
6623 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
6624 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
6625 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
6626 ; GFX940-NEXT: v_not_b32_e32 v6, v4
6627 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
6628 ; GFX940-NEXT: .LBB26_1: ; %atomicrmw.start
6629 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
6630 ; GFX940-NEXT: s_waitcnt vmcnt(0)
6631 ; GFX940-NEXT: v_lshrrev_b32_e32 v4, v3, v5
6632 ; GFX940-NEXT: v_sub_f16_e32 v4, v4, v2
6633 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, v3, v4
6634 ; GFX940-NEXT: v_and_or_b32 v4, v5, v6, v4
6635 ; GFX940-NEXT: buffer_wbl2 sc1
6636 ; GFX940-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0
6637 ; GFX940-NEXT: s_waitcnt vmcnt(0)
6638 ; GFX940-NEXT: buffer_inv sc1
6639 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
6640 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
6641 ; GFX940-NEXT: v_mov_b32_e32 v5, v4
6642 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
6643 ; GFX940-NEXT: s_cbranch_execnz .LBB26_1
6644 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
6645 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
6646 ; GFX940-NEXT: s_setpc_b64 s[30:31]
6647 ;
6648 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_pos:
6649 ; GFX11: ; %bb.0:
6650 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6651 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
6652 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
6653 ; GFX11-NEXT: s_mov_b32 s0, 0
6654 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
6655 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
6656 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
6657 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off
6658 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6659 ; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
6660 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
6661 ; GFX11-NEXT: v_not_b32_e32 v6, v3
6662 ; GFX11-NEXT: .LBB26_1: ; %atomicrmw.start
6663 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
6664 ; GFX11-NEXT: s_waitcnt vmcnt(0)
6665 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
6666 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6667 ; GFX11-NEXT: v_sub_f16_e32 v3, v3, v2
6668 ; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
6669 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6670 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
6671 ; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
6672 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
6673 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
6674 ; GFX11-NEXT: s_waitcnt vmcnt(0)
6675 ; GFX11-NEXT: buffer_gl1_inv
6676 ; GFX11-NEXT: buffer_gl0_inv
6677 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
6678 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
6679 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
6680 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
6681 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
6682 ; GFX11-NEXT: s_cbranch_execnz .LBB26_1
6683 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
6684 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
6685 ; GFX11-NEXT: s_setpc_b64 s[30:31]
6686 ;
6687 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_pos:
6688 ; GFX10: ; %bb.0:
6689 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6690 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
6691 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
6692 ; GFX10-NEXT: s_mov_b32 s4, 0
6693 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
6694 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
6695 ; GFX10-NEXT: global_load_dword v4, v[0:1], off
6696 ; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6697 ; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
6698 ; GFX10-NEXT: v_not_b32_e32 v6, v3
6699 ; GFX10-NEXT: .LBB26_1: ; %atomicrmw.start
6700 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
6701 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6702 ; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
6703 ; GFX10-NEXT: v_sub_f16_e32 v3, v3, v2
6704 ; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
6705 ; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
6706 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
6707 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
6708 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6709 ; GFX10-NEXT: buffer_gl1_inv
6710 ; GFX10-NEXT: buffer_gl0_inv
6711 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
6712 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
6713 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
6714 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
6715 ; GFX10-NEXT: s_cbranch_execnz .LBB26_1
6716 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
6717 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
6718 ; GFX10-NEXT: s_setpc_b64 s[30:31]
6719 ;
6720 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_pos:
6721 ; GFX90A: ; %bb.0:
6722 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6723 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
6724 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
6725 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
6726 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
6727 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
6728 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
6729 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
6730 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
6731 ; GFX90A-NEXT: v_not_b32_e32 v6, v4
6732 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
6733 ; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
6734 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
6735 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6736 ; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
6737 ; GFX90A-NEXT: v_sub_f16_e32 v4, v4, v2
6738 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
6739 ; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
6740 ; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
6741 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6742 ; GFX90A-NEXT: buffer_wbinvl1
6743 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
6744 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6745 ; GFX90A-NEXT: v_mov_b32_e32 v5, v4
6746 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
6747 ; GFX90A-NEXT: s_cbranch_execnz .LBB26_1
6748 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
6749 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
6750 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
6751 ;
6752 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_pos:
6753 ; GFX908: ; %bb.0:
6754 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6755 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
6756 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
6757 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
6758 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
6759 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
6760 ; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6761 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
6762 ; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
6763 ; GFX908-NEXT: v_not_b32_e32 v6, v3
6764 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
6765 ; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
6766 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
6767 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6768 ; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
6769 ; GFX908-NEXT: v_sub_f16_e32 v3, v3, v2
6770 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
6771 ; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
6772 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
6773 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6774 ; GFX908-NEXT: buffer_wbinvl1
6775 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
6776 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6777 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
6778 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
6779 ; GFX908-NEXT: s_cbranch_execnz .LBB26_1
6780 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
6781 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
6782 ; GFX908-NEXT: s_setpc_b64 s[30:31]
6783 ;
6784 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_pos:
6785 ; GFX8: ; %bb.0:
6786 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6787 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
6788 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
6789 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
6790 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
6791 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
6792 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6793 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
6794 ; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
6795 ; GFX8-NEXT: v_not_b32_e32 v6, v3
6796 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
6797 ; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start
6798 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6799 ; GFX8-NEXT: s_waitcnt vmcnt(0)
6800 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
6801 ; GFX8-NEXT: v_sub_f16_e32 v3, v3, v2
6802 ; GFX8-NEXT: v_and_b32_e32 v7, v4, v6
6803 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
6804 ; GFX8-NEXT: v_or_b32_e32 v3, v7, v3
6805 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
6806 ; GFX8-NEXT: s_waitcnt vmcnt(0)
6807 ; GFX8-NEXT: buffer_wbinvl1
6808 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
6809 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6810 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
6811 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
6812 ; GFX8-NEXT: s_cbranch_execnz .LBB26_1
6813 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6814 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
6815 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6816 ;
6817 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_pos:
6818 ; GFX7: ; %bb.0:
6819 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6820 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
6821 ; GFX7-NEXT: s_mov_b32 s6, 0
6822 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
6823 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
6824 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
6825 ; GFX7-NEXT: s_mov_b32 s4, s6
6826 ; GFX7-NEXT: s_mov_b32 s5, s6
6827 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
6828 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
6829 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
6830 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
6831 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
6832 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
6833 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
6834 ; GFX7-NEXT: v_not_b32_e32 v6, v2
6835 ; GFX7-NEXT: .LBB26_1: ; %atomicrmw.start
6836 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
6837 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6838 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
6839 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
6840 ; GFX7-NEXT: v_and_b32_e32 v7, v3, v6
6841 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v5
6842 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
6843 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v4, v2
6844 ; GFX7-NEXT: v_or_b32_e32 v2, v7, v2
6845 ; GFX7-NEXT: v_mov_b32_e32 v8, v3
6846 ; GFX7-NEXT: v_mov_b32_e32 v7, v2
6847 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
6848 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6849 ; GFX7-NEXT: buffer_wbinvl1
6850 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
6851 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
6852 ; GFX7-NEXT: v_mov_b32_e32 v3, v7
6853 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
6854 ; GFX7-NEXT: s_cbranch_execnz .LBB26_1
6855 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6856 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
6857 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6858 ;
6859 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_pos:
6860 ; GFX6: ; %bb.0:
6861 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6862 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
6863 ; GFX6-NEXT: s_mov_b32 s6, 0
6864 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
6865 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
6866 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
6867 ; GFX6-NEXT: s_mov_b32 s4, s6
6868 ; GFX6-NEXT: s_mov_b32 s5, s6
6869 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
6870 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
6871 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
6872 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
6873 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
6874 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
6875 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
6876 ; GFX6-NEXT: v_not_b32_e32 v6, v2
6877 ; GFX6-NEXT: .LBB26_1: ; %atomicrmw.start
6878 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
6879 ; GFX6-NEXT: s_waitcnt vmcnt(0)
6880 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
6881 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
6882 ; GFX6-NEXT: s_waitcnt expcnt(0)
6883 ; GFX6-NEXT: v_and_b32_e32 v7, v3, v6
6884 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v5
6885 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
6886 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v4, v2
6887 ; GFX6-NEXT: v_or_b32_e32 v2, v7, v2
6888 ; GFX6-NEXT: v_mov_b32_e32 v8, v3
6889 ; GFX6-NEXT: v_mov_b32_e32 v7, v2
6890 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
6891 ; GFX6-NEXT: s_waitcnt vmcnt(0)
6892 ; GFX6-NEXT: buffer_wbinvl1
6893 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
6894 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
6895 ; GFX6-NEXT: v_mov_b32_e32 v3, v7
6896 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
6897 ; GFX6-NEXT: s_cbranch_execnz .LBB26_1
6898 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
6899 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
6900 ; GFX6-NEXT: s_waitcnt expcnt(0)
6901 ; GFX6-NEXT: s_setpc_b64 s[30:31]
6902 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
6903 %unused = atomicrmw fsub ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst
6904 ret void
6905 }
6907 define void @global_agent_atomic_fsub_noret_f16__offset12b_neg(ptr addrspace(1) %ptr, half %val) #0 {
6908 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_neg:
6909 ; GFX12: ; %bb.0:
6910 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
6911 ; GFX12-NEXT: s_wait_expcnt 0x0
6912 ; GFX12-NEXT: s_wait_samplecnt 0x0
6913 ; GFX12-NEXT: s_wait_bvhcnt 0x0
6914 ; GFX12-NEXT: s_wait_kmcnt 0x0
6915 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
6916 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
6917 ; GFX12-NEXT: s_mov_b32 s0, 0
6918 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
6919 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
6920 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
6921 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off
6922 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6923 ; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
6924 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
6925 ; GFX12-NEXT: v_not_b32_e32 v6, v3
6926 ; GFX12-NEXT: .LBB27_1: ; %atomicrmw.start
6927 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
6928 ; GFX12-NEXT: s_wait_loadcnt 0x0
6929 ; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
6930 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6931 ; GFX12-NEXT: v_sub_f16_e32 v3, v3, v2
6932 ; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
6933 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6934 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
6935 ; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
6936 ; GFX12-NEXT: s_wait_storecnt 0x0
6937 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
6938 ; GFX12-NEXT: s_wait_loadcnt 0x0
6939 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
6940 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
6941 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
6942 ; GFX12-NEXT: s_wait_alu 0xfffe
6943 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
6944 ; GFX12-NEXT: s_wait_alu 0xfffe
6945 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
6946 ; GFX12-NEXT: s_cbranch_execnz .LBB27_1
6947 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
6948 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
6949 ; GFX12-NEXT: s_wait_alu 0xfffe
6950 ; GFX12-NEXT: s_setpc_b64 s[30:31]
6951 ;
6952 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_neg:
6953 ; GFX940: ; %bb.0:
6954 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6955 ; GFX940-NEXT: s_movk_i32 s0, 0xf800
6956 ; GFX940-NEXT: s_mov_b32 s1, -1
6957 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
6958 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
6959 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
6960 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
6961 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v4
6962 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
6963 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
6964 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
6965 ; GFX940-NEXT: v_not_b32_e32 v6, v4
6966 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
6967 ; GFX940-NEXT: .LBB27_1: ; %atomicrmw.start
6968 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
6969 ; GFX940-NEXT: s_waitcnt vmcnt(0)
6970 ; GFX940-NEXT: v_lshrrev_b32_e32 v4, v3, v5
6971 ; GFX940-NEXT: v_sub_f16_e32 v4, v4, v2
6972 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, v3, v4
6973 ; GFX940-NEXT: v_and_or_b32 v4, v5, v6, v4
6974 ; GFX940-NEXT: buffer_wbl2 sc1
6975 ; GFX940-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0
6976 ; GFX940-NEXT: s_waitcnt vmcnt(0)
6977 ; GFX940-NEXT: buffer_inv sc1
6978 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
6979 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
6980 ; GFX940-NEXT: v_mov_b32_e32 v5, v4
6981 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
6982 ; GFX940-NEXT: s_cbranch_execnz .LBB27_1
6983 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
6984 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
6985 ; GFX940-NEXT: s_setpc_b64 s[30:31]
6986 ;
6987 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_neg:
6988 ; GFX11: ; %bb.0:
6989 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6990 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
6991 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
6992 ; GFX11-NEXT: s_mov_b32 s0, 0
6993 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
6994 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
6995 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
6996 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off
6997 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6998 ; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
6999 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
7000 ; GFX11-NEXT: v_not_b32_e32 v6, v3
7001 ; GFX11-NEXT: .LBB27_1: ; %atomicrmw.start
7002 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
7003 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7004 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
7005 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7006 ; GFX11-NEXT: v_sub_f16_e32 v3, v3, v2
7007 ; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
7008 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7009 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
7010 ; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
7011 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
7012 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
7013 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7014 ; GFX11-NEXT: buffer_gl1_inv
7015 ; GFX11-NEXT: buffer_gl0_inv
7016 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
7017 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
7018 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
7019 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
7020 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
7021 ; GFX11-NEXT: s_cbranch_execnz .LBB27_1
7022 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
7023 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
7024 ; GFX11-NEXT: s_setpc_b64 s[30:31]
7025 ;
7026 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_neg:
7027 ; GFX10: ; %bb.0:
7028 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7029 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
7030 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
7031 ; GFX10-NEXT: s_mov_b32 s4, 0
7032 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
7033 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
7034 ; GFX10-NEXT: global_load_dword v4, v[0:1], off
7035 ; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
7036 ; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
7037 ; GFX10-NEXT: v_not_b32_e32 v6, v3
7038 ; GFX10-NEXT: .LBB27_1: ; %atomicrmw.start
7039 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
7040 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7041 ; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
7042 ; GFX10-NEXT: v_sub_f16_e32 v3, v3, v2
7043 ; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
7044 ; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
7045 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
7046 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
7047 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7048 ; GFX10-NEXT: buffer_gl1_inv
7049 ; GFX10-NEXT: buffer_gl0_inv
7050 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
7051 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
7052 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
7053 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
7054 ; GFX10-NEXT: s_cbranch_execnz .LBB27_1
7055 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
7056 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
7057 ; GFX10-NEXT: s_setpc_b64 s[30:31]
7058 ;
7059 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_neg:
7060 ; GFX90A: ; %bb.0:
7061 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7062 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
7063 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
7064 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
7065 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
7066 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
7067 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7068 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
7069 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
7070 ; GFX90A-NEXT: v_not_b32_e32 v6, v4
7071 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
7072 ; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
7073 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
7074 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7075 ; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
7076 ; GFX90A-NEXT: v_sub_f16_e32 v4, v4, v2
7077 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
7078 ; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
7079 ; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
7080 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7081 ; GFX90A-NEXT: buffer_wbinvl1
7082 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
7083 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7084 ; GFX90A-NEXT: v_mov_b32_e32 v5, v4
7085 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
7086 ; GFX90A-NEXT: s_cbranch_execnz .LBB27_1
7087 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
7088 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
7089 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
7090 ;
7091 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_neg:
7092 ; GFX908: ; %bb.0:
7093 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7094 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
7095 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
7096 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
7097 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
7098 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
7099 ; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
7100 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
7101 ; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
7102 ; GFX908-NEXT: v_not_b32_e32 v6, v3
7103 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
7104 ; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
7105 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
7106 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7107 ; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
7108 ; GFX908-NEXT: v_sub_f16_e32 v3, v3, v2
7109 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
7110 ; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
7111 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
7112 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7113 ; GFX908-NEXT: buffer_wbinvl1
7114 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
7115 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7116 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
7117 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
7118 ; GFX908-NEXT: s_cbranch_execnz .LBB27_1
7119 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
7120 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
7121 ; GFX908-NEXT: s_setpc_b64 s[30:31]
7122 ;
7123 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_neg:
7124 ; GFX8: ; %bb.0:
7125 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7126 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
7127 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
7128 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
7129 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
7130 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
7131 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
7132 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
7133 ; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
7134 ; GFX8-NEXT: v_not_b32_e32 v6, v3
7135 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
7136 ; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start
7137 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7138 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7139 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
7140 ; GFX8-NEXT: v_sub_f16_e32 v3, v3, v2
7141 ; GFX8-NEXT: v_and_b32_e32 v7, v4, v6
7142 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
7143 ; GFX8-NEXT: v_or_b32_e32 v3, v7, v3
7144 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
7145 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7146 ; GFX8-NEXT: buffer_wbinvl1
7147 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
7148 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7149 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
7150 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
7151 ; GFX8-NEXT: s_cbranch_execnz .LBB27_1
7152 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7153 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
7154 ; GFX8-NEXT: s_setpc_b64 s[30:31]
7155 ;
7156 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_neg:
7157 ; GFX7: ; %bb.0:
7158 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7159 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
7160 ; GFX7-NEXT: s_mov_b32 s6, 0
7161 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
7162 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
7163 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
7164 ; GFX7-NEXT: s_mov_b32 s4, s6
7165 ; GFX7-NEXT: s_mov_b32 s5, s6
7166 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
7167 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
7168 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
7169 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
7170 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
7171 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
7172 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
7173 ; GFX7-NEXT: v_not_b32_e32 v6, v2
7174 ; GFX7-NEXT: .LBB27_1: ; %atomicrmw.start
7175 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7176 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7177 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
7178 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
7179 ; GFX7-NEXT: v_and_b32_e32 v7, v3, v6
7180 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v5
7181 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
7182 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v4, v2
7183 ; GFX7-NEXT: v_or_b32_e32 v2, v7, v2
7184 ; GFX7-NEXT: v_mov_b32_e32 v8, v3
7185 ; GFX7-NEXT: v_mov_b32_e32 v7, v2
7186 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
7187 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7188 ; GFX7-NEXT: buffer_wbinvl1
7189 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
7190 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
7191 ; GFX7-NEXT: v_mov_b32_e32 v3, v7
7192 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
7193 ; GFX7-NEXT: s_cbranch_execnz .LBB27_1
7194 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
7195 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
7196 ; GFX7-NEXT: s_setpc_b64 s[30:31]
7197 ;
7198 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_neg:
7199 ; GFX6: ; %bb.0:
7200 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7201 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
7202 ; GFX6-NEXT: s_mov_b32 s6, 0
7203 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
7204 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
7205 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
7206 ; GFX6-NEXT: s_mov_b32 s4, s6
7207 ; GFX6-NEXT: s_mov_b32 s5, s6
7208 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
7209 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
7210 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
7211 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
7212 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
7213 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
7214 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
7215 ; GFX6-NEXT: v_not_b32_e32 v6, v2
7216 ; GFX6-NEXT: .LBB27_1: ; %atomicrmw.start
7217 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
7218 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7219 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
7220 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
7221 ; GFX6-NEXT: s_waitcnt expcnt(0)
7222 ; GFX6-NEXT: v_and_b32_e32 v7, v3, v6
7223 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v5
7224 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
7225 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v4, v2
7226 ; GFX6-NEXT: v_or_b32_e32 v2, v7, v2
7227 ; GFX6-NEXT: v_mov_b32_e32 v8, v3
7228 ; GFX6-NEXT: v_mov_b32_e32 v7, v2
7229 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
7230 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7231 ; GFX6-NEXT: buffer_wbinvl1
7232 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
7233 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
7234 ; GFX6-NEXT: v_mov_b32_e32 v3, v7
7235 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
7236 ; GFX6-NEXT: s_cbranch_execnz .LBB27_1
7237 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
7238 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
7239 ; GFX6-NEXT: s_waitcnt expcnt(0)
7240 ; GFX6-NEXT: s_setpc_b64 s[30:31]
7241 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 -1024
7242 %unused = atomicrmw fsub ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst
7243 ret void
7244 }
7246 define half @global_agent_atomic_fsub_ret_f16__offset12b_pos__align4(ptr addrspace(1) %ptr, half %val) #0 {
7247 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos__align4:
7248 ; GFX12: ; %bb.0:
7249 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
7250 ; GFX12-NEXT: s_wait_expcnt 0x0
7251 ; GFX12-NEXT: s_wait_samplecnt 0x0
7252 ; GFX12-NEXT: s_wait_bvhcnt 0x0
7253 ; GFX12-NEXT: s_wait_kmcnt 0x0
7254 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2046
7255 ; GFX12-NEXT: s_mov_b32 s0, 0
7256 ; GFX12-NEXT: .LBB28_1: ; %atomicrmw.start
7257 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
7258 ; GFX12-NEXT: s_wait_loadcnt 0x0
7259 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
7260 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7261 ; GFX12-NEXT: v_sub_f16_e32 v3, v4, v2
7262 ; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
7263 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
7264 ; GFX12-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
7265 ; GFX12-NEXT: s_wait_storecnt 0x0
7266 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
7267 ; GFX12-NEXT: s_wait_loadcnt 0x0
7268 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
7269 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
7270 ; GFX12-NEXT: s_wait_alu 0xfffe
7271 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
7272 ; GFX12-NEXT: s_wait_alu 0xfffe
7273 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
7274 ; GFX12-NEXT: s_cbranch_execnz .LBB28_1
7275 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
7276 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
7277 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
7278 ; GFX12-NEXT: s_wait_alu 0xfffe
7279 ; GFX12-NEXT: s_setpc_b64 s[30:31]
7280 ;
7281 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos__align4:
7282 ; GFX940: ; %bb.0:
7283 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7284 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2046
7285 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
7286 ; GFX940-NEXT: s_mov_b32 s2, 0xffff0000
7287 ; GFX940-NEXT: .LBB28_1: ; %atomicrmw.start
7288 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
7289 ; GFX940-NEXT: s_waitcnt vmcnt(0)
7290 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
7291 ; GFX940-NEXT: v_sub_f16_e32 v3, v5, v2
7292 ; GFX940-NEXT: v_and_or_b32 v4, v5, s2, v3
7293 ; GFX940-NEXT: buffer_wbl2 sc1
7294 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 sc0
7295 ; GFX940-NEXT: s_waitcnt vmcnt(0)
7296 ; GFX940-NEXT: buffer_inv sc1
7297 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
7298 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
7299 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
7300 ; GFX940-NEXT: s_cbranch_execnz .LBB28_1
7301 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
7302 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
7303 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
7304 ; GFX940-NEXT: s_setpc_b64 s[30:31]
7305 ;
7306 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos__align4:
7307 ; GFX11: ; %bb.0:
7308 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7309 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2046
7310 ; GFX11-NEXT: s_mov_b32 s0, 0
7311 ; GFX11-NEXT: .LBB28_1: ; %atomicrmw.start
7312 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
7313 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7314 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
7315 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7316 ; GFX11-NEXT: v_sub_f16_e32 v3, v4, v2
7317 ; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
7318 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
7319 ; GFX11-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
7320 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
7321 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
7322 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7323 ; GFX11-NEXT: buffer_gl1_inv
7324 ; GFX11-NEXT: buffer_gl0_inv
7325 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
7326 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
7327 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
7328 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
7329 ; GFX11-NEXT: s_cbranch_execnz .LBB28_1
7330 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
7331 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
7332 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
7333 ; GFX11-NEXT: s_setpc_b64 s[30:31]
7334 ;
7335 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos__align4:
7336 ; GFX10: ; %bb.0:
7337 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7338 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2046
7339 ; GFX10-NEXT: s_mov_b32 s4, 0
7340 ; GFX10-NEXT: .LBB28_1: ; %atomicrmw.start
7341 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
7342 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7343 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
7344 ; GFX10-NEXT: v_sub_f16_e32 v3, v4, v2
7345 ; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
7346 ; GFX10-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
7347 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
7348 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2046 glc
7349 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7350 ; GFX10-NEXT: buffer_gl1_inv
7351 ; GFX10-NEXT: buffer_gl0_inv
7352 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
7353 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
7354 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
7355 ; GFX10-NEXT: s_cbranch_execnz .LBB28_1
7356 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
7357 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
7358 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
7359 ; GFX10-NEXT: s_setpc_b64 s[30:31]
7360 ;
7361 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos__align4:
7362 ; GFX90A: ; %bb.0:
7363 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7364 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2046
7365 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
7366 ; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
7367 ; GFX90A-NEXT: .LBB28_1: ; %atomicrmw.start
7368 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
7369 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7370 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
7371 ; GFX90A-NEXT: v_sub_f16_e32 v3, v5, v2
7372 ; GFX90A-NEXT: v_and_or_b32 v4, v5, s6, v3
7373 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 glc
7374 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7375 ; GFX90A-NEXT: buffer_wbinvl1
7376 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
7377 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7378 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
7379 ; GFX90A-NEXT: s_cbranch_execnz .LBB28_1
7380 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
7381 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
7382 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
7383 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
7384 ;
7385 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos__align4:
7386 ; GFX908: ; %bb.0:
7387 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7388 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2046
7389 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
7390 ; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
7391 ; GFX908-NEXT: .LBB28_1: ; %atomicrmw.start
7392 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
7393 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7394 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
7395 ; GFX908-NEXT: v_sub_f16_e32 v3, v4, v2
7396 ; GFX908-NEXT: v_and_or_b32 v3, v4, s6, v3
7397 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2046 glc
7398 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7399 ; GFX908-NEXT: buffer_wbinvl1
7400 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
7401 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7402 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
7403 ; GFX908-NEXT: s_cbranch_execnz .LBB28_1
7404 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
7405 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
7406 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
7407 ; GFX908-NEXT: s_setpc_b64 s[30:31]
7408 ;
7409 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos__align4:
7410 ; GFX8: ; %bb.0:
7411 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7412 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
7413 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
7414 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
7415 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
7416 ; GFX8-NEXT: .LBB28_1: ; %atomicrmw.start
7417 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7418 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7419 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
7420 ; GFX8-NEXT: v_sub_f16_e32 v0, v1, v2
7421 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
7422 ; GFX8-NEXT: v_or_b32_e32 v0, v5, v0
7423 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
7424 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7425 ; GFX8-NEXT: buffer_wbinvl1
7426 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
7427 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7428 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
7429 ; GFX8-NEXT: s_cbranch_execnz .LBB28_1
7430 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7431 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
7432 ; GFX8-NEXT: s_setpc_b64 s[30:31]
7433 ;
7434 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos__align4:
7435 ; GFX7: ; %bb.0:
7436 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7437 ; GFX7-NEXT: s_mov_b32 s6, 0
7438 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
7439 ; GFX7-NEXT: s_mov_b32 s4, s6
7440 ; GFX7-NEXT: s_mov_b32 s5, s6
7441 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
7442 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
7443 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
7444 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v2
7445 ; GFX7-NEXT: .LBB28_1: ; %atomicrmw.start
7446 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7447 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7448 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
7449 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
7450 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v6
7451 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
7452 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
7453 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
7454 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
7455 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 offset:2046 glc
7456 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7457 ; GFX7-NEXT: buffer_wbinvl1
7458 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
7459 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
7460 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
7461 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
7462 ; GFX7-NEXT: s_cbranch_execnz .LBB28_1
7463 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
7464 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
7465 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v4
7466 ; GFX7-NEXT: s_setpc_b64 s[30:31]
7467 ;
7468 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos__align4:
7469 ; GFX6: ; %bb.0:
7470 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7471 ; GFX6-NEXT: s_mov_b32 s6, 0
7472 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
7473 ; GFX6-NEXT: s_mov_b32 s4, s6
7474 ; GFX6-NEXT: s_mov_b32 s5, s6
7475 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
7476 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
7477 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
7478 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v2
7479 ; GFX6-NEXT: .LBB28_1: ; %atomicrmw.start
7480 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
7481 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7482 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v3
7483 ; GFX6-NEXT: s_waitcnt expcnt(0)
7484 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
7485 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v6
7486 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
7487 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
7488 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
7489 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
7490 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 offset:2046 glc
7491 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7492 ; GFX6-NEXT: buffer_wbinvl1
7493 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
7494 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
7495 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
7496 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
7497 ; GFX6-NEXT: s_cbranch_execnz .LBB28_1
7498 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
7499 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
7500 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v4
7501 ; GFX6-NEXT: s_waitcnt expcnt(0)
7502 ; GFX6-NEXT: s_setpc_b64 s[30:31]
7503 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
7504 %result = atomicrmw fsub ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, align 4
7505 ret half %result
7506 }
7508 define void @global_agent_atomic_fsub_noret_f16__offset12b__align4_pos(ptr addrspace(1) %ptr, half %val) #0 {
7509 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f16__offset12b__align4_pos:
7510 ; GFX12: ; %bb.0:
7511 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
7512 ; GFX12-NEXT: s_wait_expcnt 0x0
7513 ; GFX12-NEXT: s_wait_samplecnt 0x0
7514 ; GFX12-NEXT: s_wait_bvhcnt 0x0
7515 ; GFX12-NEXT: s_wait_kmcnt 0x0
7516 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:2046
7517 ; GFX12-NEXT: s_mov_b32 s0, 0
7518 ; GFX12-NEXT: .LBB29_1: ; %atomicrmw.start
7519 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
7520 ; GFX12-NEXT: s_wait_loadcnt 0x0
7521 ; GFX12-NEXT: v_sub_f16_e32 v3, v4, v2
7522 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7523 ; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
7524 ; GFX12-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
7525 ; GFX12-NEXT: s_wait_storecnt 0x0
7526 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
7527 ; GFX12-NEXT: s_wait_loadcnt 0x0
7528 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
7529 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
7530 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
7531 ; GFX12-NEXT: s_wait_alu 0xfffe
7532 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
7533 ; GFX12-NEXT: s_wait_alu 0xfffe
7534 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
7535 ; GFX12-NEXT: s_cbranch_execnz .LBB29_1
7536 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
7537 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
7538 ; GFX12-NEXT: s_wait_alu 0xfffe
7539 ; GFX12-NEXT: s_setpc_b64 s[30:31]
7540 ;
7541 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f16__offset12b__align4_pos:
7542 ; GFX940: ; %bb.0:
7543 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7544 ; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:2046
7545 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
7546 ; GFX940-NEXT: s_mov_b32 s2, 0xffff0000
7547 ; GFX940-NEXT: .LBB29_1: ; %atomicrmw.start
7548 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
7549 ; GFX940-NEXT: s_waitcnt vmcnt(0)
7550 ; GFX940-NEXT: v_sub_f16_e32 v3, v5, v2
7551 ; GFX940-NEXT: v_and_or_b32 v4, v5, s2, v3
7552 ; GFX940-NEXT: buffer_wbl2 sc1
7553 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 sc0
7554 ; GFX940-NEXT: s_waitcnt vmcnt(0)
7555 ; GFX940-NEXT: buffer_inv sc1
7556 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
7557 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
7558 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
7559 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
7560 ; GFX940-NEXT: s_cbranch_execnz .LBB29_1
7561 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
7562 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
7563 ; GFX940-NEXT: s_setpc_b64 s[30:31]
7564 ;
7565 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f16__offset12b__align4_pos:
7566 ; GFX11: ; %bb.0:
7567 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7568 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:2046
7569 ; GFX11-NEXT: s_mov_b32 s0, 0
7570 ; GFX11-NEXT: .LBB29_1: ; %atomicrmw.start
7571 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
7572 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7573 ; GFX11-NEXT: v_sub_f16_e32 v3, v4, v2
7574 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7575 ; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
7576 ; GFX11-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
7577 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
7578 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
7579 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7580 ; GFX11-NEXT: buffer_gl1_inv
7581 ; GFX11-NEXT: buffer_gl0_inv
7582 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
7583 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
7584 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
7585 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
7586 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
7587 ; GFX11-NEXT: s_cbranch_execnz .LBB29_1
7588 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
7589 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
7590 ; GFX11-NEXT: s_setpc_b64 s[30:31]
7591 ;
7592 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f16__offset12b__align4_pos:
7593 ; GFX10: ; %bb.0:
7594 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7595 ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2046
7596 ; GFX10-NEXT: s_mov_b32 s4, 0
7597 ; GFX10-NEXT: .LBB29_1: ; %atomicrmw.start
7598 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
7599 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7600 ; GFX10-NEXT: v_sub_f16_e32 v3, v4, v2
7601 ; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
7602 ; GFX10-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
7603 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
7604 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2046 glc
7605 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7606 ; GFX10-NEXT: buffer_gl1_inv
7607 ; GFX10-NEXT: buffer_gl0_inv
7608 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
7609 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
7610 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
7611 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
7612 ; GFX10-NEXT: s_cbranch_execnz .LBB29_1
7613 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
7614 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
7615 ; GFX10-NEXT: s_setpc_b64 s[30:31]
7616 ;
7617 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f16__offset12b__align4_pos:
7618 ; GFX90A: ; %bb.0:
7619 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7620 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2046
7621 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
7622 ; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
7623 ; GFX90A-NEXT: .LBB29_1: ; %atomicrmw.start
7624 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
7625 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7626 ; GFX90A-NEXT: v_sub_f16_e32 v3, v5, v2
7627 ; GFX90A-NEXT: v_and_or_b32 v4, v5, s6, v3
7628 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 glc
7629 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7630 ; GFX90A-NEXT: buffer_wbinvl1
7631 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
7632 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7633 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
7634 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
7635 ; GFX90A-NEXT: s_cbranch_execnz .LBB29_1
7636 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
7637 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
7638 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
7639 ;
7640 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f16__offset12b__align4_pos:
7641 ; GFX908: ; %bb.0:
7642 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7643 ; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2046
7644 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
7645 ; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
7646 ; GFX908-NEXT: .LBB29_1: ; %atomicrmw.start
7647 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
7648 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7649 ; GFX908-NEXT: v_sub_f16_e32 v3, v4, v2
7650 ; GFX908-NEXT: v_and_or_b32 v3, v4, s6, v3
7651 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2046 glc
7652 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7653 ; GFX908-NEXT: buffer_wbinvl1
7654 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
7655 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7656 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
7657 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
7658 ; GFX908-NEXT: s_cbranch_execnz .LBB29_1
7659 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
7660 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
7661 ; GFX908-NEXT: s_setpc_b64 s[30:31]
7662 ;
7663 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f16__offset12b__align4_pos:
7664 ; GFX8: ; %bb.0:
7665 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7666 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fe, v0
7667 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
7668 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
7669 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
7670 ; GFX8-NEXT: .LBB29_1: ; %atomicrmw.start
7671 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7672 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7673 ; GFX8-NEXT: v_sub_f16_e32 v3, v4, v2
7674 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
7675 ; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
7676 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
7677 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7678 ; GFX8-NEXT: buffer_wbinvl1
7679 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
7680 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7681 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
7682 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
7683 ; GFX8-NEXT: s_cbranch_execnz .LBB29_1
7684 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7685 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
7686 ; GFX8-NEXT: s_setpc_b64 s[30:31]
7687 ;
7688 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f16__offset12b__align4_pos:
7689 ; GFX7: ; %bb.0:
7690 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7691 ; GFX7-NEXT: s_mov_b32 s6, 0
7692 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
7693 ; GFX7-NEXT: s_mov_b32 s4, s6
7694 ; GFX7-NEXT: s_mov_b32 s5, s6
7695 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
7696 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
7697 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
7698 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v2
7699 ; GFX7-NEXT: .LBB29_1: ; %atomicrmw.start
7700 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7701 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7702 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
7703 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
7704 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v4
7705 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
7706 ; GFX7-NEXT: v_or_b32_e32 v2, v5, v2
7707 ; GFX7-NEXT: v_mov_b32_e32 v6, v3
7708 ; GFX7-NEXT: v_mov_b32_e32 v5, v2
7709 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2046 glc
7710 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7711 ; GFX7-NEXT: buffer_wbinvl1
7712 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
7713 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
7714 ; GFX7-NEXT: v_mov_b32_e32 v3, v5
7715 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
7716 ; GFX7-NEXT: s_cbranch_execnz .LBB29_1
7717 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
7718 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
7719 ; GFX7-NEXT: s_setpc_b64 s[30:31]
7720 ;
7721 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f16__offset12b__align4_pos:
7722 ; GFX6: ; %bb.0:
7723 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7724 ; GFX6-NEXT: s_mov_b32 s6, 0
7725 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
7726 ; GFX6-NEXT: s_mov_b32 s4, s6
7727 ; GFX6-NEXT: s_mov_b32 s5, s6
7728 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
7729 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
7730 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
7731 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v2
7732 ; GFX6-NEXT: .LBB29_1: ; %atomicrmw.start
7733 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
7734 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7735 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v3
7736 ; GFX6-NEXT: s_waitcnt expcnt(0)
7737 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
7738 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v4
7739 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
7740 ; GFX6-NEXT: v_or_b32_e32 v2, v5, v2
7741 ; GFX6-NEXT: v_mov_b32_e32 v6, v3
7742 ; GFX6-NEXT: v_mov_b32_e32 v5, v2
7743 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2046 glc
7744 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7745 ; GFX6-NEXT: buffer_wbinvl1
7746 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
7747 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
7748 ; GFX6-NEXT: v_mov_b32_e32 v3, v5
7749 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
7750 ; GFX6-NEXT: s_cbranch_execnz .LBB29_1
7751 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
7752 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
7753 ; GFX6-NEXT: s_waitcnt expcnt(0)
7754 ; GFX6-NEXT: s_setpc_b64 s[30:31]
7755 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
7756 %unused = atomicrmw fsub ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, align 4
7757 ret void
7758 }
7760 define half @global_system_atomic_fsub_ret_f16__offset12b_pos(ptr addrspace(1) %ptr, half %val) #0 {
7761 ; GFX12-LABEL: global_system_atomic_fsub_ret_f16__offset12b_pos:
7762 ; GFX12: ; %bb.0:
7763 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
7764 ; GFX12-NEXT: s_wait_expcnt 0x0
7765 ; GFX12-NEXT: s_wait_samplecnt 0x0
7766 ; GFX12-NEXT: s_wait_bvhcnt 0x0
7767 ; GFX12-NEXT: s_wait_kmcnt 0x0
7768 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
7769 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
7770 ; GFX12-NEXT: s_mov_b32 s0, 0
7771 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
7772 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
7773 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
7774 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
7775 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7776 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
7777 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
7778 ; GFX12-NEXT: v_not_b32_e32 v4, v4
7779 ; GFX12-NEXT: .LBB30_1: ; %atomicrmw.start
7780 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
7781 ; GFX12-NEXT: s_wait_loadcnt 0x0
7782 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
7783 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7784 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
7785 ; GFX12-NEXT: v_sub_f16_e32 v5, v5, v2
7786 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7787 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff, v5
7788 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
7789 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
7790 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
7791 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
7792 ; GFX12-NEXT: s_wait_storecnt 0x0
7793 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
7794 ; GFX12-NEXT: s_wait_loadcnt 0x0
7795 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
7796 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
7797 ; GFX12-NEXT: s_wait_alu 0xfffe
7798 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
7799 ; GFX12-NEXT: s_wait_alu 0xfffe
7800 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
7801 ; GFX12-NEXT: s_cbranch_execnz .LBB30_1
7802 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
7803 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
7804 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
7805 ; GFX12-NEXT: s_wait_alu 0xfffe
7806 ; GFX12-NEXT: s_setpc_b64 s[30:31]
7807 ;
7808 ; GFX940-LABEL: global_system_atomic_fsub_ret_f16__offset12b_pos:
7809 ; GFX940: ; %bb.0:
7810 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7811 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
7812 ; GFX940-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, s[0:1]
7813 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v6
7814 ; GFX940-NEXT: v_mov_b32_e32 v1, v7
7815 ; GFX940-NEXT: global_load_dword v4, v[0:1], off
7816 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v6
7817 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7818 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
7819 ; GFX940-NEXT: v_lshlrev_b32_e64 v5, v3, s0
7820 ; GFX940-NEXT: v_not_b32_e32 v5, v5
7821 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
7822 ; GFX940-NEXT: .LBB30_1: ; %atomicrmw.start
7823 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
7824 ; GFX940-NEXT: s_waitcnt vmcnt(0)
7825 ; GFX940-NEXT: v_mov_b32_e32 v7, v4
7826 ; GFX940-NEXT: v_lshrrev_b32_e32 v4, v3, v7
7827 ; GFX940-NEXT: v_sub_f16_e32 v4, v4, v2
7828 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, v3, v4
7829 ; GFX940-NEXT: v_and_or_b32 v6, v7, v5, v4
7830 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
7831 ; GFX940-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off sc0 sc1
7832 ; GFX940-NEXT: s_waitcnt vmcnt(0)
7833 ; GFX940-NEXT: buffer_inv sc0 sc1
7834 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
7835 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
7836 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
7837 ; GFX940-NEXT: s_cbranch_execnz .LBB30_1
7838 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
7839 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
7840 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v4
7841 ; GFX940-NEXT: s_setpc_b64 s[30:31]
7842 ;
7843 ; GFX11-LABEL: global_system_atomic_fsub_ret_f16__offset12b_pos:
7844 ; GFX11: ; %bb.0:
7845 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7846 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
7847 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
7848 ; GFX11-NEXT: s_mov_b32 s0, 0
7849 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
7850 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
7851 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
7852 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
7853 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7854 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
7855 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
7856 ; GFX11-NEXT: v_not_b32_e32 v4, v4
7857 ; GFX11-NEXT: .LBB30_1: ; %atomicrmw.start
7858 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
7859 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7860 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
7861 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7862 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
7863 ; GFX11-NEXT: v_sub_f16_e32 v5, v5, v2
7864 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7865 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
7866 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
7867 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
7868 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
7869 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
7870 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
7871 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7872 ; GFX11-NEXT: buffer_gl1_inv
7873 ; GFX11-NEXT: buffer_gl0_inv
7874 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
7875 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
7876 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
7877 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
7878 ; GFX11-NEXT: s_cbranch_execnz .LBB30_1
7879 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
7880 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
7881 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
7882 ; GFX11-NEXT: s_setpc_b64 s[30:31]
7883 ;
7884 ; GFX10-LABEL: global_system_atomic_fsub_ret_f16__offset12b_pos:
7885 ; GFX10: ; %bb.0:
7886 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7887 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
7888 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
7889 ; GFX10-NEXT: s_mov_b32 s4, 0
7890 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
7891 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
7892 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
7893 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7894 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
7895 ; GFX10-NEXT: v_not_b32_e32 v4, v4
7896 ; GFX10-NEXT: .LBB30_1: ; %atomicrmw.start
7897 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
7898 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7899 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
7900 ; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
7901 ; GFX10-NEXT: v_sub_f16_e32 v5, v5, v2
7902 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
7903 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
7904 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
7905 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
7906 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7907 ; GFX10-NEXT: buffer_gl1_inv
7908 ; GFX10-NEXT: buffer_gl0_inv
7909 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
7910 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
7911 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
7912 ; GFX10-NEXT: s_cbranch_execnz .LBB30_1
7913 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
7914 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
7915 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
7916 ; GFX10-NEXT: s_setpc_b64 s[30:31]
7917 ;
7918 ; GFX90A-LABEL: global_system_atomic_fsub_ret_f16__offset12b_pos:
7919 ; GFX90A: ; %bb.0:
7920 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7921 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
7922 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
7923 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
7924 ; GFX90A-NEXT: global_load_dword v4, v[0:1], off
7925 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
7926 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7927 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
7928 ; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v3, s4
7929 ; GFX90A-NEXT: v_not_b32_e32 v5, v5
7930 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
7931 ; GFX90A-NEXT: .LBB30_1: ; %atomicrmw.start
7932 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
7933 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7934 ; GFX90A-NEXT: v_mov_b32_e32 v7, v4
7935 ; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v7
7936 ; GFX90A-NEXT: v_sub_f16_e32 v4, v4, v2
7937 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
7938 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v5, v4
7939 ; GFX90A-NEXT: buffer_wbl2
7940 ; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off glc
7941 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7942 ; GFX90A-NEXT: buffer_invl2
7943 ; GFX90A-NEXT: buffer_wbinvl1
7944 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
7945 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7946 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
7947 ; GFX90A-NEXT: s_cbranch_execnz .LBB30_1
7948 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
7949 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
7950 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v4
7951 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
7952 ;
7953 ; GFX908-LABEL: global_system_atomic_fsub_ret_f16__offset12b_pos:
7954 ; GFX908: ; %bb.0:
7955 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7956 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
7957 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
7958 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
7959 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
7960 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
7961 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7962 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
7963 ; GFX908-NEXT: v_lshlrev_b32_e64 v5, v3, s4
7964 ; GFX908-NEXT: v_not_b32_e32 v5, v5
7965 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
7966 ; GFX908-NEXT: .LBB30_1: ; %atomicrmw.start
7967 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
7968 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7969 ; GFX908-NEXT: v_mov_b32_e32 v7, v4
7970 ; GFX908-NEXT: v_lshrrev_b32_e32 v4, v3, v7
7971 ; GFX908-NEXT: v_sub_f16_e32 v4, v4, v2
7972 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, v3, v4
7973 ; GFX908-NEXT: v_and_or_b32 v6, v7, v5, v4
7974 ; GFX908-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off glc
7975 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7976 ; GFX908-NEXT: buffer_wbinvl1
7977 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
7978 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7979 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
7980 ; GFX908-NEXT: s_cbranch_execnz .LBB30_1
7981 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
7982 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
7983 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v4
7984 ; GFX908-NEXT: s_setpc_b64 s[30:31]
7985 ;
7986 ; GFX8-LABEL: global_system_atomic_fsub_ret_f16__offset12b_pos:
7987 ; GFX8: ; %bb.0:
7988 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7989 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
7990 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
7991 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
7992 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
7993 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
7994 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7995 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
7996 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
7997 ; GFX8-NEXT: v_not_b32_e32 v4, v4
7998 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
7999 ; GFX8-NEXT: .LBB30_1: ; %atomicrmw.start
8000 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
8001 ; GFX8-NEXT: s_waitcnt vmcnt(0)
8002 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
8003 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
8004 ; GFX8-NEXT: v_sub_f16_e32 v5, v5, v2
8005 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
8006 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
8007 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
8008 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
8009 ; GFX8-NEXT: s_waitcnt vmcnt(0)
8010 ; GFX8-NEXT: buffer_wbinvl1
8011 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
8012 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8013 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
8014 ; GFX8-NEXT: s_cbranch_execnz .LBB30_1
8015 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
8016 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
8017 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8018 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8019 ;
8020 ; GFX7-LABEL: global_system_atomic_fsub_ret_f16__offset12b_pos:
8021 ; GFX7: ; %bb.0:
8022 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8023 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
8024 ; GFX7-NEXT: s_mov_b32 s6, 0
8025 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8026 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
8027 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
8028 ; GFX7-NEXT: s_mov_b32 s4, s6
8029 ; GFX7-NEXT: s_mov_b32 s5, s6
8030 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
8031 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
8032 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
8033 ; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
8034 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
8035 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
8036 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
8037 ; GFX7-NEXT: v_not_b32_e32 v8, v2
8038 ; GFX7-NEXT: .LBB30_1: ; %atomicrmw.start
8039 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
8040 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8041 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
8042 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
8043 ; GFX7-NEXT: v_and_b32_e32 v4, v3, v8
8044 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v7
8045 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
8046 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v6, v2
8047 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
8048 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
8049 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
8050 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
8051 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8052 ; GFX7-NEXT: buffer_wbinvl1
8053 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
8054 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
8055 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
8056 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
8057 ; GFX7-NEXT: s_cbranch_execnz .LBB30_1
8058 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
8059 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
8060 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
8061 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
8062 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8063 ;
8064 ; GFX6-LABEL: global_system_atomic_fsub_ret_f16__offset12b_pos:
8065 ; GFX6: ; %bb.0:
8066 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8067 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
8068 ; GFX6-NEXT: s_mov_b32 s6, 0
8069 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8070 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
8071 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
8072 ; GFX6-NEXT: s_mov_b32 s4, s6
8073 ; GFX6-NEXT: s_mov_b32 s5, s6
8074 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
8075 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
8076 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
8077 ; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
8078 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
8079 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
8080 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
8081 ; GFX6-NEXT: v_not_b32_e32 v8, v2
8082 ; GFX6-NEXT: .LBB30_1: ; %atomicrmw.start
8083 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
8084 ; GFX6-NEXT: s_waitcnt vmcnt(0)
8085 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
8086 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
8087 ; GFX6-NEXT: s_waitcnt expcnt(0)
8088 ; GFX6-NEXT: v_and_b32_e32 v4, v3, v8
8089 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v7
8090 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
8091 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v6, v2
8092 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
8093 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
8094 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
8095 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
8096 ; GFX6-NEXT: s_waitcnt vmcnt(0)
8097 ; GFX6-NEXT: buffer_wbinvl1
8098 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
8099 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
8100 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
8101 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
8102 ; GFX6-NEXT: s_cbranch_execnz .LBB30_1
8103 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
8104 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
8105 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
8106 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
8107 ; GFX6-NEXT: s_waitcnt expcnt(0)
8108 ; GFX6-NEXT: s_setpc_b64 s[30:31]
8109 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
8110 %result = atomicrmw fsub ptr addrspace(1) %gep, half %val seq_cst
8111 ret half %result
8112 }
8114 define void @global_system_atomic_fsub_noret_f16__offset12b_pos(ptr addrspace(1) %ptr, half %val) #0 {
8115 ; GFX12-LABEL: global_system_atomic_fsub_noret_f16__offset12b_pos:
8116 ; GFX12: ; %bb.0:
8117 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
8118 ; GFX12-NEXT: s_wait_expcnt 0x0
8119 ; GFX12-NEXT: s_wait_samplecnt 0x0
8120 ; GFX12-NEXT: s_wait_bvhcnt 0x0
8121 ; GFX12-NEXT: s_wait_kmcnt 0x0
8122 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
8123 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
8124 ; GFX12-NEXT: s_mov_b32 s0, 0
8125 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
8126 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
8127 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
8128 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off
8129 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
8130 ; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
8131 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
8132 ; GFX12-NEXT: v_not_b32_e32 v6, v3
8133 ; GFX12-NEXT: .LBB31_1: ; %atomicrmw.start
8134 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
8135 ; GFX12-NEXT: s_wait_loadcnt 0x0
8136 ; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
8137 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8138 ; GFX12-NEXT: v_sub_f16_e32 v3, v3, v2
8139 ; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
8140 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8141 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
8142 ; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
8143 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
8144 ; GFX12-NEXT: s_wait_storecnt 0x0
8145 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
8146 ; GFX12-NEXT: s_wait_loadcnt 0x0
8147 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
8148 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
8149 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
8150 ; GFX12-NEXT: s_wait_alu 0xfffe
8151 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
8152 ; GFX12-NEXT: s_wait_alu 0xfffe
8153 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
8154 ; GFX12-NEXT: s_cbranch_execnz .LBB31_1
8155 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
8156 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
8157 ; GFX12-NEXT: s_wait_alu 0xfffe
8158 ; GFX12-NEXT: s_setpc_b64 s[30:31]
8159 ;
8160 ; GFX940-LABEL: global_system_atomic_fsub_noret_f16__offset12b_pos:
8161 ; GFX940: ; %bb.0:
8162 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8163 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
8164 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
8165 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
8166 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
8167 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
8168 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v4
8169 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8170 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
8171 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
8172 ; GFX940-NEXT: v_not_b32_e32 v6, v4
8173 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
8174 ; GFX940-NEXT: .LBB31_1: ; %atomicrmw.start
8175 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
8176 ; GFX940-NEXT: s_waitcnt vmcnt(0)
8177 ; GFX940-NEXT: v_lshrrev_b32_e32 v4, v3, v5
8178 ; GFX940-NEXT: v_sub_f16_e32 v4, v4, v2
8179 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, v3, v4
8180 ; GFX940-NEXT: v_and_or_b32 v4, v5, v6, v4
8181 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
8182 ; GFX940-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0 sc1
8183 ; GFX940-NEXT: s_waitcnt vmcnt(0)
8184 ; GFX940-NEXT: buffer_inv sc0 sc1
8185 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
8186 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
8187 ; GFX940-NEXT: v_mov_b32_e32 v5, v4
8188 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
8189 ; GFX940-NEXT: s_cbranch_execnz .LBB31_1
8190 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
8191 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
8192 ; GFX940-NEXT: s_setpc_b64 s[30:31]
8193 ;
8194 ; GFX11-LABEL: global_system_atomic_fsub_noret_f16__offset12b_pos:
8195 ; GFX11: ; %bb.0:
8196 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8197 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
8198 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
8199 ; GFX11-NEXT: s_mov_b32 s0, 0
8200 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
8201 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
8202 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
8203 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off
8204 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
8205 ; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
8206 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
8207 ; GFX11-NEXT: v_not_b32_e32 v6, v3
8208 ; GFX11-NEXT: .LBB31_1: ; %atomicrmw.start
8209 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
8210 ; GFX11-NEXT: s_waitcnt vmcnt(0)
8211 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
8212 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8213 ; GFX11-NEXT: v_sub_f16_e32 v3, v3, v2
8214 ; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
8215 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8216 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
8217 ; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
8218 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
8219 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
8220 ; GFX11-NEXT: s_waitcnt vmcnt(0)
8221 ; GFX11-NEXT: buffer_gl1_inv
8222 ; GFX11-NEXT: buffer_gl0_inv
8223 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
8224 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
8225 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
8226 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
8227 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
8228 ; GFX11-NEXT: s_cbranch_execnz .LBB31_1
8229 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
8230 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
8231 ; GFX11-NEXT: s_setpc_b64 s[30:31]
8232 ;
8233 ; GFX10-LABEL: global_system_atomic_fsub_noret_f16__offset12b_pos:
8234 ; GFX10: ; %bb.0:
8235 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8236 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
8237 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
8238 ; GFX10-NEXT: s_mov_b32 s4, 0
8239 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
8240 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
8241 ; GFX10-NEXT: global_load_dword v4, v[0:1], off
8242 ; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
8243 ; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
8244 ; GFX10-NEXT: v_not_b32_e32 v6, v3
8245 ; GFX10-NEXT: .LBB31_1: ; %atomicrmw.start
8246 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
8247 ; GFX10-NEXT: s_waitcnt vmcnt(0)
8248 ; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
8249 ; GFX10-NEXT: v_sub_f16_e32 v3, v3, v2
8250 ; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
8251 ; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
8252 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
8253 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
8254 ; GFX10-NEXT: s_waitcnt vmcnt(0)
8255 ; GFX10-NEXT: buffer_gl1_inv
8256 ; GFX10-NEXT: buffer_gl0_inv
8257 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
8258 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
8259 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
8260 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
8261 ; GFX10-NEXT: s_cbranch_execnz .LBB31_1
8262 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
8263 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
8264 ; GFX10-NEXT: s_setpc_b64 s[30:31]
8265 ;
8266 ; GFX90A-LABEL: global_system_atomic_fsub_noret_f16__offset12b_pos:
8267 ; GFX90A: ; %bb.0:
8268 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8269 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
8270 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
8271 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
8272 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
8273 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
8274 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8275 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
8276 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
8277 ; GFX90A-NEXT: v_not_b32_e32 v6, v4
8278 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
8279 ; GFX90A-NEXT: .LBB31_1: ; %atomicrmw.start
8280 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
8281 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
8282 ; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
8283 ; GFX90A-NEXT: v_sub_f16_e32 v4, v4, v2
8284 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
8285 ; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
8286 ; GFX90A-NEXT: buffer_wbl2
8287 ; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
8288 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
8289 ; GFX90A-NEXT: buffer_invl2
8290 ; GFX90A-NEXT: buffer_wbinvl1
8291 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
8292 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8293 ; GFX90A-NEXT: v_mov_b32_e32 v5, v4
8294 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
8295 ; GFX90A-NEXT: s_cbranch_execnz .LBB31_1
8296 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
8297 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
8298 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
8299 ;
8300 ; GFX908-LABEL: global_system_atomic_fsub_noret_f16__offset12b_pos:
8301 ; GFX908: ; %bb.0:
8302 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8303 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
8304 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
8305 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
8306 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
8307 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
8308 ; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
8309 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
8310 ; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
8311 ; GFX908-NEXT: v_not_b32_e32 v6, v3
8312 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
8313 ; GFX908-NEXT: .LBB31_1: ; %atomicrmw.start
8314 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
8315 ; GFX908-NEXT: s_waitcnt vmcnt(0)
8316 ; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
8317 ; GFX908-NEXT: v_sub_f16_e32 v3, v3, v2
8318 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
8319 ; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
8320 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
8321 ; GFX908-NEXT: s_waitcnt vmcnt(0)
8322 ; GFX908-NEXT: buffer_wbinvl1
8323 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
8324 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8325 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
8326 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
8327 ; GFX908-NEXT: s_cbranch_execnz .LBB31_1
8328 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
8329 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
8330 ; GFX908-NEXT: s_setpc_b64 s[30:31]
8331 ;
8332 ; GFX8-LABEL: global_system_atomic_fsub_noret_f16__offset12b_pos:
8333 ; GFX8: ; %bb.0:
8334 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8335 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
8336 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8337 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
8338 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
8339 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
8340 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
8341 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
8342 ; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
8343 ; GFX8-NEXT: v_not_b32_e32 v6, v3
8344 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
8345 ; GFX8-NEXT: .LBB31_1: ; %atomicrmw.start
8346 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
8347 ; GFX8-NEXT: s_waitcnt vmcnt(0)
8348 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
8349 ; GFX8-NEXT: v_sub_f16_e32 v3, v3, v2
8350 ; GFX8-NEXT: v_and_b32_e32 v7, v4, v6
8351 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
8352 ; GFX8-NEXT: v_or_b32_e32 v3, v7, v3
8353 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
8354 ; GFX8-NEXT: s_waitcnt vmcnt(0)
8355 ; GFX8-NEXT: buffer_wbinvl1
8356 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
8357 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8358 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
8359 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
8360 ; GFX8-NEXT: s_cbranch_execnz .LBB31_1
8361 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
8362 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
8363 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8364 ;
8365 ; GFX7-LABEL: global_system_atomic_fsub_noret_f16__offset12b_pos:
8366 ; GFX7: ; %bb.0:
8367 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8368 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
8369 ; GFX7-NEXT: s_mov_b32 s6, 0
8370 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8371 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
8372 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
8373 ; GFX7-NEXT: s_mov_b32 s4, s6
8374 ; GFX7-NEXT: s_mov_b32 s5, s6
8375 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
8376 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
8377 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
8378 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
8379 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
8380 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
8381 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
8382 ; GFX7-NEXT: v_not_b32_e32 v6, v2
8383 ; GFX7-NEXT: .LBB31_1: ; %atomicrmw.start
8384 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
8385 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8386 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
8387 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
8388 ; GFX7-NEXT: v_and_b32_e32 v7, v3, v6
8389 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v5
8390 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
8391 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v4, v2
8392 ; GFX7-NEXT: v_or_b32_e32 v2, v7, v2
8393 ; GFX7-NEXT: v_mov_b32_e32 v8, v3
8394 ; GFX7-NEXT: v_mov_b32_e32 v7, v2
8395 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
8396 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8397 ; GFX7-NEXT: buffer_wbinvl1
8398 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
8399 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
8400 ; GFX7-NEXT: v_mov_b32_e32 v3, v7
8401 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
8402 ; GFX7-NEXT: s_cbranch_execnz .LBB31_1
8403 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
8404 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
8405 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8406 ;
8407 ; GFX6-LABEL: global_system_atomic_fsub_noret_f16__offset12b_pos:
8408 ; GFX6: ; %bb.0:
8409 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8410 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
8411 ; GFX6-NEXT: s_mov_b32 s6, 0
8412 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8413 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
8414 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
8415 ; GFX6-NEXT: s_mov_b32 s4, s6
8416 ; GFX6-NEXT: s_mov_b32 s5, s6
8417 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
8418 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
8419 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
8420 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
8421 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
8422 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
8423 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
8424 ; GFX6-NEXT: v_not_b32_e32 v6, v2
8425 ; GFX6-NEXT: .LBB31_1: ; %atomicrmw.start
8426 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
8427 ; GFX6-NEXT: s_waitcnt vmcnt(0)
8428 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
8429 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
8430 ; GFX6-NEXT: s_waitcnt expcnt(0)
8431 ; GFX6-NEXT: v_and_b32_e32 v7, v3, v6
8432 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v5
8433 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
8434 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v4, v2
8435 ; GFX6-NEXT: v_or_b32_e32 v2, v7, v2
8436 ; GFX6-NEXT: v_mov_b32_e32 v8, v3
8437 ; GFX6-NEXT: v_mov_b32_e32 v7, v2
8438 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
8439 ; GFX6-NEXT: s_waitcnt vmcnt(0)
8440 ; GFX6-NEXT: buffer_wbinvl1
8441 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
8442 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
8443 ; GFX6-NEXT: v_mov_b32_e32 v3, v7
8444 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
8445 ; GFX6-NEXT: s_cbranch_execnz .LBB31_1
8446 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
8447 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
8448 ; GFX6-NEXT: s_waitcnt expcnt(0)
8449 ; GFX6-NEXT: s_setpc_b64 s[30:31]
8450 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
8451 %unused = atomicrmw fsub ptr addrspace(1) %gep, half %val seq_cst
8452 ret void
8453 }
8455 ; --------------------------------------------------------------------
8456 ; bfloat
8457 ; --------------------------------------------------------------------
8459 define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %val) #0 {
8460 ; GFX12-LABEL: global_agent_atomic_fsub_ret_bf16:
8461 ; GFX12: ; %bb.0:
8462 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
8463 ; GFX12-NEXT: s_wait_expcnt 0x0
8464 ; GFX12-NEXT: s_wait_samplecnt 0x0
8465 ; GFX12-NEXT: s_wait_bvhcnt 0x0
8466 ; GFX12-NEXT: s_wait_kmcnt 0x0
8467 ; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
8468 ; GFX12-NEXT: s_mov_b32 s0, 0
8469 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
8470 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
8471 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
8472 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
8473 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8474 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
8475 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
8476 ; GFX12-NEXT: v_not_b32_e32 v4, v4
8477 ; GFX12-NEXT: .LBB32_1: ; %atomicrmw.start
8478 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
8479 ; GFX12-NEXT: s_wait_loadcnt 0x0
8480 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
8481 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8482 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
8483 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
8484 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8485 ; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
8486 ; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
8487 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
8488 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
8489 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
8490 ; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
8491 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
8492 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8493 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
8494 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
8495 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
8496 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
8497 ; GFX12-NEXT: s_wait_storecnt 0x0
8498 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
8499 ; GFX12-NEXT: s_wait_loadcnt 0x0
8500 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
8501 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
8502 ; GFX12-NEXT: s_wait_alu 0xfffe
8503 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
8504 ; GFX12-NEXT: s_wait_alu 0xfffe
8505 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
8506 ; GFX12-NEXT: s_cbranch_execnz .LBB32_1
8507 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
8508 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
8509 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8510 ; GFX12-NEXT: s_wait_alu 0xfffe
8511 ; GFX12-NEXT: s_setpc_b64 s[30:31]
8512 ;
8513 ; GFX940-LABEL: global_agent_atomic_fsub_ret_bf16:
8514 ; GFX940: ; %bb.0:
8515 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8516 ; GFX940-NEXT: v_mov_b32_e32 v3, v0
8517 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v3
8518 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
8519 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v3
8520 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8521 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
8522 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
8523 ; GFX940-NEXT: v_not_b32_e32 v4, v4
8524 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
8525 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8526 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
8527 ; GFX940-NEXT: .LBB32_1: ; %atomicrmw.start
8528 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
8529 ; GFX940-NEXT: s_waitcnt vmcnt(0)
8530 ; GFX940-NEXT: v_mov_b32_e32 v7, v5
8531 ; GFX940-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8532 ; GFX940-NEXT: s_nop 0
8533 ; GFX940-NEXT: v_sub_f32_e32 v5, v5, v2
8534 ; GFX940-NEXT: v_bfe_u32 v6, v5, 16, 1
8535 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v5
8536 ; GFX940-NEXT: v_add3_u32 v6, v6, v5, s2
8537 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
8538 ; GFX940-NEXT: s_nop 1
8539 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
8540 ; GFX940-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8541 ; GFX940-NEXT: v_and_or_b32 v6, v7, v4, v5
8542 ; GFX940-NEXT: buffer_wbl2 sc1
8543 ; GFX940-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off sc0
8544 ; GFX940-NEXT: s_waitcnt vmcnt(0)
8545 ; GFX940-NEXT: buffer_inv sc1
8546 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
8547 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
8548 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
8549 ; GFX940-NEXT: s_cbranch_execnz .LBB32_1
8550 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
8551 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
8552 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8553 ; GFX940-NEXT: s_setpc_b64 s[30:31]
8554 ;
8555 ; GFX11-LABEL: global_agent_atomic_fsub_ret_bf16:
8556 ; GFX11: ; %bb.0:
8557 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8558 ; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
8559 ; GFX11-NEXT: s_mov_b32 s0, 0
8560 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
8561 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
8562 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
8563 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
8564 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8565 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
8566 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
8567 ; GFX11-NEXT: v_not_b32_e32 v4, v4
8568 ; GFX11-NEXT: .p2align 6
8569 ; GFX11-NEXT: .LBB32_1: ; %atomicrmw.start
8570 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
8571 ; GFX11-NEXT: s_waitcnt vmcnt(0)
8572 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
8573 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8574 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
8575 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
8576 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8577 ; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
8578 ; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
8579 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
8580 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
8581 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
8582 ; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
8583 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
8584 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8585 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
8586 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
8587 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
8588 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
8589 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
8590 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
8591 ; GFX11-NEXT: s_waitcnt vmcnt(0)
8592 ; GFX11-NEXT: buffer_gl1_inv
8593 ; GFX11-NEXT: buffer_gl0_inv
8594 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
8595 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
8596 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
8597 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
8598 ; GFX11-NEXT: s_cbranch_execnz .LBB32_1
8599 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
8600 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
8601 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8602 ; GFX11-NEXT: s_setpc_b64 s[30:31]
8603 ;
8604 ; GFX10-LABEL: global_agent_atomic_fsub_ret_bf16:
8605 ; GFX10: ; %bb.0:
8606 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8607 ; GFX10-NEXT: v_mov_b32_e32 v3, v0
8608 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8609 ; GFX10-NEXT: s_mov_b32 s4, 0
8610 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
8611 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
8612 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
8613 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8614 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
8615 ; GFX10-NEXT: v_not_b32_e32 v4, v4
8616 ; GFX10-NEXT: .LBB32_1: ; %atomicrmw.start
8617 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
8618 ; GFX10-NEXT: s_waitcnt vmcnt(0)
8619 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
8620 ; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8621 ; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
8622 ; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
8623 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v5
8624 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
8625 ; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
8626 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
8627 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8628 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
8629 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
8630 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
8631 ; GFX10-NEXT: s_waitcnt vmcnt(0)
8632 ; GFX10-NEXT: buffer_gl1_inv
8633 ; GFX10-NEXT: buffer_gl0_inv
8634 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
8635 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
8636 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
8637 ; GFX10-NEXT: s_cbranch_execnz .LBB32_1
8638 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
8639 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
8640 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8641 ; GFX10-NEXT: s_setpc_b64 s[30:31]
8642 ;
8643 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_bf16:
8644 ; GFX90A: ; %bb.0:
8645 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8646 ; GFX90A-NEXT: v_mov_b32_e32 v3, v0
8647 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
8648 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
8649 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
8650 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8651 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
8652 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
8653 ; GFX90A-NEXT: v_not_b32_e32 v4, v4
8654 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
8655 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8656 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
8657 ; GFX90A-NEXT: .LBB32_1: ; %atomicrmw.start
8658 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
8659 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
8660 ; GFX90A-NEXT: v_mov_b32_e32 v7, v5
8661 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8662 ; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v2
8663 ; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
8664 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v5
8665 ; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
8666 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
8667 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
8668 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8669 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v4, v5
8670 ; GFX90A-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off glc
8671 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
8672 ; GFX90A-NEXT: buffer_wbinvl1
8673 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
8674 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8675 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
8676 ; GFX90A-NEXT: s_cbranch_execnz .LBB32_1
8677 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
8678 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
8679 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8680 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
8681 ;
8682 ; GFX908-LABEL: global_agent_atomic_fsub_ret_bf16:
8683 ; GFX908: ; %bb.0:
8684 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8685 ; GFX908-NEXT: v_mov_b32_e32 v3, v0
8686 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
8687 ; GFX908-NEXT: global_load_dword v5, v[0:1], off
8688 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
8689 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8690 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
8691 ; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
8692 ; GFX908-NEXT: v_not_b32_e32 v4, v4
8693 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
8694 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8695 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
8696 ; GFX908-NEXT: .LBB32_1: ; %atomicrmw.start
8697 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
8698 ; GFX908-NEXT: s_waitcnt vmcnt(0)
8699 ; GFX908-NEXT: v_mov_b32_e32 v6, v5
8700 ; GFX908-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8701 ; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
8702 ; GFX908-NEXT: v_bfe_u32 v7, v5, 16, 1
8703 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v5
8704 ; GFX908-NEXT: v_add3_u32 v7, v7, v5, s6
8705 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
8706 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc
8707 ; GFX908-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8708 ; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
8709 ; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
8710 ; GFX908-NEXT: s_waitcnt vmcnt(0)
8711 ; GFX908-NEXT: buffer_wbinvl1
8712 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
8713 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8714 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
8715 ; GFX908-NEXT: s_cbranch_execnz .LBB32_1
8716 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
8717 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
8718 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8719 ; GFX908-NEXT: s_setpc_b64 s[30:31]
8720 ;
8721 ; GFX8-LABEL: global_agent_atomic_fsub_ret_bf16:
8722 ; GFX8: ; %bb.0:
8723 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8724 ; GFX8-NEXT: v_mov_b32_e32 v3, v0
8725 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
8726 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
8727 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
8728 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8729 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
8730 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
8731 ; GFX8-NEXT: v_not_b32_e32 v4, v4
8732 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
8733 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8734 ; GFX8-NEXT: .LBB32_1: ; %atomicrmw.start
8735 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
8736 ; GFX8-NEXT: s_waitcnt vmcnt(0)
8737 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
8738 ; GFX8-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8739 ; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
8740 ; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
8741 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
8742 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
8743 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
8744 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
8745 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
8746 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
8747 ; GFX8-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8748 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
8749 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
8750 ; GFX8-NEXT: s_waitcnt vmcnt(0)
8751 ; GFX8-NEXT: buffer_wbinvl1
8752 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
8753 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8754 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
8755 ; GFX8-NEXT: s_cbranch_execnz .LBB32_1
8756 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
8757 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
8758 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8759 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8760 ;
8761 ; GFX7-LABEL: global_agent_atomic_fsub_ret_bf16:
8762 ; GFX7: ; %bb.0:
8763 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8764 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
8765 ; GFX7-NEXT: s_mov_b32 s6, 0
8766 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
8767 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
8768 ; GFX7-NEXT: s_mov_b32 s4, s6
8769 ; GFX7-NEXT: s_mov_b32 s5, s6
8770 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
8771 ; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
8772 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 3, v3
8773 ; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
8774 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
8775 ; GFX7-NEXT: v_not_b32_e32 v6, v3
8776 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
8777 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
8778 ; GFX7-NEXT: .LBB32_1: ; %atomicrmw.start
8779 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
8780 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8781 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v5, v4
8782 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8783 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v7
8784 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
8785 ; GFX7-NEXT: v_and_b32_e32 v3, v4, v6
8786 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v5, v2
8787 ; GFX7-NEXT: v_or_b32_e32 v3, v3, v2
8788 ; GFX7-NEXT: v_mov_b32_e32 v2, v3
8789 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
8790 ; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v[0:1], s[4:7], 0 addr64 glc
8791 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8792 ; GFX7-NEXT: buffer_wbinvl1
8793 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
8794 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
8795 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
8796 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
8797 ; GFX7-NEXT: s_cbranch_execnz .LBB32_1
8798 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
8799 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
8800 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v5, v2
8801 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
8802 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8803 ;
8804 ; GFX6-LABEL: global_agent_atomic_fsub_ret_bf16:
8805 ; GFX6: ; %bb.0:
8806 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8807 ; GFX6-NEXT: v_mov_b32_e32 v3, v0
8808 ; GFX6-NEXT: s_mov_b32 s6, 0
8809 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v3
8810 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
8811 ; GFX6-NEXT: s_mov_b32 s4, s6
8812 ; GFX6-NEXT: s_mov_b32 s5, s6
8813 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
8814 ; GFX6-NEXT: v_and_b32_e32 v3, 3, v3
8815 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 3, v3
8816 ; GFX6-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
8817 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
8818 ; GFX6-NEXT: v_not_b32_e32 v6, v3
8819 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
8820 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
8821 ; GFX6-NEXT: .LBB32_1: ; %atomicrmw.start
8822 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
8823 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
8824 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v5, v4
8825 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8826 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v7
8827 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
8828 ; GFX6-NEXT: v_and_b32_e32 v3, v4, v6
8829 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v5, v2
8830 ; GFX6-NEXT: v_or_b32_e32 v3, v3, v2
8831 ; GFX6-NEXT: v_mov_b32_e32 v2, v3
8832 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
8833 ; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v[0:1], s[4:7], 0 addr64 glc
8834 ; GFX6-NEXT: s_waitcnt vmcnt(0)
8835 ; GFX6-NEXT: buffer_wbinvl1
8836 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
8837 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
8838 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
8839 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
8840 ; GFX6-NEXT: s_cbranch_execnz .LBB32_1
8841 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
8842 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
8843 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v5, v2
8844 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
8845 ; GFX6-NEXT: s_waitcnt expcnt(0)
8846 ; GFX6-NEXT: s_setpc_b64 s[30:31]
8847 %result = atomicrmw fsub ptr addrspace(1) %ptr, bfloat %val syncscope("agent") seq_cst
8848 ret bfloat %result
8849 }
8851 define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
8852 ; GFX12-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
8853 ; GFX12: ; %bb.0:
8854 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
8855 ; GFX12-NEXT: s_wait_expcnt 0x0
8856 ; GFX12-NEXT: s_wait_samplecnt 0x0
8857 ; GFX12-NEXT: s_wait_bvhcnt 0x0
8858 ; GFX12-NEXT: s_wait_kmcnt 0x0
8859 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
8860 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
8861 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8862 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
8863 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
8864 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
8865 ; GFX12-NEXT: s_mov_b32 s0, 0
8866 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
8867 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8868 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
8869 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
8870 ; GFX12-NEXT: v_not_b32_e32 v4, v4
8871 ; GFX12-NEXT: .LBB33_1: ; %atomicrmw.start
8872 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
8873 ; GFX12-NEXT: s_wait_loadcnt 0x0
8874 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
8875 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8876 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
8877 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
8878 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8879 ; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
8880 ; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
8881 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
8882 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
8883 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
8884 ; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
8885 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
8886 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8887 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
8888 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
8889 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
8890 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
8891 ; GFX12-NEXT: s_wait_storecnt 0x0
8892 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
8893 ; GFX12-NEXT: s_wait_loadcnt 0x0
8894 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
8895 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
8896 ; GFX12-NEXT: s_wait_alu 0xfffe
8897 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
8898 ; GFX12-NEXT: s_wait_alu 0xfffe
8899 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
8900 ; GFX12-NEXT: s_cbranch_execnz .LBB33_1
8901 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
8902 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
8903 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8904 ; GFX12-NEXT: s_wait_alu 0xfffe
8905 ; GFX12-NEXT: s_setpc_b64 s[30:31]
8906 ;
8907 ; GFX940-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
8908 ; GFX940: ; %bb.0:
8909 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8910 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
8911 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
8912 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
8913 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
8914 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
8915 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v4
8916 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8917 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
8918 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
8919 ; GFX940-NEXT: v_not_b32_e32 v4, v4
8920 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
8921 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8922 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
8923 ; GFX940-NEXT: .LBB33_1: ; %atomicrmw.start
8924 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
8925 ; GFX940-NEXT: s_waitcnt vmcnt(0)
8926 ; GFX940-NEXT: v_mov_b32_e32 v7, v5
8927 ; GFX940-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8928 ; GFX940-NEXT: s_nop 0
8929 ; GFX940-NEXT: v_sub_f32_e32 v5, v5, v2
8930 ; GFX940-NEXT: v_bfe_u32 v6, v5, 16, 1
8931 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v5
8932 ; GFX940-NEXT: v_add3_u32 v6, v6, v5, s2
8933 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
8934 ; GFX940-NEXT: s_nop 1
8935 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
8936 ; GFX940-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8937 ; GFX940-NEXT: v_and_or_b32 v6, v7, v4, v5
8938 ; GFX940-NEXT: buffer_wbl2 sc1
8939 ; GFX940-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off sc0
8940 ; GFX940-NEXT: s_waitcnt vmcnt(0)
8941 ; GFX940-NEXT: buffer_inv sc1
8942 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
8943 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
8944 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
8945 ; GFX940-NEXT: s_cbranch_execnz .LBB33_1
8946 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
8947 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
8948 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8949 ; GFX940-NEXT: s_setpc_b64 s[30:31]
8950 ;
8951 ; GFX11-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
8952 ; GFX11: ; %bb.0:
8953 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8954 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
8955 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
8956 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8957 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
8958 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
8959 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
8960 ; GFX11-NEXT: s_mov_b32 s0, 0
8961 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
8962 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8963 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
8964 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
8965 ; GFX11-NEXT: v_not_b32_e32 v4, v4
8966 ; GFX11-NEXT: .p2align 6
8967 ; GFX11-NEXT: .LBB33_1: ; %atomicrmw.start
8968 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
8969 ; GFX11-NEXT: s_waitcnt vmcnt(0)
8970 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
8971 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8972 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
8973 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
8974 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8975 ; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
8976 ; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
8977 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
8978 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
8979 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
8980 ; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
8981 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
8982 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8983 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
8984 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
8985 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
8986 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
8987 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
8988 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
8989 ; GFX11-NEXT: s_waitcnt vmcnt(0)
8990 ; GFX11-NEXT: buffer_gl1_inv
8991 ; GFX11-NEXT: buffer_gl0_inv
8992 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
8993 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
8994 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
8995 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
8996 ; GFX11-NEXT: s_cbranch_execnz .LBB33_1
8997 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
8998 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
8999 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
9000 ; GFX11-NEXT: s_setpc_b64 s[30:31]
9001 ;
9002 ; GFX10-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
9003 ; GFX10: ; %bb.0:
9004 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9005 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
9006 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
9007 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9008 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
9009 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
9010 ; GFX10-NEXT: s_mov_b32 s4, 0
9011 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
9012 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9013 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
9014 ; GFX10-NEXT: v_not_b32_e32 v4, v4
9015 ; GFX10-NEXT: .LBB33_1: ; %atomicrmw.start
9016 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
9017 ; GFX10-NEXT: s_waitcnt vmcnt(0)
9018 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
9019 ; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9020 ; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
9021 ; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
9022 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v5
9023 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
9024 ; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
9025 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
9026 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9027 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
9028 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
9029 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
9030 ; GFX10-NEXT: s_waitcnt vmcnt(0)
9031 ; GFX10-NEXT: buffer_gl1_inv
9032 ; GFX10-NEXT: buffer_gl0_inv
9033 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
9034 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
9035 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
9036 ; GFX10-NEXT: s_cbranch_execnz .LBB33_1
9037 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
9038 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
9039 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
9040 ; GFX10-NEXT: s_setpc_b64 s[30:31]
9041 ;
9042 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
9043 ; GFX90A: ; %bb.0:
9044 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9045 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
9046 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
9047 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
9048 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
9049 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
9050 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9051 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
9052 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
9053 ; GFX90A-NEXT: v_not_b32_e32 v4, v4
9054 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
9055 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9056 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
9057 ; GFX90A-NEXT: .LBB33_1: ; %atomicrmw.start
9058 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
9059 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
9060 ; GFX90A-NEXT: v_mov_b32_e32 v7, v5
9061 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9062 ; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v2
9063 ; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
9064 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v5
9065 ; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
9066 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
9067 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
9068 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9069 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v4, v5
9070 ; GFX90A-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off glc
9071 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
9072 ; GFX90A-NEXT: buffer_wbinvl1
9073 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
9074 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9075 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
9076 ; GFX90A-NEXT: s_cbranch_execnz .LBB33_1
9077 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
9078 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
9079 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
9080 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
9081 ;
9082 ; GFX908-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
9083 ; GFX908: ; %bb.0:
9084 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9085 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
9086 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
9087 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
9088 ; GFX908-NEXT: global_load_dword v5, v[0:1], off
9089 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
9090 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9091 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
9092 ; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
9093 ; GFX908-NEXT: v_not_b32_e32 v4, v4
9094 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
9095 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9096 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
9097 ; GFX908-NEXT: .LBB33_1: ; %atomicrmw.start
9098 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
9099 ; GFX908-NEXT: s_waitcnt vmcnt(0)
9100 ; GFX908-NEXT: v_mov_b32_e32 v6, v5
9101 ; GFX908-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9102 ; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
9103 ; GFX908-NEXT: v_bfe_u32 v7, v5, 16, 1
9104 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v5
9105 ; GFX908-NEXT: v_add3_u32 v7, v7, v5, s6
9106 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
9107 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc
9108 ; GFX908-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9109 ; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
9110 ; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
9111 ; GFX908-NEXT: s_waitcnt vmcnt(0)
9112 ; GFX908-NEXT: buffer_wbinvl1
9113 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
9114 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9115 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
9116 ; GFX908-NEXT: s_cbranch_execnz .LBB33_1
9117 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
9118 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
9119 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
9120 ; GFX908-NEXT: s_setpc_b64 s[30:31]
9121 ;
9122 ; GFX8-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
9123 ; GFX8: ; %bb.0:
9124 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9125 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
9126 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
9127 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
9128 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
9129 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
9130 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9131 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
9132 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
9133 ; GFX8-NEXT: v_not_b32_e32 v4, v4
9134 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
9135 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9136 ; GFX8-NEXT: .LBB33_1: ; %atomicrmw.start
9137 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
9138 ; GFX8-NEXT: s_waitcnt vmcnt(0)
9139 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
9140 ; GFX8-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9141 ; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
9142 ; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
9143 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
9144 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
9145 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
9146 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
9147 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
9148 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
9149 ; GFX8-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9150 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
9151 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
9152 ; GFX8-NEXT: s_waitcnt vmcnt(0)
9153 ; GFX8-NEXT: buffer_wbinvl1
9154 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
9155 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9156 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
9157 ; GFX8-NEXT: s_cbranch_execnz .LBB33_1
9158 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
9159 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
9160 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
9161 ; GFX8-NEXT: s_setpc_b64 s[30:31]
9162 ;
9163 ; GFX7-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
9164 ; GFX7: ; %bb.0:
9165 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9166 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
9167 ; GFX7-NEXT: s_mov_b32 s6, 0
9168 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
9169 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
9170 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
9171 ; GFX7-NEXT: s_mov_b32 s4, s6
9172 ; GFX7-NEXT: s_mov_b32 s5, s6
9173 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
9174 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
9175 ; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
9176 ; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
9177 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
9178 ; GFX7-NEXT: v_not_b32_e32 v7, v4
9179 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
9180 ; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
9181 ; GFX7-NEXT: .LBB33_1: ; %atomicrmw.start
9182 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
9183 ; GFX7-NEXT: s_waitcnt vmcnt(0)
9184 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
9185 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9186 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v8
9187 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
9188 ; GFX7-NEXT: v_and_b32_e32 v4, v3, v7
9189 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v6, v2
9190 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
9191 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
9192 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
9193 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
9194 ; GFX7-NEXT: s_waitcnt vmcnt(0)
9195 ; GFX7-NEXT: buffer_wbinvl1
9196 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
9197 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
9198 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
9199 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
9200 ; GFX7-NEXT: s_cbranch_execnz .LBB33_1
9201 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
9202 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
9203 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
9204 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
9205 ; GFX7-NEXT: s_setpc_b64 s[30:31]
9206 ;
9207 ; GFX6-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
9208 ; GFX6: ; %bb.0:
9209 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9210 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
9211 ; GFX6-NEXT: s_mov_b32 s6, 0
9212 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
9213 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
9214 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
9215 ; GFX6-NEXT: s_mov_b32 s4, s6
9216 ; GFX6-NEXT: s_mov_b32 s5, s6
9217 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
9218 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
9219 ; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
9220 ; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
9221 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
9222 ; GFX6-NEXT: v_not_b32_e32 v7, v4
9223 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
9224 ; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
9225 ; GFX6-NEXT: .LBB33_1: ; %atomicrmw.start
9226 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
9227 ; GFX6-NEXT: s_waitcnt vmcnt(0)
9228 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
9229 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9230 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v8
9231 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
9232 ; GFX6-NEXT: s_waitcnt expcnt(0)
9233 ; GFX6-NEXT: v_and_b32_e32 v4, v3, v7
9234 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v6, v2
9235 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
9236 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
9237 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
9238 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
9239 ; GFX6-NEXT: s_waitcnt vmcnt(0)
9240 ; GFX6-NEXT: buffer_wbinvl1
9241 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
9242 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
9243 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
9244 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
9245 ; GFX6-NEXT: s_cbranch_execnz .LBB33_1
9246 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
9247 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
9248 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
9249 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
9250 ; GFX6-NEXT: s_waitcnt expcnt(0)
9251 ; GFX6-NEXT: s_setpc_b64 s[30:31]
9252 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
9253 %result = atomicrmw fsub ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst
9254 ret bfloat %result
9255 }
9257 define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1) %ptr, bfloat %val) #0 {
9258 ; GFX12-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
9259 ; GFX12: ; %bb.0:
9260 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
9261 ; GFX12-NEXT: s_wait_expcnt 0x0
9262 ; GFX12-NEXT: s_wait_samplecnt 0x0
9263 ; GFX12-NEXT: s_wait_bvhcnt 0x0
9264 ; GFX12-NEXT: s_wait_kmcnt 0x0
9265 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
9266 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
9267 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9268 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
9269 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
9270 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
9271 ; GFX12-NEXT: s_mov_b32 s0, 0
9272 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
9273 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9274 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
9275 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
9276 ; GFX12-NEXT: v_not_b32_e32 v4, v4
9277 ; GFX12-NEXT: .LBB34_1: ; %atomicrmw.start
9278 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
9279 ; GFX12-NEXT: s_wait_loadcnt 0x0
9280 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
9281 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9282 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
9283 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
9284 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9285 ; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
9286 ; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
9287 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
9288 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
9289 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
9290 ; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
9291 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
9292 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9293 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
9294 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
9295 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
9296 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
9297 ; GFX12-NEXT: s_wait_storecnt 0x0
9298 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
9299 ; GFX12-NEXT: s_wait_loadcnt 0x0
9300 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
9301 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
9302 ; GFX12-NEXT: s_wait_alu 0xfffe
9303 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
9304 ; GFX12-NEXT: s_wait_alu 0xfffe
9305 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
9306 ; GFX12-NEXT: s_cbranch_execnz .LBB34_1
9307 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
9308 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
9309 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
9310 ; GFX12-NEXT: s_wait_alu 0xfffe
9311 ; GFX12-NEXT: s_setpc_b64 s[30:31]
9312 ;
9313 ; GFX940-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
9314 ; GFX940: ; %bb.0:
9315 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9316 ; GFX940-NEXT: s_movk_i32 s0, 0xf800
9317 ; GFX940-NEXT: s_mov_b32 s1, -1
9318 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
9319 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
9320 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
9321 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
9322 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v4
9323 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9324 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
9325 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
9326 ; GFX940-NEXT: v_not_b32_e32 v4, v4
9327 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
9328 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9329 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
9330 ; GFX940-NEXT: .LBB34_1: ; %atomicrmw.start
9331 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
9332 ; GFX940-NEXT: s_waitcnt vmcnt(0)
9333 ; GFX940-NEXT: v_mov_b32_e32 v7, v5
9334 ; GFX940-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9335 ; GFX940-NEXT: s_nop 0
9336 ; GFX940-NEXT: v_sub_f32_e32 v5, v5, v2
9337 ; GFX940-NEXT: v_bfe_u32 v6, v5, 16, 1
9338 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v5
9339 ; GFX940-NEXT: v_add3_u32 v6, v6, v5, s2
9340 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
9341 ; GFX940-NEXT: s_nop 1
9342 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
9343 ; GFX940-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9344 ; GFX940-NEXT: v_and_or_b32 v6, v7, v4, v5
9345 ; GFX940-NEXT: buffer_wbl2 sc1
9346 ; GFX940-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off sc0
9347 ; GFX940-NEXT: s_waitcnt vmcnt(0)
9348 ; GFX940-NEXT: buffer_inv sc1
9349 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
9350 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
9351 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
9352 ; GFX940-NEXT: s_cbranch_execnz .LBB34_1
9353 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
9354 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
9355 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
9356 ; GFX940-NEXT: s_setpc_b64 s[30:31]
9357 ;
9358 ; GFX11-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
9359 ; GFX11: ; %bb.0:
9360 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9361 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
9362 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
9363 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9364 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
9365 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
9366 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
9367 ; GFX11-NEXT: s_mov_b32 s0, 0
9368 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
9369 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9370 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
9371 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
9372 ; GFX11-NEXT: v_not_b32_e32 v4, v4
9373 ; GFX11-NEXT: .p2align 6
9374 ; GFX11-NEXT: .LBB34_1: ; %atomicrmw.start
9375 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
9376 ; GFX11-NEXT: s_waitcnt vmcnt(0)
9377 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
9378 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9379 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
9380 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
9381 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9382 ; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
9383 ; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
9384 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
9385 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
9386 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
9387 ; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
9388 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
9389 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9390 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
9391 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
9392 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
9393 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
9394 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
9395 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
9396 ; GFX11-NEXT: s_waitcnt vmcnt(0)
9397 ; GFX11-NEXT: buffer_gl1_inv
9398 ; GFX11-NEXT: buffer_gl0_inv
9399 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
9400 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
9401 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
9402 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
9403 ; GFX11-NEXT: s_cbranch_execnz .LBB34_1
9404 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
9405 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
9406 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
9407 ; GFX11-NEXT: s_setpc_b64 s[30:31]
9408 ;
9409 ; GFX10-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
9410 ; GFX10: ; %bb.0:
9411 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9412 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
9413 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
9414 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9415 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
9416 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
9417 ; GFX10-NEXT: s_mov_b32 s4, 0
9418 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
9419 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9420 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
9421 ; GFX10-NEXT: v_not_b32_e32 v4, v4
9422 ; GFX10-NEXT: .LBB34_1: ; %atomicrmw.start
9423 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
9424 ; GFX10-NEXT: s_waitcnt vmcnt(0)
9425 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
9426 ; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9427 ; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
9428 ; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
9429 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v5
9430 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
9431 ; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
9432 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
9433 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9434 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
9435 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
9436 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
9437 ; GFX10-NEXT: s_waitcnt vmcnt(0)
9438 ; GFX10-NEXT: buffer_gl1_inv
9439 ; GFX10-NEXT: buffer_gl0_inv
9440 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
9441 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
9442 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
9443 ; GFX10-NEXT: s_cbranch_execnz .LBB34_1
9444 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
9445 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
9446 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
9447 ; GFX10-NEXT: s_setpc_b64 s[30:31]
9448 ;
9449 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
9450 ; GFX90A: ; %bb.0:
9451 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9452 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
9453 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
9454 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
9455 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
9456 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
9457 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9458 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
9459 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
9460 ; GFX90A-NEXT: v_not_b32_e32 v4, v4
9461 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
9462 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9463 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
9464 ; GFX90A-NEXT: .LBB34_1: ; %atomicrmw.start
9465 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
9466 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
9467 ; GFX90A-NEXT: v_mov_b32_e32 v7, v5
9468 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9469 ; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v2
9470 ; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
9471 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v5
9472 ; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
9473 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
9474 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
9475 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9476 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v4, v5
9477 ; GFX90A-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off glc
9478 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
9479 ; GFX90A-NEXT: buffer_wbinvl1
9480 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
9481 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9482 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
9483 ; GFX90A-NEXT: s_cbranch_execnz .LBB34_1
9484 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
9485 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
9486 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
9487 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
9488 ;
9489 ; GFX908-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
9490 ; GFX908: ; %bb.0:
9491 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9492 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
9493 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
9494 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
9495 ; GFX908-NEXT: global_load_dword v5, v[0:1], off
9496 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
9497 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9498 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
9499 ; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
9500 ; GFX908-NEXT: v_not_b32_e32 v4, v4
9501 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
9502 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9503 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
9504 ; GFX908-NEXT: .LBB34_1: ; %atomicrmw.start
9505 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
9506 ; GFX908-NEXT: s_waitcnt vmcnt(0)
9507 ; GFX908-NEXT: v_mov_b32_e32 v6, v5
9508 ; GFX908-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9509 ; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
9510 ; GFX908-NEXT: v_bfe_u32 v7, v5, 16, 1
9511 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v5
9512 ; GFX908-NEXT: v_add3_u32 v7, v7, v5, s6
9513 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
9514 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc
9515 ; GFX908-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9516 ; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
9517 ; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
9518 ; GFX908-NEXT: s_waitcnt vmcnt(0)
9519 ; GFX908-NEXT: buffer_wbinvl1
9520 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
9521 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9522 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
9523 ; GFX908-NEXT: s_cbranch_execnz .LBB34_1
9524 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
9525 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
9526 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
9527 ; GFX908-NEXT: s_setpc_b64 s[30:31]
9528 ;
9529 ; GFX8-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
9530 ; GFX8: ; %bb.0:
9531 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9532 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
9533 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
9534 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
9535 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
9536 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
9537 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9538 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
9539 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
9540 ; GFX8-NEXT: v_not_b32_e32 v4, v4
9541 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
9542 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9543 ; GFX8-NEXT: .LBB34_1: ; %atomicrmw.start
9544 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
9545 ; GFX8-NEXT: s_waitcnt vmcnt(0)
9546 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
9547 ; GFX8-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9548 ; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
9549 ; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
9550 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
9551 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
9552 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
9553 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
9554 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
9555 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
9556 ; GFX8-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9557 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
9558 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
9559 ; GFX8-NEXT: s_waitcnt vmcnt(0)
9560 ; GFX8-NEXT: buffer_wbinvl1
9561 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
9562 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9563 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
9564 ; GFX8-NEXT: s_cbranch_execnz .LBB34_1
9565 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
9566 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
9567 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
9568 ; GFX8-NEXT: s_setpc_b64 s[30:31]
9569 ;
9570 ; GFX7-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
9571 ; GFX7: ; %bb.0:
9572 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9573 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
9574 ; GFX7-NEXT: s_mov_b32 s6, 0
9575 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
9576 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
9577 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
9578 ; GFX7-NEXT: s_mov_b32 s4, s6
9579 ; GFX7-NEXT: s_mov_b32 s5, s6
9580 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
9581 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
9582 ; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
9583 ; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
9584 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
9585 ; GFX7-NEXT: v_not_b32_e32 v7, v4
9586 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
9587 ; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
9588 ; GFX7-NEXT: .LBB34_1: ; %atomicrmw.start
9589 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
9590 ; GFX7-NEXT: s_waitcnt vmcnt(0)
9591 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
9592 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9593 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v8
9594 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
9595 ; GFX7-NEXT: v_and_b32_e32 v4, v3, v7
9596 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v6, v2
9597 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
9598 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
9599 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
9600 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
9601 ; GFX7-NEXT: s_waitcnt vmcnt(0)
9602 ; GFX7-NEXT: buffer_wbinvl1
9603 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
9604 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
9605 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
9606 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
9607 ; GFX7-NEXT: s_cbranch_execnz .LBB34_1
9608 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
9609 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
9610 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
9611 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
9612 ; GFX7-NEXT: s_setpc_b64 s[30:31]
9613 ;
9614 ; GFX6-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
9615 ; GFX6: ; %bb.0:
9616 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9617 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
9618 ; GFX6-NEXT: s_mov_b32 s6, 0
9619 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
9620 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
9621 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
9622 ; GFX6-NEXT: s_mov_b32 s4, s6
9623 ; GFX6-NEXT: s_mov_b32 s5, s6
9624 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
9625 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
9626 ; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
9627 ; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
9628 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
9629 ; GFX6-NEXT: v_not_b32_e32 v7, v4
9630 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
9631 ; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
9632 ; GFX6-NEXT: .LBB34_1: ; %atomicrmw.start
9633 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
9634 ; GFX6-NEXT: s_waitcnt vmcnt(0)
9635 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
9636 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9637 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v8
9638 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
9639 ; GFX6-NEXT: s_waitcnt expcnt(0)
9640 ; GFX6-NEXT: v_and_b32_e32 v4, v3, v7
9641 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v6, v2
9642 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
9643 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
9644 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
9645 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
9646 ; GFX6-NEXT: s_waitcnt vmcnt(0)
9647 ; GFX6-NEXT: buffer_wbinvl1
9648 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
9649 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
9650 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
9651 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
9652 ; GFX6-NEXT: s_cbranch_execnz .LBB34_1
9653 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
9654 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
9655 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
9656 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
9657 ; GFX6-NEXT: s_waitcnt expcnt(0)
9658 ; GFX6-NEXT: s_setpc_b64 s[30:31]
9659 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 -1024
9660 %result = atomicrmw fsub ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst
9661 ret bfloat %result
9662 }
9664 define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat %val) #0 {
9665 ; GFX12-LABEL: global_agent_atomic_fsub_noret_bf16:
9666 ; GFX12: ; %bb.0:
9667 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
9668 ; GFX12-NEXT: s_wait_expcnt 0x0
9669 ; GFX12-NEXT: s_wait_samplecnt 0x0
9670 ; GFX12-NEXT: s_wait_bvhcnt 0x0
9671 ; GFX12-NEXT: s_wait_kmcnt 0x0
9672 ; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
9673 ; GFX12-NEXT: s_mov_b32 s0, 0
9674 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
9675 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
9676 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
9677 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off
9678 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
9679 ; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
9680 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
9681 ; GFX12-NEXT: v_not_b32_e32 v6, v3
9682 ; GFX12-NEXT: .LBB35_1: ; %atomicrmw.start
9683 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
9684 ; GFX12-NEXT: s_wait_loadcnt 0x0
9685 ; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
9686 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9687 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
9688 ; GFX12-NEXT: v_sub_f32_e32 v3, v3, v2
9689 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
9690 ; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
9691 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
9692 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
9693 ; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
9694 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9695 ; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
9696 ; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
9697 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9698 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
9699 ; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
9700 ; GFX12-NEXT: s_wait_storecnt 0x0
9701 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
9702 ; GFX12-NEXT: s_wait_loadcnt 0x0
9703 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
9704 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
9705 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
9706 ; GFX12-NEXT: s_wait_alu 0xfffe
9707 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
9708 ; GFX12-NEXT: s_wait_alu 0xfffe
9709 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
9710 ; GFX12-NEXT: s_cbranch_execnz .LBB35_1
9711 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
9712 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
9713 ; GFX12-NEXT: s_wait_alu 0xfffe
9714 ; GFX12-NEXT: s_setpc_b64 s[30:31]
9715 ;
9716 ; GFX940-LABEL: global_agent_atomic_fsub_noret_bf16:
9717 ; GFX940: ; %bb.0:
9718 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9719 ; GFX940-NEXT: v_mov_b32_e32 v3, v0
9720 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v3
9721 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
9722 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v3
9723 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9724 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
9725 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
9726 ; GFX940-NEXT: v_not_b32_e32 v6, v4
9727 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
9728 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9729 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
9730 ; GFX940-NEXT: .LBB35_1: ; %atomicrmw.start
9731 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
9732 ; GFX940-NEXT: s_waitcnt vmcnt(0)
9733 ; GFX940-NEXT: v_lshrrev_b32_sdwa v4, v3, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9734 ; GFX940-NEXT: s_nop 0
9735 ; GFX940-NEXT: v_sub_f32_e32 v4, v4, v2
9736 ; GFX940-NEXT: v_bfe_u32 v7, v4, 16, 1
9737 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v4
9738 ; GFX940-NEXT: v_add3_u32 v7, v7, v4, s2
9739 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
9740 ; GFX940-NEXT: s_nop 1
9741 ; GFX940-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
9742 ; GFX940-NEXT: v_lshlrev_b32_sdwa v4, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9743 ; GFX940-NEXT: v_and_or_b32 v4, v5, v6, v4
9744 ; GFX940-NEXT: buffer_wbl2 sc1
9745 ; GFX940-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0
9746 ; GFX940-NEXT: s_waitcnt vmcnt(0)
9747 ; GFX940-NEXT: buffer_inv sc1
9748 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
9749 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
9750 ; GFX940-NEXT: v_mov_b32_e32 v5, v4
9751 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
9752 ; GFX940-NEXT: s_cbranch_execnz .LBB35_1
9753 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
9754 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
9755 ; GFX940-NEXT: s_setpc_b64 s[30:31]
9756 ;
9757 ; GFX11-LABEL: global_agent_atomic_fsub_noret_bf16:
9758 ; GFX11: ; %bb.0:
9759 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9760 ; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
9761 ; GFX11-NEXT: s_mov_b32 s0, 0
9762 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
9763 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
9764 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
9765 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off
9766 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
9767 ; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
9768 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
9769 ; GFX11-NEXT: v_not_b32_e32 v6, v3
9770 ; GFX11-NEXT: .p2align 6
9771 ; GFX11-NEXT: .LBB35_1: ; %atomicrmw.start
9772 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
9773 ; GFX11-NEXT: s_waitcnt vmcnt(0)
9774 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
9775 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9776 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
9777 ; GFX11-NEXT: v_sub_f32_e32 v3, v3, v2
9778 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
9779 ; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
9780 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
9781 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
9782 ; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
9783 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9784 ; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
9785 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
9786 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9787 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
9788 ; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
9789 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
9790 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
9791 ; GFX11-NEXT: s_waitcnt vmcnt(0)
9792 ; GFX11-NEXT: buffer_gl1_inv
9793 ; GFX11-NEXT: buffer_gl0_inv
9794 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
9795 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
9796 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
9797 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
9798 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
9799 ; GFX11-NEXT: s_cbranch_execnz .LBB35_1
9800 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
9801 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
9802 ; GFX11-NEXT: s_setpc_b64 s[30:31]
9803 ;
9804 ; GFX10-LABEL: global_agent_atomic_fsub_noret_bf16:
9805 ; GFX10: ; %bb.0:
9806 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9807 ; GFX10-NEXT: v_mov_b32_e32 v3, v0
9808 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9809 ; GFX10-NEXT: s_mov_b32 s4, 0
9810 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
9811 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
9812 ; GFX10-NEXT: global_load_dword v4, v[0:1], off
9813 ; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
9814 ; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
9815 ; GFX10-NEXT: v_not_b32_e32 v6, v3
9816 ; GFX10-NEXT: .LBB35_1: ; %atomicrmw.start
9817 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
9818 ; GFX10-NEXT: s_waitcnt vmcnt(0)
9819 ; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9820 ; GFX10-NEXT: v_sub_f32_e32 v3, v3, v2
9821 ; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
9822 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
9823 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
9824 ; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
9825 ; GFX10-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
9826 ; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9827 ; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
9828 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
9829 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
9830 ; GFX10-NEXT: s_waitcnt vmcnt(0)
9831 ; GFX10-NEXT: buffer_gl1_inv
9832 ; GFX10-NEXT: buffer_gl0_inv
9833 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
9834 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
9835 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
9836 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
9837 ; GFX10-NEXT: s_cbranch_execnz .LBB35_1
9838 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
9839 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
9840 ; GFX10-NEXT: s_setpc_b64 s[30:31]
9841 ;
9842 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_bf16:
9843 ; GFX90A: ; %bb.0:
9844 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9845 ; GFX90A-NEXT: v_mov_b32_e32 v3, v0
9846 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
9847 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
9848 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
9849 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9850 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
9851 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
9852 ; GFX90A-NEXT: v_not_b32_e32 v6, v4
9853 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
9854 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9855 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
9856 ; GFX90A-NEXT: .LBB35_1: ; %atomicrmw.start
9857 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
9858 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
9859 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v4, v3, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9860 ; GFX90A-NEXT: v_sub_f32_e32 v4, v4, v2
9861 ; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
9862 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
9863 ; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s6
9864 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
9865 ; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
9866 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v4, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9867 ; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
9868 ; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
9869 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
9870 ; GFX90A-NEXT: buffer_wbinvl1
9871 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
9872 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9873 ; GFX90A-NEXT: v_mov_b32_e32 v5, v4
9874 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
9875 ; GFX90A-NEXT: s_cbranch_execnz .LBB35_1
9876 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
9877 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
9878 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
9879 ;
9880 ; GFX908-LABEL: global_agent_atomic_fsub_noret_bf16:
9881 ; GFX908: ; %bb.0:
9882 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9883 ; GFX908-NEXT: v_mov_b32_e32 v3, v0
9884 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
9885 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
9886 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
9887 ; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
9888 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
9889 ; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
9890 ; GFX908-NEXT: v_not_b32_e32 v6, v3
9891 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
9892 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9893 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
9894 ; GFX908-NEXT: .LBB35_1: ; %atomicrmw.start
9895 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
9896 ; GFX908-NEXT: s_waitcnt vmcnt(0)
9897 ; GFX908-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9898 ; GFX908-NEXT: v_sub_f32_e32 v3, v3, v2
9899 ; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
9900 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
9901 ; GFX908-NEXT: v_add3_u32 v7, v7, v3, s6
9902 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
9903 ; GFX908-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
9904 ; GFX908-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9905 ; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
9906 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
9907 ; GFX908-NEXT: s_waitcnt vmcnt(0)
9908 ; GFX908-NEXT: buffer_wbinvl1
9909 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
9910 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9911 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
9912 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
9913 ; GFX908-NEXT: s_cbranch_execnz .LBB35_1
9914 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
9915 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
9916 ; GFX908-NEXT: s_setpc_b64 s[30:31]
9917 ;
9918 ; GFX8-LABEL: global_agent_atomic_fsub_noret_bf16:
9919 ; GFX8: ; %bb.0:
9920 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9921 ; GFX8-NEXT: v_mov_b32_e32 v3, v0
9922 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
9923 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
9924 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
9925 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
9926 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
9927 ; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
9928 ; GFX8-NEXT: v_not_b32_e32 v6, v3
9929 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
9930 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9931 ; GFX8-NEXT: .LBB35_1: ; %atomicrmw.start
9932 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
9933 ; GFX8-NEXT: s_waitcnt vmcnt(0)
9934 ; GFX8-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9935 ; GFX8-NEXT: v_sub_f32_e32 v3, v3, v2
9936 ; GFX8-NEXT: v_bfe_u32 v8, v3, 16, 1
9937 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v3
9938 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
9939 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v3
9940 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
9941 ; GFX8-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc
9942 ; GFX8-NEXT: v_and_b32_e32 v7, v4, v6
9943 ; GFX8-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9944 ; GFX8-NEXT: v_or_b32_e32 v3, v7, v3
9945 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
9946 ; GFX8-NEXT: s_waitcnt vmcnt(0)
9947 ; GFX8-NEXT: buffer_wbinvl1
9948 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
9949 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9950 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
9951 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
9952 ; GFX8-NEXT: s_cbranch_execnz .LBB35_1
9953 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
9954 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
9955 ; GFX8-NEXT: s_setpc_b64 s[30:31]
9956 ;
9957 ; GFX7-LABEL: global_agent_atomic_fsub_noret_bf16:
9958 ; GFX7: ; %bb.0:
9959 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9960 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
9961 ; GFX7-NEXT: s_mov_b32 s6, 0
9962 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
9963 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
9964 ; GFX7-NEXT: s_mov_b32 s4, s6
9965 ; GFX7-NEXT: s_mov_b32 s5, s6
9966 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
9967 ; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
9968 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 3, v3
9969 ; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
9970 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
9971 ; GFX7-NEXT: v_not_b32_e32 v6, v3
9972 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
9973 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
9974 ; GFX7-NEXT: .LBB35_1: ; %atomicrmw.start
9975 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
9976 ; GFX7-NEXT: s_waitcnt vmcnt(0)
9977 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, v5, v4
9978 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
9979 ; GFX7-NEXT: v_sub_f32_e32 v3, v3, v2
9980 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
9981 ; GFX7-NEXT: v_and_b32_e32 v7, v4, v6
9982 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, v5, v3
9983 ; GFX7-NEXT: v_or_b32_e32 v3, v7, v3
9984 ; GFX7-NEXT: v_mov_b32_e32 v8, v4
9985 ; GFX7-NEXT: v_mov_b32_e32 v7, v3
9986 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
9987 ; GFX7-NEXT: s_waitcnt vmcnt(0)
9988 ; GFX7-NEXT: buffer_wbinvl1
9989 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v4
9990 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
9991 ; GFX7-NEXT: v_mov_b32_e32 v4, v7
9992 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
9993 ; GFX7-NEXT: s_cbranch_execnz .LBB35_1
9994 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
9995 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
9996 ; GFX7-NEXT: s_setpc_b64 s[30:31]
9997 ;
9998 ; GFX6-LABEL: global_agent_atomic_fsub_noret_bf16:
9999 ; GFX6: ; %bb.0:
10000 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10001 ; GFX6-NEXT: v_mov_b32_e32 v3, v0
10002 ; GFX6-NEXT: s_mov_b32 s6, 0
10003 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v3
10004 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
10005 ; GFX6-NEXT: s_mov_b32 s4, s6
10006 ; GFX6-NEXT: s_mov_b32 s5, s6
10007 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
10008 ; GFX6-NEXT: v_and_b32_e32 v3, 3, v3
10009 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 3, v3
10010 ; GFX6-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
10011 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
10012 ; GFX6-NEXT: v_not_b32_e32 v6, v3
10013 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
10014 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
10015 ; GFX6-NEXT: .LBB35_1: ; %atomicrmw.start
10016 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
10017 ; GFX6-NEXT: s_waitcnt vmcnt(0)
10018 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, v5, v4
10019 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
10020 ; GFX6-NEXT: v_sub_f32_e32 v3, v3, v2
10021 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
10022 ; GFX6-NEXT: s_waitcnt expcnt(0)
10023 ; GFX6-NEXT: v_and_b32_e32 v7, v4, v6
10024 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, v5, v3
10025 ; GFX6-NEXT: v_or_b32_e32 v3, v7, v3
10026 ; GFX6-NEXT: v_mov_b32_e32 v8, v4
10027 ; GFX6-NEXT: v_mov_b32_e32 v7, v3
10028 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
10029 ; GFX6-NEXT: s_waitcnt vmcnt(0)
10030 ; GFX6-NEXT: buffer_wbinvl1
10031 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v4
10032 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
10033 ; GFX6-NEXT: v_mov_b32_e32 v4, v7
10034 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
10035 ; GFX6-NEXT: s_cbranch_execnz .LBB35_1
10036 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
10037 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
10038 ; GFX6-NEXT: s_waitcnt expcnt(0)
10039 ; GFX6-NEXT: s_setpc_b64 s[30:31]
10040 %unused = atomicrmw fsub ptr addrspace(1) %ptr, bfloat %val syncscope("agent") seq_cst
10041 ret void
10042 }
10044 define void @global_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
10045 ; GFX12-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_pos:
10046 ; GFX12: ; %bb.0:
10047 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
10048 ; GFX12-NEXT: s_wait_expcnt 0x0
10049 ; GFX12-NEXT: s_wait_samplecnt 0x0
10050 ; GFX12-NEXT: s_wait_bvhcnt 0x0
10051 ; GFX12-NEXT: s_wait_kmcnt 0x0
10052 ; GFX12-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
10053 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
10054 ; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10055 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
10056 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v4
10057 ; GFX12-NEXT: v_and_b32_e32 v4, 3, v4
10058 ; GFX12-NEXT: s_mov_b32 s0, 0
10059 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
10060 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10061 ; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
10062 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
10063 ; GFX12-NEXT: v_not_b32_e32 v5, v5
10064 ; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
10065 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
10066 ; GFX12-NEXT: s_wait_loadcnt 0x0
10067 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
10068 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10069 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10070 ; GFX12-NEXT: v_sub_f32_e32 v2, v2, v6
10071 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
10072 ; GFX12-NEXT: v_bfe_u32 v7, v2, 16, 1
10073 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v2
10074 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
10075 ; GFX12-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
10076 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10077 ; GFX12-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
10078 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10079 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10080 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, v4, v2
10081 ; GFX12-NEXT: v_and_or_b32 v2, v3, v5, v2
10082 ; GFX12-NEXT: s_wait_storecnt 0x0
10083 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
10084 ; GFX12-NEXT: s_wait_loadcnt 0x0
10085 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
10086 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
10087 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
10088 ; GFX12-NEXT: s_wait_alu 0xfffe
10089 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
10090 ; GFX12-NEXT: s_wait_alu 0xfffe
10091 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
10092 ; GFX12-NEXT: s_cbranch_execnz .LBB36_1
10093 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
10094 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
10095 ; GFX12-NEXT: s_wait_alu 0xfffe
10096 ; GFX12-NEXT: s_setpc_b64 s[30:31]
10097 ;
10098 ; GFX940-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_pos:
10099 ; GFX940: ; %bb.0:
10100 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10101 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
10102 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
10103 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
10104 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
10105 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
10106 ; GFX940-NEXT: v_and_b32_e32 v4, 3, v4
10107 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10108 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
10109 ; GFX940-NEXT: v_lshlrev_b32_e64 v5, v4, s0
10110 ; GFX940-NEXT: v_not_b32_e32 v5, v5
10111 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
10112 ; GFX940-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10113 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
10114 ; GFX940-NEXT: .LBB36_1: ; %atomicrmw.start
10115 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
10116 ; GFX940-NEXT: s_waitcnt vmcnt(0)
10117 ; GFX940-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10118 ; GFX940-NEXT: s_nop 0
10119 ; GFX940-NEXT: v_sub_f32_e32 v2, v2, v6
10120 ; GFX940-NEXT: v_bfe_u32 v7, v2, 16, 1
10121 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v2
10122 ; GFX940-NEXT: v_add3_u32 v7, v7, v2, s2
10123 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
10124 ; GFX940-NEXT: s_nop 1
10125 ; GFX940-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
10126 ; GFX940-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10127 ; GFX940-NEXT: v_and_or_b32 v2, v3, v5, v2
10128 ; GFX940-NEXT: buffer_wbl2 sc1
10129 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
10130 ; GFX940-NEXT: s_waitcnt vmcnt(0)
10131 ; GFX940-NEXT: buffer_inv sc1
10132 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
10133 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
10134 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
10135 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
10136 ; GFX940-NEXT: s_cbranch_execnz .LBB36_1
10137 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
10138 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
10139 ; GFX940-NEXT: s_setpc_b64 s[30:31]
10140 ;
10141 ; GFX11-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_pos:
10142 ; GFX11: ; %bb.0:
10143 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10144 ; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
10145 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
10146 ; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10147 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
10148 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v4
10149 ; GFX11-NEXT: v_and_b32_e32 v4, 3, v4
10150 ; GFX11-NEXT: s_mov_b32 s0, 0
10151 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
10152 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10153 ; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
10154 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
10155 ; GFX11-NEXT: v_not_b32_e32 v5, v5
10156 ; GFX11-NEXT: .p2align 6
10157 ; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
10158 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
10159 ; GFX11-NEXT: s_waitcnt vmcnt(0)
10160 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
10161 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10162 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10163 ; GFX11-NEXT: v_sub_f32_e32 v2, v2, v6
10164 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
10165 ; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
10166 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v2
10167 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
10168 ; GFX11-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
10169 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10170 ; GFX11-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
10171 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10172 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10173 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, v4, v2
10174 ; GFX11-NEXT: v_and_or_b32 v2, v3, v5, v2
10175 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
10176 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
10177 ; GFX11-NEXT: s_waitcnt vmcnt(0)
10178 ; GFX11-NEXT: buffer_gl1_inv
10179 ; GFX11-NEXT: buffer_gl0_inv
10180 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
10181 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
10182 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
10183 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
10184 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
10185 ; GFX11-NEXT: s_cbranch_execnz .LBB36_1
10186 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
10187 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
10188 ; GFX11-NEXT: s_setpc_b64 s[30:31]
10189 ;
10190 ; GFX10-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_pos:
10191 ; GFX10: ; %bb.0:
10192 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10193 ; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
10194 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
10195 ; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10196 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
10197 ; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
10198 ; GFX10-NEXT: s_mov_b32 s4, 0
10199 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
10200 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10201 ; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
10202 ; GFX10-NEXT: v_not_b32_e32 v5, v5
10203 ; GFX10-NEXT: .LBB36_1: ; %atomicrmw.start
10204 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
10205 ; GFX10-NEXT: s_waitcnt vmcnt(0)
10206 ; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10207 ; GFX10-NEXT: v_sub_f32_e32 v2, v2, v6
10208 ; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
10209 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v2
10210 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
10211 ; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
10212 ; GFX10-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
10213 ; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10214 ; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
10215 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
10216 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
10217 ; GFX10-NEXT: s_waitcnt vmcnt(0)
10218 ; GFX10-NEXT: buffer_gl1_inv
10219 ; GFX10-NEXT: buffer_gl0_inv
10220 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
10221 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
10222 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
10223 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
10224 ; GFX10-NEXT: s_cbranch_execnz .LBB36_1
10225 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
10226 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
10227 ; GFX10-NEXT: s_setpc_b64 s[30:31]
10228 ;
10229 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_pos:
10230 ; GFX90A: ; %bb.0:
10231 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10232 ; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
10233 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
10234 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
10235 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
10236 ; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
10237 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10238 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
10239 ; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
10240 ; GFX90A-NEXT: v_not_b32_e32 v5, v5
10241 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
10242 ; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10243 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
10244 ; GFX90A-NEXT: .LBB36_1: ; %atomicrmw.start
10245 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
10246 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
10247 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10248 ; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v6
10249 ; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
10250 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
10251 ; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s6
10252 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
10253 ; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
10254 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10255 ; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
10256 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
10257 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
10258 ; GFX90A-NEXT: buffer_wbinvl1
10259 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
10260 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
10261 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
10262 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
10263 ; GFX90A-NEXT: s_cbranch_execnz .LBB36_1
10264 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
10265 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
10266 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
10267 ;
10268 ; GFX908-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_pos:
10269 ; GFX908: ; %bb.0:
10270 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10271 ; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
10272 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
10273 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
10274 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
10275 ; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
10276 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10277 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
10278 ; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
10279 ; GFX908-NEXT: v_not_b32_e32 v5, v5
10280 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
10281 ; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10282 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
10283 ; GFX908-NEXT: .LBB36_1: ; %atomicrmw.start
10284 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
10285 ; GFX908-NEXT: s_waitcnt vmcnt(0)
10286 ; GFX908-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10287 ; GFX908-NEXT: v_sub_f32_e32 v2, v2, v6
10288 ; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
10289 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
10290 ; GFX908-NEXT: v_add3_u32 v7, v7, v2, s6
10291 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
10292 ; GFX908-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
10293 ; GFX908-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10294 ; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
10295 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
10296 ; GFX908-NEXT: s_waitcnt vmcnt(0)
10297 ; GFX908-NEXT: buffer_wbinvl1
10298 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
10299 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
10300 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
10301 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
10302 ; GFX908-NEXT: s_cbranch_execnz .LBB36_1
10303 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
10304 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
10305 ; GFX908-NEXT: s_setpc_b64 s[30:31]
10306 ;
10307 ; GFX8-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_pos:
10308 ; GFX8: ; %bb.0:
10309 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10310 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
10311 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
10312 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
10313 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
10314 ; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
10315 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10316 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
10317 ; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
10318 ; GFX8-NEXT: v_not_b32_e32 v5, v5
10319 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
10320 ; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10321 ; GFX8-NEXT: .LBB36_1: ; %atomicrmw.start
10322 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
10323 ; GFX8-NEXT: s_waitcnt vmcnt(0)
10324 ; GFX8-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10325 ; GFX8-NEXT: v_sub_f32_e32 v2, v2, v6
10326 ; GFX8-NEXT: v_bfe_u32 v8, v2, 16, 1
10327 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v2
10328 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
10329 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v2
10330 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
10331 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v8, v9, vcc
10332 ; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
10333 ; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10334 ; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
10335 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10336 ; GFX8-NEXT: s_waitcnt vmcnt(0)
10337 ; GFX8-NEXT: buffer_wbinvl1
10338 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
10339 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
10340 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
10341 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
10342 ; GFX8-NEXT: s_cbranch_execnz .LBB36_1
10343 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
10344 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
10345 ; GFX8-NEXT: s_setpc_b64 s[30:31]
10346 ;
10347 ; GFX7-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_pos:
10348 ; GFX7: ; %bb.0:
10349 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10350 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
10351 ; GFX7-NEXT: s_mov_b32 s6, 0
10352 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
10353 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
10354 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
10355 ; GFX7-NEXT: s_mov_b32 s4, s6
10356 ; GFX7-NEXT: s_mov_b32 s5, s6
10357 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
10358 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
10359 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10360 ; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
10361 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
10362 ; GFX7-NEXT: v_not_b32_e32 v5, v5
10363 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
10364 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
10365 ; GFX7-NEXT: .LBB36_1: ; %atomicrmw.start
10366 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
10367 ; GFX7-NEXT: s_waitcnt vmcnt(0)
10368 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
10369 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10370 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v6
10371 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10372 ; GFX7-NEXT: v_and_b32_e32 v7, v3, v5
10373 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v4, v2
10374 ; GFX7-NEXT: v_or_b32_e32 v2, v7, v2
10375 ; GFX7-NEXT: v_mov_b32_e32 v8, v3
10376 ; GFX7-NEXT: v_mov_b32_e32 v7, v2
10377 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
10378 ; GFX7-NEXT: s_waitcnt vmcnt(0)
10379 ; GFX7-NEXT: buffer_wbinvl1
10380 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
10381 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
10382 ; GFX7-NEXT: v_mov_b32_e32 v3, v7
10383 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
10384 ; GFX7-NEXT: s_cbranch_execnz .LBB36_1
10385 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
10386 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
10387 ; GFX7-NEXT: s_setpc_b64 s[30:31]
10388 ;
10389 ; GFX6-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_pos:
10390 ; GFX6: ; %bb.0:
10391 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10392 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
10393 ; GFX6-NEXT: s_mov_b32 s6, 0
10394 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
10395 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
10396 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
10397 ; GFX6-NEXT: s_mov_b32 s4, s6
10398 ; GFX6-NEXT: s_mov_b32 s5, s6
10399 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
10400 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
10401 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10402 ; GFX6-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
10403 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
10404 ; GFX6-NEXT: v_not_b32_e32 v5, v5
10405 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
10406 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
10407 ; GFX6-NEXT: .LBB36_1: ; %atomicrmw.start
10408 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
10409 ; GFX6-NEXT: s_waitcnt vmcnt(0)
10410 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
10411 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10412 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v6
10413 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10414 ; GFX6-NEXT: s_waitcnt expcnt(0)
10415 ; GFX6-NEXT: v_and_b32_e32 v7, v3, v5
10416 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v4, v2
10417 ; GFX6-NEXT: v_or_b32_e32 v2, v7, v2
10418 ; GFX6-NEXT: v_mov_b32_e32 v8, v3
10419 ; GFX6-NEXT: v_mov_b32_e32 v7, v2
10420 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
10421 ; GFX6-NEXT: s_waitcnt vmcnt(0)
10422 ; GFX6-NEXT: buffer_wbinvl1
10423 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
10424 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
10425 ; GFX6-NEXT: v_mov_b32_e32 v3, v7
10426 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
10427 ; GFX6-NEXT: s_cbranch_execnz .LBB36_1
10428 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
10429 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
10430 ; GFX6-NEXT: s_waitcnt expcnt(0)
10431 ; GFX6-NEXT: s_setpc_b64 s[30:31]
10432 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
10433 %unused = atomicrmw fsub ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst
10434 ret void
10435 }
10437 define void @global_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr addrspace(1) %ptr, bfloat %val) #0 {
10438 ; GFX12-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_neg:
10439 ; GFX12: ; %bb.0:
10440 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
10441 ; GFX12-NEXT: s_wait_expcnt 0x0
10442 ; GFX12-NEXT: s_wait_samplecnt 0x0
10443 ; GFX12-NEXT: s_wait_bvhcnt 0x0
10444 ; GFX12-NEXT: s_wait_kmcnt 0x0
10445 ; GFX12-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
10446 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
10447 ; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10448 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
10449 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v4
10450 ; GFX12-NEXT: v_and_b32_e32 v4, 3, v4
10451 ; GFX12-NEXT: s_mov_b32 s0, 0
10452 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
10453 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10454 ; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
10455 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
10456 ; GFX12-NEXT: v_not_b32_e32 v5, v5
10457 ; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
10458 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
10459 ; GFX12-NEXT: s_wait_loadcnt 0x0
10460 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
10461 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10462 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10463 ; GFX12-NEXT: v_sub_f32_e32 v2, v2, v6
10464 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
10465 ; GFX12-NEXT: v_bfe_u32 v7, v2, 16, 1
10466 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v2
10467 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
10468 ; GFX12-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
10469 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10470 ; GFX12-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
10471 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10472 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10473 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, v4, v2
10474 ; GFX12-NEXT: v_and_or_b32 v2, v3, v5, v2
10475 ; GFX12-NEXT: s_wait_storecnt 0x0
10476 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
10477 ; GFX12-NEXT: s_wait_loadcnt 0x0
10478 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
10479 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
10480 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
10481 ; GFX12-NEXT: s_wait_alu 0xfffe
10482 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
10483 ; GFX12-NEXT: s_wait_alu 0xfffe
10484 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
10485 ; GFX12-NEXT: s_cbranch_execnz .LBB37_1
10486 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
10487 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
10488 ; GFX12-NEXT: s_wait_alu 0xfffe
10489 ; GFX12-NEXT: s_setpc_b64 s[30:31]
10490 ;
10491 ; GFX940-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_neg:
10492 ; GFX940: ; %bb.0:
10493 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10494 ; GFX940-NEXT: s_movk_i32 s0, 0xf800
10495 ; GFX940-NEXT: s_mov_b32 s1, -1
10496 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
10497 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
10498 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
10499 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
10500 ; GFX940-NEXT: v_and_b32_e32 v4, 3, v4
10501 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10502 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
10503 ; GFX940-NEXT: v_lshlrev_b32_e64 v5, v4, s0
10504 ; GFX940-NEXT: v_not_b32_e32 v5, v5
10505 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
10506 ; GFX940-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10507 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
10508 ; GFX940-NEXT: .LBB37_1: ; %atomicrmw.start
10509 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
10510 ; GFX940-NEXT: s_waitcnt vmcnt(0)
10511 ; GFX940-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10512 ; GFX940-NEXT: s_nop 0
10513 ; GFX940-NEXT: v_sub_f32_e32 v2, v2, v6
10514 ; GFX940-NEXT: v_bfe_u32 v7, v2, 16, 1
10515 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v2
10516 ; GFX940-NEXT: v_add3_u32 v7, v7, v2, s2
10517 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
10518 ; GFX940-NEXT: s_nop 1
10519 ; GFX940-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
10520 ; GFX940-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10521 ; GFX940-NEXT: v_and_or_b32 v2, v3, v5, v2
10522 ; GFX940-NEXT: buffer_wbl2 sc1
10523 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
10524 ; GFX940-NEXT: s_waitcnt vmcnt(0)
10525 ; GFX940-NEXT: buffer_inv sc1
10526 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
10527 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
10528 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
10529 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
10530 ; GFX940-NEXT: s_cbranch_execnz .LBB37_1
10531 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
10532 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
10533 ; GFX940-NEXT: s_setpc_b64 s[30:31]
10534 ;
10535 ; GFX11-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_neg:
10536 ; GFX11: ; %bb.0:
10537 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10538 ; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
10539 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
10540 ; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10541 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
10542 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v4
10543 ; GFX11-NEXT: v_and_b32_e32 v4, 3, v4
10544 ; GFX11-NEXT: s_mov_b32 s0, 0
10545 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
10546 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10547 ; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
10548 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
10549 ; GFX11-NEXT: v_not_b32_e32 v5, v5
10550 ; GFX11-NEXT: .p2align 6
10551 ; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
10552 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
10553 ; GFX11-NEXT: s_waitcnt vmcnt(0)
10554 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
10555 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10556 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10557 ; GFX11-NEXT: v_sub_f32_e32 v2, v2, v6
10558 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
10559 ; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
10560 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v2
10561 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
10562 ; GFX11-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
10563 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10564 ; GFX11-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
10565 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10566 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10567 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, v4, v2
10568 ; GFX11-NEXT: v_and_or_b32 v2, v3, v5, v2
10569 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
10570 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
10571 ; GFX11-NEXT: s_waitcnt vmcnt(0)
10572 ; GFX11-NEXT: buffer_gl1_inv
10573 ; GFX11-NEXT: buffer_gl0_inv
10574 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
10575 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
10576 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
10577 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
10578 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
10579 ; GFX11-NEXT: s_cbranch_execnz .LBB37_1
10580 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
10581 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
10582 ; GFX11-NEXT: s_setpc_b64 s[30:31]
10583 ;
10584 ; GFX10-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_neg:
10585 ; GFX10: ; %bb.0:
10586 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10587 ; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
10588 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
10589 ; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10590 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
10591 ; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
10592 ; GFX10-NEXT: s_mov_b32 s4, 0
10593 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
10594 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10595 ; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
10596 ; GFX10-NEXT: v_not_b32_e32 v5, v5
10597 ; GFX10-NEXT: .LBB37_1: ; %atomicrmw.start
10598 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
10599 ; GFX10-NEXT: s_waitcnt vmcnt(0)
10600 ; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10601 ; GFX10-NEXT: v_sub_f32_e32 v2, v2, v6
10602 ; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
10603 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v2
10604 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
10605 ; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
10606 ; GFX10-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
10607 ; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10608 ; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
10609 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
10610 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
10611 ; GFX10-NEXT: s_waitcnt vmcnt(0)
10612 ; GFX10-NEXT: buffer_gl1_inv
10613 ; GFX10-NEXT: buffer_gl0_inv
10614 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
10615 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
10616 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
10617 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
10618 ; GFX10-NEXT: s_cbranch_execnz .LBB37_1
10619 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
10620 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
10621 ; GFX10-NEXT: s_setpc_b64 s[30:31]
10622 ;
10623 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_neg:
10624 ; GFX90A: ; %bb.0:
10625 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10626 ; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
10627 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
10628 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
10629 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
10630 ; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
10631 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10632 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
10633 ; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
10634 ; GFX90A-NEXT: v_not_b32_e32 v5, v5
10635 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
10636 ; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10637 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
10638 ; GFX90A-NEXT: .LBB37_1: ; %atomicrmw.start
10639 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
10640 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
10641 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10642 ; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v6
10643 ; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
10644 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
10645 ; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s6
10646 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
10647 ; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
10648 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10649 ; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
10650 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
10651 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
10652 ; GFX90A-NEXT: buffer_wbinvl1
10653 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
10654 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
10655 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
10656 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
10657 ; GFX90A-NEXT: s_cbranch_execnz .LBB37_1
10658 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
10659 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
10660 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
10661 ;
10662 ; GFX908-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_neg:
10663 ; GFX908: ; %bb.0:
10664 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10665 ; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
10666 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
10667 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
10668 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
10669 ; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
10670 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10671 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
10672 ; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
10673 ; GFX908-NEXT: v_not_b32_e32 v5, v5
10674 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
10675 ; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10676 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
10677 ; GFX908-NEXT: .LBB37_1: ; %atomicrmw.start
10678 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
10679 ; GFX908-NEXT: s_waitcnt vmcnt(0)
10680 ; GFX908-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10681 ; GFX908-NEXT: v_sub_f32_e32 v2, v2, v6
10682 ; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
10683 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
10684 ; GFX908-NEXT: v_add3_u32 v7, v7, v2, s6
10685 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
10686 ; GFX908-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
10687 ; GFX908-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10688 ; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
10689 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
10690 ; GFX908-NEXT: s_waitcnt vmcnt(0)
10691 ; GFX908-NEXT: buffer_wbinvl1
10692 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
10693 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
10694 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
10695 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
10696 ; GFX908-NEXT: s_cbranch_execnz .LBB37_1
10697 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
10698 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
10699 ; GFX908-NEXT: s_setpc_b64 s[30:31]
10700 ;
10701 ; GFX8-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_neg:
10702 ; GFX8: ; %bb.0:
10703 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10704 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
10705 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
10706 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
10707 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
10708 ; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
10709 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10710 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
10711 ; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
10712 ; GFX8-NEXT: v_not_b32_e32 v5, v5
10713 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
10714 ; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10715 ; GFX8-NEXT: .LBB37_1: ; %atomicrmw.start
10716 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
10717 ; GFX8-NEXT: s_waitcnt vmcnt(0)
10718 ; GFX8-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10719 ; GFX8-NEXT: v_sub_f32_e32 v2, v2, v6
10720 ; GFX8-NEXT: v_bfe_u32 v8, v2, 16, 1
10721 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v2
10722 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
10723 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v2
10724 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
10725 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v8, v9, vcc
10726 ; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
10727 ; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10728 ; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
10729 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10730 ; GFX8-NEXT: s_waitcnt vmcnt(0)
10731 ; GFX8-NEXT: buffer_wbinvl1
10732 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
10733 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
10734 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
10735 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
10736 ; GFX8-NEXT: s_cbranch_execnz .LBB37_1
10737 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
10738 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
10739 ; GFX8-NEXT: s_setpc_b64 s[30:31]
10740 ;
10741 ; GFX7-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_neg:
10742 ; GFX7: ; %bb.0:
10743 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10744 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
10745 ; GFX7-NEXT: s_mov_b32 s6, 0
10746 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
10747 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
10748 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
10749 ; GFX7-NEXT: s_mov_b32 s4, s6
10750 ; GFX7-NEXT: s_mov_b32 s5, s6
10751 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
10752 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
10753 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10754 ; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
10755 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
10756 ; GFX7-NEXT: v_not_b32_e32 v5, v5
10757 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
10758 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
10759 ; GFX7-NEXT: .LBB37_1: ; %atomicrmw.start
10760 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
10761 ; GFX7-NEXT: s_waitcnt vmcnt(0)
10762 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
10763 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10764 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v6
10765 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10766 ; GFX7-NEXT: v_and_b32_e32 v7, v3, v5
10767 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v4, v2
10768 ; GFX7-NEXT: v_or_b32_e32 v2, v7, v2
10769 ; GFX7-NEXT: v_mov_b32_e32 v8, v3
10770 ; GFX7-NEXT: v_mov_b32_e32 v7, v2
10771 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
10772 ; GFX7-NEXT: s_waitcnt vmcnt(0)
10773 ; GFX7-NEXT: buffer_wbinvl1
10774 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
10775 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
10776 ; GFX7-NEXT: v_mov_b32_e32 v3, v7
10777 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
10778 ; GFX7-NEXT: s_cbranch_execnz .LBB37_1
10779 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
10780 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
10781 ; GFX7-NEXT: s_setpc_b64 s[30:31]
10782 ;
10783 ; GFX6-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_neg:
10784 ; GFX6: ; %bb.0:
10785 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10786 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
10787 ; GFX6-NEXT: s_mov_b32 s6, 0
10788 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
10789 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
10790 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
10791 ; GFX6-NEXT: s_mov_b32 s4, s6
10792 ; GFX6-NEXT: s_mov_b32 s5, s6
10793 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
10794 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
10795 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10796 ; GFX6-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
10797 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
10798 ; GFX6-NEXT: v_not_b32_e32 v5, v5
10799 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
10800 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
10801 ; GFX6-NEXT: .LBB37_1: ; %atomicrmw.start
10802 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
10803 ; GFX6-NEXT: s_waitcnt vmcnt(0)
10804 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
10805 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10806 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v6
10807 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10808 ; GFX6-NEXT: s_waitcnt expcnt(0)
10809 ; GFX6-NEXT: v_and_b32_e32 v7, v3, v5
10810 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v4, v2
10811 ; GFX6-NEXT: v_or_b32_e32 v2, v7, v2
10812 ; GFX6-NEXT: v_mov_b32_e32 v8, v3
10813 ; GFX6-NEXT: v_mov_b32_e32 v7, v2
10814 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
10815 ; GFX6-NEXT: s_waitcnt vmcnt(0)
10816 ; GFX6-NEXT: buffer_wbinvl1
10817 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
10818 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
10819 ; GFX6-NEXT: v_mov_b32_e32 v3, v7
10820 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
10821 ; GFX6-NEXT: s_cbranch_execnz .LBB37_1
10822 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
10823 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
10824 ; GFX6-NEXT: s_waitcnt expcnt(0)
10825 ; GFX6-NEXT: s_setpc_b64 s[30:31]
10826 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 -1024
10827 %unused = atomicrmw fsub ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst
10828 ret void
10829 }
10831 define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr addrspace(1) %ptr, bfloat %val) #0 {
10832 ; GFX12-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
10833 ; GFX12: ; %bb.0:
10834 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
10835 ; GFX12-NEXT: s_wait_expcnt 0x0
10836 ; GFX12-NEXT: s_wait_samplecnt 0x0
10837 ; GFX12-NEXT: s_wait_bvhcnt 0x0
10838 ; GFX12-NEXT: s_wait_kmcnt 0x0
10839 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2046
10840 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10841 ; GFX12-NEXT: s_mov_b32 s0, 0
10842 ; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
10843 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
10844 ; GFX12-NEXT: s_wait_loadcnt 0x0
10845 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
10846 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10847 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v4
10848 ; GFX12-NEXT: v_sub_f32_e32 v3, v3, v2
10849 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
10850 ; GFX12-NEXT: v_bfe_u32 v5, v3, 16, 1
10851 ; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v3
10852 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
10853 ; GFX12-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
10854 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10855 ; GFX12-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
10856 ; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
10857 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
10858 ; GFX12-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
10859 ; GFX12-NEXT: s_wait_storecnt 0x0
10860 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
10861 ; GFX12-NEXT: s_wait_loadcnt 0x0
10862 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
10863 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
10864 ; GFX12-NEXT: s_wait_alu 0xfffe
10865 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
10866 ; GFX12-NEXT: s_wait_alu 0xfffe
10867 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
10868 ; GFX12-NEXT: s_cbranch_execnz .LBB38_1
10869 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
10870 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
10871 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
10872 ; GFX12-NEXT: s_wait_alu 0xfffe
10873 ; GFX12-NEXT: s_setpc_b64 s[30:31]
10874 ;
10875 ; GFX940-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
10876 ; GFX940: ; %bb.0:
10877 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10878 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2046
10879 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
10880 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10881 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
10882 ; GFX940-NEXT: s_mov_b32 s3, 0xffff0000
10883 ; GFX940-NEXT: .LBB38_1: ; %atomicrmw.start
10884 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
10885 ; GFX940-NEXT: s_waitcnt vmcnt(0)
10886 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
10887 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 16, v5
10888 ; GFX940-NEXT: v_sub_f32_e32 v3, v3, v2
10889 ; GFX940-NEXT: v_bfe_u32 v4, v3, 16, 1
10890 ; GFX940-NEXT: v_or_b32_e32 v6, 0x400000, v3
10891 ; GFX940-NEXT: v_add3_u32 v4, v4, v3, s2
10892 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
10893 ; GFX940-NEXT: s_nop 1
10894 ; GFX940-NEXT: v_cndmask_b32_e32 v3, v4, v6, vcc
10895 ; GFX940-NEXT: v_lshrrev_b32_e32 v3, 16, v3
10896 ; GFX940-NEXT: v_and_or_b32 v4, v5, s3, v3
10897 ; GFX940-NEXT: buffer_wbl2 sc1
10898 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 sc0
10899 ; GFX940-NEXT: s_waitcnt vmcnt(0)
10900 ; GFX940-NEXT: buffer_inv sc1
10901 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
10902 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
10903 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
10904 ; GFX940-NEXT: s_cbranch_execnz .LBB38_1
10905 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
10906 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
10907 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
10908 ; GFX940-NEXT: s_setpc_b64 s[30:31]
10909 ;
10910 ; GFX11-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
10911 ; GFX11: ; %bb.0:
10912 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10913 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2046
10914 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10915 ; GFX11-NEXT: s_mov_b32 s0, 0
10916 ; GFX11-NEXT: .p2align 6
10917 ; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
10918 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
10919 ; GFX11-NEXT: s_waitcnt vmcnt(0)
10920 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
10921 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10922 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v4
10923 ; GFX11-NEXT: v_sub_f32_e32 v3, v3, v2
10924 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
10925 ; GFX11-NEXT: v_bfe_u32 v5, v3, 16, 1
10926 ; GFX11-NEXT: v_or_b32_e32 v6, 0x400000, v3
10927 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
10928 ; GFX11-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
10929 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10930 ; GFX11-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
10931 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
10932 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
10933 ; GFX11-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
10934 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
10935 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
10936 ; GFX11-NEXT: s_waitcnt vmcnt(0)
10937 ; GFX11-NEXT: buffer_gl1_inv
10938 ; GFX11-NEXT: buffer_gl0_inv
10939 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
10940 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
10941 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
10942 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
10943 ; GFX11-NEXT: s_cbranch_execnz .LBB38_1
10944 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
10945 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
10946 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
10947 ; GFX11-NEXT: s_setpc_b64 s[30:31]
10948 ;
10949 ; GFX10-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
10950 ; GFX10: ; %bb.0:
10951 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10952 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2046
10953 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10954 ; GFX10-NEXT: s_mov_b32 s4, 0
10955 ; GFX10-NEXT: .LBB38_1: ; %atomicrmw.start
10956 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
10957 ; GFX10-NEXT: s_waitcnt vmcnt(0)
10958 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
10959 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v4
10960 ; GFX10-NEXT: v_sub_f32_e32 v3, v3, v2
10961 ; GFX10-NEXT: v_bfe_u32 v5, v3, 16, 1
10962 ; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v3
10963 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
10964 ; GFX10-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
10965 ; GFX10-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
10966 ; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v3
10967 ; GFX10-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
10968 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
10969 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2046 glc
10970 ; GFX10-NEXT: s_waitcnt vmcnt(0)
10971 ; GFX10-NEXT: buffer_gl1_inv
10972 ; GFX10-NEXT: buffer_gl0_inv
10973 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
10974 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
10975 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
10976 ; GFX10-NEXT: s_cbranch_execnz .LBB38_1
10977 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
10978 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
10979 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
10980 ; GFX10-NEXT: s_setpc_b64 s[30:31]
10981 ;
10982 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
10983 ; GFX90A: ; %bb.0:
10984 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10985 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2046
10986 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
10987 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10988 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
10989 ; GFX90A-NEXT: s_mov_b32 s7, 0xffff0000
10990 ; GFX90A-NEXT: .LBB38_1: ; %atomicrmw.start
10991 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
10992 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
10993 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
10994 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v5
10995 ; GFX90A-NEXT: v_sub_f32_e32 v3, v3, v2
10996 ; GFX90A-NEXT: v_bfe_u32 v4, v3, 16, 1
10997 ; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v3
10998 ; GFX90A-NEXT: v_add3_u32 v4, v4, v3, s6
10999 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
11000 ; GFX90A-NEXT: v_cndmask_b32_e32 v3, v4, v6, vcc
11001 ; GFX90A-NEXT: v_lshrrev_b32_e32 v3, 16, v3
11002 ; GFX90A-NEXT: v_and_or_b32 v4, v5, s7, v3
11003 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 glc
11004 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
11005 ; GFX90A-NEXT: buffer_wbinvl1
11006 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
11007 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11008 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
11009 ; GFX90A-NEXT: s_cbranch_execnz .LBB38_1
11010 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
11011 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
11012 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
11013 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
11014 ;
11015 ; GFX908-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
11016 ; GFX908: ; %bb.0:
11017 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11018 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2046
11019 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
11020 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11021 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
11022 ; GFX908-NEXT: s_mov_b32 s7, 0xffff0000
11023 ; GFX908-NEXT: .LBB38_1: ; %atomicrmw.start
11024 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
11025 ; GFX908-NEXT: s_waitcnt vmcnt(0)
11026 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
11027 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v4
11028 ; GFX908-NEXT: v_sub_f32_e32 v3, v3, v2
11029 ; GFX908-NEXT: v_bfe_u32 v5, v3, 16, 1
11030 ; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v3
11031 ; GFX908-NEXT: v_add3_u32 v5, v5, v3, s6
11032 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
11033 ; GFX908-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
11034 ; GFX908-NEXT: v_lshrrev_b32_e32 v3, 16, v3
11035 ; GFX908-NEXT: v_and_or_b32 v3, v4, s7, v3
11036 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2046 glc
11037 ; GFX908-NEXT: s_waitcnt vmcnt(0)
11038 ; GFX908-NEXT: buffer_wbinvl1
11039 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
11040 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11041 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
11042 ; GFX908-NEXT: s_cbranch_execnz .LBB38_1
11043 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
11044 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
11045 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
11046 ; GFX908-NEXT: s_setpc_b64 s[30:31]
11047 ;
11048 ; GFX8-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
11049 ; GFX8: ; %bb.0:
11050 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11051 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
11052 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
11053 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
11054 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
11055 ; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
11056 ; GFX8-NEXT: .LBB38_1: ; %atomicrmw.start
11057 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
11058 ; GFX8-NEXT: s_waitcnt vmcnt(0)
11059 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
11060 ; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
11061 ; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1
11062 ; GFX8-NEXT: v_bfe_u32 v5, v0, 16, 1
11063 ; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v0
11064 ; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
11065 ; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v0
11066 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
11067 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
11068 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v5, v7, vcc
11069 ; GFX8-NEXT: v_or_b32_sdwa v5, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11070 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
11071 ; GFX8-NEXT: s_waitcnt vmcnt(0)
11072 ; GFX8-NEXT: buffer_wbinvl1
11073 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
11074 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11075 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
11076 ; GFX8-NEXT: s_cbranch_execnz .LBB38_1
11077 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
11078 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
11079 ; GFX8-NEXT: s_setpc_b64 s[30:31]
11080 ;
11081 ; GFX7-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
11082 ; GFX7: ; %bb.0:
11083 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11084 ; GFX7-NEXT: s_mov_b32 s6, 0
11085 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
11086 ; GFX7-NEXT: s_mov_b32 s4, s6
11087 ; GFX7-NEXT: s_mov_b32 s5, s6
11088 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
11089 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
11090 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
11091 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
11092 ; GFX7-NEXT: .LBB38_1: ; %atomicrmw.start
11093 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
11094 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11095 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
11096 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v6
11097 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
11098 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11099 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
11100 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
11101 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
11102 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 offset:2046 glc
11103 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11104 ; GFX7-NEXT: buffer_wbinvl1
11105 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
11106 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
11107 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
11108 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
11109 ; GFX7-NEXT: s_cbranch_execnz .LBB38_1
11110 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
11111 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
11112 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v4
11113 ; GFX7-NEXT: s_setpc_b64 s[30:31]
11114 ;
11115 ; GFX6-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
11116 ; GFX6: ; %bb.0:
11117 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11118 ; GFX6-NEXT: s_mov_b32 s6, 0
11119 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
11120 ; GFX6-NEXT: s_mov_b32 s4, s6
11121 ; GFX6-NEXT: s_mov_b32 s5, s6
11122 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
11123 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
11124 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
11125 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
11126 ; GFX6-NEXT: .LBB38_1: ; %atomicrmw.start
11127 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
11128 ; GFX6-NEXT: s_waitcnt vmcnt(0)
11129 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
11130 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v6
11131 ; GFX6-NEXT: s_waitcnt expcnt(0)
11132 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
11133 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11134 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
11135 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
11136 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
11137 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 offset:2046 glc
11138 ; GFX6-NEXT: s_waitcnt vmcnt(0)
11139 ; GFX6-NEXT: buffer_wbinvl1
11140 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
11141 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
11142 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
11143 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
11144 ; GFX6-NEXT: s_cbranch_execnz .LBB38_1
11145 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
11146 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
11147 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v4
11148 ; GFX6-NEXT: s_waitcnt expcnt(0)
11149 ; GFX6-NEXT: s_setpc_b64 s[30:31]
11150 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
11151 %result = atomicrmw fsub ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, align 4
11152 ret bfloat %result
11153 }
11155 define void @global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
11156 ; GFX12-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos:
11157 ; GFX12: ; %bb.0:
11158 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
11159 ; GFX12-NEXT: s_wait_expcnt 0x0
11160 ; GFX12-NEXT: s_wait_samplecnt 0x0
11161 ; GFX12-NEXT: s_wait_bvhcnt 0x0
11162 ; GFX12-NEXT: s_wait_kmcnt 0x0
11163 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2046
11164 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
11165 ; GFX12-NEXT: s_mov_b32 s0, 0
11166 ; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
11167 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
11168 ; GFX12-NEXT: s_wait_loadcnt 0x0
11169 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
11170 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11171 ; GFX12-NEXT: v_sub_f32_e32 v2, v2, v4
11172 ; GFX12-NEXT: v_bfe_u32 v5, v2, 16, 1
11173 ; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v2
11174 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
11175 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
11176 ; GFX12-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
11177 ; GFX12-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
11178 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11179 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11180 ; GFX12-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
11181 ; GFX12-NEXT: s_wait_storecnt 0x0
11182 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
11183 ; GFX12-NEXT: s_wait_loadcnt 0x0
11184 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
11185 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
11186 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
11187 ; GFX12-NEXT: s_wait_alu 0xfffe
11188 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
11189 ; GFX12-NEXT: s_wait_alu 0xfffe
11190 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
11191 ; GFX12-NEXT: s_cbranch_execnz .LBB39_1
11192 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
11193 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
11194 ; GFX12-NEXT: s_wait_alu 0xfffe
11195 ; GFX12-NEXT: s_setpc_b64 s[30:31]
11196 ;
11197 ; GFX940-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos:
11198 ; GFX940: ; %bb.0:
11199 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11200 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2046
11201 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
11202 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
11203 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
11204 ; GFX940-NEXT: s_mov_b32 s3, 0xffff0000
11205 ; GFX940-NEXT: .LBB39_1: ; %atomicrmw.start
11206 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
11207 ; GFX940-NEXT: s_waitcnt vmcnt(0)
11208 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
11209 ; GFX940-NEXT: v_sub_f32_e32 v2, v2, v4
11210 ; GFX940-NEXT: v_bfe_u32 v5, v2, 16, 1
11211 ; GFX940-NEXT: v_or_b32_e32 v6, 0x400000, v2
11212 ; GFX940-NEXT: v_add3_u32 v5, v5, v2, s2
11213 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
11214 ; GFX940-NEXT: s_nop 1
11215 ; GFX940-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
11216 ; GFX940-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11217 ; GFX940-NEXT: v_and_or_b32 v2, v3, s3, v2
11218 ; GFX940-NEXT: buffer_wbl2 sc1
11219 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 sc0
11220 ; GFX940-NEXT: s_waitcnt vmcnt(0)
11221 ; GFX940-NEXT: buffer_inv sc1
11222 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
11223 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
11224 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
11225 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
11226 ; GFX940-NEXT: s_cbranch_execnz .LBB39_1
11227 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
11228 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
11229 ; GFX940-NEXT: s_setpc_b64 s[30:31]
11230 ;
11231 ; GFX11-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos:
11232 ; GFX11: ; %bb.0:
11233 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11234 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2046
11235 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
11236 ; GFX11-NEXT: s_mov_b32 s0, 0
11237 ; GFX11-NEXT: .p2align 6
11238 ; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
11239 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
11240 ; GFX11-NEXT: s_waitcnt vmcnt(0)
11241 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
11242 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11243 ; GFX11-NEXT: v_sub_f32_e32 v2, v2, v4
11244 ; GFX11-NEXT: v_bfe_u32 v5, v2, 16, 1
11245 ; GFX11-NEXT: v_or_b32_e32 v6, 0x400000, v2
11246 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
11247 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
11248 ; GFX11-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
11249 ; GFX11-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
11250 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11251 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11252 ; GFX11-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
11253 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
11254 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 glc
11255 ; GFX11-NEXT: s_waitcnt vmcnt(0)
11256 ; GFX11-NEXT: buffer_gl1_inv
11257 ; GFX11-NEXT: buffer_gl0_inv
11258 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
11259 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
11260 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
11261 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
11262 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
11263 ; GFX11-NEXT: s_cbranch_execnz .LBB39_1
11264 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
11265 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
11266 ; GFX11-NEXT: s_setpc_b64 s[30:31]
11267 ;
11268 ; GFX10-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos:
11269 ; GFX10: ; %bb.0:
11270 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11271 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2046
11272 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
11273 ; GFX10-NEXT: s_mov_b32 s4, 0
11274 ; GFX10-NEXT: .LBB39_1: ; %atomicrmw.start
11275 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
11276 ; GFX10-NEXT: s_waitcnt vmcnt(0)
11277 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
11278 ; GFX10-NEXT: v_sub_f32_e32 v2, v2, v4
11279 ; GFX10-NEXT: v_bfe_u32 v5, v2, 16, 1
11280 ; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v2
11281 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
11282 ; GFX10-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
11283 ; GFX10-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
11284 ; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11285 ; GFX10-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
11286 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
11287 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 glc
11288 ; GFX10-NEXT: s_waitcnt vmcnt(0)
11289 ; GFX10-NEXT: buffer_gl1_inv
11290 ; GFX10-NEXT: buffer_gl0_inv
11291 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
11292 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
11293 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
11294 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
11295 ; GFX10-NEXT: s_cbranch_execnz .LBB39_1
11296 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
11297 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
11298 ; GFX10-NEXT: s_setpc_b64 s[30:31]
11299 ;
11300 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos:
11301 ; GFX90A: ; %bb.0:
11302 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11303 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2046
11304 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
11305 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
11306 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
11307 ; GFX90A-NEXT: s_mov_b32 s7, 0xffff0000
11308 ; GFX90A-NEXT: .LBB39_1: ; %atomicrmw.start
11309 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
11310 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
11311 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
11312 ; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
11313 ; GFX90A-NEXT: v_bfe_u32 v5, v2, 16, 1
11314 ; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v2
11315 ; GFX90A-NEXT: v_add3_u32 v5, v5, v2, s6
11316 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
11317 ; GFX90A-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
11318 ; GFX90A-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11319 ; GFX90A-NEXT: v_and_or_b32 v2, v3, s7, v2
11320 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 glc
11321 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
11322 ; GFX90A-NEXT: buffer_wbinvl1
11323 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
11324 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11325 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
11326 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
11327 ; GFX90A-NEXT: s_cbranch_execnz .LBB39_1
11328 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
11329 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
11330 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
11331 ;
11332 ; GFX908-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos:
11333 ; GFX908: ; %bb.0:
11334 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11335 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2046
11336 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
11337 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
11338 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
11339 ; GFX908-NEXT: s_mov_b32 s7, 0xffff0000
11340 ; GFX908-NEXT: .LBB39_1: ; %atomicrmw.start
11341 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
11342 ; GFX908-NEXT: s_waitcnt vmcnt(0)
11343 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
11344 ; GFX908-NEXT: v_sub_f32_e32 v2, v2, v4
11345 ; GFX908-NEXT: v_bfe_u32 v5, v2, 16, 1
11346 ; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v2
11347 ; GFX908-NEXT: v_add3_u32 v5, v5, v2, s6
11348 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
11349 ; GFX908-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
11350 ; GFX908-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11351 ; GFX908-NEXT: v_and_or_b32 v2, v3, s7, v2
11352 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 glc
11353 ; GFX908-NEXT: s_waitcnt vmcnt(0)
11354 ; GFX908-NEXT: buffer_wbinvl1
11355 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
11356 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11357 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
11358 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
11359 ; GFX908-NEXT: s_cbranch_execnz .LBB39_1
11360 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
11361 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
11362 ; GFX908-NEXT: s_setpc_b64 s[30:31]
11363 ;
11364 ; GFX8-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos:
11365 ; GFX8: ; %bb.0:
11366 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11367 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fe, v0
11368 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
11369 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
11370 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
11371 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
11372 ; GFX8-NEXT: .LBB39_1: ; %atomicrmw.start
11373 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
11374 ; GFX8-NEXT: s_waitcnt vmcnt(0)
11375 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
11376 ; GFX8-NEXT: v_sub_f32_e32 v2, v2, v4
11377 ; GFX8-NEXT: v_bfe_u32 v6, v2, 16, 1
11378 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v2
11379 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
11380 ; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v2
11381 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
11382 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
11383 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
11384 ; GFX8-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11385 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11386 ; GFX8-NEXT: s_waitcnt vmcnt(0)
11387 ; GFX8-NEXT: buffer_wbinvl1
11388 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
11389 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11390 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
11391 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
11392 ; GFX8-NEXT: s_cbranch_execnz .LBB39_1
11393 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
11394 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
11395 ; GFX8-NEXT: s_setpc_b64 s[30:31]
11396 ;
11397 ; GFX7-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos:
11398 ; GFX7: ; %bb.0:
11399 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11400 ; GFX7-NEXT: s_mov_b32 s6, 0
11401 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
11402 ; GFX7-NEXT: s_mov_b32 s4, s6
11403 ; GFX7-NEXT: s_mov_b32 s5, s6
11404 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
11405 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
11406 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
11407 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
11408 ; GFX7-NEXT: .LBB39_1: ; %atomicrmw.start
11409 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
11410 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11411 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
11412 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v4
11413 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
11414 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11415 ; GFX7-NEXT: v_or_b32_e32 v2, v5, v2
11416 ; GFX7-NEXT: v_mov_b32_e32 v6, v3
11417 ; GFX7-NEXT: v_mov_b32_e32 v5, v2
11418 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2046 glc
11419 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11420 ; GFX7-NEXT: buffer_wbinvl1
11421 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
11422 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
11423 ; GFX7-NEXT: v_mov_b32_e32 v3, v5
11424 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
11425 ; GFX7-NEXT: s_cbranch_execnz .LBB39_1
11426 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
11427 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
11428 ; GFX7-NEXT: s_setpc_b64 s[30:31]
11429 ;
11430 ; GFX6-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos:
11431 ; GFX6: ; %bb.0:
11432 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11433 ; GFX6-NEXT: s_mov_b32 s6, 0
11434 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
11435 ; GFX6-NEXT: s_mov_b32 s4, s6
11436 ; GFX6-NEXT: s_mov_b32 s5, s6
11437 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
11438 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
11439 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
11440 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
11441 ; GFX6-NEXT: .LBB39_1: ; %atomicrmw.start
11442 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
11443 ; GFX6-NEXT: s_waitcnt vmcnt(0)
11444 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
11445 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v4
11446 ; GFX6-NEXT: s_waitcnt expcnt(0)
11447 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
11448 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11449 ; GFX6-NEXT: v_or_b32_e32 v2, v5, v2
11450 ; GFX6-NEXT: v_mov_b32_e32 v6, v3
11451 ; GFX6-NEXT: v_mov_b32_e32 v5, v2
11452 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2046 glc
11453 ; GFX6-NEXT: s_waitcnt vmcnt(0)
11454 ; GFX6-NEXT: buffer_wbinvl1
11455 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
11456 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
11457 ; GFX6-NEXT: v_mov_b32_e32 v3, v5
11458 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
11459 ; GFX6-NEXT: s_cbranch_execnz .LBB39_1
11460 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
11461 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
11462 ; GFX6-NEXT: s_waitcnt expcnt(0)
11463 ; GFX6-NEXT: s_setpc_b64 s[30:31]
11464 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
11465 %unused = atomicrmw fsub ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, align 4
11466 ret void
11467 }
11469 define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
11470 ; GFX12-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
11471 ; GFX12: ; %bb.0:
11472 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
11473 ; GFX12-NEXT: s_wait_expcnt 0x0
11474 ; GFX12-NEXT: s_wait_samplecnt 0x0
11475 ; GFX12-NEXT: s_wait_bvhcnt 0x0
11476 ; GFX12-NEXT: s_wait_kmcnt 0x0
11477 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
11478 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
11479 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11480 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
11481 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
11482 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
11483 ; GFX12-NEXT: s_mov_b32 s0, 0
11484 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
11485 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
11486 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
11487 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
11488 ; GFX12-NEXT: v_not_b32_e32 v4, v4
11489 ; GFX12-NEXT: .LBB40_1: ; %atomicrmw.start
11490 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
11491 ; GFX12-NEXT: s_wait_loadcnt 0x0
11492 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
11493 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11494 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
11495 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
11496 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11497 ; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
11498 ; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
11499 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
11500 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
11501 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
11502 ; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
11503 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
11504 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11505 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
11506 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
11507 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
11508 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
11509 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
11510 ; GFX12-NEXT: s_wait_storecnt 0x0
11511 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
11512 ; GFX12-NEXT: s_wait_loadcnt 0x0
11513 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
11514 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
11515 ; GFX12-NEXT: s_wait_alu 0xfffe
11516 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
11517 ; GFX12-NEXT: s_wait_alu 0xfffe
11518 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
11519 ; GFX12-NEXT: s_cbranch_execnz .LBB40_1
11520 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
11521 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
11522 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
11523 ; GFX12-NEXT: s_wait_alu 0xfffe
11524 ; GFX12-NEXT: s_setpc_b64 s[30:31]
11525 ;
11526 ; GFX940-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
11527 ; GFX940: ; %bb.0:
11528 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11529 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
11530 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
11531 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
11532 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
11533 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
11534 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v4
11535 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
11536 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
11537 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
11538 ; GFX940-NEXT: v_not_b32_e32 v4, v4
11539 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
11540 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11541 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
11542 ; GFX940-NEXT: .LBB40_1: ; %atomicrmw.start
11543 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
11544 ; GFX940-NEXT: s_waitcnt vmcnt(0)
11545 ; GFX940-NEXT: v_mov_b32_e32 v7, v5
11546 ; GFX940-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
11547 ; GFX940-NEXT: s_nop 0
11548 ; GFX940-NEXT: v_sub_f32_e32 v5, v5, v2
11549 ; GFX940-NEXT: v_bfe_u32 v6, v5, 16, 1
11550 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v5
11551 ; GFX940-NEXT: v_add3_u32 v6, v6, v5, s2
11552 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
11553 ; GFX940-NEXT: s_nop 1
11554 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
11555 ; GFX940-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11556 ; GFX940-NEXT: v_and_or_b32 v6, v7, v4, v5
11557 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
11558 ; GFX940-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off sc0 sc1
11559 ; GFX940-NEXT: s_waitcnt vmcnt(0)
11560 ; GFX940-NEXT: buffer_inv sc0 sc1
11561 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
11562 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
11563 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
11564 ; GFX940-NEXT: s_cbranch_execnz .LBB40_1
11565 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
11566 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
11567 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
11568 ; GFX940-NEXT: s_setpc_b64 s[30:31]
11569 ;
11570 ; GFX11-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
11571 ; GFX11: ; %bb.0:
11572 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11573 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
11574 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
11575 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11576 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
11577 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
11578 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
11579 ; GFX11-NEXT: s_mov_b32 s0, 0
11580 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
11581 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
11582 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
11583 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
11584 ; GFX11-NEXT: v_not_b32_e32 v4, v4
11585 ; GFX11-NEXT: .p2align 6
11586 ; GFX11-NEXT: .LBB40_1: ; %atomicrmw.start
11587 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
11588 ; GFX11-NEXT: s_waitcnt vmcnt(0)
11589 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
11590 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11591 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
11592 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
11593 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11594 ; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
11595 ; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
11596 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
11597 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
11598 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
11599 ; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
11600 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
11601 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11602 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
11603 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
11604 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
11605 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
11606 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
11607 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
11608 ; GFX11-NEXT: s_waitcnt vmcnt(0)
11609 ; GFX11-NEXT: buffer_gl1_inv
11610 ; GFX11-NEXT: buffer_gl0_inv
11611 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
11612 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
11613 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
11614 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
11615 ; GFX11-NEXT: s_cbranch_execnz .LBB40_1
11616 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
11617 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
11618 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
11619 ; GFX11-NEXT: s_setpc_b64 s[30:31]
11620 ;
11621 ; GFX10-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
11622 ; GFX10: ; %bb.0:
11623 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11624 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
11625 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
11626 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11627 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
11628 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
11629 ; GFX10-NEXT: s_mov_b32 s4, 0
11630 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
11631 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
11632 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
11633 ; GFX10-NEXT: v_not_b32_e32 v4, v4
11634 ; GFX10-NEXT: .LBB40_1: ; %atomicrmw.start
11635 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
11636 ; GFX10-NEXT: s_waitcnt vmcnt(0)
11637 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
11638 ; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
11639 ; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
11640 ; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
11641 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v5
11642 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
11643 ; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
11644 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
11645 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11646 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
11647 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
11648 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
11649 ; GFX10-NEXT: s_waitcnt vmcnt(0)
11650 ; GFX10-NEXT: buffer_gl1_inv
11651 ; GFX10-NEXT: buffer_gl0_inv
11652 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
11653 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
11654 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
11655 ; GFX10-NEXT: s_cbranch_execnz .LBB40_1
11656 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
11657 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
11658 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
11659 ; GFX10-NEXT: s_setpc_b64 s[30:31]
11660 ;
11661 ; GFX90A-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
11662 ; GFX90A: ; %bb.0:
11663 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11664 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
11665 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
11666 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
11667 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
11668 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
11669 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
11670 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
11671 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
11672 ; GFX90A-NEXT: v_not_b32_e32 v4, v4
11673 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
11674 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11675 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
11676 ; GFX90A-NEXT: .LBB40_1: ; %atomicrmw.start
11677 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
11678 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
11679 ; GFX90A-NEXT: v_mov_b32_e32 v7, v5
11680 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
11681 ; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v2
11682 ; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
11683 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v5
11684 ; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
11685 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
11686 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
11687 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11688 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v4, v5
11689 ; GFX90A-NEXT: buffer_wbl2
11690 ; GFX90A-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off glc
11691 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
11692 ; GFX90A-NEXT: buffer_invl2
11693 ; GFX90A-NEXT: buffer_wbinvl1
11694 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
11695 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11696 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
11697 ; GFX90A-NEXT: s_cbranch_execnz .LBB40_1
11698 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
11699 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
11700 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
11701 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
11702 ;
11703 ; GFX908-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
11704 ; GFX908: ; %bb.0:
11705 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11706 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
11707 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
11708 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
11709 ; GFX908-NEXT: global_load_dword v5, v[0:1], off
11710 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
11711 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
11712 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
11713 ; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
11714 ; GFX908-NEXT: v_not_b32_e32 v4, v4
11715 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
11716 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11717 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
11718 ; GFX908-NEXT: .LBB40_1: ; %atomicrmw.start
11719 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
11720 ; GFX908-NEXT: s_waitcnt vmcnt(0)
11721 ; GFX908-NEXT: v_mov_b32_e32 v6, v5
11722 ; GFX908-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
11723 ; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
11724 ; GFX908-NEXT: v_bfe_u32 v7, v5, 16, 1
11725 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v5
11726 ; GFX908-NEXT: v_add3_u32 v7, v7, v5, s6
11727 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
11728 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc
11729 ; GFX908-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11730 ; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
11731 ; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
11732 ; GFX908-NEXT: s_waitcnt vmcnt(0)
11733 ; GFX908-NEXT: buffer_wbinvl1
11734 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
11735 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11736 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
11737 ; GFX908-NEXT: s_cbranch_execnz .LBB40_1
11738 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
11739 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
11740 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
11741 ; GFX908-NEXT: s_setpc_b64 s[30:31]
11742 ;
11743 ; GFX8-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
11744 ; GFX8: ; %bb.0:
11745 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11746 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
11747 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
11748 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
11749 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
11750 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
11751 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
11752 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
11753 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
11754 ; GFX8-NEXT: v_not_b32_e32 v4, v4
11755 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
11756 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11757 ; GFX8-NEXT: .LBB40_1: ; %atomicrmw.start
11758 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
11759 ; GFX8-NEXT: s_waitcnt vmcnt(0)
11760 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
11761 ; GFX8-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
11762 ; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
11763 ; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
11764 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
11765 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
11766 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
11767 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
11768 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
11769 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
11770 ; GFX8-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11771 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
11772 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
11773 ; GFX8-NEXT: s_waitcnt vmcnt(0)
11774 ; GFX8-NEXT: buffer_wbinvl1
11775 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
11776 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11777 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
11778 ; GFX8-NEXT: s_cbranch_execnz .LBB40_1
11779 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
11780 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
11781 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
11782 ; GFX8-NEXT: s_setpc_b64 s[30:31]
11783 ;
11784 ; GFX7-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
11785 ; GFX7: ; %bb.0:
11786 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11787 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
11788 ; GFX7-NEXT: s_mov_b32 s6, 0
11789 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
11790 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
11791 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
11792 ; GFX7-NEXT: s_mov_b32 s4, s6
11793 ; GFX7-NEXT: s_mov_b32 s5, s6
11794 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
11795 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
11796 ; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
11797 ; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
11798 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
11799 ; GFX7-NEXT: v_not_b32_e32 v7, v4
11800 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
11801 ; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
11802 ; GFX7-NEXT: .LBB40_1: ; %atomicrmw.start
11803 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
11804 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11805 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
11806 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11807 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v8
11808 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11809 ; GFX7-NEXT: v_and_b32_e32 v4, v3, v7
11810 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v6, v2
11811 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
11812 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
11813 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
11814 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
11815 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11816 ; GFX7-NEXT: buffer_wbinvl1
11817 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
11818 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
11819 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
11820 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
11821 ; GFX7-NEXT: s_cbranch_execnz .LBB40_1
11822 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
11823 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
11824 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
11825 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
11826 ; GFX7-NEXT: s_setpc_b64 s[30:31]
11827 ;
11828 ; GFX6-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
11829 ; GFX6: ; %bb.0:
11830 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11831 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
11832 ; GFX6-NEXT: s_mov_b32 s6, 0
11833 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
11834 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
11835 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
11836 ; GFX6-NEXT: s_mov_b32 s4, s6
11837 ; GFX6-NEXT: s_mov_b32 s5, s6
11838 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
11839 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
11840 ; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
11841 ; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
11842 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
11843 ; GFX6-NEXT: v_not_b32_e32 v7, v4
11844 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
11845 ; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
11846 ; GFX6-NEXT: .LBB40_1: ; %atomicrmw.start
11847 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
11848 ; GFX6-NEXT: s_waitcnt vmcnt(0)
11849 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
11850 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11851 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v8
11852 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11853 ; GFX6-NEXT: s_waitcnt expcnt(0)
11854 ; GFX6-NEXT: v_and_b32_e32 v4, v3, v7
11855 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v6, v2
11856 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
11857 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
11858 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
11859 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
11860 ; GFX6-NEXT: s_waitcnt vmcnt(0)
11861 ; GFX6-NEXT: buffer_wbinvl1
11862 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
11863 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
11864 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
11865 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
11866 ; GFX6-NEXT: s_cbranch_execnz .LBB40_1
11867 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
11868 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
11869 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
11870 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
11871 ; GFX6-NEXT: s_waitcnt expcnt(0)
11872 ; GFX6-NEXT: s_setpc_b64 s[30:31]
11873 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
11874 %result = atomicrmw fsub ptr addrspace(1) %gep, bfloat %val seq_cst
11875 ret bfloat %result
11876 }
11878 define void @global_system_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
11879 ; GFX12-LABEL: global_system_atomic_fsub_noret_bf16__offset12b_pos:
11880 ; GFX12: ; %bb.0:
11881 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
11882 ; GFX12-NEXT: s_wait_expcnt 0x0
11883 ; GFX12-NEXT: s_wait_samplecnt 0x0
11884 ; GFX12-NEXT: s_wait_bvhcnt 0x0
11885 ; GFX12-NEXT: s_wait_kmcnt 0x0
11886 ; GFX12-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
11887 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
11888 ; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v2
11889 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
11890 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v4
11891 ; GFX12-NEXT: v_and_b32_e32 v4, 3, v4
11892 ; GFX12-NEXT: s_mov_b32 s0, 0
11893 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
11894 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 3, v4
11895 ; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
11896 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
11897 ; GFX12-NEXT: v_not_b32_e32 v5, v5
11898 ; GFX12-NEXT: .LBB41_1: ; %atomicrmw.start
11899 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
11900 ; GFX12-NEXT: s_wait_loadcnt 0x0
11901 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
11902 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11903 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11904 ; GFX12-NEXT: v_sub_f32_e32 v2, v2, v6
11905 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
11906 ; GFX12-NEXT: v_bfe_u32 v7, v2, 16, 1
11907 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v2
11908 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
11909 ; GFX12-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
11910 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11911 ; GFX12-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
11912 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11913 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11914 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, v4, v2
11915 ; GFX12-NEXT: v_and_or_b32 v2, v3, v5, v2
11916 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
11917 ; GFX12-NEXT: s_wait_storecnt 0x0
11918 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
11919 ; GFX12-NEXT: s_wait_loadcnt 0x0
11920 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
11921 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
11922 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
11923 ; GFX12-NEXT: s_wait_alu 0xfffe
11924 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
11925 ; GFX12-NEXT: s_wait_alu 0xfffe
11926 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
11927 ; GFX12-NEXT: s_cbranch_execnz .LBB41_1
11928 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
11929 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
11930 ; GFX12-NEXT: s_wait_alu 0xfffe
11931 ; GFX12-NEXT: s_setpc_b64 s[30:31]
11932 ;
11933 ; GFX940-LABEL: global_system_atomic_fsub_noret_bf16__offset12b_pos:
11934 ; GFX940: ; %bb.0:
11935 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11936 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
11937 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
11938 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
11939 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
11940 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
11941 ; GFX940-NEXT: v_and_b32_e32 v4, 3, v4
11942 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 3, v4
11943 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
11944 ; GFX940-NEXT: v_lshlrev_b32_e64 v5, v4, s0
11945 ; GFX940-NEXT: v_not_b32_e32 v5, v5
11946 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
11947 ; GFX940-NEXT: v_lshlrev_b32_e32 v6, 16, v2
11948 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
11949 ; GFX940-NEXT: .LBB41_1: ; %atomicrmw.start
11950 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
11951 ; GFX940-NEXT: s_waitcnt vmcnt(0)
11952 ; GFX940-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
11953 ; GFX940-NEXT: s_nop 0
11954 ; GFX940-NEXT: v_sub_f32_e32 v2, v2, v6
11955 ; GFX940-NEXT: v_bfe_u32 v7, v2, 16, 1
11956 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v2
11957 ; GFX940-NEXT: v_add3_u32 v7, v7, v2, s2
11958 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
11959 ; GFX940-NEXT: s_nop 1
11960 ; GFX940-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
11961 ; GFX940-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11962 ; GFX940-NEXT: v_and_or_b32 v2, v3, v5, v2
11963 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
11964 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0 sc1
11965 ; GFX940-NEXT: s_waitcnt vmcnt(0)
11966 ; GFX940-NEXT: buffer_inv sc0 sc1
11967 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
11968 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
11969 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
11970 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
11971 ; GFX940-NEXT: s_cbranch_execnz .LBB41_1
11972 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
11973 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
11974 ; GFX940-NEXT: s_setpc_b64 s[30:31]
11975 ;
11976 ; GFX11-LABEL: global_system_atomic_fsub_noret_bf16__offset12b_pos:
11977 ; GFX11: ; %bb.0:
11978 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11979 ; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
11980 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
11981 ; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v2
11982 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
11983 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v4
11984 ; GFX11-NEXT: v_and_b32_e32 v4, 3, v4
11985 ; GFX11-NEXT: s_mov_b32 s0, 0
11986 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
11987 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v4
11988 ; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
11989 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
11990 ; GFX11-NEXT: v_not_b32_e32 v5, v5
11991 ; GFX11-NEXT: .p2align 6
11992 ; GFX11-NEXT: .LBB41_1: ; %atomicrmw.start
11993 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
11994 ; GFX11-NEXT: s_waitcnt vmcnt(0)
11995 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
11996 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11997 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11998 ; GFX11-NEXT: v_sub_f32_e32 v2, v2, v6
11999 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
12000 ; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
12001 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v2
12002 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
12003 ; GFX11-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
12004 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
12005 ; GFX11-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
12006 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v2
12007 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
12008 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, v4, v2
12009 ; GFX11-NEXT: v_and_or_b32 v2, v3, v5, v2
12010 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
12011 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
12012 ; GFX11-NEXT: s_waitcnt vmcnt(0)
12013 ; GFX11-NEXT: buffer_gl1_inv
12014 ; GFX11-NEXT: buffer_gl0_inv
12015 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
12016 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
12017 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
12018 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
12019 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
12020 ; GFX11-NEXT: s_cbranch_execnz .LBB41_1
12021 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
12022 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
12023 ; GFX11-NEXT: s_setpc_b64 s[30:31]
12024 ;
12025 ; GFX10-LABEL: global_system_atomic_fsub_noret_bf16__offset12b_pos:
12026 ; GFX10: ; %bb.0:
12027 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12028 ; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
12029 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
12030 ; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v2
12031 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
12032 ; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
12033 ; GFX10-NEXT: s_mov_b32 s4, 0
12034 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
12035 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
12036 ; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
12037 ; GFX10-NEXT: v_not_b32_e32 v5, v5
12038 ; GFX10-NEXT: .LBB41_1: ; %atomicrmw.start
12039 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
12040 ; GFX10-NEXT: s_waitcnt vmcnt(0)
12041 ; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
12042 ; GFX10-NEXT: v_sub_f32_e32 v2, v2, v6
12043 ; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
12044 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v2
12045 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
12046 ; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
12047 ; GFX10-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
12048 ; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
12049 ; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
12050 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
12051 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
12052 ; GFX10-NEXT: s_waitcnt vmcnt(0)
12053 ; GFX10-NEXT: buffer_gl1_inv
12054 ; GFX10-NEXT: buffer_gl0_inv
12055 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
12056 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
12057 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
12058 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
12059 ; GFX10-NEXT: s_cbranch_execnz .LBB41_1
12060 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
12061 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
12062 ; GFX10-NEXT: s_setpc_b64 s[30:31]
12063 ;
12064 ; GFX90A-LABEL: global_system_atomic_fsub_noret_bf16__offset12b_pos:
12065 ; GFX90A: ; %bb.0:
12066 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12067 ; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
12068 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
12069 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
12070 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
12071 ; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
12072 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
12073 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
12074 ; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
12075 ; GFX90A-NEXT: v_not_b32_e32 v5, v5
12076 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
12077 ; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v2
12078 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
12079 ; GFX90A-NEXT: .LBB41_1: ; %atomicrmw.start
12080 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
12081 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12082 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
12083 ; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v6
12084 ; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
12085 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
12086 ; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s6
12087 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
12088 ; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
12089 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
12090 ; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
12091 ; GFX90A-NEXT: buffer_wbl2
12092 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
12093 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12094 ; GFX90A-NEXT: buffer_invl2
12095 ; GFX90A-NEXT: buffer_wbinvl1
12096 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
12097 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12098 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
12099 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
12100 ; GFX90A-NEXT: s_cbranch_execnz .LBB41_1
12101 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
12102 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
12103 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
12104 ;
12105 ; GFX908-LABEL: global_system_atomic_fsub_noret_bf16__offset12b_pos:
12106 ; GFX908: ; %bb.0:
12107 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12108 ; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
12109 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
12110 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
12111 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
12112 ; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
12113 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
12114 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
12115 ; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
12116 ; GFX908-NEXT: v_not_b32_e32 v5, v5
12117 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
12118 ; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v2
12119 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
12120 ; GFX908-NEXT: .LBB41_1: ; %atomicrmw.start
12121 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
12122 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12123 ; GFX908-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
12124 ; GFX908-NEXT: v_sub_f32_e32 v2, v2, v6
12125 ; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
12126 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
12127 ; GFX908-NEXT: v_add3_u32 v7, v7, v2, s6
12128 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
12129 ; GFX908-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
12130 ; GFX908-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
12131 ; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
12132 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
12133 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12134 ; GFX908-NEXT: buffer_wbinvl1
12135 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
12136 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12137 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
12138 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
12139 ; GFX908-NEXT: s_cbranch_execnz .LBB41_1
12140 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
12141 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
12142 ; GFX908-NEXT: s_setpc_b64 s[30:31]
12143 ;
12144 ; GFX8-LABEL: global_system_atomic_fsub_noret_bf16__offset12b_pos:
12145 ; GFX8: ; %bb.0:
12146 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12147 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
12148 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
12149 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
12150 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
12151 ; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
12152 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
12153 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
12154 ; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
12155 ; GFX8-NEXT: v_not_b32_e32 v5, v5
12156 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
12157 ; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v2
12158 ; GFX8-NEXT: .LBB41_1: ; %atomicrmw.start
12159 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
12160 ; GFX8-NEXT: s_waitcnt vmcnt(0)
12161 ; GFX8-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
12162 ; GFX8-NEXT: v_sub_f32_e32 v2, v2, v6
12163 ; GFX8-NEXT: v_bfe_u32 v8, v2, 16, 1
12164 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v2
12165 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
12166 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v2
12167 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
12168 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v8, v9, vcc
12169 ; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
12170 ; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
12171 ; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
12172 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12173 ; GFX8-NEXT: s_waitcnt vmcnt(0)
12174 ; GFX8-NEXT: buffer_wbinvl1
12175 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
12176 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12177 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
12178 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
12179 ; GFX8-NEXT: s_cbranch_execnz .LBB41_1
12180 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
12181 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
12182 ; GFX8-NEXT: s_setpc_b64 s[30:31]
12183 ;
12184 ; GFX7-LABEL: global_system_atomic_fsub_noret_bf16__offset12b_pos:
12185 ; GFX7: ; %bb.0:
12186 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12187 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
12188 ; GFX7-NEXT: s_mov_b32 s6, 0
12189 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
12190 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
12191 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
12192 ; GFX7-NEXT: s_mov_b32 s4, s6
12193 ; GFX7-NEXT: s_mov_b32 s5, s6
12194 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
12195 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
12196 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
12197 ; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
12198 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
12199 ; GFX7-NEXT: v_not_b32_e32 v5, v5
12200 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
12201 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
12202 ; GFX7-NEXT: .LBB41_1: ; %atomicrmw.start
12203 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
12204 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12205 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
12206 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
12207 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v6
12208 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
12209 ; GFX7-NEXT: v_and_b32_e32 v7, v3, v5
12210 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v4, v2
12211 ; GFX7-NEXT: v_or_b32_e32 v2, v7, v2
12212 ; GFX7-NEXT: v_mov_b32_e32 v8, v3
12213 ; GFX7-NEXT: v_mov_b32_e32 v7, v2
12214 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
12215 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12216 ; GFX7-NEXT: buffer_wbinvl1
12217 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
12218 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
12219 ; GFX7-NEXT: v_mov_b32_e32 v3, v7
12220 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
12221 ; GFX7-NEXT: s_cbranch_execnz .LBB41_1
12222 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
12223 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
12224 ; GFX7-NEXT: s_setpc_b64 s[30:31]
12225 ;
12226 ; GFX6-LABEL: global_system_atomic_fsub_noret_bf16__offset12b_pos:
12227 ; GFX6: ; %bb.0:
12228 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12229 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
12230 ; GFX6-NEXT: s_mov_b32 s6, 0
12231 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
12232 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
12233 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
12234 ; GFX6-NEXT: s_mov_b32 s4, s6
12235 ; GFX6-NEXT: s_mov_b32 s5, s6
12236 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
12237 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
12238 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
12239 ; GFX6-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
12240 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
12241 ; GFX6-NEXT: v_not_b32_e32 v5, v5
12242 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
12243 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
12244 ; GFX6-NEXT: .LBB41_1: ; %atomicrmw.start
12245 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
12246 ; GFX6-NEXT: s_waitcnt vmcnt(0)
12247 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
12248 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
12249 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v6
12250 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
12251 ; GFX6-NEXT: s_waitcnt expcnt(0)
12252 ; GFX6-NEXT: v_and_b32_e32 v7, v3, v5
12253 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v4, v2
12254 ; GFX6-NEXT: v_or_b32_e32 v2, v7, v2
12255 ; GFX6-NEXT: v_mov_b32_e32 v8, v3
12256 ; GFX6-NEXT: v_mov_b32_e32 v7, v2
12257 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
12258 ; GFX6-NEXT: s_waitcnt vmcnt(0)
12259 ; GFX6-NEXT: buffer_wbinvl1
12260 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
12261 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
12262 ; GFX6-NEXT: v_mov_b32_e32 v3, v7
12263 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
12264 ; GFX6-NEXT: s_cbranch_execnz .LBB41_1
12265 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
12266 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
12267 ; GFX6-NEXT: s_waitcnt expcnt(0)
12268 ; GFX6-NEXT: s_setpc_b64 s[30:31]
12269 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
12270 %unused = atomicrmw fsub ptr addrspace(1) %gep, bfloat %val seq_cst
12271 ret void
12272 }
12274 ; --------------------------------------------------------------------
12275 ; <2 x half>
12276 ; --------------------------------------------------------------------
12278 define <2 x half> @global_agent_atomic_fsub_ret_v2f16(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
12279 ; GFX12-LABEL: global_agent_atomic_fsub_ret_v2f16:
12280 ; GFX12: ; %bb.0:
12281 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
12282 ; GFX12-NEXT: s_wait_expcnt 0x0
12283 ; GFX12-NEXT: s_wait_samplecnt 0x0
12284 ; GFX12-NEXT: s_wait_bvhcnt 0x0
12285 ; GFX12-NEXT: s_wait_kmcnt 0x0
12286 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
12287 ; GFX12-NEXT: s_mov_b32 s0, 0
12288 ; GFX12-NEXT: .LBB42_1: ; %atomicrmw.start
12289 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
12290 ; GFX12-NEXT: s_wait_loadcnt 0x0
12291 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
12292 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
12293 ; GFX12-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12294 ; GFX12-NEXT: s_wait_storecnt 0x0
12295 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
12296 ; GFX12-NEXT: s_wait_loadcnt 0x0
12297 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
12298 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
12299 ; GFX12-NEXT: s_wait_alu 0xfffe
12300 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
12301 ; GFX12-NEXT: s_wait_alu 0xfffe
12302 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
12303 ; GFX12-NEXT: s_cbranch_execnz .LBB42_1
12304 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
12305 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
12306 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
12307 ; GFX12-NEXT: s_wait_alu 0xfffe
12308 ; GFX12-NEXT: s_setpc_b64 s[30:31]
12309 ;
12310 ; GFX940-LABEL: global_agent_atomic_fsub_ret_v2f16:
12311 ; GFX940: ; %bb.0:
12312 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12313 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
12314 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
12315 ; GFX940-NEXT: .LBB42_1: ; %atomicrmw.start
12316 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
12317 ; GFX940-NEXT: s_waitcnt vmcnt(0)
12318 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
12319 ; GFX940-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
12320 ; GFX940-NEXT: buffer_wbl2 sc1
12321 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
12322 ; GFX940-NEXT: s_waitcnt vmcnt(0)
12323 ; GFX940-NEXT: buffer_inv sc1
12324 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
12325 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
12326 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
12327 ; GFX940-NEXT: s_cbranch_execnz .LBB42_1
12328 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
12329 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
12330 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
12331 ; GFX940-NEXT: s_setpc_b64 s[30:31]
12332 ;
12333 ; GFX11-LABEL: global_agent_atomic_fsub_ret_v2f16:
12334 ; GFX11: ; %bb.0:
12335 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12336 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
12337 ; GFX11-NEXT: s_mov_b32 s0, 0
12338 ; GFX11-NEXT: .LBB42_1: ; %atomicrmw.start
12339 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
12340 ; GFX11-NEXT: s_waitcnt vmcnt(0)
12341 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
12342 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
12343 ; GFX11-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12344 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
12345 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
12346 ; GFX11-NEXT: s_waitcnt vmcnt(0)
12347 ; GFX11-NEXT: buffer_gl1_inv
12348 ; GFX11-NEXT: buffer_gl0_inv
12349 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
12350 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
12351 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
12352 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
12353 ; GFX11-NEXT: s_cbranch_execnz .LBB42_1
12354 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
12355 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
12356 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
12357 ; GFX11-NEXT: s_setpc_b64 s[30:31]
12358 ;
12359 ; GFX10-LABEL: global_agent_atomic_fsub_ret_v2f16:
12360 ; GFX10: ; %bb.0:
12361 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12362 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
12363 ; GFX10-NEXT: s_mov_b32 s4, 0
12364 ; GFX10-NEXT: .LBB42_1: ; %atomicrmw.start
12365 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
12366 ; GFX10-NEXT: s_waitcnt vmcnt(0)
12367 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
12368 ; GFX10-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12369 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
12370 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
12371 ; GFX10-NEXT: s_waitcnt vmcnt(0)
12372 ; GFX10-NEXT: buffer_gl1_inv
12373 ; GFX10-NEXT: buffer_gl0_inv
12374 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
12375 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
12376 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
12377 ; GFX10-NEXT: s_cbranch_execnz .LBB42_1
12378 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
12379 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
12380 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
12381 ; GFX10-NEXT: s_setpc_b64 s[30:31]
12382 ;
12383 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_v2f16:
12384 ; GFX90A: ; %bb.0:
12385 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12386 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
12387 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
12388 ; GFX90A-NEXT: .LBB42_1: ; %atomicrmw.start
12389 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
12390 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12391 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
12392 ; GFX90A-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
12393 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
12394 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12395 ; GFX90A-NEXT: buffer_wbinvl1
12396 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
12397 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12398 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
12399 ; GFX90A-NEXT: s_cbranch_execnz .LBB42_1
12400 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
12401 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
12402 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
12403 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
12404 ;
12405 ; GFX908-LABEL: global_agent_atomic_fsub_ret_v2f16:
12406 ; GFX908: ; %bb.0:
12407 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12408 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
12409 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
12410 ; GFX908-NEXT: .LBB42_1: ; %atomicrmw.start
12411 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
12412 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12413 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
12414 ; GFX908-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12415 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
12416 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12417 ; GFX908-NEXT: buffer_wbinvl1
12418 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
12419 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12420 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
12421 ; GFX908-NEXT: s_cbranch_execnz .LBB42_1
12422 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
12423 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
12424 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
12425 ; GFX908-NEXT: s_setpc_b64 s[30:31]
12426 ;
12427 ; GFX8-LABEL: global_agent_atomic_fsub_ret_v2f16:
12428 ; GFX8: ; %bb.0:
12429 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12430 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
12431 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
12432 ; GFX8-NEXT: .LBB42_1: ; %atomicrmw.start
12433 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
12434 ; GFX8-NEXT: s_waitcnt vmcnt(0)
12435 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
12436 ; GFX8-NEXT: v_sub_f16_sdwa v3, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
12437 ; GFX8-NEXT: v_sub_f16_e32 v5, v4, v2
12438 ; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
12439 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
12440 ; GFX8-NEXT: s_waitcnt vmcnt(0)
12441 ; GFX8-NEXT: buffer_wbinvl1
12442 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
12443 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12444 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
12445 ; GFX8-NEXT: s_cbranch_execnz .LBB42_1
12446 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
12447 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
12448 ; GFX8-NEXT: v_mov_b32_e32 v0, v3
12449 ; GFX8-NEXT: s_setpc_b64 s[30:31]
12450 ;
12451 ; GFX7-LABEL: global_agent_atomic_fsub_ret_v2f16:
12452 ; GFX7: ; %bb.0:
12453 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12454 ; GFX7-NEXT: s_mov_b32 s6, 0
12455 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
12456 ; GFX7-NEXT: s_mov_b32 s4, s6
12457 ; GFX7-NEXT: s_mov_b32 s5, s6
12458 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
12459 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
12460 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v2
12461 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
12462 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v3
12463 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12464 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
12465 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v5
12466 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
12467 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v6
12468 ; GFX7-NEXT: .LBB42_1: ; %atomicrmw.start
12469 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
12470 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
12471 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
12472 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v3
12473 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
12474 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
12475 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v4
12476 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v5
12477 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6
12478 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v7
12479 ; GFX7-NEXT: v_or_b32_e32 v7, v2, v3
12480 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
12481 ; GFX7-NEXT: v_or_b32_e32 v6, v8, v2
12482 ; GFX7-NEXT: v_mov_b32_e32 v9, v7
12483 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
12484 ; GFX7-NEXT: buffer_atomic_cmpswap v[8:9], v[0:1], s[4:7], 0 addr64 glc
12485 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12486 ; GFX7-NEXT: buffer_wbinvl1
12487 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
12488 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v8
12489 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
12490 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
12491 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
12492 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
12493 ; GFX7-NEXT: s_cbranch_execnz .LBB42_1
12494 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
12495 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
12496 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
12497 ; GFX7-NEXT: v_mov_b32_e32 v1, v3
12498 ; GFX7-NEXT: s_setpc_b64 s[30:31]
12499 ;
12500 ; GFX6-LABEL: global_agent_atomic_fsub_ret_v2f16:
12501 ; GFX6: ; %bb.0:
12502 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12503 ; GFX6-NEXT: s_mov_b32 s6, 0
12504 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
12505 ; GFX6-NEXT: s_mov_b32 s4, s6
12506 ; GFX6-NEXT: s_mov_b32 s5, s6
12507 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
12508 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
12509 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v2
12510 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
12511 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v3
12512 ; GFX6-NEXT: s_waitcnt vmcnt(0)
12513 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
12514 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v5
12515 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
12516 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v6
12517 ; GFX6-NEXT: .LBB42_1: ; %atomicrmw.start
12518 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
12519 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
12520 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
12521 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v3
12522 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
12523 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
12524 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v4
12525 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v5
12526 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
12527 ; GFX6-NEXT: s_waitcnt expcnt(0)
12528 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v7
12529 ; GFX6-NEXT: v_or_b32_e32 v7, v2, v3
12530 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v6
12531 ; GFX6-NEXT: v_or_b32_e32 v6, v8, v2
12532 ; GFX6-NEXT: v_mov_b32_e32 v9, v7
12533 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
12534 ; GFX6-NEXT: buffer_atomic_cmpswap v[8:9], v[0:1], s[4:7], 0 addr64 glc
12535 ; GFX6-NEXT: s_waitcnt vmcnt(0)
12536 ; GFX6-NEXT: buffer_wbinvl1
12537 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
12538 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v8
12539 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
12540 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
12541 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
12542 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
12543 ; GFX6-NEXT: s_cbranch_execnz .LBB42_1
12544 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
12545 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
12546 ; GFX6-NEXT: v_mov_b32_e32 v0, v2
12547 ; GFX6-NEXT: v_mov_b32_e32 v1, v3
12548 ; GFX6-NEXT: s_waitcnt expcnt(0)
12549 ; GFX6-NEXT: s_setpc_b64 s[30:31]
12550 %result = atomicrmw fsub ptr addrspace(1) %ptr, <2 x half> %val syncscope("agent") seq_cst
12551 ret <2 x half> %result
12552 }
12554 define <2 x half> @global_agent_atomic_fsub_ret_v2f16__offset12b_pos(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
12555 ; GFX12-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_pos:
12556 ; GFX12: ; %bb.0:
12557 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
12558 ; GFX12-NEXT: s_wait_expcnt 0x0
12559 ; GFX12-NEXT: s_wait_samplecnt 0x0
12560 ; GFX12-NEXT: s_wait_bvhcnt 0x0
12561 ; GFX12-NEXT: s_wait_kmcnt 0x0
12562 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
12563 ; GFX12-NEXT: s_mov_b32 s0, 0
12564 ; GFX12-NEXT: .LBB43_1: ; %atomicrmw.start
12565 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
12566 ; GFX12-NEXT: s_wait_loadcnt 0x0
12567 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
12568 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
12569 ; GFX12-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12570 ; GFX12-NEXT: s_wait_storecnt 0x0
12571 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
12572 ; GFX12-NEXT: s_wait_loadcnt 0x0
12573 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
12574 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
12575 ; GFX12-NEXT: s_wait_alu 0xfffe
12576 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
12577 ; GFX12-NEXT: s_wait_alu 0xfffe
12578 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
12579 ; GFX12-NEXT: s_cbranch_execnz .LBB43_1
12580 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
12581 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
12582 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
12583 ; GFX12-NEXT: s_wait_alu 0xfffe
12584 ; GFX12-NEXT: s_setpc_b64 s[30:31]
12585 ;
12586 ; GFX940-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_pos:
12587 ; GFX940: ; %bb.0:
12588 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12589 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
12590 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
12591 ; GFX940-NEXT: .LBB43_1: ; %atomicrmw.start
12592 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
12593 ; GFX940-NEXT: s_waitcnt vmcnt(0)
12594 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
12595 ; GFX940-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
12596 ; GFX940-NEXT: buffer_wbl2 sc1
12597 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
12598 ; GFX940-NEXT: s_waitcnt vmcnt(0)
12599 ; GFX940-NEXT: buffer_inv sc1
12600 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
12601 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
12602 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
12603 ; GFX940-NEXT: s_cbranch_execnz .LBB43_1
12604 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
12605 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
12606 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
12607 ; GFX940-NEXT: s_setpc_b64 s[30:31]
12608 ;
12609 ; GFX11-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_pos:
12610 ; GFX11: ; %bb.0:
12611 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12612 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
12613 ; GFX11-NEXT: s_mov_b32 s0, 0
12614 ; GFX11-NEXT: .LBB43_1: ; %atomicrmw.start
12615 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
12616 ; GFX11-NEXT: s_waitcnt vmcnt(0)
12617 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
12618 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
12619 ; GFX11-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12620 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
12621 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
12622 ; GFX11-NEXT: s_waitcnt vmcnt(0)
12623 ; GFX11-NEXT: buffer_gl1_inv
12624 ; GFX11-NEXT: buffer_gl0_inv
12625 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
12626 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
12627 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
12628 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
12629 ; GFX11-NEXT: s_cbranch_execnz .LBB43_1
12630 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
12631 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
12632 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
12633 ; GFX11-NEXT: s_setpc_b64 s[30:31]
12634 ;
12635 ; GFX10-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_pos:
12636 ; GFX10: ; %bb.0:
12637 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12638 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
12639 ; GFX10-NEXT: s_mov_b32 s4, 0
12640 ; GFX10-NEXT: .LBB43_1: ; %atomicrmw.start
12641 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
12642 ; GFX10-NEXT: s_waitcnt vmcnt(0)
12643 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
12644 ; GFX10-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12645 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
12646 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
12647 ; GFX10-NEXT: s_waitcnt vmcnt(0)
12648 ; GFX10-NEXT: buffer_gl1_inv
12649 ; GFX10-NEXT: buffer_gl0_inv
12650 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
12651 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
12652 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
12653 ; GFX10-NEXT: s_cbranch_execnz .LBB43_1
12654 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
12655 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
12656 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
12657 ; GFX10-NEXT: s_setpc_b64 s[30:31]
12658 ;
12659 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_pos:
12660 ; GFX90A: ; %bb.0:
12661 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12662 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
12663 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
12664 ; GFX90A-NEXT: .LBB43_1: ; %atomicrmw.start
12665 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
12666 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12667 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
12668 ; GFX90A-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
12669 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
12670 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12671 ; GFX90A-NEXT: buffer_wbinvl1
12672 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
12673 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12674 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
12675 ; GFX90A-NEXT: s_cbranch_execnz .LBB43_1
12676 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
12677 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
12678 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
12679 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
12680 ;
12681 ; GFX908-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_pos:
12682 ; GFX908: ; %bb.0:
12683 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12684 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
12685 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
12686 ; GFX908-NEXT: .LBB43_1: ; %atomicrmw.start
12687 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
12688 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12689 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
12690 ; GFX908-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12691 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
12692 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12693 ; GFX908-NEXT: buffer_wbinvl1
12694 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
12695 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12696 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
12697 ; GFX908-NEXT: s_cbranch_execnz .LBB43_1
12698 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
12699 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
12700 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
12701 ; GFX908-NEXT: s_setpc_b64 s[30:31]
12702 ;
12703 ; GFX8-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_pos:
12704 ; GFX8: ; %bb.0:
12705 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12706 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
12707 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
12708 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
12709 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
12710 ; GFX8-NEXT: .LBB43_1: ; %atomicrmw.start
12711 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
12712 ; GFX8-NEXT: s_waitcnt vmcnt(0)
12713 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
12714 ; GFX8-NEXT: v_sub_f16_sdwa v0, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
12715 ; GFX8-NEXT: v_sub_f16_e32 v5, v1, v2
12716 ; GFX8-NEXT: v_or_b32_e32 v0, v5, v0
12717 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
12718 ; GFX8-NEXT: s_waitcnt vmcnt(0)
12719 ; GFX8-NEXT: buffer_wbinvl1
12720 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
12721 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12722 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
12723 ; GFX8-NEXT: s_cbranch_execnz .LBB43_1
12724 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
12725 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
12726 ; GFX8-NEXT: s_setpc_b64 s[30:31]
12727 ;
12728 ; GFX7-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_pos:
12729 ; GFX7: ; %bb.0:
12730 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12731 ; GFX7-NEXT: s_mov_b32 s6, 0
12732 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
12733 ; GFX7-NEXT: s_mov_b32 s4, s6
12734 ; GFX7-NEXT: s_mov_b32 s5, s6
12735 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
12736 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
12737 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v2
12738 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
12739 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v3
12740 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12741 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
12742 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v5
12743 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
12744 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v6
12745 ; GFX7-NEXT: .LBB43_1: ; %atomicrmw.start
12746 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
12747 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
12748 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
12749 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v3
12750 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
12751 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
12752 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v4
12753 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v5
12754 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6
12755 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v7
12756 ; GFX7-NEXT: v_or_b32_e32 v7, v2, v3
12757 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
12758 ; GFX7-NEXT: v_or_b32_e32 v6, v8, v2
12759 ; GFX7-NEXT: v_mov_b32_e32 v9, v7
12760 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
12761 ; GFX7-NEXT: buffer_atomic_cmpswap v[8:9], v[0:1], s[4:7], 0 addr64 offset:2044 glc
12762 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12763 ; GFX7-NEXT: buffer_wbinvl1
12764 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
12765 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v8
12766 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
12767 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
12768 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
12769 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
12770 ; GFX7-NEXT: s_cbranch_execnz .LBB43_1
12771 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
12772 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
12773 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
12774 ; GFX7-NEXT: v_mov_b32_e32 v1, v3
12775 ; GFX7-NEXT: s_setpc_b64 s[30:31]
12776 ;
12777 ; GFX6-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_pos:
12778 ; GFX6: ; %bb.0:
12779 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12780 ; GFX6-NEXT: s_mov_b32 s6, 0
12781 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
12782 ; GFX6-NEXT: s_mov_b32 s4, s6
12783 ; GFX6-NEXT: s_mov_b32 s5, s6
12784 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
12785 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
12786 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v2
12787 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
12788 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v3
12789 ; GFX6-NEXT: s_waitcnt vmcnt(0)
12790 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
12791 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v5
12792 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
12793 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v6
12794 ; GFX6-NEXT: .LBB43_1: ; %atomicrmw.start
12795 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
12796 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
12797 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
12798 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v3
12799 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
12800 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
12801 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v4
12802 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v5
12803 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
12804 ; GFX6-NEXT: s_waitcnt expcnt(0)
12805 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v7
12806 ; GFX6-NEXT: v_or_b32_e32 v7, v2, v3
12807 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v6
12808 ; GFX6-NEXT: v_or_b32_e32 v6, v8, v2
12809 ; GFX6-NEXT: v_mov_b32_e32 v9, v7
12810 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
12811 ; GFX6-NEXT: buffer_atomic_cmpswap v[8:9], v[0:1], s[4:7], 0 addr64 offset:2044 glc
12812 ; GFX6-NEXT: s_waitcnt vmcnt(0)
12813 ; GFX6-NEXT: buffer_wbinvl1
12814 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
12815 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v8
12816 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
12817 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
12818 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
12819 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
12820 ; GFX6-NEXT: s_cbranch_execnz .LBB43_1
12821 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
12822 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
12823 ; GFX6-NEXT: v_mov_b32_e32 v0, v2
12824 ; GFX6-NEXT: v_mov_b32_e32 v1, v3
12825 ; GFX6-NEXT: s_waitcnt expcnt(0)
12826 ; GFX6-NEXT: s_setpc_b64 s[30:31]
12827 %gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 511
12828 %result = atomicrmw fsub ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst
12829 ret <2 x half> %result
12830 }
12832 define <2 x half> @global_agent_atomic_fsub_ret_v2f16__offset12b_neg(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
12833 ; GFX12-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_neg:
12834 ; GFX12: ; %bb.0:
12835 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
12836 ; GFX12-NEXT: s_wait_expcnt 0x0
12837 ; GFX12-NEXT: s_wait_samplecnt 0x0
12838 ; GFX12-NEXT: s_wait_bvhcnt 0x0
12839 ; GFX12-NEXT: s_wait_kmcnt 0x0
12840 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
12841 ; GFX12-NEXT: s_mov_b32 s0, 0
12842 ; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
12843 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
12844 ; GFX12-NEXT: s_wait_loadcnt 0x0
12845 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
12846 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
12847 ; GFX12-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12848 ; GFX12-NEXT: s_wait_storecnt 0x0
12849 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
12850 ; GFX12-NEXT: s_wait_loadcnt 0x0
12851 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
12852 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
12853 ; GFX12-NEXT: s_wait_alu 0xfffe
12854 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
12855 ; GFX12-NEXT: s_wait_alu 0xfffe
12856 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
12857 ; GFX12-NEXT: s_cbranch_execnz .LBB44_1
12858 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
12859 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
12860 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
12861 ; GFX12-NEXT: s_wait_alu 0xfffe
12862 ; GFX12-NEXT: s_setpc_b64 s[30:31]
12863 ;
12864 ; GFX940-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_neg:
12865 ; GFX940: ; %bb.0:
12866 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12867 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
12868 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
12869 ; GFX940-NEXT: .LBB44_1: ; %atomicrmw.start
12870 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
12871 ; GFX940-NEXT: s_waitcnt vmcnt(0)
12872 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
12873 ; GFX940-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
12874 ; GFX940-NEXT: buffer_wbl2 sc1
12875 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
12876 ; GFX940-NEXT: s_waitcnt vmcnt(0)
12877 ; GFX940-NEXT: buffer_inv sc1
12878 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
12879 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
12880 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
12881 ; GFX940-NEXT: s_cbranch_execnz .LBB44_1
12882 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
12883 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
12884 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
12885 ; GFX940-NEXT: s_setpc_b64 s[30:31]
12886 ;
12887 ; GFX11-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_neg:
12888 ; GFX11: ; %bb.0:
12889 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12890 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
12891 ; GFX11-NEXT: s_mov_b32 s0, 0
12892 ; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
12893 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
12894 ; GFX11-NEXT: s_waitcnt vmcnt(0)
12895 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
12896 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
12897 ; GFX11-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12898 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
12899 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
12900 ; GFX11-NEXT: s_waitcnt vmcnt(0)
12901 ; GFX11-NEXT: buffer_gl1_inv
12902 ; GFX11-NEXT: buffer_gl0_inv
12903 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
12904 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
12905 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
12906 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
12907 ; GFX11-NEXT: s_cbranch_execnz .LBB44_1
12908 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
12909 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
12910 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
12911 ; GFX11-NEXT: s_setpc_b64 s[30:31]
12912 ;
12913 ; GFX10-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_neg:
12914 ; GFX10: ; %bb.0:
12915 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12916 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
12917 ; GFX10-NEXT: s_mov_b32 s4, 0
12918 ; GFX10-NEXT: .LBB44_1: ; %atomicrmw.start
12919 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
12920 ; GFX10-NEXT: s_waitcnt vmcnt(0)
12921 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
12922 ; GFX10-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12923 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
12924 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
12925 ; GFX10-NEXT: s_waitcnt vmcnt(0)
12926 ; GFX10-NEXT: buffer_gl1_inv
12927 ; GFX10-NEXT: buffer_gl0_inv
12928 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
12929 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
12930 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
12931 ; GFX10-NEXT: s_cbranch_execnz .LBB44_1
12932 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
12933 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
12934 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
12935 ; GFX10-NEXT: s_setpc_b64 s[30:31]
12936 ;
12937 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_neg:
12938 ; GFX90A: ; %bb.0:
12939 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12940 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
12941 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
12942 ; GFX90A-NEXT: .LBB44_1: ; %atomicrmw.start
12943 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
12944 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12945 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
12946 ; GFX90A-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
12947 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
12948 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12949 ; GFX90A-NEXT: buffer_wbinvl1
12950 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
12951 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12952 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
12953 ; GFX90A-NEXT: s_cbranch_execnz .LBB44_1
12954 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
12955 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
12956 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
12957 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
12958 ;
12959 ; GFX908-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_neg:
12960 ; GFX908: ; %bb.0:
12961 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12962 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
12963 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
12964 ; GFX908-NEXT: .LBB44_1: ; %atomicrmw.start
12965 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
12966 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12967 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
12968 ; GFX908-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12969 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
12970 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12971 ; GFX908-NEXT: buffer_wbinvl1
12972 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
12973 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12974 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
12975 ; GFX908-NEXT: s_cbranch_execnz .LBB44_1
12976 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
12977 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
12978 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
12979 ; GFX908-NEXT: s_setpc_b64 s[30:31]
12980 ;
12981 ; GFX8-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_neg:
12982 ; GFX8: ; %bb.0:
12983 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12984 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
12985 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
12986 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
12987 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
12988 ; GFX8-NEXT: .LBB44_1: ; %atomicrmw.start
12989 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
12990 ; GFX8-NEXT: s_waitcnt vmcnt(0)
12991 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
12992 ; GFX8-NEXT: v_sub_f16_sdwa v0, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
12993 ; GFX8-NEXT: v_sub_f16_e32 v5, v1, v2
12994 ; GFX8-NEXT: v_or_b32_e32 v0, v5, v0
12995 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
12996 ; GFX8-NEXT: s_waitcnt vmcnt(0)
12997 ; GFX8-NEXT: buffer_wbinvl1
12998 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
12999 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13000 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
13001 ; GFX8-NEXT: s_cbranch_execnz .LBB44_1
13002 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
13003 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
13004 ; GFX8-NEXT: s_setpc_b64 s[30:31]
13005 ;
13006 ; GFX7-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_neg:
13007 ; GFX7: ; %bb.0:
13008 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13009 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
13010 ; GFX7-NEXT: s_mov_b32 s5, -1
13011 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
13012 ; GFX7-NEXT: s_mov_b32 s6, 0
13013 ; GFX7-NEXT: buffer_load_dword v6, v[0:1], s[4:7], 0 addr64
13014 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
13015 ; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v3
13016 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v2
13017 ; GFX7-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
13018 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v0
13019 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
13020 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
13021 ; GFX7-NEXT: s_mov_b32 s4, s6
13022 ; GFX7-NEXT: s_mov_b32 s5, s6
13023 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13024 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v6
13025 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v6
13026 ; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
13027 ; GFX7-NEXT: .LBB44_1: ; %atomicrmw.start
13028 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
13029 ; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
13030 ; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
13031 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v1
13032 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v0
13033 ; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
13034 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
13035 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
13036 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6
13037 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v7
13038 ; GFX7-NEXT: v_or_b32_e32 v7, v0, v1
13039 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v6
13040 ; GFX7-NEXT: v_or_b32_e32 v6, v8, v0
13041 ; GFX7-NEXT: v_mov_b32_e32 v9, v7
13042 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
13043 ; GFX7-NEXT: buffer_atomic_cmpswap v[8:9], v[4:5], s[4:7], 0 addr64 glc
13044 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13045 ; GFX7-NEXT: buffer_wbinvl1
13046 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v8
13047 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v8
13048 ; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
13049 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
13050 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
13051 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
13052 ; GFX7-NEXT: s_cbranch_execnz .LBB44_1
13053 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
13054 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
13055 ; GFX7-NEXT: s_setpc_b64 s[30:31]
13056 ;
13057 ; GFX6-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_neg:
13058 ; GFX6: ; %bb.0:
13059 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13060 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
13061 ; GFX6-NEXT: s_mov_b32 s5, -1
13062 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
13063 ; GFX6-NEXT: s_mov_b32 s6, 0
13064 ; GFX6-NEXT: buffer_load_dword v6, v[0:1], s[4:7], 0 addr64
13065 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
13066 ; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v3
13067 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v2
13068 ; GFX6-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
13069 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v0
13070 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
13071 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
13072 ; GFX6-NEXT: s_mov_b32 s4, s6
13073 ; GFX6-NEXT: s_mov_b32 s5, s6
13074 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13075 ; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v6
13076 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v6
13077 ; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
13078 ; GFX6-NEXT: .LBB44_1: ; %atomicrmw.start
13079 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
13080 ; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
13081 ; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
13082 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v1
13083 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v0
13084 ; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
13085 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
13086 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
13087 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
13088 ; GFX6-NEXT: s_waitcnt expcnt(0)
13089 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v7
13090 ; GFX6-NEXT: v_or_b32_e32 v7, v0, v1
13091 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v6
13092 ; GFX6-NEXT: v_or_b32_e32 v6, v8, v0
13093 ; GFX6-NEXT: v_mov_b32_e32 v9, v7
13094 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
13095 ; GFX6-NEXT: buffer_atomic_cmpswap v[8:9], v[4:5], s[4:7], 0 addr64 glc
13096 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13097 ; GFX6-NEXT: buffer_wbinvl1
13098 ; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v8
13099 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v8
13100 ; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
13101 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
13102 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
13103 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
13104 ; GFX6-NEXT: s_cbranch_execnz .LBB44_1
13105 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
13106 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
13107 ; GFX6-NEXT: s_waitcnt expcnt(0)
13108 ; GFX6-NEXT: s_setpc_b64 s[30:31]
13109 %gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 -512
13110 %result = atomicrmw fsub ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst
13111 ret <2 x half> %result
13112 }
13114 define void @global_agent_atomic_fsub_noret_v2f16(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
13115 ; GFX12-LABEL: global_agent_atomic_fsub_noret_v2f16:
13116 ; GFX12: ; %bb.0:
13117 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
13118 ; GFX12-NEXT: s_wait_expcnt 0x0
13119 ; GFX12-NEXT: s_wait_samplecnt 0x0
13120 ; GFX12-NEXT: s_wait_bvhcnt 0x0
13121 ; GFX12-NEXT: s_wait_kmcnt 0x0
13122 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off
13123 ; GFX12-NEXT: s_mov_b32 s0, 0
13124 ; GFX12-NEXT: .LBB45_1: ; %atomicrmw.start
13125 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
13126 ; GFX12-NEXT: s_wait_loadcnt 0x0
13127 ; GFX12-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13128 ; GFX12-NEXT: s_wait_storecnt 0x0
13129 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
13130 ; GFX12-NEXT: s_wait_loadcnt 0x0
13131 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
13132 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13133 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
13134 ; GFX12-NEXT: s_wait_alu 0xfffe
13135 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
13136 ; GFX12-NEXT: s_wait_alu 0xfffe
13137 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
13138 ; GFX12-NEXT: s_cbranch_execnz .LBB45_1
13139 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
13140 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
13141 ; GFX12-NEXT: s_wait_alu 0xfffe
13142 ; GFX12-NEXT: s_setpc_b64 s[30:31]
13143 ;
13144 ; GFX940-LABEL: global_agent_atomic_fsub_noret_v2f16:
13145 ; GFX940: ; %bb.0:
13146 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13147 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
13148 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
13149 ; GFX940-NEXT: .LBB45_1: ; %atomicrmw.start
13150 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
13151 ; GFX940-NEXT: s_waitcnt vmcnt(0)
13152 ; GFX940-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
13153 ; GFX940-NEXT: buffer_wbl2 sc1
13154 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
13155 ; GFX940-NEXT: s_waitcnt vmcnt(0)
13156 ; GFX940-NEXT: buffer_inv sc1
13157 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
13158 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
13159 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
13160 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
13161 ; GFX940-NEXT: s_cbranch_execnz .LBB45_1
13162 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
13163 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
13164 ; GFX940-NEXT: s_setpc_b64 s[30:31]
13165 ;
13166 ; GFX11-LABEL: global_agent_atomic_fsub_noret_v2f16:
13167 ; GFX11: ; %bb.0:
13168 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13169 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off
13170 ; GFX11-NEXT: s_mov_b32 s0, 0
13171 ; GFX11-NEXT: .LBB45_1: ; %atomicrmw.start
13172 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
13173 ; GFX11-NEXT: s_waitcnt vmcnt(0)
13174 ; GFX11-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13175 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
13176 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
13177 ; GFX11-NEXT: s_waitcnt vmcnt(0)
13178 ; GFX11-NEXT: buffer_gl1_inv
13179 ; GFX11-NEXT: buffer_gl0_inv
13180 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13181 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
13182 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
13183 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
13184 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
13185 ; GFX11-NEXT: s_cbranch_execnz .LBB45_1
13186 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
13187 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
13188 ; GFX11-NEXT: s_setpc_b64 s[30:31]
13189 ;
13190 ; GFX10-LABEL: global_agent_atomic_fsub_noret_v2f16:
13191 ; GFX10: ; %bb.0:
13192 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13193 ; GFX10-NEXT: global_load_dword v4, v[0:1], off
13194 ; GFX10-NEXT: s_mov_b32 s4, 0
13195 ; GFX10-NEXT: .LBB45_1: ; %atomicrmw.start
13196 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
13197 ; GFX10-NEXT: s_waitcnt vmcnt(0)
13198 ; GFX10-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13199 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
13200 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
13201 ; GFX10-NEXT: s_waitcnt vmcnt(0)
13202 ; GFX10-NEXT: buffer_gl1_inv
13203 ; GFX10-NEXT: buffer_gl0_inv
13204 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13205 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
13206 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
13207 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
13208 ; GFX10-NEXT: s_cbranch_execnz .LBB45_1
13209 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
13210 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
13211 ; GFX10-NEXT: s_setpc_b64 s[30:31]
13212 ;
13213 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_v2f16:
13214 ; GFX90A: ; %bb.0:
13215 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13216 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
13217 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
13218 ; GFX90A-NEXT: .LBB45_1: ; %atomicrmw.start
13219 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
13220 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
13221 ; GFX90A-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
13222 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
13223 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
13224 ; GFX90A-NEXT: buffer_wbinvl1
13225 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
13226 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13227 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
13228 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
13229 ; GFX90A-NEXT: s_cbranch_execnz .LBB45_1
13230 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
13231 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
13232 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
13233 ;
13234 ; GFX908-LABEL: global_agent_atomic_fsub_noret_v2f16:
13235 ; GFX908: ; %bb.0:
13236 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13237 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
13238 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
13239 ; GFX908-NEXT: .LBB45_1: ; %atomicrmw.start
13240 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
13241 ; GFX908-NEXT: s_waitcnt vmcnt(0)
13242 ; GFX908-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13243 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
13244 ; GFX908-NEXT: s_waitcnt vmcnt(0)
13245 ; GFX908-NEXT: buffer_wbinvl1
13246 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
13247 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13248 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
13249 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
13250 ; GFX908-NEXT: s_cbranch_execnz .LBB45_1
13251 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
13252 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
13253 ; GFX908-NEXT: s_setpc_b64 s[30:31]
13254 ;
13255 ; GFX8-LABEL: global_agent_atomic_fsub_noret_v2f16:
13256 ; GFX8: ; %bb.0:
13257 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13258 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
13259 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
13260 ; GFX8-NEXT: .LBB45_1: ; %atomicrmw.start
13261 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
13262 ; GFX8-NEXT: s_waitcnt vmcnt(0)
13263 ; GFX8-NEXT: v_sub_f16_sdwa v3, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
13264 ; GFX8-NEXT: v_sub_f16_e32 v5, v4, v2
13265 ; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
13266 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
13267 ; GFX8-NEXT: s_waitcnt vmcnt(0)
13268 ; GFX8-NEXT: buffer_wbinvl1
13269 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
13270 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13271 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
13272 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
13273 ; GFX8-NEXT: s_cbranch_execnz .LBB45_1
13274 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
13275 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
13276 ; GFX8-NEXT: s_setpc_b64 s[30:31]
13277 ;
13278 ; GFX7-LABEL: global_agent_atomic_fsub_noret_v2f16:
13279 ; GFX7: ; %bb.0:
13280 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13281 ; GFX7-NEXT: s_mov_b32 s6, 0
13282 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
13283 ; GFX7-NEXT: s_mov_b32 s4, s6
13284 ; GFX7-NEXT: s_mov_b32 s5, s6
13285 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
13286 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
13287 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v2
13288 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
13289 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
13290 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13291 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
13292 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
13293 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
13294 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
13295 ; GFX7-NEXT: .LBB45_1: ; %atomicrmw.start
13296 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
13297 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
13298 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
13299 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v5
13300 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v4
13301 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
13302 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
13303 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
13304 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v6
13305 ; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v7
13306 ; GFX7-NEXT: v_or_b32_e32 v6, v4, v5
13307 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v8
13308 ; GFX7-NEXT: v_or_b32_e32 v5, v7, v4
13309 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
13310 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
13311 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
13312 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13313 ; GFX7-NEXT: buffer_wbinvl1
13314 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v7
13315 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v7
13316 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
13317 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
13318 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
13319 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
13320 ; GFX7-NEXT: s_cbranch_execnz .LBB45_1
13321 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
13322 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
13323 ; GFX7-NEXT: s_setpc_b64 s[30:31]
13324 ;
13325 ; GFX6-LABEL: global_agent_atomic_fsub_noret_v2f16:
13326 ; GFX6: ; %bb.0:
13327 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13328 ; GFX6-NEXT: s_mov_b32 s6, 0
13329 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
13330 ; GFX6-NEXT: s_mov_b32 s4, s6
13331 ; GFX6-NEXT: s_mov_b32 s5, s6
13332 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
13333 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
13334 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v2
13335 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
13336 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v3
13337 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13338 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
13339 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v5
13340 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v3
13341 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v6
13342 ; GFX6-NEXT: .LBB45_1: ; %atomicrmw.start
13343 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
13344 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
13345 ; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
13346 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v5
13347 ; GFX6-NEXT: s_waitcnt expcnt(0)
13348 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v4
13349 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
13350 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
13351 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
13352 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v6
13353 ; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v7
13354 ; GFX6-NEXT: v_or_b32_e32 v6, v4, v5
13355 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v8
13356 ; GFX6-NEXT: v_or_b32_e32 v5, v7, v4
13357 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
13358 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
13359 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
13360 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13361 ; GFX6-NEXT: buffer_wbinvl1
13362 ; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v7
13363 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v7
13364 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
13365 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
13366 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
13367 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
13368 ; GFX6-NEXT: s_cbranch_execnz .LBB45_1
13369 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
13370 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
13371 ; GFX6-NEXT: s_waitcnt expcnt(0)
13372 ; GFX6-NEXT: s_setpc_b64 s[30:31]
13373 %unused = atomicrmw fsub ptr addrspace(1) %ptr, <2 x half> %val syncscope("agent") seq_cst
13374 ret void
13375 }
13377 define void @global_agent_atomic_fsub_noret_v2f16__offset12b_pos(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
13378 ; GFX12-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_pos:
13379 ; GFX12: ; %bb.0:
13380 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
13381 ; GFX12-NEXT: s_wait_expcnt 0x0
13382 ; GFX12-NEXT: s_wait_samplecnt 0x0
13383 ; GFX12-NEXT: s_wait_bvhcnt 0x0
13384 ; GFX12-NEXT: s_wait_kmcnt 0x0
13385 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:2044
13386 ; GFX12-NEXT: s_mov_b32 s0, 0
13387 ; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start
13388 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
13389 ; GFX12-NEXT: s_wait_loadcnt 0x0
13390 ; GFX12-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13391 ; GFX12-NEXT: s_wait_storecnt 0x0
13392 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
13393 ; GFX12-NEXT: s_wait_loadcnt 0x0
13394 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
13395 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13396 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
13397 ; GFX12-NEXT: s_wait_alu 0xfffe
13398 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
13399 ; GFX12-NEXT: s_wait_alu 0xfffe
13400 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
13401 ; GFX12-NEXT: s_cbranch_execnz .LBB46_1
13402 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
13403 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
13404 ; GFX12-NEXT: s_wait_alu 0xfffe
13405 ; GFX12-NEXT: s_setpc_b64 s[30:31]
13406 ;
13407 ; GFX940-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_pos:
13408 ; GFX940: ; %bb.0:
13409 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13410 ; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:2044
13411 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
13412 ; GFX940-NEXT: .LBB46_1: ; %atomicrmw.start
13413 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
13414 ; GFX940-NEXT: s_waitcnt vmcnt(0)
13415 ; GFX940-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
13416 ; GFX940-NEXT: buffer_wbl2 sc1
13417 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
13418 ; GFX940-NEXT: s_waitcnt vmcnt(0)
13419 ; GFX940-NEXT: buffer_inv sc1
13420 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
13421 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
13422 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
13423 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
13424 ; GFX940-NEXT: s_cbranch_execnz .LBB46_1
13425 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
13426 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
13427 ; GFX940-NEXT: s_setpc_b64 s[30:31]
13428 ;
13429 ; GFX11-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_pos:
13430 ; GFX11: ; %bb.0:
13431 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13432 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:2044
13433 ; GFX11-NEXT: s_mov_b32 s0, 0
13434 ; GFX11-NEXT: .LBB46_1: ; %atomicrmw.start
13435 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
13436 ; GFX11-NEXT: s_waitcnt vmcnt(0)
13437 ; GFX11-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13438 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
13439 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
13440 ; GFX11-NEXT: s_waitcnt vmcnt(0)
13441 ; GFX11-NEXT: buffer_gl1_inv
13442 ; GFX11-NEXT: buffer_gl0_inv
13443 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13444 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
13445 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
13446 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
13447 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
13448 ; GFX11-NEXT: s_cbranch_execnz .LBB46_1
13449 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
13450 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
13451 ; GFX11-NEXT: s_setpc_b64 s[30:31]
13452 ;
13453 ; GFX10-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_pos:
13454 ; GFX10: ; %bb.0:
13455 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13456 ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
13457 ; GFX10-NEXT: s_mov_b32 s4, 0
13458 ; GFX10-NEXT: .LBB46_1: ; %atomicrmw.start
13459 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
13460 ; GFX10-NEXT: s_waitcnt vmcnt(0)
13461 ; GFX10-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13462 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
13463 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
13464 ; GFX10-NEXT: s_waitcnt vmcnt(0)
13465 ; GFX10-NEXT: buffer_gl1_inv
13466 ; GFX10-NEXT: buffer_gl0_inv
13467 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13468 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
13469 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
13470 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
13471 ; GFX10-NEXT: s_cbranch_execnz .LBB46_1
13472 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
13473 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
13474 ; GFX10-NEXT: s_setpc_b64 s[30:31]
13475 ;
13476 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_pos:
13477 ; GFX90A: ; %bb.0:
13478 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13479 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
13480 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
13481 ; GFX90A-NEXT: .LBB46_1: ; %atomicrmw.start
13482 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
13483 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
13484 ; GFX90A-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
13485 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
13486 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
13487 ; GFX90A-NEXT: buffer_wbinvl1
13488 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
13489 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13490 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
13491 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
13492 ; GFX90A-NEXT: s_cbranch_execnz .LBB46_1
13493 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
13494 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
13495 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
13496 ;
13497 ; GFX908-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_pos:
13498 ; GFX908: ; %bb.0:
13499 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13500 ; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
13501 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
13502 ; GFX908-NEXT: .LBB46_1: ; %atomicrmw.start
13503 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
13504 ; GFX908-NEXT: s_waitcnt vmcnt(0)
13505 ; GFX908-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13506 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
13507 ; GFX908-NEXT: s_waitcnt vmcnt(0)
13508 ; GFX908-NEXT: buffer_wbinvl1
13509 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
13510 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13511 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
13512 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
13513 ; GFX908-NEXT: s_cbranch_execnz .LBB46_1
13514 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
13515 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
13516 ; GFX908-NEXT: s_setpc_b64 s[30:31]
13517 ;
13518 ; GFX8-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_pos:
13519 ; GFX8: ; %bb.0:
13520 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13521 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
13522 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
13523 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
13524 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
13525 ; GFX8-NEXT: .LBB46_1: ; %atomicrmw.start
13526 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
13527 ; GFX8-NEXT: s_waitcnt vmcnt(0)
13528 ; GFX8-NEXT: v_sub_f16_sdwa v3, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
13529 ; GFX8-NEXT: v_sub_f16_e32 v5, v4, v2
13530 ; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
13531 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
13532 ; GFX8-NEXT: s_waitcnt vmcnt(0)
13533 ; GFX8-NEXT: buffer_wbinvl1
13534 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
13535 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13536 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
13537 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
13538 ; GFX8-NEXT: s_cbranch_execnz .LBB46_1
13539 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
13540 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
13541 ; GFX8-NEXT: s_setpc_b64 s[30:31]
13542 ;
13543 ; GFX7-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_pos:
13544 ; GFX7: ; %bb.0:
13545 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13546 ; GFX7-NEXT: s_mov_b32 s6, 0
13547 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
13548 ; GFX7-NEXT: s_mov_b32 s4, s6
13549 ; GFX7-NEXT: s_mov_b32 s5, s6
13550 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
13551 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
13552 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v2
13553 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
13554 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
13555 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13556 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
13557 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
13558 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
13559 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
13560 ; GFX7-NEXT: .LBB46_1: ; %atomicrmw.start
13561 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
13562 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
13563 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
13564 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v5
13565 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v4
13566 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
13567 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
13568 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
13569 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v6
13570 ; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v7
13571 ; GFX7-NEXT: v_or_b32_e32 v6, v4, v5
13572 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v8
13573 ; GFX7-NEXT: v_or_b32_e32 v5, v7, v4
13574 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
13575 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
13576 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 offset:2044 glc
13577 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13578 ; GFX7-NEXT: buffer_wbinvl1
13579 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v7
13580 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v7
13581 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
13582 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
13583 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
13584 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
13585 ; GFX7-NEXT: s_cbranch_execnz .LBB46_1
13586 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
13587 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
13588 ; GFX7-NEXT: s_setpc_b64 s[30:31]
13589 ;
13590 ; GFX6-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_pos:
13591 ; GFX6: ; %bb.0:
13592 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13593 ; GFX6-NEXT: s_mov_b32 s6, 0
13594 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
13595 ; GFX6-NEXT: s_mov_b32 s4, s6
13596 ; GFX6-NEXT: s_mov_b32 s5, s6
13597 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
13598 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
13599 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v2
13600 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
13601 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v3
13602 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13603 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
13604 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v5
13605 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v3
13606 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v6
13607 ; GFX6-NEXT: .LBB46_1: ; %atomicrmw.start
13608 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
13609 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
13610 ; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
13611 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v5
13612 ; GFX6-NEXT: s_waitcnt expcnt(0)
13613 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v4
13614 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
13615 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
13616 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
13617 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v6
13618 ; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v7
13619 ; GFX6-NEXT: v_or_b32_e32 v6, v4, v5
13620 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v8
13621 ; GFX6-NEXT: v_or_b32_e32 v5, v7, v4
13622 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
13623 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
13624 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 offset:2044 glc
13625 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13626 ; GFX6-NEXT: buffer_wbinvl1
13627 ; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v7
13628 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v7
13629 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
13630 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
13631 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
13632 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
13633 ; GFX6-NEXT: s_cbranch_execnz .LBB46_1
13634 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
13635 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
13636 ; GFX6-NEXT: s_waitcnt expcnt(0)
13637 ; GFX6-NEXT: s_setpc_b64 s[30:31]
13638 %gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 511
13639 %unused = atomicrmw fsub ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst
13640 ret void
13641 }
13643 define void @global_agent_atomic_fsub_noret_v2f16__offset12b_neg(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
13644 ; GFX12-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_neg:
13645 ; GFX12: ; %bb.0:
13646 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
13647 ; GFX12-NEXT: s_wait_expcnt 0x0
13648 ; GFX12-NEXT: s_wait_samplecnt 0x0
13649 ; GFX12-NEXT: s_wait_bvhcnt 0x0
13650 ; GFX12-NEXT: s_wait_kmcnt 0x0
13651 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
13652 ; GFX12-NEXT: s_mov_b32 s0, 0
13653 ; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start
13654 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
13655 ; GFX12-NEXT: s_wait_loadcnt 0x0
13656 ; GFX12-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13657 ; GFX12-NEXT: s_wait_storecnt 0x0
13658 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
13659 ; GFX12-NEXT: s_wait_loadcnt 0x0
13660 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
13661 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13662 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
13663 ; GFX12-NEXT: s_wait_alu 0xfffe
13664 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
13665 ; GFX12-NEXT: s_wait_alu 0xfffe
13666 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
13667 ; GFX12-NEXT: s_cbranch_execnz .LBB47_1
13668 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
13669 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
13670 ; GFX12-NEXT: s_wait_alu 0xfffe
13671 ; GFX12-NEXT: s_setpc_b64 s[30:31]
13672 ;
13673 ; GFX940-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_neg:
13674 ; GFX940: ; %bb.0:
13675 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13676 ; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:-2048
13677 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
13678 ; GFX940-NEXT: .LBB47_1: ; %atomicrmw.start
13679 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
13680 ; GFX940-NEXT: s_waitcnt vmcnt(0)
13681 ; GFX940-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
13682 ; GFX940-NEXT: buffer_wbl2 sc1
13683 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
13684 ; GFX940-NEXT: s_waitcnt vmcnt(0)
13685 ; GFX940-NEXT: buffer_inv sc1
13686 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
13687 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
13688 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
13689 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
13690 ; GFX940-NEXT: s_cbranch_execnz .LBB47_1
13691 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
13692 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
13693 ; GFX940-NEXT: s_setpc_b64 s[30:31]
13694 ;
13695 ; GFX11-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_neg:
13696 ; GFX11: ; %bb.0:
13697 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13698 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
13699 ; GFX11-NEXT: s_mov_b32 s0, 0
13700 ; GFX11-NEXT: .LBB47_1: ; %atomicrmw.start
13701 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
13702 ; GFX11-NEXT: s_waitcnt vmcnt(0)
13703 ; GFX11-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13704 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
13705 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
13706 ; GFX11-NEXT: s_waitcnt vmcnt(0)
13707 ; GFX11-NEXT: buffer_gl1_inv
13708 ; GFX11-NEXT: buffer_gl0_inv
13709 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13710 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
13711 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
13712 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
13713 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
13714 ; GFX11-NEXT: s_cbranch_execnz .LBB47_1
13715 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
13716 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
13717 ; GFX11-NEXT: s_setpc_b64 s[30:31]
13718 ;
13719 ; GFX10-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_neg:
13720 ; GFX10: ; %bb.0:
13721 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13722 ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:-2048
13723 ; GFX10-NEXT: s_mov_b32 s4, 0
13724 ; GFX10-NEXT: .LBB47_1: ; %atomicrmw.start
13725 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
13726 ; GFX10-NEXT: s_waitcnt vmcnt(0)
13727 ; GFX10-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13728 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
13729 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
13730 ; GFX10-NEXT: s_waitcnt vmcnt(0)
13731 ; GFX10-NEXT: buffer_gl1_inv
13732 ; GFX10-NEXT: buffer_gl0_inv
13733 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13734 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
13735 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
13736 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
13737 ; GFX10-NEXT: s_cbranch_execnz .LBB47_1
13738 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
13739 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
13740 ; GFX10-NEXT: s_setpc_b64 s[30:31]
13741 ;
13742 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_neg:
13743 ; GFX90A: ; %bb.0:
13744 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13745 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
13746 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
13747 ; GFX90A-NEXT: .LBB47_1: ; %atomicrmw.start
13748 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
13749 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
13750 ; GFX90A-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
13751 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
13752 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
13753 ; GFX90A-NEXT: buffer_wbinvl1
13754 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
13755 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13756 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
13757 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
13758 ; GFX90A-NEXT: s_cbranch_execnz .LBB47_1
13759 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
13760 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
13761 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
13762 ;
13763 ; GFX908-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_neg:
13764 ; GFX908: ; %bb.0:
13765 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13766 ; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:-2048
13767 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
13768 ; GFX908-NEXT: .LBB47_1: ; %atomicrmw.start
13769 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
13770 ; GFX908-NEXT: s_waitcnt vmcnt(0)
13771 ; GFX908-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13772 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
13773 ; GFX908-NEXT: s_waitcnt vmcnt(0)
13774 ; GFX908-NEXT: buffer_wbinvl1
13775 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
13776 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13777 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
13778 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
13779 ; GFX908-NEXT: s_cbranch_execnz .LBB47_1
13780 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
13781 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
13782 ; GFX908-NEXT: s_setpc_b64 s[30:31]
13783 ;
13784 ; GFX8-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_neg:
13785 ; GFX8: ; %bb.0:
13786 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13787 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
13788 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
13789 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
13790 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
13791 ; GFX8-NEXT: .LBB47_1: ; %atomicrmw.start
13792 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
13793 ; GFX8-NEXT: s_waitcnt vmcnt(0)
13794 ; GFX8-NEXT: v_sub_f16_sdwa v3, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
13795 ; GFX8-NEXT: v_sub_f16_e32 v5, v4, v2
13796 ; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
13797 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
13798 ; GFX8-NEXT: s_waitcnt vmcnt(0)
13799 ; GFX8-NEXT: buffer_wbinvl1
13800 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
13801 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13802 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
13803 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
13804 ; GFX8-NEXT: s_cbranch_execnz .LBB47_1
13805 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
13806 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
13807 ; GFX8-NEXT: s_setpc_b64 s[30:31]
13808 ;
13809 ; GFX7-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_neg:
13810 ; GFX7: ; %bb.0:
13811 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13812 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
13813 ; GFX7-NEXT: s_mov_b32 s5, -1
13814 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
13815 ; GFX7-NEXT: s_mov_b32 s6, 0
13816 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
13817 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
13818 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v2
13819 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
13820 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
13821 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
13822 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
13823 ; GFX7-NEXT: s_mov_b32 s4, s6
13824 ; GFX7-NEXT: s_mov_b32 s5, s6
13825 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13826 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
13827 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
13828 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
13829 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
13830 ; GFX7-NEXT: .LBB47_1: ; %atomicrmw.start
13831 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
13832 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
13833 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
13834 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v5
13835 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v4
13836 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
13837 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
13838 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
13839 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v6
13840 ; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v7
13841 ; GFX7-NEXT: v_or_b32_e32 v6, v4, v5
13842 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v8
13843 ; GFX7-NEXT: v_or_b32_e32 v5, v7, v4
13844 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
13845 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
13846 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
13847 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13848 ; GFX7-NEXT: buffer_wbinvl1
13849 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v7
13850 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v7
13851 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
13852 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
13853 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
13854 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
13855 ; GFX7-NEXT: s_cbranch_execnz .LBB47_1
13856 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
13857 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
13858 ; GFX7-NEXT: s_setpc_b64 s[30:31]
13859 ;
13860 ; GFX6-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_neg:
13861 ; GFX6: ; %bb.0:
13862 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13863 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
13864 ; GFX6-NEXT: s_mov_b32 s5, -1
13865 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
13866 ; GFX6-NEXT: s_mov_b32 s6, 0
13867 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
13868 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
13869 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v2
13870 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
13871 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v3
13872 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
13873 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
13874 ; GFX6-NEXT: s_mov_b32 s4, s6
13875 ; GFX6-NEXT: s_mov_b32 s5, s6
13876 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13877 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
13878 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v5
13879 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v3
13880 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v6
13881 ; GFX6-NEXT: .LBB47_1: ; %atomicrmw.start
13882 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
13883 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
13884 ; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
13885 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v5
13886 ; GFX6-NEXT: s_waitcnt expcnt(0)
13887 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v4
13888 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
13889 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
13890 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
13891 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v6
13892 ; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v7
13893 ; GFX6-NEXT: v_or_b32_e32 v6, v4, v5
13894 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v8
13895 ; GFX6-NEXT: v_or_b32_e32 v5, v7, v4
13896 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
13897 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
13898 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
13899 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13900 ; GFX6-NEXT: buffer_wbinvl1
13901 ; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v7
13902 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v7
13903 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
13904 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
13905 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
13906 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
13907 ; GFX6-NEXT: s_cbranch_execnz .LBB47_1
13908 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
13909 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
13910 ; GFX6-NEXT: s_waitcnt expcnt(0)
13911 ; GFX6-NEXT: s_setpc_b64 s[30:31]
13912 %gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 -512
13913 %unused = atomicrmw fsub ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst
13914 ret void
13915 }
13917 define <2 x half> @global_system_atomic_fsub_ret_v2f16__offset12b_pos(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
13918 ; GFX12-LABEL: global_system_atomic_fsub_ret_v2f16__offset12b_pos:
13919 ; GFX12: ; %bb.0:
13920 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
13921 ; GFX12-NEXT: s_wait_expcnt 0x0
13922 ; GFX12-NEXT: s_wait_samplecnt 0x0
13923 ; GFX12-NEXT: s_wait_bvhcnt 0x0
13924 ; GFX12-NEXT: s_wait_kmcnt 0x0
13925 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
13926 ; GFX12-NEXT: s_mov_b32 s0, 0
13927 ; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start
13928 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
13929 ; GFX12-NEXT: s_wait_loadcnt 0x0
13930 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
13931 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
13932 ; GFX12-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13933 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
13934 ; GFX12-NEXT: s_wait_storecnt 0x0
13935 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
13936 ; GFX12-NEXT: s_wait_loadcnt 0x0
13937 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
13938 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13939 ; GFX12-NEXT: s_wait_alu 0xfffe
13940 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
13941 ; GFX12-NEXT: s_wait_alu 0xfffe
13942 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
13943 ; GFX12-NEXT: s_cbranch_execnz .LBB48_1
13944 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
13945 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
13946 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
13947 ; GFX12-NEXT: s_wait_alu 0xfffe
13948 ; GFX12-NEXT: s_setpc_b64 s[30:31]
13949 ;
13950 ; GFX940-LABEL: global_system_atomic_fsub_ret_v2f16__offset12b_pos:
13951 ; GFX940: ; %bb.0:
13952 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13953 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
13954 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
13955 ; GFX940-NEXT: .LBB48_1: ; %atomicrmw.start
13956 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
13957 ; GFX940-NEXT: s_waitcnt vmcnt(0)
13958 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
13959 ; GFX940-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
13960 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
13961 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
13962 ; GFX940-NEXT: s_waitcnt vmcnt(0)
13963 ; GFX940-NEXT: buffer_inv sc0 sc1
13964 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
13965 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
13966 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
13967 ; GFX940-NEXT: s_cbranch_execnz .LBB48_1
13968 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
13969 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
13970 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
13971 ; GFX940-NEXT: s_setpc_b64 s[30:31]
13972 ;
13973 ; GFX11-LABEL: global_system_atomic_fsub_ret_v2f16__offset12b_pos:
13974 ; GFX11: ; %bb.0:
13975 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13976 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
13977 ; GFX11-NEXT: s_mov_b32 s0, 0
13978 ; GFX11-NEXT: .LBB48_1: ; %atomicrmw.start
13979 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
13980 ; GFX11-NEXT: s_waitcnt vmcnt(0)
13981 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
13982 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
13983 ; GFX11-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13984 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
13985 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
13986 ; GFX11-NEXT: s_waitcnt vmcnt(0)
13987 ; GFX11-NEXT: buffer_gl1_inv
13988 ; GFX11-NEXT: buffer_gl0_inv
13989 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13990 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
13991 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
13992 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
13993 ; GFX11-NEXT: s_cbranch_execnz .LBB48_1
13994 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
13995 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
13996 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
13997 ; GFX11-NEXT: s_setpc_b64 s[30:31]
13998 ;
13999 ; GFX10-LABEL: global_system_atomic_fsub_ret_v2f16__offset12b_pos:
14000 ; GFX10: ; %bb.0:
14001 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14002 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
14003 ; GFX10-NEXT: s_mov_b32 s4, 0
14004 ; GFX10-NEXT: .LBB48_1: ; %atomicrmw.start
14005 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
14006 ; GFX10-NEXT: s_waitcnt vmcnt(0)
14007 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
14008 ; GFX10-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
14009 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
14010 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
14011 ; GFX10-NEXT: s_waitcnt vmcnt(0)
14012 ; GFX10-NEXT: buffer_gl1_inv
14013 ; GFX10-NEXT: buffer_gl0_inv
14014 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
14015 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
14016 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
14017 ; GFX10-NEXT: s_cbranch_execnz .LBB48_1
14018 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
14019 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
14020 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
14021 ; GFX10-NEXT: s_setpc_b64 s[30:31]
14022 ;
14023 ; GFX90A-LABEL: global_system_atomic_fsub_ret_v2f16__offset12b_pos:
14024 ; GFX90A: ; %bb.0:
14025 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14026 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
14027 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
14028 ; GFX90A-NEXT: .LBB48_1: ; %atomicrmw.start
14029 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
14030 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
14031 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
14032 ; GFX90A-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
14033 ; GFX90A-NEXT: buffer_wbl2
14034 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
14035 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
14036 ; GFX90A-NEXT: buffer_invl2
14037 ; GFX90A-NEXT: buffer_wbinvl1
14038 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
14039 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
14040 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
14041 ; GFX90A-NEXT: s_cbranch_execnz .LBB48_1
14042 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
14043 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
14044 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
14045 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
14046 ;
14047 ; GFX908-LABEL: global_system_atomic_fsub_ret_v2f16__offset12b_pos:
14048 ; GFX908: ; %bb.0:
14049 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14050 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
14051 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
14052 ; GFX908-NEXT: .LBB48_1: ; %atomicrmw.start
14053 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
14054 ; GFX908-NEXT: s_waitcnt vmcnt(0)
14055 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
14056 ; GFX908-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
14057 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
14058 ; GFX908-NEXT: s_waitcnt vmcnt(0)
14059 ; GFX908-NEXT: buffer_wbinvl1
14060 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
14061 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
14062 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
14063 ; GFX908-NEXT: s_cbranch_execnz .LBB48_1
14064 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
14065 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
14066 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
14067 ; GFX908-NEXT: s_setpc_b64 s[30:31]
14068 ;
14069 ; GFX8-LABEL: global_system_atomic_fsub_ret_v2f16__offset12b_pos:
14070 ; GFX8: ; %bb.0:
14071 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14072 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
14073 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
14074 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
14075 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
14076 ; GFX8-NEXT: .LBB48_1: ; %atomicrmw.start
14077 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
14078 ; GFX8-NEXT: s_waitcnt vmcnt(0)
14079 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
14080 ; GFX8-NEXT: v_sub_f16_sdwa v0, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
14081 ; GFX8-NEXT: v_sub_f16_e32 v5, v1, v2
14082 ; GFX8-NEXT: v_or_b32_e32 v0, v5, v0
14083 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
14084 ; GFX8-NEXT: s_waitcnt vmcnt(0)
14085 ; GFX8-NEXT: buffer_wbinvl1
14086 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
14087 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
14088 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
14089 ; GFX8-NEXT: s_cbranch_execnz .LBB48_1
14090 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
14091 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
14092 ; GFX8-NEXT: s_setpc_b64 s[30:31]
14093 ;
14094 ; GFX7-LABEL: global_system_atomic_fsub_ret_v2f16__offset12b_pos:
14095 ; GFX7: ; %bb.0:
14096 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14097 ; GFX7-NEXT: s_mov_b32 s6, 0
14098 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
14099 ; GFX7-NEXT: s_mov_b32 s4, s6
14100 ; GFX7-NEXT: s_mov_b32 s5, s6
14101 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
14102 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
14103 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v2
14104 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
14105 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v3
14106 ; GFX7-NEXT: s_waitcnt vmcnt(0)
14107 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
14108 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v5
14109 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
14110 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v6
14111 ; GFX7-NEXT: .LBB48_1: ; %atomicrmw.start
14112 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
14113 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
14114 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
14115 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v3
14116 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
14117 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
14118 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v4
14119 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v5
14120 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6
14121 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v7
14122 ; GFX7-NEXT: v_or_b32_e32 v7, v2, v3
14123 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
14124 ; GFX7-NEXT: v_or_b32_e32 v6, v8, v2
14125 ; GFX7-NEXT: v_mov_b32_e32 v9, v7
14126 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
14127 ; GFX7-NEXT: buffer_atomic_cmpswap v[8:9], v[0:1], s[4:7], 0 addr64 offset:2044 glc
14128 ; GFX7-NEXT: s_waitcnt vmcnt(0)
14129 ; GFX7-NEXT: buffer_wbinvl1
14130 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
14131 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v8
14132 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
14133 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
14134 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
14135 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
14136 ; GFX7-NEXT: s_cbranch_execnz .LBB48_1
14137 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
14138 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
14139 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
14140 ; GFX7-NEXT: v_mov_b32_e32 v1, v3
14141 ; GFX7-NEXT: s_setpc_b64 s[30:31]
14142 ;
14143 ; GFX6-LABEL: global_system_atomic_fsub_ret_v2f16__offset12b_pos:
14144 ; GFX6: ; %bb.0:
14145 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14146 ; GFX6-NEXT: s_mov_b32 s6, 0
14147 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
14148 ; GFX6-NEXT: s_mov_b32 s4, s6
14149 ; GFX6-NEXT: s_mov_b32 s5, s6
14150 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
14151 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
14152 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v2
14153 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
14154 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v3
14155 ; GFX6-NEXT: s_waitcnt vmcnt(0)
14156 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
14157 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v5
14158 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
14159 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v6
14160 ; GFX6-NEXT: .LBB48_1: ; %atomicrmw.start
14161 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
14162 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
14163 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
14164 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v3
14165 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
14166 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
14167 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v4
14168 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v5
14169 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
14170 ; GFX6-NEXT: s_waitcnt expcnt(0)
14171 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v7
14172 ; GFX6-NEXT: v_or_b32_e32 v7, v2, v3
14173 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v6
14174 ; GFX6-NEXT: v_or_b32_e32 v6, v8, v2
14175 ; GFX6-NEXT: v_mov_b32_e32 v9, v7
14176 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
14177 ; GFX6-NEXT: buffer_atomic_cmpswap v[8:9], v[0:1], s[4:7], 0 addr64 offset:2044 glc
14178 ; GFX6-NEXT: s_waitcnt vmcnt(0)
14179 ; GFX6-NEXT: buffer_wbinvl1
14180 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
14181 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v8
14182 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
14183 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
14184 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
14185 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
14186 ; GFX6-NEXT: s_cbranch_execnz .LBB48_1
14187 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
14188 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
14189 ; GFX6-NEXT: v_mov_b32_e32 v0, v2
14190 ; GFX6-NEXT: v_mov_b32_e32 v1, v3
14191 ; GFX6-NEXT: s_waitcnt expcnt(0)
14192 ; GFX6-NEXT: s_setpc_b64 s[30:31]
14193 %gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 511
14194 %result = atomicrmw fsub ptr addrspace(1) %gep, <2 x half> %val seq_cst
14195 ret <2 x half> %result
14196 }
14198 define void @global_system_atomic_fsub_noret_v2f16__offset12b_pos(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
14199 ; GFX12-LABEL: global_system_atomic_fsub_noret_v2f16__offset12b_pos:
14200 ; GFX12: ; %bb.0:
14201 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
14202 ; GFX12-NEXT: s_wait_expcnt 0x0
14203 ; GFX12-NEXT: s_wait_samplecnt 0x0
14204 ; GFX12-NEXT: s_wait_bvhcnt 0x0
14205 ; GFX12-NEXT: s_wait_kmcnt 0x0
14206 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:2044
14207 ; GFX12-NEXT: s_mov_b32 s0, 0
14208 ; GFX12-NEXT: .LBB49_1: ; %atomicrmw.start
14209 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
14210 ; GFX12-NEXT: s_wait_loadcnt 0x0
14211 ; GFX12-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
14212 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
14213 ; GFX12-NEXT: s_wait_storecnt 0x0
14214 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
14215 ; GFX12-NEXT: s_wait_loadcnt 0x0
14216 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
14217 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
14218 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
14219 ; GFX12-NEXT: s_wait_alu 0xfffe
14220 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
14221 ; GFX12-NEXT: s_wait_alu 0xfffe
14222 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
14223 ; GFX12-NEXT: s_cbranch_execnz .LBB49_1
14224 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
14225 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
14226 ; GFX12-NEXT: s_wait_alu 0xfffe
14227 ; GFX12-NEXT: s_setpc_b64 s[30:31]
14228 ;
14229 ; GFX940-LABEL: global_system_atomic_fsub_noret_v2f16__offset12b_pos:
14230 ; GFX940: ; %bb.0:
14231 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14232 ; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:2044
14233 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
14234 ; GFX940-NEXT: .LBB49_1: ; %atomicrmw.start
14235 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
14236 ; GFX940-NEXT: s_waitcnt vmcnt(0)
14237 ; GFX940-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
14238 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
14239 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
14240 ; GFX940-NEXT: s_waitcnt vmcnt(0)
14241 ; GFX940-NEXT: buffer_inv sc0 sc1
14242 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
14243 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
14244 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
14245 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
14246 ; GFX940-NEXT: s_cbranch_execnz .LBB49_1
14247 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
14248 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
14249 ; GFX940-NEXT: s_setpc_b64 s[30:31]
14250 ;
14251 ; GFX11-LABEL: global_system_atomic_fsub_noret_v2f16__offset12b_pos:
14252 ; GFX11: ; %bb.0:
14253 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14254 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:2044
14255 ; GFX11-NEXT: s_mov_b32 s0, 0
14256 ; GFX11-NEXT: .LBB49_1: ; %atomicrmw.start
14257 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
14258 ; GFX11-NEXT: s_waitcnt vmcnt(0)
14259 ; GFX11-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
14260 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
14261 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
14262 ; GFX11-NEXT: s_waitcnt vmcnt(0)
14263 ; GFX11-NEXT: buffer_gl1_inv
14264 ; GFX11-NEXT: buffer_gl0_inv
14265 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
14266 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
14267 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
14268 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
14269 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
14270 ; GFX11-NEXT: s_cbranch_execnz .LBB49_1
14271 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
14272 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
14273 ; GFX11-NEXT: s_setpc_b64 s[30:31]
14274 ;
14275 ; GFX10-LABEL: global_system_atomic_fsub_noret_v2f16__offset12b_pos:
14276 ; GFX10: ; %bb.0:
14277 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14278 ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
14279 ; GFX10-NEXT: s_mov_b32 s4, 0
14280 ; GFX10-NEXT: .LBB49_1: ; %atomicrmw.start
14281 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
14282 ; GFX10-NEXT: s_waitcnt vmcnt(0)
14283 ; GFX10-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
14284 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
14285 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
14286 ; GFX10-NEXT: s_waitcnt vmcnt(0)
14287 ; GFX10-NEXT: buffer_gl1_inv
14288 ; GFX10-NEXT: buffer_gl0_inv
14289 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
14290 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
14291 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
14292 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
14293 ; GFX10-NEXT: s_cbranch_execnz .LBB49_1
14294 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
14295 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
14296 ; GFX10-NEXT: s_setpc_b64 s[30:31]
14297 ;
14298 ; GFX90A-LABEL: global_system_atomic_fsub_noret_v2f16__offset12b_pos:
14299 ; GFX90A: ; %bb.0:
14300 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14301 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
14302 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
14303 ; GFX90A-NEXT: .LBB49_1: ; %atomicrmw.start
14304 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
14305 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
14306 ; GFX90A-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
14307 ; GFX90A-NEXT: buffer_wbl2
14308 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
14309 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
14310 ; GFX90A-NEXT: buffer_invl2
14311 ; GFX90A-NEXT: buffer_wbinvl1
14312 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
14313 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
14314 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
14315 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
14316 ; GFX90A-NEXT: s_cbranch_execnz .LBB49_1
14317 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
14318 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
14319 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
14320 ;
14321 ; GFX908-LABEL: global_system_atomic_fsub_noret_v2f16__offset12b_pos:
14322 ; GFX908: ; %bb.0:
14323 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14324 ; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
14325 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
14326 ; GFX908-NEXT: .LBB49_1: ; %atomicrmw.start
14327 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
14328 ; GFX908-NEXT: s_waitcnt vmcnt(0)
14329 ; GFX908-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
14330 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
14331 ; GFX908-NEXT: s_waitcnt vmcnt(0)
14332 ; GFX908-NEXT: buffer_wbinvl1
14333 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
14334 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
14335 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
14336 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
14337 ; GFX908-NEXT: s_cbranch_execnz .LBB49_1
14338 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
14339 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
14340 ; GFX908-NEXT: s_setpc_b64 s[30:31]
14341 ;
14342 ; GFX8-LABEL: global_system_atomic_fsub_noret_v2f16__offset12b_pos:
14343 ; GFX8: ; %bb.0:
14344 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14345 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
14346 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
14347 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
14348 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
14349 ; GFX8-NEXT: .LBB49_1: ; %atomicrmw.start
14350 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
14351 ; GFX8-NEXT: s_waitcnt vmcnt(0)
14352 ; GFX8-NEXT: v_sub_f16_sdwa v3, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
14353 ; GFX8-NEXT: v_sub_f16_e32 v5, v4, v2
14354 ; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
14355 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
14356 ; GFX8-NEXT: s_waitcnt vmcnt(0)
14357 ; GFX8-NEXT: buffer_wbinvl1
14358 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
14359 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
14360 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
14361 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
14362 ; GFX8-NEXT: s_cbranch_execnz .LBB49_1
14363 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
14364 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
14365 ; GFX8-NEXT: s_setpc_b64 s[30:31]
14366 ;
14367 ; GFX7-LABEL: global_system_atomic_fsub_noret_v2f16__offset12b_pos:
14368 ; GFX7: ; %bb.0:
14369 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14370 ; GFX7-NEXT: s_mov_b32 s6, 0
14371 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
14372 ; GFX7-NEXT: s_mov_b32 s4, s6
14373 ; GFX7-NEXT: s_mov_b32 s5, s6
14374 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
14375 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
14376 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v2
14377 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
14378 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
14379 ; GFX7-NEXT: s_waitcnt vmcnt(0)
14380 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
14381 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
14382 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
14383 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
14384 ; GFX7-NEXT: .LBB49_1: ; %atomicrmw.start
14385 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
14386 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
14387 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
14388 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v5
14389 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v4
14390 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
14391 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
14392 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
14393 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v6
14394 ; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v7
14395 ; GFX7-NEXT: v_or_b32_e32 v6, v4, v5
14396 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v8
14397 ; GFX7-NEXT: v_or_b32_e32 v5, v7, v4
14398 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
14399 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
14400 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 offset:2044 glc
14401 ; GFX7-NEXT: s_waitcnt vmcnt(0)
14402 ; GFX7-NEXT: buffer_wbinvl1
14403 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v7
14404 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v7
14405 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
14406 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
14407 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
14408 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
14409 ; GFX7-NEXT: s_cbranch_execnz .LBB49_1
14410 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
14411 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
14412 ; GFX7-NEXT: s_setpc_b64 s[30:31]
14413 ;
14414 ; GFX6-LABEL: global_system_atomic_fsub_noret_v2f16__offset12b_pos:
14415 ; GFX6: ; %bb.0:
14416 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14417 ; GFX6-NEXT: s_mov_b32 s6, 0
14418 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
14419 ; GFX6-NEXT: s_mov_b32 s4, s6
14420 ; GFX6-NEXT: s_mov_b32 s5, s6
14421 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
14422 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
14423 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v2
14424 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
14425 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v3
14426 ; GFX6-NEXT: s_waitcnt vmcnt(0)
14427 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
14428 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v5
14429 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v3
14430 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v6
14431 ; GFX6-NEXT: .LBB49_1: ; %atomicrmw.start
14432 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
14433 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
14434 ; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
14435 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v5
14436 ; GFX6-NEXT: s_waitcnt expcnt(0)
14437 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v4
14438 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
14439 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
14440 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
14441 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v6
14442 ; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v7
14443 ; GFX6-NEXT: v_or_b32_e32 v6, v4, v5
14444 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v8
14445 ; GFX6-NEXT: v_or_b32_e32 v5, v7, v4
14446 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
14447 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
14448 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 offset:2044 glc
14449 ; GFX6-NEXT: s_waitcnt vmcnt(0)
14450 ; GFX6-NEXT: buffer_wbinvl1
14451 ; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v7
14452 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v7
14453 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
14454 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
14455 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
14456 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
14457 ; GFX6-NEXT: s_cbranch_execnz .LBB49_1
14458 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
14459 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
14460 ; GFX6-NEXT: s_waitcnt expcnt(0)
14461 ; GFX6-NEXT: s_setpc_b64 s[30:31]
14462 %gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 511
14463 %unused = atomicrmw fsub ptr addrspace(1) %gep, <2 x half> %val seq_cst
14464 ret void
14465 }
14467 ; --------------------------------------------------------------------
14468 ; <2 x bfloat>
14469 ; --------------------------------------------------------------------
14471 define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
14472 ; GFX12-LABEL: global_agent_atomic_fsub_ret_v2bf16:
14473 ; GFX12: ; %bb.0:
14474 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
14475 ; GFX12-NEXT: s_wait_expcnt 0x0
14476 ; GFX12-NEXT: s_wait_samplecnt 0x0
14477 ; GFX12-NEXT: s_wait_bvhcnt 0x0
14478 ; GFX12-NEXT: s_wait_kmcnt 0x0
14479 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
14480 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14481 ; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14482 ; GFX12-NEXT: s_mov_b32 s1, 0
14483 ; GFX12-NEXT: .LBB50_1: ; %atomicrmw.start
14484 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
14485 ; GFX12-NEXT: s_wait_loadcnt 0x0
14486 ; GFX12-NEXT: v_mov_b32_e32 v6, v3
14487 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14488 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14489 ; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
14490 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14491 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
14492 ; GFX12-NEXT: v_bfe_u32 v8, v5, 16, 1
14493 ; GFX12-NEXT: v_sub_f32_e32 v3, v3, v4
14494 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v5
14495 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
14496 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14497 ; GFX12-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
14498 ; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
14499 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v3
14500 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v3, v3
14501 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14502 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
14503 ; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
14504 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14505 ; GFX12-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
14506 ; GFX12-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
14507 ; GFX12-NEXT: s_wait_storecnt 0x0
14508 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
14509 ; GFX12-NEXT: s_wait_loadcnt 0x0
14510 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
14511 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
14512 ; GFX12-NEXT: s_wait_alu 0xfffe
14513 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
14514 ; GFX12-NEXT: s_wait_alu 0xfffe
14515 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
14516 ; GFX12-NEXT: s_cbranch_execnz .LBB50_1
14517 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
14518 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
14519 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
14520 ; GFX12-NEXT: s_wait_alu 0xfffe
14521 ; GFX12-NEXT: s_setpc_b64 s[30:31]
14522 ;
14523 ; GFX940-LABEL: global_agent_atomic_fsub_ret_v2bf16:
14524 ; GFX940: ; %bb.0:
14525 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14526 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
14527 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
14528 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14529 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
14530 ; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14531 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
14532 ; GFX940-NEXT: .LBB50_1: ; %atomicrmw.start
14533 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
14534 ; GFX940-NEXT: s_waitcnt vmcnt(0)
14535 ; GFX940-NEXT: v_mov_b32_e32 v7, v3
14536 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 16, v7
14537 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
14538 ; GFX940-NEXT: v_sub_f32_e32 v3, v3, v4
14539 ; GFX940-NEXT: v_sub_f32_e32 v5, v5, v2
14540 ; GFX940-NEXT: v_bfe_u32 v6, v3, 16, 1
14541 ; GFX940-NEXT: v_bfe_u32 v9, v5, 16, 1
14542 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v3
14543 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v5
14544 ; GFX940-NEXT: v_add3_u32 v6, v6, v3, s4
14545 ; GFX940-NEXT: v_add3_u32 v9, v9, v5, s4
14546 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
14547 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
14548 ; GFX940-NEXT: s_nop 0
14549 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
14550 ; GFX940-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[0:1]
14551 ; GFX940-NEXT: v_perm_b32 v6, v5, v3, s5
14552 ; GFX940-NEXT: buffer_wbl2 sc1
14553 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off sc0
14554 ; GFX940-NEXT: s_waitcnt vmcnt(0)
14555 ; GFX940-NEXT: buffer_inv sc1
14556 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
14557 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
14558 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
14559 ; GFX940-NEXT: s_cbranch_execnz .LBB50_1
14560 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
14561 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
14562 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
14563 ; GFX940-NEXT: s_setpc_b64 s[30:31]
14564 ;
14565 ; GFX11-LABEL: global_agent_atomic_fsub_ret_v2bf16:
14566 ; GFX11: ; %bb.0:
14567 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14568 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
14569 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14570 ; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14571 ; GFX11-NEXT: s_mov_b32 s1, 0
14572 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
14573 ; GFX11-NEXT: .p2align 6
14574 ; GFX11-NEXT: .LBB50_1: ; %atomicrmw.start
14575 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
14576 ; GFX11-NEXT: s_waitcnt vmcnt(0)
14577 ; GFX11-NEXT: v_mov_b32_e32 v6, v3
14578 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14579 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14580 ; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
14581 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14582 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
14583 ; GFX11-NEXT: v_bfe_u32 v8, v5, 16, 1
14584 ; GFX11-NEXT: v_sub_f32_e32 v3, v3, v4
14585 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v5
14586 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
14587 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14588 ; GFX11-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
14589 ; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
14590 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v3
14591 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v3, v3
14592 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14593 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
14594 ; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
14595 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14596 ; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
14597 ; GFX11-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
14598 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
14599 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
14600 ; GFX11-NEXT: s_waitcnt vmcnt(0)
14601 ; GFX11-NEXT: buffer_gl1_inv
14602 ; GFX11-NEXT: buffer_gl0_inv
14603 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
14604 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
14605 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
14606 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
14607 ; GFX11-NEXT: s_cbranch_execnz .LBB50_1
14608 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
14609 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
14610 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
14611 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
14612 ; GFX11-NEXT: s_setpc_b64 s[30:31]
14613 ;
14614 ; GFX10-LABEL: global_agent_atomic_fsub_ret_v2bf16:
14615 ; GFX10: ; %bb.0:
14616 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14617 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
14618 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14619 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14620 ; GFX10-NEXT: s_mov_b32 s5, 0
14621 ; GFX10-NEXT: .LBB50_1: ; %atomicrmw.start
14622 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
14623 ; GFX10-NEXT: s_waitcnt vmcnt(0)
14624 ; GFX10-NEXT: v_mov_b32_e32 v6, v3
14625 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14626 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14627 ; GFX10-NEXT: v_sub_f32_e32 v3, v3, v4
14628 ; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
14629 ; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
14630 ; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
14631 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
14632 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
14633 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
14634 ; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
14635 ; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
14636 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
14637 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
14638 ; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
14639 ; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
14640 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
14641 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
14642 ; GFX10-NEXT: s_waitcnt vmcnt(0)
14643 ; GFX10-NEXT: buffer_gl1_inv
14644 ; GFX10-NEXT: buffer_gl0_inv
14645 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
14646 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
14647 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
14648 ; GFX10-NEXT: s_cbranch_execnz .LBB50_1
14649 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
14650 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
14651 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
14652 ; GFX10-NEXT: s_setpc_b64 s[30:31]
14653 ;
14654 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_v2bf16:
14655 ; GFX90A: ; %bb.0:
14656 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14657 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
14658 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
14659 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14660 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
14661 ; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14662 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
14663 ; GFX90A-NEXT: .LBB50_1: ; %atomicrmw.start
14664 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
14665 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
14666 ; GFX90A-NEXT: v_mov_b32_e32 v7, v3
14667 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v7
14668 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
14669 ; GFX90A-NEXT: v_sub_f32_e32 v3, v3, v4
14670 ; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v2
14671 ; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
14672 ; GFX90A-NEXT: v_bfe_u32 v9, v5, 16, 1
14673 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v3
14674 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v5
14675 ; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
14676 ; GFX90A-NEXT: v_add3_u32 v9, v9, v5, s8
14677 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
14678 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
14679 ; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[4:5]
14680 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
14681 ; GFX90A-NEXT: v_perm_b32 v6, v5, v3, s9
14682 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off glc
14683 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
14684 ; GFX90A-NEXT: buffer_wbinvl1
14685 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
14686 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
14687 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
14688 ; GFX90A-NEXT: s_cbranch_execnz .LBB50_1
14689 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
14690 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
14691 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
14692 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
14693 ;
14694 ; GFX908-LABEL: global_agent_atomic_fsub_ret_v2bf16:
14695 ; GFX908: ; %bb.0:
14696 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14697 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
14698 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
14699 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14700 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
14701 ; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14702 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
14703 ; GFX908-NEXT: .LBB50_1: ; %atomicrmw.start
14704 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
14705 ; GFX908-NEXT: s_waitcnt vmcnt(0)
14706 ; GFX908-NEXT: v_mov_b32_e32 v6, v3
14707 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14708 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14709 ; GFX908-NEXT: v_sub_f32_e32 v3, v3, v4
14710 ; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
14711 ; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
14712 ; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
14713 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
14714 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
14715 ; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
14716 ; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
14717 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
14718 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
14719 ; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
14720 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
14721 ; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
14722 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
14723 ; GFX908-NEXT: s_waitcnt vmcnt(0)
14724 ; GFX908-NEXT: buffer_wbinvl1
14725 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
14726 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
14727 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
14728 ; GFX908-NEXT: s_cbranch_execnz .LBB50_1
14729 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
14730 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
14731 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
14732 ; GFX908-NEXT: s_setpc_b64 s[30:31]
14733 ;
14734 ; GFX8-LABEL: global_agent_atomic_fsub_ret_v2bf16:
14735 ; GFX8: ; %bb.0:
14736 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14737 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
14738 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
14739 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14740 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14741 ; GFX8-NEXT: .LBB50_1: ; %atomicrmw.start
14742 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
14743 ; GFX8-NEXT: s_waitcnt vmcnt(0)
14744 ; GFX8-NEXT: v_mov_b32_e32 v6, v3
14745 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14746 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14747 ; GFX8-NEXT: v_sub_f32_e32 v3, v3, v4
14748 ; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
14749 ; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
14750 ; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
14751 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
14752 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
14753 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
14754 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
14755 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
14756 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
14757 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v3
14758 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
14759 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
14760 ; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
14761 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
14762 ; GFX8-NEXT: v_alignbit_b32 v5, v5, v3, 16
14763 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
14764 ; GFX8-NEXT: s_waitcnt vmcnt(0)
14765 ; GFX8-NEXT: buffer_wbinvl1
14766 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
14767 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
14768 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
14769 ; GFX8-NEXT: s_cbranch_execnz .LBB50_1
14770 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
14771 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
14772 ; GFX8-NEXT: v_mov_b32_e32 v0, v3
14773 ; GFX8-NEXT: s_setpc_b64 s[30:31]
14774 ;
14775 ; GFX7-LABEL: global_agent_atomic_fsub_ret_v2bf16:
14776 ; GFX7: ; %bb.0:
14777 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14778 ; GFX7-NEXT: s_mov_b32 s6, 0
14779 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
14780 ; GFX7-NEXT: s_mov_b32 s4, s6
14781 ; GFX7-NEXT: s_mov_b32 s5, s6
14782 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
14783 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
14784 ; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v3
14785 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
14786 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
14787 ; GFX7-NEXT: s_waitcnt vmcnt(0)
14788 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
14789 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
14790 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14791 ; GFX7-NEXT: .LBB50_1: ; %atomicrmw.start
14792 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
14793 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
14794 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
14795 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
14796 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
14797 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
14798 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v5
14799 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v4
14800 ; GFX7-NEXT: v_alignbit_b32 v3, v2, v3, 16
14801 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v7
14802 ; GFX7-NEXT: v_alignbit_b32 v2, v2, v6, 16
14803 ; GFX7-NEXT: v_mov_b32_e32 v7, v3
14804 ; GFX7-NEXT: v_mov_b32_e32 v6, v2
14805 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
14806 ; GFX7-NEXT: s_waitcnt vmcnt(0)
14807 ; GFX7-NEXT: buffer_wbinvl1
14808 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v3
14809 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
14810 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
14811 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14812 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
14813 ; GFX7-NEXT: s_cbranch_execnz .LBB50_1
14814 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
14815 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
14816 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
14817 ; GFX7-NEXT: v_mov_b32_e32 v1, v2
14818 ; GFX7-NEXT: s_setpc_b64 s[30:31]
14819 ;
14820 ; GFX6-LABEL: global_agent_atomic_fsub_ret_v2bf16:
14821 ; GFX6: ; %bb.0:
14822 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14823 ; GFX6-NEXT: s_mov_b32 s6, 0
14824 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
14825 ; GFX6-NEXT: s_mov_b32 s4, s6
14826 ; GFX6-NEXT: s_mov_b32 s5, s6
14827 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
14828 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
14829 ; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v3
14830 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
14831 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
14832 ; GFX6-NEXT: s_waitcnt vmcnt(0)
14833 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
14834 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
14835 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14836 ; GFX6-NEXT: .LBB50_1: ; %atomicrmw.start
14837 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
14838 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
14839 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
14840 ; GFX6-NEXT: s_waitcnt expcnt(0)
14841 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
14842 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
14843 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
14844 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v5
14845 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v4
14846 ; GFX6-NEXT: v_alignbit_b32 v3, v2, v3, 16
14847 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v7
14848 ; GFX6-NEXT: v_alignbit_b32 v2, v2, v6, 16
14849 ; GFX6-NEXT: v_mov_b32_e32 v7, v3
14850 ; GFX6-NEXT: v_mov_b32_e32 v6, v2
14851 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
14852 ; GFX6-NEXT: s_waitcnt vmcnt(0)
14853 ; GFX6-NEXT: buffer_wbinvl1
14854 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v3
14855 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
14856 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
14857 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14858 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
14859 ; GFX6-NEXT: s_cbranch_execnz .LBB50_1
14860 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
14861 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
14862 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
14863 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
14864 ; GFX6-NEXT: s_waitcnt expcnt(0)
14865 ; GFX6-NEXT: s_setpc_b64 s[30:31]
14866 %result = atomicrmw fsub ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst
14867 ret <2 x bfloat> %result
14868 }
14870 define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
14871 ; GFX12-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
14872 ; GFX12: ; %bb.0:
14873 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
14874 ; GFX12-NEXT: s_wait_expcnt 0x0
14875 ; GFX12-NEXT: s_wait_samplecnt 0x0
14876 ; GFX12-NEXT: s_wait_bvhcnt 0x0
14877 ; GFX12-NEXT: s_wait_kmcnt 0x0
14878 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
14879 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14880 ; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14881 ; GFX12-NEXT: s_mov_b32 s1, 0
14882 ; GFX12-NEXT: .LBB51_1: ; %atomicrmw.start
14883 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
14884 ; GFX12-NEXT: s_wait_loadcnt 0x0
14885 ; GFX12-NEXT: v_mov_b32_e32 v6, v3
14886 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14887 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14888 ; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
14889 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14890 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
14891 ; GFX12-NEXT: v_bfe_u32 v8, v5, 16, 1
14892 ; GFX12-NEXT: v_sub_f32_e32 v3, v3, v4
14893 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v5
14894 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
14895 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14896 ; GFX12-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
14897 ; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
14898 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v3
14899 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v3, v3
14900 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14901 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
14902 ; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
14903 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14904 ; GFX12-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
14905 ; GFX12-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
14906 ; GFX12-NEXT: s_wait_storecnt 0x0
14907 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
14908 ; GFX12-NEXT: s_wait_loadcnt 0x0
14909 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
14910 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
14911 ; GFX12-NEXT: s_wait_alu 0xfffe
14912 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
14913 ; GFX12-NEXT: s_wait_alu 0xfffe
14914 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
14915 ; GFX12-NEXT: s_cbranch_execnz .LBB51_1
14916 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
14917 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
14918 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
14919 ; GFX12-NEXT: s_wait_alu 0xfffe
14920 ; GFX12-NEXT: s_setpc_b64 s[30:31]
14921 ;
14922 ; GFX940-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
14923 ; GFX940: ; %bb.0:
14924 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14925 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
14926 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
14927 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14928 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
14929 ; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14930 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
14931 ; GFX940-NEXT: .LBB51_1: ; %atomicrmw.start
14932 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
14933 ; GFX940-NEXT: s_waitcnt vmcnt(0)
14934 ; GFX940-NEXT: v_mov_b32_e32 v7, v3
14935 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 16, v7
14936 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
14937 ; GFX940-NEXT: v_sub_f32_e32 v3, v3, v4
14938 ; GFX940-NEXT: v_sub_f32_e32 v5, v5, v2
14939 ; GFX940-NEXT: v_bfe_u32 v6, v3, 16, 1
14940 ; GFX940-NEXT: v_bfe_u32 v9, v5, 16, 1
14941 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v3
14942 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v5
14943 ; GFX940-NEXT: v_add3_u32 v6, v6, v3, s4
14944 ; GFX940-NEXT: v_add3_u32 v9, v9, v5, s4
14945 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
14946 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
14947 ; GFX940-NEXT: s_nop 0
14948 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
14949 ; GFX940-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[0:1]
14950 ; GFX940-NEXT: v_perm_b32 v6, v5, v3, s5
14951 ; GFX940-NEXT: buffer_wbl2 sc1
14952 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off offset:2044 sc0
14953 ; GFX940-NEXT: s_waitcnt vmcnt(0)
14954 ; GFX940-NEXT: buffer_inv sc1
14955 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
14956 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
14957 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
14958 ; GFX940-NEXT: s_cbranch_execnz .LBB51_1
14959 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
14960 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
14961 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
14962 ; GFX940-NEXT: s_setpc_b64 s[30:31]
14963 ;
14964 ; GFX11-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
14965 ; GFX11: ; %bb.0:
14966 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14967 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
14968 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14969 ; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14970 ; GFX11-NEXT: s_mov_b32 s1, 0
14971 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
14972 ; GFX11-NEXT: .p2align 6
14973 ; GFX11-NEXT: .LBB51_1: ; %atomicrmw.start
14974 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
14975 ; GFX11-NEXT: s_waitcnt vmcnt(0)
14976 ; GFX11-NEXT: v_mov_b32_e32 v6, v3
14977 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14978 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14979 ; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
14980 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14981 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
14982 ; GFX11-NEXT: v_bfe_u32 v8, v5, 16, 1
14983 ; GFX11-NEXT: v_sub_f32_e32 v3, v3, v4
14984 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v5
14985 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
14986 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14987 ; GFX11-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
14988 ; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
14989 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v3
14990 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v3, v3
14991 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14992 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
14993 ; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
14994 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14995 ; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
14996 ; GFX11-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
14997 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
14998 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
14999 ; GFX11-NEXT: s_waitcnt vmcnt(0)
15000 ; GFX11-NEXT: buffer_gl1_inv
15001 ; GFX11-NEXT: buffer_gl0_inv
15002 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
15003 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
15004 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
15005 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
15006 ; GFX11-NEXT: s_cbranch_execnz .LBB51_1
15007 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
15008 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
15009 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
15010 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
15011 ; GFX11-NEXT: s_setpc_b64 s[30:31]
15012 ;
15013 ; GFX10-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
15014 ; GFX10: ; %bb.0:
15015 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15016 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
15017 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15018 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15019 ; GFX10-NEXT: s_mov_b32 s5, 0
15020 ; GFX10-NEXT: .LBB51_1: ; %atomicrmw.start
15021 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
15022 ; GFX10-NEXT: s_waitcnt vmcnt(0)
15023 ; GFX10-NEXT: v_mov_b32_e32 v6, v3
15024 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
15025 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15026 ; GFX10-NEXT: v_sub_f32_e32 v3, v3, v4
15027 ; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
15028 ; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
15029 ; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
15030 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
15031 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
15032 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
15033 ; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
15034 ; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
15035 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
15036 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
15037 ; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
15038 ; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
15039 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
15040 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
15041 ; GFX10-NEXT: s_waitcnt vmcnt(0)
15042 ; GFX10-NEXT: buffer_gl1_inv
15043 ; GFX10-NEXT: buffer_gl0_inv
15044 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
15045 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
15046 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
15047 ; GFX10-NEXT: s_cbranch_execnz .LBB51_1
15048 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
15049 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
15050 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
15051 ; GFX10-NEXT: s_setpc_b64 s[30:31]
15052 ;
15053 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
15054 ; GFX90A: ; %bb.0:
15055 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15056 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
15057 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
15058 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15059 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
15060 ; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15061 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
15062 ; GFX90A-NEXT: .LBB51_1: ; %atomicrmw.start
15063 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
15064 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
15065 ; GFX90A-NEXT: v_mov_b32_e32 v7, v3
15066 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v7
15067 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
15068 ; GFX90A-NEXT: v_sub_f32_e32 v3, v3, v4
15069 ; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v2
15070 ; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
15071 ; GFX90A-NEXT: v_bfe_u32 v9, v5, 16, 1
15072 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v3
15073 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v5
15074 ; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
15075 ; GFX90A-NEXT: v_add3_u32 v9, v9, v5, s8
15076 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
15077 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
15078 ; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[4:5]
15079 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
15080 ; GFX90A-NEXT: v_perm_b32 v6, v5, v3, s9
15081 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off offset:2044 glc
15082 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
15083 ; GFX90A-NEXT: buffer_wbinvl1
15084 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
15085 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15086 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
15087 ; GFX90A-NEXT: s_cbranch_execnz .LBB51_1
15088 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
15089 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
15090 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
15091 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
15092 ;
15093 ; GFX908-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
15094 ; GFX908: ; %bb.0:
15095 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15096 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
15097 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
15098 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15099 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
15100 ; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15101 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
15102 ; GFX908-NEXT: .LBB51_1: ; %atomicrmw.start
15103 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
15104 ; GFX908-NEXT: s_waitcnt vmcnt(0)
15105 ; GFX908-NEXT: v_mov_b32_e32 v6, v3
15106 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
15107 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15108 ; GFX908-NEXT: v_sub_f32_e32 v3, v3, v4
15109 ; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
15110 ; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
15111 ; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
15112 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
15113 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
15114 ; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
15115 ; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
15116 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
15117 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
15118 ; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
15119 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
15120 ; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
15121 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
15122 ; GFX908-NEXT: s_waitcnt vmcnt(0)
15123 ; GFX908-NEXT: buffer_wbinvl1
15124 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
15125 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15126 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
15127 ; GFX908-NEXT: s_cbranch_execnz .LBB51_1
15128 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
15129 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
15130 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
15131 ; GFX908-NEXT: s_setpc_b64 s[30:31]
15132 ;
15133 ; GFX8-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
15134 ; GFX8: ; %bb.0:
15135 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15136 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
15137 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
15138 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
15139 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
15140 ; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
15141 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15142 ; GFX8-NEXT: .LBB51_1: ; %atomicrmw.start
15143 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
15144 ; GFX8-NEXT: s_waitcnt vmcnt(0)
15145 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
15146 ; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
15147 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15148 ; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1
15149 ; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
15150 ; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
15151 ; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
15152 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
15153 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
15154 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
15155 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
15156 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
15157 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
15158 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
15159 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
15160 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
15161 ; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
15162 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
15163 ; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
15164 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
15165 ; GFX8-NEXT: s_waitcnt vmcnt(0)
15166 ; GFX8-NEXT: buffer_wbinvl1
15167 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
15168 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15169 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
15170 ; GFX8-NEXT: s_cbranch_execnz .LBB51_1
15171 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
15172 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
15173 ; GFX8-NEXT: s_setpc_b64 s[30:31]
15174 ;
15175 ; GFX7-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
15176 ; GFX7: ; %bb.0:
15177 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15178 ; GFX7-NEXT: s_mov_b32 s6, 0
15179 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
15180 ; GFX7-NEXT: s_mov_b32 s4, s6
15181 ; GFX7-NEXT: s_mov_b32 s5, s6
15182 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
15183 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
15184 ; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v3
15185 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
15186 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
15187 ; GFX7-NEXT: s_waitcnt vmcnt(0)
15188 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
15189 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
15190 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15191 ; GFX7-NEXT: .LBB51_1: ; %atomicrmw.start
15192 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
15193 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
15194 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
15195 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
15196 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15197 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
15198 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v5
15199 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v4
15200 ; GFX7-NEXT: v_alignbit_b32 v3, v2, v3, 16
15201 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v7
15202 ; GFX7-NEXT: v_alignbit_b32 v2, v2, v6, 16
15203 ; GFX7-NEXT: v_mov_b32_e32 v7, v3
15204 ; GFX7-NEXT: v_mov_b32_e32 v6, v2
15205 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
15206 ; GFX7-NEXT: s_waitcnt vmcnt(0)
15207 ; GFX7-NEXT: buffer_wbinvl1
15208 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v3
15209 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
15210 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
15211 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v6
15212 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
15213 ; GFX7-NEXT: s_cbranch_execnz .LBB51_1
15214 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
15215 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
15216 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
15217 ; GFX7-NEXT: v_mov_b32_e32 v1, v2
15218 ; GFX7-NEXT: s_setpc_b64 s[30:31]
15219 ;
15220 ; GFX6-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
15221 ; GFX6: ; %bb.0:
15222 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15223 ; GFX6-NEXT: s_mov_b32 s6, 0
15224 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
15225 ; GFX6-NEXT: s_mov_b32 s4, s6
15226 ; GFX6-NEXT: s_mov_b32 s5, s6
15227 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
15228 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
15229 ; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v3
15230 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
15231 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
15232 ; GFX6-NEXT: s_waitcnt vmcnt(0)
15233 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
15234 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
15235 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15236 ; GFX6-NEXT: .LBB51_1: ; %atomicrmw.start
15237 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
15238 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
15239 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
15240 ; GFX6-NEXT: s_waitcnt expcnt(0)
15241 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
15242 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15243 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
15244 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v5
15245 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v4
15246 ; GFX6-NEXT: v_alignbit_b32 v3, v2, v3, 16
15247 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v7
15248 ; GFX6-NEXT: v_alignbit_b32 v2, v2, v6, 16
15249 ; GFX6-NEXT: v_mov_b32_e32 v7, v3
15250 ; GFX6-NEXT: v_mov_b32_e32 v6, v2
15251 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
15252 ; GFX6-NEXT: s_waitcnt vmcnt(0)
15253 ; GFX6-NEXT: buffer_wbinvl1
15254 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v3
15255 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
15256 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
15257 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v6
15258 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
15259 ; GFX6-NEXT: s_cbranch_execnz .LBB51_1
15260 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
15261 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
15262 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
15263 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
15264 ; GFX6-NEXT: s_waitcnt expcnt(0)
15265 ; GFX6-NEXT: s_setpc_b64 s[30:31]
15266 %gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
15267 %result = atomicrmw fsub ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
15268 ret <2 x bfloat> %result
15269 }
15271 define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
15272 ; GFX12-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
15273 ; GFX12: ; %bb.0:
15274 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
15275 ; GFX12-NEXT: s_wait_expcnt 0x0
15276 ; GFX12-NEXT: s_wait_samplecnt 0x0
15277 ; GFX12-NEXT: s_wait_bvhcnt 0x0
15278 ; GFX12-NEXT: s_wait_kmcnt 0x0
15279 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
15280 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15281 ; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15282 ; GFX12-NEXT: s_mov_b32 s1, 0
15283 ; GFX12-NEXT: .LBB52_1: ; %atomicrmw.start
15284 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
15285 ; GFX12-NEXT: s_wait_loadcnt 0x0
15286 ; GFX12-NEXT: v_mov_b32_e32 v6, v3
15287 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
15288 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15289 ; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
15290 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v6
15291 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15292 ; GFX12-NEXT: v_bfe_u32 v8, v5, 16, 1
15293 ; GFX12-NEXT: v_sub_f32_e32 v3, v3, v4
15294 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v5
15295 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
15296 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
15297 ; GFX12-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
15298 ; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
15299 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v3
15300 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v3, v3
15301 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
15302 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
15303 ; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
15304 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
15305 ; GFX12-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
15306 ; GFX12-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
15307 ; GFX12-NEXT: s_wait_storecnt 0x0
15308 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
15309 ; GFX12-NEXT: s_wait_loadcnt 0x0
15310 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
15311 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
15312 ; GFX12-NEXT: s_wait_alu 0xfffe
15313 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
15314 ; GFX12-NEXT: s_wait_alu 0xfffe
15315 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
15316 ; GFX12-NEXT: s_cbranch_execnz .LBB52_1
15317 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
15318 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
15319 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
15320 ; GFX12-NEXT: s_wait_alu 0xfffe
15321 ; GFX12-NEXT: s_setpc_b64 s[30:31]
15322 ;
15323 ; GFX940-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
15324 ; GFX940: ; %bb.0:
15325 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15326 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
15327 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
15328 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15329 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
15330 ; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15331 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
15332 ; GFX940-NEXT: .LBB52_1: ; %atomicrmw.start
15333 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
15334 ; GFX940-NEXT: s_waitcnt vmcnt(0)
15335 ; GFX940-NEXT: v_mov_b32_e32 v7, v3
15336 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 16, v7
15337 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
15338 ; GFX940-NEXT: v_sub_f32_e32 v3, v3, v4
15339 ; GFX940-NEXT: v_sub_f32_e32 v5, v5, v2
15340 ; GFX940-NEXT: v_bfe_u32 v6, v3, 16, 1
15341 ; GFX940-NEXT: v_bfe_u32 v9, v5, 16, 1
15342 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v3
15343 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v5
15344 ; GFX940-NEXT: v_add3_u32 v6, v6, v3, s4
15345 ; GFX940-NEXT: v_add3_u32 v9, v9, v5, s4
15346 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
15347 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
15348 ; GFX940-NEXT: s_nop 0
15349 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
15350 ; GFX940-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[0:1]
15351 ; GFX940-NEXT: v_perm_b32 v6, v5, v3, s5
15352 ; GFX940-NEXT: buffer_wbl2 sc1
15353 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off offset:-2048 sc0
15354 ; GFX940-NEXT: s_waitcnt vmcnt(0)
15355 ; GFX940-NEXT: buffer_inv sc1
15356 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
15357 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
15358 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
15359 ; GFX940-NEXT: s_cbranch_execnz .LBB52_1
15360 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
15361 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
15362 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
15363 ; GFX940-NEXT: s_setpc_b64 s[30:31]
15364 ;
15365 ; GFX11-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
15366 ; GFX11: ; %bb.0:
15367 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15368 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
15369 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15370 ; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15371 ; GFX11-NEXT: s_mov_b32 s1, 0
15372 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
15373 ; GFX11-NEXT: .p2align 6
15374 ; GFX11-NEXT: .LBB52_1: ; %atomicrmw.start
15375 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
15376 ; GFX11-NEXT: s_waitcnt vmcnt(0)
15377 ; GFX11-NEXT: v_mov_b32_e32 v6, v3
15378 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
15379 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15380 ; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
15381 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v6
15382 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15383 ; GFX11-NEXT: v_bfe_u32 v8, v5, 16, 1
15384 ; GFX11-NEXT: v_sub_f32_e32 v3, v3, v4
15385 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v5
15386 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
15387 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
15388 ; GFX11-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
15389 ; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
15390 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v3
15391 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v3, v3
15392 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
15393 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
15394 ; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
15395 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
15396 ; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
15397 ; GFX11-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
15398 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
15399 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 glc
15400 ; GFX11-NEXT: s_waitcnt vmcnt(0)
15401 ; GFX11-NEXT: buffer_gl1_inv
15402 ; GFX11-NEXT: buffer_gl0_inv
15403 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
15404 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
15405 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
15406 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
15407 ; GFX11-NEXT: s_cbranch_execnz .LBB52_1
15408 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
15409 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
15410 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
15411 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
15412 ; GFX11-NEXT: s_setpc_b64 s[30:31]
15413 ;
15414 ; GFX10-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
15415 ; GFX10: ; %bb.0:
15416 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15417 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
15418 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15419 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15420 ; GFX10-NEXT: s_mov_b32 s5, 0
15421 ; GFX10-NEXT: .LBB52_1: ; %atomicrmw.start
15422 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
15423 ; GFX10-NEXT: s_waitcnt vmcnt(0)
15424 ; GFX10-NEXT: v_mov_b32_e32 v6, v3
15425 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
15426 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15427 ; GFX10-NEXT: v_sub_f32_e32 v3, v3, v4
15428 ; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
15429 ; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
15430 ; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
15431 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
15432 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
15433 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
15434 ; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
15435 ; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
15436 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
15437 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
15438 ; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
15439 ; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
15440 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
15441 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:-2048 glc
15442 ; GFX10-NEXT: s_waitcnt vmcnt(0)
15443 ; GFX10-NEXT: buffer_gl1_inv
15444 ; GFX10-NEXT: buffer_gl0_inv
15445 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
15446 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
15447 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
15448 ; GFX10-NEXT: s_cbranch_execnz .LBB52_1
15449 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
15450 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
15451 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
15452 ; GFX10-NEXT: s_setpc_b64 s[30:31]
15453 ;
15454 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
15455 ; GFX90A: ; %bb.0:
15456 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15457 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
15458 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
15459 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15460 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
15461 ; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15462 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
15463 ; GFX90A-NEXT: .LBB52_1: ; %atomicrmw.start
15464 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
15465 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
15466 ; GFX90A-NEXT: v_mov_b32_e32 v7, v3
15467 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v7
15468 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
15469 ; GFX90A-NEXT: v_sub_f32_e32 v3, v3, v4
15470 ; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v2
15471 ; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
15472 ; GFX90A-NEXT: v_bfe_u32 v9, v5, 16, 1
15473 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v3
15474 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v5
15475 ; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
15476 ; GFX90A-NEXT: v_add3_u32 v9, v9, v5, s8
15477 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
15478 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
15479 ; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[4:5]
15480 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
15481 ; GFX90A-NEXT: v_perm_b32 v6, v5, v3, s9
15482 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off offset:-2048 glc
15483 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
15484 ; GFX90A-NEXT: buffer_wbinvl1
15485 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
15486 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15487 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
15488 ; GFX90A-NEXT: s_cbranch_execnz .LBB52_1
15489 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
15490 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
15491 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
15492 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
15493 ;
15494 ; GFX908-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
15495 ; GFX908: ; %bb.0:
15496 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15497 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
15498 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
15499 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15500 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
15501 ; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15502 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
15503 ; GFX908-NEXT: .LBB52_1: ; %atomicrmw.start
15504 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
15505 ; GFX908-NEXT: s_waitcnt vmcnt(0)
15506 ; GFX908-NEXT: v_mov_b32_e32 v6, v3
15507 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
15508 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15509 ; GFX908-NEXT: v_sub_f32_e32 v3, v3, v4
15510 ; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
15511 ; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
15512 ; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
15513 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
15514 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
15515 ; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
15516 ; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
15517 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
15518 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
15519 ; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
15520 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
15521 ; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
15522 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:-2048 glc
15523 ; GFX908-NEXT: s_waitcnt vmcnt(0)
15524 ; GFX908-NEXT: buffer_wbinvl1
15525 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
15526 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15527 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
15528 ; GFX908-NEXT: s_cbranch_execnz .LBB52_1
15529 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
15530 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
15531 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
15532 ; GFX908-NEXT: s_setpc_b64 s[30:31]
15533 ;
15534 ; GFX8-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
15535 ; GFX8: ; %bb.0:
15536 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15537 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
15538 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
15539 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
15540 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
15541 ; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
15542 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15543 ; GFX8-NEXT: .LBB52_1: ; %atomicrmw.start
15544 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
15545 ; GFX8-NEXT: s_waitcnt vmcnt(0)
15546 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
15547 ; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
15548 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15549 ; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1
15550 ; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
15551 ; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
15552 ; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
15553 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
15554 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
15555 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
15556 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
15557 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
15558 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
15559 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
15560 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
15561 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
15562 ; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
15563 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
15564 ; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
15565 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
15566 ; GFX8-NEXT: s_waitcnt vmcnt(0)
15567 ; GFX8-NEXT: buffer_wbinvl1
15568 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
15569 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15570 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
15571 ; GFX8-NEXT: s_cbranch_execnz .LBB52_1
15572 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
15573 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
15574 ; GFX8-NEXT: s_setpc_b64 s[30:31]
15575 ;
15576 ; GFX7-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
15577 ; GFX7: ; %bb.0:
15578 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15579 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
15580 ; GFX7-NEXT: s_mov_b32 s5, -1
15581 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
15582 ; GFX7-NEXT: s_mov_b32 s6, 0
15583 ; GFX7-NEXT: buffer_load_dword v6, v[0:1], s[4:7], 0 addr64
15584 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
15585 ; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v2
15586 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
15587 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
15588 ; GFX7-NEXT: s_mov_b32 s4, s6
15589 ; GFX7-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
15590 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
15591 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
15592 ; GFX7-NEXT: s_mov_b32 s5, s6
15593 ; GFX7-NEXT: s_waitcnt vmcnt(0)
15594 ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
15595 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v6
15596 ; GFX7-NEXT: .LBB52_1: ; %atomicrmw.start
15597 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
15598 ; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
15599 ; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
15600 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
15601 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
15602 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
15603 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
15604 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
15605 ; GFX7-NEXT: v_alignbit_b32 v1, v1, v0, 16
15606 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v7
15607 ; GFX7-NEXT: v_alignbit_b32 v0, v0, v6, 16
15608 ; GFX7-NEXT: v_mov_b32_e32 v7, v1
15609 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
15610 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[4:5], s[4:7], 0 addr64 glc
15611 ; GFX7-NEXT: s_waitcnt vmcnt(0)
15612 ; GFX7-NEXT: buffer_wbinvl1
15613 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v1
15614 ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
15615 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
15616 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v6
15617 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
15618 ; GFX7-NEXT: s_cbranch_execnz .LBB52_1
15619 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
15620 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
15621 ; GFX7-NEXT: s_setpc_b64 s[30:31]
15622 ;
15623 ; GFX6-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
15624 ; GFX6: ; %bb.0:
15625 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15626 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
15627 ; GFX6-NEXT: s_mov_b32 s5, -1
15628 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
15629 ; GFX6-NEXT: s_mov_b32 s6, 0
15630 ; GFX6-NEXT: buffer_load_dword v6, v[0:1], s[4:7], 0 addr64
15631 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
15632 ; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v2
15633 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
15634 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
15635 ; GFX6-NEXT: s_mov_b32 s4, s6
15636 ; GFX6-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
15637 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
15638 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
15639 ; GFX6-NEXT: s_mov_b32 s5, s6
15640 ; GFX6-NEXT: s_waitcnt vmcnt(0)
15641 ; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
15642 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v6
15643 ; GFX6-NEXT: .LBB52_1: ; %atomicrmw.start
15644 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
15645 ; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
15646 ; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
15647 ; GFX6-NEXT: s_waitcnt expcnt(0)
15648 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
15649 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
15650 ; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
15651 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
15652 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
15653 ; GFX6-NEXT: v_alignbit_b32 v1, v1, v0, 16
15654 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v7
15655 ; GFX6-NEXT: v_alignbit_b32 v0, v0, v6, 16
15656 ; GFX6-NEXT: v_mov_b32_e32 v7, v1
15657 ; GFX6-NEXT: v_mov_b32_e32 v6, v0
15658 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[4:5], s[4:7], 0 addr64 glc
15659 ; GFX6-NEXT: s_waitcnt vmcnt(0)
15660 ; GFX6-NEXT: buffer_wbinvl1
15661 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v1
15662 ; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
15663 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
15664 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v6
15665 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
15666 ; GFX6-NEXT: s_cbranch_execnz .LBB52_1
15667 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
15668 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
15669 ; GFX6-NEXT: s_waitcnt expcnt(0)
15670 ; GFX6-NEXT: s_setpc_b64 s[30:31]
15671 %gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 -512
15672 %result = atomicrmw fsub ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
15673 ret <2 x bfloat> %result
15674 }
15676 define void @global_agent_atomic_fsub_noret_v2bf16(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
15677 ; GFX12-LABEL: global_agent_atomic_fsub_noret_v2bf16:
15678 ; GFX12: ; %bb.0:
15679 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
15680 ; GFX12-NEXT: s_wait_expcnt 0x0
15681 ; GFX12-NEXT: s_wait_samplecnt 0x0
15682 ; GFX12-NEXT: s_wait_bvhcnt 0x0
15683 ; GFX12-NEXT: s_wait_kmcnt 0x0
15684 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
15685 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15686 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15687 ; GFX12-NEXT: s_mov_b32 s1, 0
15688 ; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start
15689 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
15690 ; GFX12-NEXT: s_wait_loadcnt 0x0
15691 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15692 ; GFX12-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15693 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15694 ; GFX12-NEXT: v_sub_f32_e32 v2, v2, v4
15695 ; GFX12-NEXT: v_sub_f32_e32 v6, v6, v5
15696 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15697 ; GFX12-NEXT: v_bfe_u32 v7, v2, 16, 1
15698 ; GFX12-NEXT: v_bfe_u32 v8, v6, 16, 1
15699 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v2
15700 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v6
15701 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
15702 ; GFX12-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
15703 ; GFX12-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
15704 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v2, v2
15705 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15706 ; GFX12-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
15707 ; GFX12-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
15708 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
15709 ; GFX12-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
15710 ; GFX12-NEXT: s_wait_storecnt 0x0
15711 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
15712 ; GFX12-NEXT: s_wait_loadcnt 0x0
15713 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
15714 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
15715 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
15716 ; GFX12-NEXT: s_wait_alu 0xfffe
15717 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
15718 ; GFX12-NEXT: s_wait_alu 0xfffe
15719 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
15720 ; GFX12-NEXT: s_cbranch_execnz .LBB53_1
15721 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
15722 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
15723 ; GFX12-NEXT: s_wait_alu 0xfffe
15724 ; GFX12-NEXT: s_setpc_b64 s[30:31]
15725 ;
15726 ; GFX940-LABEL: global_agent_atomic_fsub_noret_v2bf16:
15727 ; GFX940: ; %bb.0:
15728 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15729 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
15730 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
15731 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15732 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
15733 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15734 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
15735 ; GFX940-NEXT: .LBB53_1: ; %atomicrmw.start
15736 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
15737 ; GFX940-NEXT: s_waitcnt vmcnt(0)
15738 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15739 ; GFX940-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15740 ; GFX940-NEXT: v_sub_f32_e32 v2, v2, v4
15741 ; GFX940-NEXT: v_sub_f32_e32 v6, v6, v5
15742 ; GFX940-NEXT: v_bfe_u32 v7, v2, 16, 1
15743 ; GFX940-NEXT: v_bfe_u32 v9, v6, 16, 1
15744 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v2
15745 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v6
15746 ; GFX940-NEXT: v_add3_u32 v7, v7, v2, s4
15747 ; GFX940-NEXT: v_add3_u32 v9, v9, v6, s4
15748 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
15749 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
15750 ; GFX940-NEXT: s_nop 0
15751 ; GFX940-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
15752 ; GFX940-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
15753 ; GFX940-NEXT: v_perm_b32 v2, v6, v2, s5
15754 ; GFX940-NEXT: buffer_wbl2 sc1
15755 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
15756 ; GFX940-NEXT: s_waitcnt vmcnt(0)
15757 ; GFX940-NEXT: buffer_inv sc1
15758 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
15759 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
15760 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
15761 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
15762 ; GFX940-NEXT: s_cbranch_execnz .LBB53_1
15763 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
15764 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
15765 ; GFX940-NEXT: s_setpc_b64 s[30:31]
15766 ;
15767 ; GFX11-LABEL: global_agent_atomic_fsub_noret_v2bf16:
15768 ; GFX11: ; %bb.0:
15769 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15770 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
15771 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15772 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15773 ; GFX11-NEXT: s_mov_b32 s1, 0
15774 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
15775 ; GFX11-NEXT: .p2align 6
15776 ; GFX11-NEXT: .LBB53_1: ; %atomicrmw.start
15777 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
15778 ; GFX11-NEXT: s_waitcnt vmcnt(0)
15779 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15780 ; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15781 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15782 ; GFX11-NEXT: v_sub_f32_e32 v2, v2, v4
15783 ; GFX11-NEXT: v_sub_f32_e32 v6, v6, v5
15784 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15785 ; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
15786 ; GFX11-NEXT: v_bfe_u32 v8, v6, 16, 1
15787 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v2
15788 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v6
15789 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
15790 ; GFX11-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
15791 ; GFX11-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
15792 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v2, v2
15793 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15794 ; GFX11-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
15795 ; GFX11-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
15796 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
15797 ; GFX11-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
15798 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
15799 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
15800 ; GFX11-NEXT: s_waitcnt vmcnt(0)
15801 ; GFX11-NEXT: buffer_gl1_inv
15802 ; GFX11-NEXT: buffer_gl0_inv
15803 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
15804 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
15805 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
15806 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
15807 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
15808 ; GFX11-NEXT: s_cbranch_execnz .LBB53_1
15809 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
15810 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
15811 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
15812 ; GFX11-NEXT: s_setpc_b64 s[30:31]
15813 ;
15814 ; GFX10-LABEL: global_agent_atomic_fsub_noret_v2bf16:
15815 ; GFX10: ; %bb.0:
15816 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15817 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
15818 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15819 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15820 ; GFX10-NEXT: s_mov_b32 s5, 0
15821 ; GFX10-NEXT: .LBB53_1: ; %atomicrmw.start
15822 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
15823 ; GFX10-NEXT: s_waitcnt vmcnt(0)
15824 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15825 ; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15826 ; GFX10-NEXT: v_sub_f32_e32 v2, v2, v4
15827 ; GFX10-NEXT: v_sub_f32_e32 v6, v6, v5
15828 ; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
15829 ; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
15830 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
15831 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
15832 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
15833 ; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
15834 ; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
15835 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
15836 ; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
15837 ; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
15838 ; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
15839 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
15840 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
15841 ; GFX10-NEXT: s_waitcnt vmcnt(0)
15842 ; GFX10-NEXT: buffer_gl1_inv
15843 ; GFX10-NEXT: buffer_gl0_inv
15844 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
15845 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
15846 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
15847 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
15848 ; GFX10-NEXT: s_cbranch_execnz .LBB53_1
15849 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
15850 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
15851 ; GFX10-NEXT: s_setpc_b64 s[30:31]
15852 ;
15853 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_v2bf16:
15854 ; GFX90A: ; %bb.0:
15855 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15856 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
15857 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
15858 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15859 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
15860 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15861 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
15862 ; GFX90A-NEXT: .LBB53_1: ; %atomicrmw.start
15863 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
15864 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
15865 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15866 ; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15867 ; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
15868 ; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
15869 ; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
15870 ; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
15871 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
15872 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
15873 ; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
15874 ; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
15875 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
15876 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
15877 ; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
15878 ; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
15879 ; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
15880 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
15881 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
15882 ; GFX90A-NEXT: buffer_wbinvl1
15883 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
15884 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15885 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
15886 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
15887 ; GFX90A-NEXT: s_cbranch_execnz .LBB53_1
15888 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
15889 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
15890 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
15891 ;
15892 ; GFX908-LABEL: global_agent_atomic_fsub_noret_v2bf16:
15893 ; GFX908: ; %bb.0:
15894 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15895 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
15896 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
15897 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15898 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
15899 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15900 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
15901 ; GFX908-NEXT: .LBB53_1: ; %atomicrmw.start
15902 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
15903 ; GFX908-NEXT: s_waitcnt vmcnt(0)
15904 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15905 ; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15906 ; GFX908-NEXT: v_sub_f32_e32 v2, v2, v4
15907 ; GFX908-NEXT: v_sub_f32_e32 v6, v6, v5
15908 ; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
15909 ; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
15910 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
15911 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
15912 ; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
15913 ; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
15914 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
15915 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
15916 ; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
15917 ; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
15918 ; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
15919 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
15920 ; GFX908-NEXT: s_waitcnt vmcnt(0)
15921 ; GFX908-NEXT: buffer_wbinvl1
15922 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
15923 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15924 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
15925 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
15926 ; GFX908-NEXT: s_cbranch_execnz .LBB53_1
15927 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
15928 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
15929 ; GFX908-NEXT: s_setpc_b64 s[30:31]
15930 ;
15931 ; GFX8-LABEL: global_agent_atomic_fsub_noret_v2bf16:
15932 ; GFX8: ; %bb.0:
15933 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15934 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
15935 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
15936 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15937 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15938 ; GFX8-NEXT: .LBB53_1: ; %atomicrmw.start
15939 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
15940 ; GFX8-NEXT: s_waitcnt vmcnt(0)
15941 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15942 ; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15943 ; GFX8-NEXT: v_sub_f32_e32 v2, v2, v4
15944 ; GFX8-NEXT: v_sub_f32_e32 v6, v6, v5
15945 ; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
15946 ; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
15947 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
15948 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
15949 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
15950 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
15951 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
15952 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
15953 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
15954 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
15955 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
15956 ; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
15957 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
15958 ; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
15959 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
15960 ; GFX8-NEXT: s_waitcnt vmcnt(0)
15961 ; GFX8-NEXT: buffer_wbinvl1
15962 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
15963 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15964 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
15965 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
15966 ; GFX8-NEXT: s_cbranch_execnz .LBB53_1
15967 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
15968 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
15969 ; GFX8-NEXT: s_setpc_b64 s[30:31]
15970 ;
15971 ; GFX7-LABEL: global_agent_atomic_fsub_noret_v2bf16:
15972 ; GFX7: ; %bb.0:
15973 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15974 ; GFX7-NEXT: s_mov_b32 s6, 0
15975 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
15976 ; GFX7-NEXT: s_mov_b32 s4, s6
15977 ; GFX7-NEXT: s_mov_b32 s5, s6
15978 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
15979 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
15980 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
15981 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
15982 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15983 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
15984 ; GFX7-NEXT: s_waitcnt vmcnt(0)
15985 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
15986 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
15987 ; GFX7-NEXT: .LBB53_1: ; %atomicrmw.start
15988 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
15989 ; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
15990 ; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
15991 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
15992 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
15993 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
15994 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
15995 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
15996 ; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
15997 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
15998 ; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
15999 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
16000 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
16001 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
16002 ; GFX7-NEXT: s_waitcnt vmcnt(0)
16003 ; GFX7-NEXT: buffer_wbinvl1
16004 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
16005 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
16006 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
16007 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
16008 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
16009 ; GFX7-NEXT: s_cbranch_execnz .LBB53_1
16010 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
16011 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
16012 ; GFX7-NEXT: s_setpc_b64 s[30:31]
16013 ;
16014 ; GFX6-LABEL: global_agent_atomic_fsub_noret_v2bf16:
16015 ; GFX6: ; %bb.0:
16016 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16017 ; GFX6-NEXT: s_mov_b32 s6, 0
16018 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
16019 ; GFX6-NEXT: s_mov_b32 s4, s6
16020 ; GFX6-NEXT: s_mov_b32 s5, s6
16021 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
16022 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
16023 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
16024 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
16025 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16026 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
16027 ; GFX6-NEXT: s_waitcnt vmcnt(0)
16028 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
16029 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
16030 ; GFX6-NEXT: .LBB53_1: ; %atomicrmw.start
16031 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
16032 ; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
16033 ; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
16034 ; GFX6-NEXT: s_waitcnt expcnt(0)
16035 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
16036 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
16037 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
16038 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
16039 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
16040 ; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
16041 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
16042 ; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
16043 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
16044 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
16045 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
16046 ; GFX6-NEXT: s_waitcnt vmcnt(0)
16047 ; GFX6-NEXT: buffer_wbinvl1
16048 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
16049 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
16050 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
16051 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
16052 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
16053 ; GFX6-NEXT: s_cbranch_execnz .LBB53_1
16054 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
16055 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
16056 ; GFX6-NEXT: s_waitcnt expcnt(0)
16057 ; GFX6-NEXT: s_setpc_b64 s[30:31]
16058 %unused = atomicrmw fsub ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst
16059 ret void
16060 }
16062 define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
16063 ; GFX12-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
16064 ; GFX12: ; %bb.0:
16065 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
16066 ; GFX12-NEXT: s_wait_expcnt 0x0
16067 ; GFX12-NEXT: s_wait_samplecnt 0x0
16068 ; GFX12-NEXT: s_wait_bvhcnt 0x0
16069 ; GFX12-NEXT: s_wait_kmcnt 0x0
16070 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
16071 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16072 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16073 ; GFX12-NEXT: s_mov_b32 s1, 0
16074 ; GFX12-NEXT: .LBB54_1: ; %atomicrmw.start
16075 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
16076 ; GFX12-NEXT: s_wait_loadcnt 0x0
16077 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16078 ; GFX12-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16079 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16080 ; GFX12-NEXT: v_sub_f32_e32 v2, v2, v4
16081 ; GFX12-NEXT: v_sub_f32_e32 v6, v6, v5
16082 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16083 ; GFX12-NEXT: v_bfe_u32 v7, v2, 16, 1
16084 ; GFX12-NEXT: v_bfe_u32 v8, v6, 16, 1
16085 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v2
16086 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v6
16087 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
16088 ; GFX12-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
16089 ; GFX12-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
16090 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v2, v2
16091 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16092 ; GFX12-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
16093 ; GFX12-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
16094 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
16095 ; GFX12-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
16096 ; GFX12-NEXT: s_wait_storecnt 0x0
16097 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
16098 ; GFX12-NEXT: s_wait_loadcnt 0x0
16099 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
16100 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
16101 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
16102 ; GFX12-NEXT: s_wait_alu 0xfffe
16103 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
16104 ; GFX12-NEXT: s_wait_alu 0xfffe
16105 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
16106 ; GFX12-NEXT: s_cbranch_execnz .LBB54_1
16107 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
16108 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
16109 ; GFX12-NEXT: s_wait_alu 0xfffe
16110 ; GFX12-NEXT: s_setpc_b64 s[30:31]
16111 ;
16112 ; GFX940-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
16113 ; GFX940: ; %bb.0:
16114 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16115 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
16116 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
16117 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16118 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
16119 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16120 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
16121 ; GFX940-NEXT: .LBB54_1: ; %atomicrmw.start
16122 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
16123 ; GFX940-NEXT: s_waitcnt vmcnt(0)
16124 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16125 ; GFX940-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16126 ; GFX940-NEXT: v_sub_f32_e32 v2, v2, v4
16127 ; GFX940-NEXT: v_sub_f32_e32 v6, v6, v5
16128 ; GFX940-NEXT: v_bfe_u32 v7, v2, 16, 1
16129 ; GFX940-NEXT: v_bfe_u32 v9, v6, 16, 1
16130 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v2
16131 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v6
16132 ; GFX940-NEXT: v_add3_u32 v7, v7, v2, s4
16133 ; GFX940-NEXT: v_add3_u32 v9, v9, v6, s4
16134 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
16135 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
16136 ; GFX940-NEXT: s_nop 0
16137 ; GFX940-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
16138 ; GFX940-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
16139 ; GFX940-NEXT: v_perm_b32 v2, v6, v2, s5
16140 ; GFX940-NEXT: buffer_wbl2 sc1
16141 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
16142 ; GFX940-NEXT: s_waitcnt vmcnt(0)
16143 ; GFX940-NEXT: buffer_inv sc1
16144 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
16145 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
16146 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
16147 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
16148 ; GFX940-NEXT: s_cbranch_execnz .LBB54_1
16149 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
16150 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
16151 ; GFX940-NEXT: s_setpc_b64 s[30:31]
16152 ;
16153 ; GFX11-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
16154 ; GFX11: ; %bb.0:
16155 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16156 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
16157 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16158 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16159 ; GFX11-NEXT: s_mov_b32 s1, 0
16160 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
16161 ; GFX11-NEXT: .p2align 6
16162 ; GFX11-NEXT: .LBB54_1: ; %atomicrmw.start
16163 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
16164 ; GFX11-NEXT: s_waitcnt vmcnt(0)
16165 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16166 ; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16167 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16168 ; GFX11-NEXT: v_sub_f32_e32 v2, v2, v4
16169 ; GFX11-NEXT: v_sub_f32_e32 v6, v6, v5
16170 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16171 ; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
16172 ; GFX11-NEXT: v_bfe_u32 v8, v6, 16, 1
16173 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v2
16174 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v6
16175 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
16176 ; GFX11-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
16177 ; GFX11-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
16178 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v2, v2
16179 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16180 ; GFX11-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
16181 ; GFX11-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
16182 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
16183 ; GFX11-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
16184 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
16185 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
16186 ; GFX11-NEXT: s_waitcnt vmcnt(0)
16187 ; GFX11-NEXT: buffer_gl1_inv
16188 ; GFX11-NEXT: buffer_gl0_inv
16189 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
16190 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
16191 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
16192 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
16193 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
16194 ; GFX11-NEXT: s_cbranch_execnz .LBB54_1
16195 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
16196 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
16197 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
16198 ; GFX11-NEXT: s_setpc_b64 s[30:31]
16199 ;
16200 ; GFX10-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
16201 ; GFX10: ; %bb.0:
16202 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16203 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
16204 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16205 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16206 ; GFX10-NEXT: s_mov_b32 s5, 0
16207 ; GFX10-NEXT: .LBB54_1: ; %atomicrmw.start
16208 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
16209 ; GFX10-NEXT: s_waitcnt vmcnt(0)
16210 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16211 ; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16212 ; GFX10-NEXT: v_sub_f32_e32 v2, v2, v4
16213 ; GFX10-NEXT: v_sub_f32_e32 v6, v6, v5
16214 ; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
16215 ; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
16216 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
16217 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
16218 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
16219 ; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
16220 ; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
16221 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
16222 ; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
16223 ; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
16224 ; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
16225 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
16226 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
16227 ; GFX10-NEXT: s_waitcnt vmcnt(0)
16228 ; GFX10-NEXT: buffer_gl1_inv
16229 ; GFX10-NEXT: buffer_gl0_inv
16230 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
16231 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
16232 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
16233 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
16234 ; GFX10-NEXT: s_cbranch_execnz .LBB54_1
16235 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
16236 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
16237 ; GFX10-NEXT: s_setpc_b64 s[30:31]
16238 ;
16239 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
16240 ; GFX90A: ; %bb.0:
16241 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16242 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
16243 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
16244 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16245 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
16246 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16247 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
16248 ; GFX90A-NEXT: .LBB54_1: ; %atomicrmw.start
16249 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
16250 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
16251 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16252 ; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16253 ; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
16254 ; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
16255 ; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
16256 ; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
16257 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
16258 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
16259 ; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
16260 ; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
16261 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
16262 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
16263 ; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
16264 ; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
16265 ; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
16266 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
16267 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
16268 ; GFX90A-NEXT: buffer_wbinvl1
16269 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
16270 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
16271 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
16272 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
16273 ; GFX90A-NEXT: s_cbranch_execnz .LBB54_1
16274 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
16275 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
16276 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
16277 ;
16278 ; GFX908-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
16279 ; GFX908: ; %bb.0:
16280 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16281 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
16282 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
16283 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16284 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
16285 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16286 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
16287 ; GFX908-NEXT: .LBB54_1: ; %atomicrmw.start
16288 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
16289 ; GFX908-NEXT: s_waitcnt vmcnt(0)
16290 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16291 ; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16292 ; GFX908-NEXT: v_sub_f32_e32 v2, v2, v4
16293 ; GFX908-NEXT: v_sub_f32_e32 v6, v6, v5
16294 ; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
16295 ; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
16296 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
16297 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
16298 ; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
16299 ; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
16300 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
16301 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
16302 ; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
16303 ; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
16304 ; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
16305 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
16306 ; GFX908-NEXT: s_waitcnt vmcnt(0)
16307 ; GFX908-NEXT: buffer_wbinvl1
16308 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
16309 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
16310 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
16311 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
16312 ; GFX908-NEXT: s_cbranch_execnz .LBB54_1
16313 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
16314 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
16315 ; GFX908-NEXT: s_setpc_b64 s[30:31]
16316 ;
16317 ; GFX8-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
16318 ; GFX8: ; %bb.0:
16319 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16320 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
16321 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
16322 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
16323 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
16324 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16325 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16326 ; GFX8-NEXT: .LBB54_1: ; %atomicrmw.start
16327 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
16328 ; GFX8-NEXT: s_waitcnt vmcnt(0)
16329 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16330 ; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16331 ; GFX8-NEXT: v_sub_f32_e32 v2, v2, v4
16332 ; GFX8-NEXT: v_sub_f32_e32 v6, v6, v5
16333 ; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
16334 ; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
16335 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
16336 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
16337 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
16338 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
16339 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
16340 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
16341 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
16342 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
16343 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
16344 ; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
16345 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
16346 ; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
16347 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
16348 ; GFX8-NEXT: s_waitcnt vmcnt(0)
16349 ; GFX8-NEXT: buffer_wbinvl1
16350 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
16351 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
16352 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
16353 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
16354 ; GFX8-NEXT: s_cbranch_execnz .LBB54_1
16355 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
16356 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
16357 ; GFX8-NEXT: s_setpc_b64 s[30:31]
16358 ;
16359 ; GFX7-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
16360 ; GFX7: ; %bb.0:
16361 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16362 ; GFX7-NEXT: s_mov_b32 s6, 0
16363 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
16364 ; GFX7-NEXT: s_mov_b32 s4, s6
16365 ; GFX7-NEXT: s_mov_b32 s5, s6
16366 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
16367 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
16368 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
16369 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
16370 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16371 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
16372 ; GFX7-NEXT: s_waitcnt vmcnt(0)
16373 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
16374 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
16375 ; GFX7-NEXT: .LBB54_1: ; %atomicrmw.start
16376 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
16377 ; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
16378 ; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
16379 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
16380 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
16381 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
16382 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
16383 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
16384 ; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
16385 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
16386 ; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
16387 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
16388 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
16389 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
16390 ; GFX7-NEXT: s_waitcnt vmcnt(0)
16391 ; GFX7-NEXT: buffer_wbinvl1
16392 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
16393 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
16394 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
16395 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
16396 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
16397 ; GFX7-NEXT: s_cbranch_execnz .LBB54_1
16398 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
16399 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
16400 ; GFX7-NEXT: s_setpc_b64 s[30:31]
16401 ;
16402 ; GFX6-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
16403 ; GFX6: ; %bb.0:
16404 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16405 ; GFX6-NEXT: s_mov_b32 s6, 0
16406 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
16407 ; GFX6-NEXT: s_mov_b32 s4, s6
16408 ; GFX6-NEXT: s_mov_b32 s5, s6
16409 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
16410 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
16411 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
16412 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
16413 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16414 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
16415 ; GFX6-NEXT: s_waitcnt vmcnt(0)
16416 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
16417 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
16418 ; GFX6-NEXT: .LBB54_1: ; %atomicrmw.start
16419 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
16420 ; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
16421 ; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
16422 ; GFX6-NEXT: s_waitcnt expcnt(0)
16423 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
16424 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
16425 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
16426 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
16427 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
16428 ; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
16429 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
16430 ; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
16431 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
16432 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
16433 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
16434 ; GFX6-NEXT: s_waitcnt vmcnt(0)
16435 ; GFX6-NEXT: buffer_wbinvl1
16436 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
16437 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
16438 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
16439 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
16440 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
16441 ; GFX6-NEXT: s_cbranch_execnz .LBB54_1
16442 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
16443 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
16444 ; GFX6-NEXT: s_waitcnt expcnt(0)
16445 ; GFX6-NEXT: s_setpc_b64 s[30:31]
16446 %gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
16447 %unused = atomicrmw fsub ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
16448 ret void
16449 }
16451 define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
16452 ; GFX12-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
16453 ; GFX12: ; %bb.0:
16454 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
16455 ; GFX12-NEXT: s_wait_expcnt 0x0
16456 ; GFX12-NEXT: s_wait_samplecnt 0x0
16457 ; GFX12-NEXT: s_wait_bvhcnt 0x0
16458 ; GFX12-NEXT: s_wait_kmcnt 0x0
16459 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
16460 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16461 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16462 ; GFX12-NEXT: s_mov_b32 s1, 0
16463 ; GFX12-NEXT: .LBB55_1: ; %atomicrmw.start
16464 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
16465 ; GFX12-NEXT: s_wait_loadcnt 0x0
16466 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16467 ; GFX12-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16468 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16469 ; GFX12-NEXT: v_sub_f32_e32 v2, v2, v4
16470 ; GFX12-NEXT: v_sub_f32_e32 v6, v6, v5
16471 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16472 ; GFX12-NEXT: v_bfe_u32 v7, v2, 16, 1
16473 ; GFX12-NEXT: v_bfe_u32 v8, v6, 16, 1
16474 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v2
16475 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v6
16476 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
16477 ; GFX12-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
16478 ; GFX12-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
16479 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v2, v2
16480 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16481 ; GFX12-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
16482 ; GFX12-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
16483 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
16484 ; GFX12-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
16485 ; GFX12-NEXT: s_wait_storecnt 0x0
16486 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
16487 ; GFX12-NEXT: s_wait_loadcnt 0x0
16488 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
16489 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
16490 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
16491 ; GFX12-NEXT: s_wait_alu 0xfffe
16492 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
16493 ; GFX12-NEXT: s_wait_alu 0xfffe
16494 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
16495 ; GFX12-NEXT: s_cbranch_execnz .LBB55_1
16496 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
16497 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
16498 ; GFX12-NEXT: s_wait_alu 0xfffe
16499 ; GFX12-NEXT: s_setpc_b64 s[30:31]
16500 ;
16501 ; GFX940-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
16502 ; GFX940: ; %bb.0:
16503 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16504 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
16505 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
16506 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16507 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
16508 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16509 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
16510 ; GFX940-NEXT: .LBB55_1: ; %atomicrmw.start
16511 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
16512 ; GFX940-NEXT: s_waitcnt vmcnt(0)
16513 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16514 ; GFX940-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16515 ; GFX940-NEXT: v_sub_f32_e32 v2, v2, v4
16516 ; GFX940-NEXT: v_sub_f32_e32 v6, v6, v5
16517 ; GFX940-NEXT: v_bfe_u32 v7, v2, 16, 1
16518 ; GFX940-NEXT: v_bfe_u32 v9, v6, 16, 1
16519 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v2
16520 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v6
16521 ; GFX940-NEXT: v_add3_u32 v7, v7, v2, s4
16522 ; GFX940-NEXT: v_add3_u32 v9, v9, v6, s4
16523 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
16524 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
16525 ; GFX940-NEXT: s_nop 0
16526 ; GFX940-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
16527 ; GFX940-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
16528 ; GFX940-NEXT: v_perm_b32 v2, v6, v2, s5
16529 ; GFX940-NEXT: buffer_wbl2 sc1
16530 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
16531 ; GFX940-NEXT: s_waitcnt vmcnt(0)
16532 ; GFX940-NEXT: buffer_inv sc1
16533 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
16534 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
16535 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
16536 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
16537 ; GFX940-NEXT: s_cbranch_execnz .LBB55_1
16538 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
16539 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
16540 ; GFX940-NEXT: s_setpc_b64 s[30:31]
16541 ;
16542 ; GFX11-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
16543 ; GFX11: ; %bb.0:
16544 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16545 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
16546 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16547 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16548 ; GFX11-NEXT: s_mov_b32 s1, 0
16549 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
16550 ; GFX11-NEXT: .p2align 6
16551 ; GFX11-NEXT: .LBB55_1: ; %atomicrmw.start
16552 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
16553 ; GFX11-NEXT: s_waitcnt vmcnt(0)
16554 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16555 ; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16556 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16557 ; GFX11-NEXT: v_sub_f32_e32 v2, v2, v4
16558 ; GFX11-NEXT: v_sub_f32_e32 v6, v6, v5
16559 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16560 ; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
16561 ; GFX11-NEXT: v_bfe_u32 v8, v6, 16, 1
16562 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v2
16563 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v6
16564 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
16565 ; GFX11-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
16566 ; GFX11-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
16567 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v2, v2
16568 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16569 ; GFX11-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
16570 ; GFX11-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
16571 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
16572 ; GFX11-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
16573 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
16574 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 glc
16575 ; GFX11-NEXT: s_waitcnt vmcnt(0)
16576 ; GFX11-NEXT: buffer_gl1_inv
16577 ; GFX11-NEXT: buffer_gl0_inv
16578 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
16579 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
16580 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
16581 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
16582 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
16583 ; GFX11-NEXT: s_cbranch_execnz .LBB55_1
16584 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
16585 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
16586 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
16587 ; GFX11-NEXT: s_setpc_b64 s[30:31]
16588 ;
16589 ; GFX10-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
16590 ; GFX10: ; %bb.0:
16591 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16592 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
16593 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16594 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16595 ; GFX10-NEXT: s_mov_b32 s5, 0
16596 ; GFX10-NEXT: .LBB55_1: ; %atomicrmw.start
16597 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
16598 ; GFX10-NEXT: s_waitcnt vmcnt(0)
16599 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16600 ; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16601 ; GFX10-NEXT: v_sub_f32_e32 v2, v2, v4
16602 ; GFX10-NEXT: v_sub_f32_e32 v6, v6, v5
16603 ; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
16604 ; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
16605 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
16606 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
16607 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
16608 ; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
16609 ; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
16610 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
16611 ; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
16612 ; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
16613 ; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
16614 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
16615 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
16616 ; GFX10-NEXT: s_waitcnt vmcnt(0)
16617 ; GFX10-NEXT: buffer_gl1_inv
16618 ; GFX10-NEXT: buffer_gl0_inv
16619 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
16620 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
16621 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
16622 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
16623 ; GFX10-NEXT: s_cbranch_execnz .LBB55_1
16624 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
16625 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
16626 ; GFX10-NEXT: s_setpc_b64 s[30:31]
16627 ;
16628 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
16629 ; GFX90A: ; %bb.0:
16630 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16631 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
16632 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
16633 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16634 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
16635 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16636 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
16637 ; GFX90A-NEXT: .LBB55_1: ; %atomicrmw.start
16638 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
16639 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
16640 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16641 ; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16642 ; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
16643 ; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
16644 ; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
16645 ; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
16646 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
16647 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
16648 ; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
16649 ; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
16650 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
16651 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
16652 ; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
16653 ; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
16654 ; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
16655 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
16656 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
16657 ; GFX90A-NEXT: buffer_wbinvl1
16658 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
16659 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
16660 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
16661 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
16662 ; GFX90A-NEXT: s_cbranch_execnz .LBB55_1
16663 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
16664 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
16665 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
16666 ;
16667 ; GFX908-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
16668 ; GFX908: ; %bb.0:
16669 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16670 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
16671 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
16672 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16673 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
16674 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16675 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
16676 ; GFX908-NEXT: .LBB55_1: ; %atomicrmw.start
16677 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
16678 ; GFX908-NEXT: s_waitcnt vmcnt(0)
16679 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16680 ; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16681 ; GFX908-NEXT: v_sub_f32_e32 v2, v2, v4
16682 ; GFX908-NEXT: v_sub_f32_e32 v6, v6, v5
16683 ; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
16684 ; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
16685 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
16686 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
16687 ; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
16688 ; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
16689 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
16690 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
16691 ; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
16692 ; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
16693 ; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
16694 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
16695 ; GFX908-NEXT: s_waitcnt vmcnt(0)
16696 ; GFX908-NEXT: buffer_wbinvl1
16697 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
16698 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
16699 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
16700 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
16701 ; GFX908-NEXT: s_cbranch_execnz .LBB55_1
16702 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
16703 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
16704 ; GFX908-NEXT: s_setpc_b64 s[30:31]
16705 ;
16706 ; GFX8-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
16707 ; GFX8: ; %bb.0:
16708 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16709 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
16710 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
16711 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
16712 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
16713 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16714 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16715 ; GFX8-NEXT: .LBB55_1: ; %atomicrmw.start
16716 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
16717 ; GFX8-NEXT: s_waitcnt vmcnt(0)
16718 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16719 ; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16720 ; GFX8-NEXT: v_sub_f32_e32 v2, v2, v4
16721 ; GFX8-NEXT: v_sub_f32_e32 v6, v6, v5
16722 ; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
16723 ; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
16724 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
16725 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
16726 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
16727 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
16728 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
16729 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
16730 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
16731 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
16732 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
16733 ; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
16734 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
16735 ; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
16736 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
16737 ; GFX8-NEXT: s_waitcnt vmcnt(0)
16738 ; GFX8-NEXT: buffer_wbinvl1
16739 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
16740 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
16741 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
16742 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
16743 ; GFX8-NEXT: s_cbranch_execnz .LBB55_1
16744 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
16745 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
16746 ; GFX8-NEXT: s_setpc_b64 s[30:31]
16747 ;
16748 ; GFX7-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
16749 ; GFX7: ; %bb.0:
16750 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16751 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
16752 ; GFX7-NEXT: s_mov_b32 s5, -1
16753 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
16754 ; GFX7-NEXT: s_mov_b32 s6, 0
16755 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
16756 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
16757 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
16758 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
16759 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
16760 ; GFX7-NEXT: s_mov_b32 s4, s6
16761 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
16762 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16763 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
16764 ; GFX7-NEXT: s_mov_b32 s5, s6
16765 ; GFX7-NEXT: s_waitcnt vmcnt(0)
16766 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
16767 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
16768 ; GFX7-NEXT: .LBB55_1: ; %atomicrmw.start
16769 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
16770 ; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
16771 ; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
16772 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
16773 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
16774 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
16775 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
16776 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
16777 ; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
16778 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
16779 ; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
16780 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
16781 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
16782 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
16783 ; GFX7-NEXT: s_waitcnt vmcnt(0)
16784 ; GFX7-NEXT: buffer_wbinvl1
16785 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
16786 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
16787 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
16788 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
16789 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
16790 ; GFX7-NEXT: s_cbranch_execnz .LBB55_1
16791 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
16792 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
16793 ; GFX7-NEXT: s_setpc_b64 s[30:31]
16794 ;
16795 ; GFX6-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
16796 ; GFX6: ; %bb.0:
16797 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16798 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
16799 ; GFX6-NEXT: s_mov_b32 s5, -1
16800 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
16801 ; GFX6-NEXT: s_mov_b32 s6, 0
16802 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
16803 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
16804 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
16805 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
16806 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
16807 ; GFX6-NEXT: s_mov_b32 s4, s6
16808 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
16809 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16810 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
16811 ; GFX6-NEXT: s_mov_b32 s5, s6
16812 ; GFX6-NEXT: s_waitcnt vmcnt(0)
16813 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
16814 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
16815 ; GFX6-NEXT: .LBB55_1: ; %atomicrmw.start
16816 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
16817 ; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
16818 ; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
16819 ; GFX6-NEXT: s_waitcnt expcnt(0)
16820 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
16821 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
16822 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
16823 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
16824 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
16825 ; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
16826 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
16827 ; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
16828 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
16829 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
16830 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
16831 ; GFX6-NEXT: s_waitcnt vmcnt(0)
16832 ; GFX6-NEXT: buffer_wbinvl1
16833 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
16834 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
16835 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
16836 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
16837 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
16838 ; GFX6-NEXT: s_cbranch_execnz .LBB55_1
16839 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
16840 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
16841 ; GFX6-NEXT: s_waitcnt expcnt(0)
16842 ; GFX6-NEXT: s_setpc_b64 s[30:31]
16843 %gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 -512
16844 %unused = atomicrmw fsub ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
16845 ret void
16846 }
16848 define <2 x bfloat> @global_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
16849 ; GFX12-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
16850 ; GFX12: ; %bb.0:
16851 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
16852 ; GFX12-NEXT: s_wait_expcnt 0x0
16853 ; GFX12-NEXT: s_wait_samplecnt 0x0
16854 ; GFX12-NEXT: s_wait_bvhcnt 0x0
16855 ; GFX12-NEXT: s_wait_kmcnt 0x0
16856 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
16857 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16858 ; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16859 ; GFX12-NEXT: s_mov_b32 s1, 0
16860 ; GFX12-NEXT: .LBB56_1: ; %atomicrmw.start
16861 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
16862 ; GFX12-NEXT: s_wait_loadcnt 0x0
16863 ; GFX12-NEXT: v_mov_b32_e32 v6, v3
16864 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
16865 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
16866 ; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
16867 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v6
16868 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16869 ; GFX12-NEXT: v_bfe_u32 v8, v5, 16, 1
16870 ; GFX12-NEXT: v_sub_f32_e32 v3, v3, v4
16871 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v5
16872 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
16873 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
16874 ; GFX12-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
16875 ; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
16876 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v3
16877 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v3, v3
16878 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
16879 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
16880 ; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
16881 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
16882 ; GFX12-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
16883 ; GFX12-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
16884 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
16885 ; GFX12-NEXT: s_wait_storecnt 0x0
16886 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
16887 ; GFX12-NEXT: s_wait_loadcnt 0x0
16888 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
16889 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
16890 ; GFX12-NEXT: s_wait_alu 0xfffe
16891 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
16892 ; GFX12-NEXT: s_wait_alu 0xfffe
16893 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
16894 ; GFX12-NEXT: s_cbranch_execnz .LBB56_1
16895 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
16896 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
16897 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
16898 ; GFX12-NEXT: s_wait_alu 0xfffe
16899 ; GFX12-NEXT: s_setpc_b64 s[30:31]
16900 ;
16901 ; GFX940-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
16902 ; GFX940: ; %bb.0:
16903 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16904 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
16905 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
16906 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16907 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
16908 ; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16909 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
16910 ; GFX940-NEXT: .LBB56_1: ; %atomicrmw.start
16911 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
16912 ; GFX940-NEXT: s_waitcnt vmcnt(0)
16913 ; GFX940-NEXT: v_mov_b32_e32 v7, v3
16914 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 16, v7
16915 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
16916 ; GFX940-NEXT: v_sub_f32_e32 v3, v3, v4
16917 ; GFX940-NEXT: v_sub_f32_e32 v5, v5, v2
16918 ; GFX940-NEXT: v_bfe_u32 v6, v3, 16, 1
16919 ; GFX940-NEXT: v_bfe_u32 v9, v5, 16, 1
16920 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v3
16921 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v5
16922 ; GFX940-NEXT: v_add3_u32 v6, v6, v3, s4
16923 ; GFX940-NEXT: v_add3_u32 v9, v9, v5, s4
16924 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
16925 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
16926 ; GFX940-NEXT: s_nop 0
16927 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
16928 ; GFX940-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[0:1]
16929 ; GFX940-NEXT: v_perm_b32 v6, v5, v3, s5
16930 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
16931 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off offset:2044 sc0 sc1
16932 ; GFX940-NEXT: s_waitcnt vmcnt(0)
16933 ; GFX940-NEXT: buffer_inv sc0 sc1
16934 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
16935 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
16936 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
16937 ; GFX940-NEXT: s_cbranch_execnz .LBB56_1
16938 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
16939 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
16940 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
16941 ; GFX940-NEXT: s_setpc_b64 s[30:31]
16942 ;
16943 ; GFX11-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
16944 ; GFX11: ; %bb.0:
16945 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16946 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
16947 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16948 ; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16949 ; GFX11-NEXT: s_mov_b32 s1, 0
16950 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
16951 ; GFX11-NEXT: .p2align 6
16952 ; GFX11-NEXT: .LBB56_1: ; %atomicrmw.start
16953 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
16954 ; GFX11-NEXT: s_waitcnt vmcnt(0)
16955 ; GFX11-NEXT: v_mov_b32_e32 v6, v3
16956 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
16957 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
16958 ; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
16959 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v6
16960 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16961 ; GFX11-NEXT: v_bfe_u32 v8, v5, 16, 1
16962 ; GFX11-NEXT: v_sub_f32_e32 v3, v3, v4
16963 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v5
16964 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
16965 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
16966 ; GFX11-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
16967 ; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
16968 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v3
16969 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v3, v3
16970 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
16971 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
16972 ; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
16973 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
16974 ; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
16975 ; GFX11-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
16976 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
16977 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
16978 ; GFX11-NEXT: s_waitcnt vmcnt(0)
16979 ; GFX11-NEXT: buffer_gl1_inv
16980 ; GFX11-NEXT: buffer_gl0_inv
16981 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
16982 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
16983 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
16984 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
16985 ; GFX11-NEXT: s_cbranch_execnz .LBB56_1
16986 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
16987 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
16988 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
16989 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
16990 ; GFX11-NEXT: s_setpc_b64 s[30:31]
16991 ;
16992 ; GFX10-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
16993 ; GFX10: ; %bb.0:
16994 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16995 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
16996 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16997 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16998 ; GFX10-NEXT: s_mov_b32 s5, 0
16999 ; GFX10-NEXT: .LBB56_1: ; %atomicrmw.start
17000 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
17001 ; GFX10-NEXT: s_waitcnt vmcnt(0)
17002 ; GFX10-NEXT: v_mov_b32_e32 v6, v3
17003 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
17004 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
17005 ; GFX10-NEXT: v_sub_f32_e32 v3, v3, v4
17006 ; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
17007 ; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
17008 ; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
17009 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
17010 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
17011 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
17012 ; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
17013 ; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
17014 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
17015 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
17016 ; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
17017 ; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
17018 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
17019 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
17020 ; GFX10-NEXT: s_waitcnt vmcnt(0)
17021 ; GFX10-NEXT: buffer_gl1_inv
17022 ; GFX10-NEXT: buffer_gl0_inv
17023 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
17024 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
17025 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
17026 ; GFX10-NEXT: s_cbranch_execnz .LBB56_1
17027 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
17028 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
17029 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
17030 ; GFX10-NEXT: s_setpc_b64 s[30:31]
17031 ;
17032 ; GFX90A-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
17033 ; GFX90A: ; %bb.0:
17034 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17035 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
17036 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
17037 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
17038 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
17039 ; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
17040 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
17041 ; GFX90A-NEXT: .LBB56_1: ; %atomicrmw.start
17042 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
17043 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
17044 ; GFX90A-NEXT: v_mov_b32_e32 v7, v3
17045 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v7
17046 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
17047 ; GFX90A-NEXT: v_sub_f32_e32 v3, v3, v4
17048 ; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v2
17049 ; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
17050 ; GFX90A-NEXT: v_bfe_u32 v9, v5, 16, 1
17051 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v3
17052 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v5
17053 ; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
17054 ; GFX90A-NEXT: v_add3_u32 v9, v9, v5, s8
17055 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
17056 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
17057 ; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[4:5]
17058 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
17059 ; GFX90A-NEXT: v_perm_b32 v6, v5, v3, s9
17060 ; GFX90A-NEXT: buffer_wbl2
17061 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off offset:2044 glc
17062 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
17063 ; GFX90A-NEXT: buffer_invl2
17064 ; GFX90A-NEXT: buffer_wbinvl1
17065 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
17066 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
17067 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
17068 ; GFX90A-NEXT: s_cbranch_execnz .LBB56_1
17069 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
17070 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
17071 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
17072 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
17073 ;
17074 ; GFX908-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
17075 ; GFX908: ; %bb.0:
17076 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17077 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
17078 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
17079 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
17080 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
17081 ; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
17082 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
17083 ; GFX908-NEXT: .LBB56_1: ; %atomicrmw.start
17084 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
17085 ; GFX908-NEXT: s_waitcnt vmcnt(0)
17086 ; GFX908-NEXT: v_mov_b32_e32 v6, v3
17087 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
17088 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
17089 ; GFX908-NEXT: v_sub_f32_e32 v3, v3, v4
17090 ; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
17091 ; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
17092 ; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
17093 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
17094 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
17095 ; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
17096 ; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
17097 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
17098 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
17099 ; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
17100 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
17101 ; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
17102 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
17103 ; GFX908-NEXT: s_waitcnt vmcnt(0)
17104 ; GFX908-NEXT: buffer_wbinvl1
17105 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
17106 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
17107 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
17108 ; GFX908-NEXT: s_cbranch_execnz .LBB56_1
17109 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
17110 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
17111 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
17112 ; GFX908-NEXT: s_setpc_b64 s[30:31]
17113 ;
17114 ; GFX8-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
17115 ; GFX8: ; %bb.0:
17116 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17117 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
17118 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
17119 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
17120 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
17121 ; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
17122 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
17123 ; GFX8-NEXT: .LBB56_1: ; %atomicrmw.start
17124 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
17125 ; GFX8-NEXT: s_waitcnt vmcnt(0)
17126 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
17127 ; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
17128 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
17129 ; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1
17130 ; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
17131 ; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
17132 ; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
17133 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
17134 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
17135 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
17136 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
17137 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
17138 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
17139 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
17140 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
17141 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
17142 ; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
17143 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
17144 ; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
17145 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
17146 ; GFX8-NEXT: s_waitcnt vmcnt(0)
17147 ; GFX8-NEXT: buffer_wbinvl1
17148 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
17149 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
17150 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
17151 ; GFX8-NEXT: s_cbranch_execnz .LBB56_1
17152 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
17153 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
17154 ; GFX8-NEXT: s_setpc_b64 s[30:31]
17155 ;
17156 ; GFX7-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
17157 ; GFX7: ; %bb.0:
17158 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17159 ; GFX7-NEXT: s_mov_b32 s6, 0
17160 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
17161 ; GFX7-NEXT: s_mov_b32 s4, s6
17162 ; GFX7-NEXT: s_mov_b32 s5, s6
17163 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
17164 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
17165 ; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v3
17166 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
17167 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
17168 ; GFX7-NEXT: s_waitcnt vmcnt(0)
17169 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
17170 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
17171 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
17172 ; GFX7-NEXT: .LBB56_1: ; %atomicrmw.start
17173 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
17174 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
17175 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
17176 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
17177 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
17178 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
17179 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v5
17180 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v4
17181 ; GFX7-NEXT: v_alignbit_b32 v3, v2, v3, 16
17182 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v7
17183 ; GFX7-NEXT: v_alignbit_b32 v2, v2, v6, 16
17184 ; GFX7-NEXT: v_mov_b32_e32 v7, v3
17185 ; GFX7-NEXT: v_mov_b32_e32 v6, v2
17186 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
17187 ; GFX7-NEXT: s_waitcnt vmcnt(0)
17188 ; GFX7-NEXT: buffer_wbinvl1
17189 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v3
17190 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
17191 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
17192 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v6
17193 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
17194 ; GFX7-NEXT: s_cbranch_execnz .LBB56_1
17195 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
17196 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
17197 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
17198 ; GFX7-NEXT: v_mov_b32_e32 v1, v2
17199 ; GFX7-NEXT: s_setpc_b64 s[30:31]
17200 ;
17201 ; GFX6-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
17202 ; GFX6: ; %bb.0:
17203 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17204 ; GFX6-NEXT: s_mov_b32 s6, 0
17205 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
17206 ; GFX6-NEXT: s_mov_b32 s4, s6
17207 ; GFX6-NEXT: s_mov_b32 s5, s6
17208 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
17209 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
17210 ; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v3
17211 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
17212 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
17213 ; GFX6-NEXT: s_waitcnt vmcnt(0)
17214 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
17215 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
17216 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
17217 ; GFX6-NEXT: .LBB56_1: ; %atomicrmw.start
17218 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
17219 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
17220 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
17221 ; GFX6-NEXT: s_waitcnt expcnt(0)
17222 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
17223 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
17224 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
17225 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v5
17226 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v4
17227 ; GFX6-NEXT: v_alignbit_b32 v3, v2, v3, 16
17228 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v7
17229 ; GFX6-NEXT: v_alignbit_b32 v2, v2, v6, 16
17230 ; GFX6-NEXT: v_mov_b32_e32 v7, v3
17231 ; GFX6-NEXT: v_mov_b32_e32 v6, v2
17232 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
17233 ; GFX6-NEXT: s_waitcnt vmcnt(0)
17234 ; GFX6-NEXT: buffer_wbinvl1
17235 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v3
17236 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
17237 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
17238 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v6
17239 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
17240 ; GFX6-NEXT: s_cbranch_execnz .LBB56_1
17241 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
17242 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
17243 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
17244 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
17245 ; GFX6-NEXT: s_waitcnt expcnt(0)
17246 ; GFX6-NEXT: s_setpc_b64 s[30:31]
17247 %gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
17248 %result = atomicrmw fsub ptr addrspace(1) %gep, <2 x bfloat> %val seq_cst
17249 ret <2 x bfloat> %result
17250 }
17252 define void @global_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
17253 ; GFX12-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
17254 ; GFX12: ; %bb.0:
17255 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
17256 ; GFX12-NEXT: s_wait_expcnt 0x0
17257 ; GFX12-NEXT: s_wait_samplecnt 0x0
17258 ; GFX12-NEXT: s_wait_bvhcnt 0x0
17259 ; GFX12-NEXT: s_wait_kmcnt 0x0
17260 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
17261 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
17262 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
17263 ; GFX12-NEXT: s_mov_b32 s1, 0
17264 ; GFX12-NEXT: .LBB57_1: ; %atomicrmw.start
17265 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
17266 ; GFX12-NEXT: s_wait_loadcnt 0x0
17267 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
17268 ; GFX12-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
17269 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
17270 ; GFX12-NEXT: v_sub_f32_e32 v2, v2, v4
17271 ; GFX12-NEXT: v_sub_f32_e32 v6, v6, v5
17272 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
17273 ; GFX12-NEXT: v_bfe_u32 v7, v2, 16, 1
17274 ; GFX12-NEXT: v_bfe_u32 v8, v6, 16, 1
17275 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v2
17276 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v6
17277 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
17278 ; GFX12-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
17279 ; GFX12-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
17280 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v2, v2
17281 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
17282 ; GFX12-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
17283 ; GFX12-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
17284 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
17285 ; GFX12-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
17286 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
17287 ; GFX12-NEXT: s_wait_storecnt 0x0
17288 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
17289 ; GFX12-NEXT: s_wait_loadcnt 0x0
17290 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
17291 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
17292 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
17293 ; GFX12-NEXT: s_wait_alu 0xfffe
17294 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
17295 ; GFX12-NEXT: s_wait_alu 0xfffe
17296 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
17297 ; GFX12-NEXT: s_cbranch_execnz .LBB57_1
17298 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
17299 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
17300 ; GFX12-NEXT: s_wait_alu 0xfffe
17301 ; GFX12-NEXT: s_setpc_b64 s[30:31]
17302 ;
17303 ; GFX940-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
17304 ; GFX940: ; %bb.0:
17305 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17306 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
17307 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
17308 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
17309 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
17310 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
17311 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
17312 ; GFX940-NEXT: .LBB57_1: ; %atomicrmw.start
17313 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
17314 ; GFX940-NEXT: s_waitcnt vmcnt(0)
17315 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
17316 ; GFX940-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
17317 ; GFX940-NEXT: v_sub_f32_e32 v2, v2, v4
17318 ; GFX940-NEXT: v_sub_f32_e32 v6, v6, v5
17319 ; GFX940-NEXT: v_bfe_u32 v7, v2, 16, 1
17320 ; GFX940-NEXT: v_bfe_u32 v9, v6, 16, 1
17321 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v2
17322 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v6
17323 ; GFX940-NEXT: v_add3_u32 v7, v7, v2, s4
17324 ; GFX940-NEXT: v_add3_u32 v9, v9, v6, s4
17325 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
17326 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
17327 ; GFX940-NEXT: s_nop 0
17328 ; GFX940-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
17329 ; GFX940-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
17330 ; GFX940-NEXT: v_perm_b32 v2, v6, v2, s5
17331 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
17332 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
17333 ; GFX940-NEXT: s_waitcnt vmcnt(0)
17334 ; GFX940-NEXT: buffer_inv sc0 sc1
17335 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
17336 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
17337 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
17338 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
17339 ; GFX940-NEXT: s_cbranch_execnz .LBB57_1
17340 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
17341 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
17342 ; GFX940-NEXT: s_setpc_b64 s[30:31]
17343 ;
17344 ; GFX11-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
17345 ; GFX11: ; %bb.0:
17346 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17347 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
17348 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
17349 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
17350 ; GFX11-NEXT: s_mov_b32 s1, 0
17351 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
17352 ; GFX11-NEXT: .p2align 6
17353 ; GFX11-NEXT: .LBB57_1: ; %atomicrmw.start
17354 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
17355 ; GFX11-NEXT: s_waitcnt vmcnt(0)
17356 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
17357 ; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
17358 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
17359 ; GFX11-NEXT: v_sub_f32_e32 v2, v2, v4
17360 ; GFX11-NEXT: v_sub_f32_e32 v6, v6, v5
17361 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
17362 ; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
17363 ; GFX11-NEXT: v_bfe_u32 v8, v6, 16, 1
17364 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v2
17365 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v6
17366 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
17367 ; GFX11-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
17368 ; GFX11-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
17369 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v2, v2
17370 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
17371 ; GFX11-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
17372 ; GFX11-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
17373 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
17374 ; GFX11-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
17375 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
17376 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
17377 ; GFX11-NEXT: s_waitcnt vmcnt(0)
17378 ; GFX11-NEXT: buffer_gl1_inv
17379 ; GFX11-NEXT: buffer_gl0_inv
17380 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
17381 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
17382 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
17383 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
17384 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
17385 ; GFX11-NEXT: s_cbranch_execnz .LBB57_1
17386 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
17387 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
17388 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
17389 ; GFX11-NEXT: s_setpc_b64 s[30:31]
17390 ;
17391 ; GFX10-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
17392 ; GFX10: ; %bb.0:
17393 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17394 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
17395 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
17396 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
17397 ; GFX10-NEXT: s_mov_b32 s5, 0
17398 ; GFX10-NEXT: .LBB57_1: ; %atomicrmw.start
17399 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
17400 ; GFX10-NEXT: s_waitcnt vmcnt(0)
17401 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
17402 ; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
17403 ; GFX10-NEXT: v_sub_f32_e32 v2, v2, v4
17404 ; GFX10-NEXT: v_sub_f32_e32 v6, v6, v5
17405 ; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
17406 ; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
17407 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
17408 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
17409 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
17410 ; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
17411 ; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
17412 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
17413 ; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
17414 ; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
17415 ; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
17416 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
17417 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
17418 ; GFX10-NEXT: s_waitcnt vmcnt(0)
17419 ; GFX10-NEXT: buffer_gl1_inv
17420 ; GFX10-NEXT: buffer_gl0_inv
17421 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
17422 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
17423 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
17424 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
17425 ; GFX10-NEXT: s_cbranch_execnz .LBB57_1
17426 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
17427 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
17428 ; GFX10-NEXT: s_setpc_b64 s[30:31]
17429 ;
17430 ; GFX90A-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
17431 ; GFX90A: ; %bb.0:
17432 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17433 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
17434 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
17435 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
17436 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
17437 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
17438 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
17439 ; GFX90A-NEXT: .LBB57_1: ; %atomicrmw.start
17440 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
17441 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
17442 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
17443 ; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
17444 ; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
17445 ; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
17446 ; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
17447 ; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
17448 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
17449 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
17450 ; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
17451 ; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
17452 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
17453 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
17454 ; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
17455 ; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
17456 ; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
17457 ; GFX90A-NEXT: buffer_wbl2
17458 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
17459 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
17460 ; GFX90A-NEXT: buffer_invl2
17461 ; GFX90A-NEXT: buffer_wbinvl1
17462 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
17463 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
17464 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
17465 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
17466 ; GFX90A-NEXT: s_cbranch_execnz .LBB57_1
17467 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
17468 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
17469 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
17470 ;
17471 ; GFX908-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
17472 ; GFX908: ; %bb.0:
17473 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17474 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
17475 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
17476 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
17477 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
17478 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
17479 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
17480 ; GFX908-NEXT: .LBB57_1: ; %atomicrmw.start
17481 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
17482 ; GFX908-NEXT: s_waitcnt vmcnt(0)
17483 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
17484 ; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
17485 ; GFX908-NEXT: v_sub_f32_e32 v2, v2, v4
17486 ; GFX908-NEXT: v_sub_f32_e32 v6, v6, v5
17487 ; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
17488 ; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
17489 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
17490 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
17491 ; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
17492 ; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
17493 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
17494 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
17495 ; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
17496 ; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
17497 ; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
17498 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
17499 ; GFX908-NEXT: s_waitcnt vmcnt(0)
17500 ; GFX908-NEXT: buffer_wbinvl1
17501 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
17502 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
17503 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
17504 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
17505 ; GFX908-NEXT: s_cbranch_execnz .LBB57_1
17506 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
17507 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
17508 ; GFX908-NEXT: s_setpc_b64 s[30:31]
17509 ;
17510 ; GFX8-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
17511 ; GFX8: ; %bb.0:
17512 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17513 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
17514 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
17515 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
17516 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
17517 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
17518 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
17519 ; GFX8-NEXT: .LBB57_1: ; %atomicrmw.start
17520 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
17521 ; GFX8-NEXT: s_waitcnt vmcnt(0)
17522 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
17523 ; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
17524 ; GFX8-NEXT: v_sub_f32_e32 v2, v2, v4
17525 ; GFX8-NEXT: v_sub_f32_e32 v6, v6, v5
17526 ; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
17527 ; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
17528 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
17529 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
17530 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
17531 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
17532 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
17533 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
17534 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
17535 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
17536 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
17537 ; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
17538 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
17539 ; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
17540 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
17541 ; GFX8-NEXT: s_waitcnt vmcnt(0)
17542 ; GFX8-NEXT: buffer_wbinvl1
17543 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
17544 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
17545 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
17546 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
17547 ; GFX8-NEXT: s_cbranch_execnz .LBB57_1
17548 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
17549 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
17550 ; GFX8-NEXT: s_setpc_b64 s[30:31]
17551 ;
17552 ; GFX7-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
17553 ; GFX7: ; %bb.0:
17554 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17555 ; GFX7-NEXT: s_mov_b32 s6, 0
17556 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
17557 ; GFX7-NEXT: s_mov_b32 s4, s6
17558 ; GFX7-NEXT: s_mov_b32 s5, s6
17559 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
17560 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
17561 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
17562 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
17563 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
17564 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
17565 ; GFX7-NEXT: s_waitcnt vmcnt(0)
17566 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
17567 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
17568 ; GFX7-NEXT: .LBB57_1: ; %atomicrmw.start
17569 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
17570 ; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
17571 ; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
17572 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
17573 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
17574 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
17575 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
17576 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
17577 ; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
17578 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
17579 ; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
17580 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
17581 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
17582 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
17583 ; GFX7-NEXT: s_waitcnt vmcnt(0)
17584 ; GFX7-NEXT: buffer_wbinvl1
17585 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
17586 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
17587 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
17588 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
17589 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
17590 ; GFX7-NEXT: s_cbranch_execnz .LBB57_1
17591 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
17592 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
17593 ; GFX7-NEXT: s_setpc_b64 s[30:31]
17594 ;
17595 ; GFX6-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
17596 ; GFX6: ; %bb.0:
17597 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17598 ; GFX6-NEXT: s_mov_b32 s6, 0
17599 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
17600 ; GFX6-NEXT: s_mov_b32 s4, s6
17601 ; GFX6-NEXT: s_mov_b32 s5, s6
17602 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
17603 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
17604 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
17605 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
17606 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
17607 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
17608 ; GFX6-NEXT: s_waitcnt vmcnt(0)
17609 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
17610 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
17611 ; GFX6-NEXT: .LBB57_1: ; %atomicrmw.start
17612 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
17613 ; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
17614 ; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
17615 ; GFX6-NEXT: s_waitcnt expcnt(0)
17616 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
17617 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
17618 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
17619 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
17620 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
17621 ; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
17622 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
17623 ; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
17624 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
17625 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
17626 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
17627 ; GFX6-NEXT: s_waitcnt vmcnt(0)
17628 ; GFX6-NEXT: buffer_wbinvl1
17629 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
17630 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
17631 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
17632 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
17633 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
17634 ; GFX6-NEXT: s_cbranch_execnz .LBB57_1
17635 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
17636 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
17637 ; GFX6-NEXT: s_waitcnt expcnt(0)
17638 ; GFX6-NEXT: s_setpc_b64 s[30:31]
17639 %gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
17640 %unused = atomicrmw fsub ptr addrspace(1) %gep, <2 x bfloat> %val seq_cst
17641 ret void
17642 }
17644 attributes #0 = { nounwind }
17645 attributes #1 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }