| blob | 8e58f309dd9ae9adf7b5832238adc5ad1e938703 |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
2 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
3 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx940 < %s | FileCheck -check-prefix=GFX940 %s
4 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 < %s | FileCheck -check-prefix=GFX11 %s
5 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
6 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx90a < %s | FileCheck -check-prefix=GFX90A %s
7 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx908 < %s | FileCheck -check-prefix=GFX908 %s
8 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=tonga < %s | FileCheck -check-prefix=GFX8 %s
9 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=hawaii < %s | FileCheck -check-prefix=GFX7 %s
10 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=tahiti < %s | FileCheck -check-prefix=GFX6 %s
12 ; --------------------------------------------------------------------
13 ; float
14 ; --------------------------------------------------------------------
16 define float @global_agent_atomic_fsub_ret_f32(ptr addrspace(1) %ptr, float %val) #0 {
17 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f32:
18 ; GFX12: ; %bb.0:
19 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
20 ; GFX12-NEXT: s_wait_expcnt 0x0
21 ; GFX12-NEXT: s_wait_samplecnt 0x0
22 ; GFX12-NEXT: s_wait_bvhcnt 0x0
23 ; GFX12-NEXT: s_wait_kmcnt 0x0
24 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
25 ; GFX12-NEXT: s_mov_b32 s0, 0
26 ; GFX12-NEXT: .LBB0_1: ; %atomicrmw.start
27 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
28 ; GFX12-NEXT: s_wait_loadcnt 0x0
29 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
30 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
31 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
32 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
33 ; GFX12-NEXT: s_wait_storecnt 0x0
34 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
35 ; GFX12-NEXT: s_wait_loadcnt 0x0
36 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
37 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
38 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
39 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
40 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
41 ; GFX12-NEXT: s_cbranch_execnz .LBB0_1
42 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
43 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
44 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
45 ; GFX12-NEXT: s_setpc_b64 s[30:31]
46 ;
47 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f32:
48 ; GFX940: ; %bb.0:
49 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
50 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
51 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
52 ; GFX940-NEXT: .LBB0_1: ; %atomicrmw.start
53 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
54 ; GFX940-NEXT: s_waitcnt vmcnt(0)
55 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
56 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
57 ; GFX940-NEXT: buffer_wbl2 sc1
58 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
59 ; GFX940-NEXT: s_waitcnt vmcnt(0)
60 ; GFX940-NEXT: buffer_inv sc1
61 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
62 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
63 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
64 ; GFX940-NEXT: s_cbranch_execnz .LBB0_1
65 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
66 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
67 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
68 ; GFX940-NEXT: s_setpc_b64 s[30:31]
69 ;
70 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f32:
71 ; GFX11: ; %bb.0:
72 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
73 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
74 ; GFX11-NEXT: s_mov_b32 s0, 0
75 ; GFX11-NEXT: .LBB0_1: ; %atomicrmw.start
76 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
77 ; GFX11-NEXT: s_waitcnt vmcnt(0)
78 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
79 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
80 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
81 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
82 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
83 ; GFX11-NEXT: s_waitcnt vmcnt(0)
84 ; GFX11-NEXT: buffer_gl1_inv
85 ; GFX11-NEXT: buffer_gl0_inv
86 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
87 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
88 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
89 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
90 ; GFX11-NEXT: s_cbranch_execnz .LBB0_1
91 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
92 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
93 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
94 ; GFX11-NEXT: s_setpc_b64 s[30:31]
95 ;
96 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f32:
97 ; GFX10: ; %bb.0:
98 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
99 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
100 ; GFX10-NEXT: s_mov_b32 s4, 0
101 ; GFX10-NEXT: .LBB0_1: ; %atomicrmw.start
102 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
103 ; GFX10-NEXT: s_waitcnt vmcnt(0)
104 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
105 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
106 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
107 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
108 ; GFX10-NEXT: s_waitcnt vmcnt(0)
109 ; GFX10-NEXT: buffer_gl1_inv
110 ; GFX10-NEXT: buffer_gl0_inv
111 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
112 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
113 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
114 ; GFX10-NEXT: s_cbranch_execnz .LBB0_1
115 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
116 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
117 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
118 ; GFX10-NEXT: s_setpc_b64 s[30:31]
119 ;
120 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f32:
121 ; GFX90A: ; %bb.0:
122 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
123 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
124 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
125 ; GFX90A-NEXT: .LBB0_1: ; %atomicrmw.start
126 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
127 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
128 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
129 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
130 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
131 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
132 ; GFX90A-NEXT: buffer_wbinvl1
133 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
134 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
135 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
136 ; GFX90A-NEXT: s_cbranch_execnz .LBB0_1
137 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
138 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
139 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
140 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
141 ;
142 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f32:
143 ; GFX908: ; %bb.0:
144 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
145 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
146 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
147 ; GFX908-NEXT: .LBB0_1: ; %atomicrmw.start
148 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
149 ; GFX908-NEXT: s_waitcnt vmcnt(0)
150 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
151 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
152 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
153 ; GFX908-NEXT: s_waitcnt vmcnt(0)
154 ; GFX908-NEXT: buffer_wbinvl1
155 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
156 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
157 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
158 ; GFX908-NEXT: s_cbranch_execnz .LBB0_1
159 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
160 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
161 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
162 ; GFX908-NEXT: s_setpc_b64 s[30:31]
163 ;
164 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f32:
165 ; GFX8: ; %bb.0:
166 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
167 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
168 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
169 ; GFX8-NEXT: .LBB0_1: ; %atomicrmw.start
170 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
171 ; GFX8-NEXT: s_waitcnt vmcnt(0)
172 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
173 ; GFX8-NEXT: v_sub_f32_e32 v3, v4, v2
174 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
175 ; GFX8-NEXT: s_waitcnt vmcnt(0)
176 ; GFX8-NEXT: buffer_wbinvl1
177 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
178 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
179 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
180 ; GFX8-NEXT: s_cbranch_execnz .LBB0_1
181 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
182 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
183 ; GFX8-NEXT: v_mov_b32_e32 v0, v3
184 ; GFX8-NEXT: s_setpc_b64 s[30:31]
185 ;
186 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f32:
187 ; GFX7: ; %bb.0:
188 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
189 ; GFX7-NEXT: s_mov_b32 s6, 0
190 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
191 ; GFX7-NEXT: s_mov_b32 s4, s6
192 ; GFX7-NEXT: s_mov_b32 s5, s6
193 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
194 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
195 ; GFX7-NEXT: .LBB0_1: ; %atomicrmw.start
196 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
197 ; GFX7-NEXT: s_waitcnt vmcnt(0)
198 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
199 ; GFX7-NEXT: v_sub_f32_e32 v4, v5, v2
200 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
201 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
202 ; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
203 ; GFX7-NEXT: s_waitcnt vmcnt(0)
204 ; GFX7-NEXT: buffer_wbinvl1
205 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
206 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
207 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
208 ; GFX7-NEXT: s_cbranch_execnz .LBB0_1
209 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
210 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
211 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
212 ; GFX7-NEXT: s_setpc_b64 s[30:31]
213 ;
214 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f32:
215 ; GFX6: ; %bb.0:
216 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
217 ; GFX6-NEXT: s_mov_b32 s6, 0
218 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
219 ; GFX6-NEXT: s_mov_b32 s4, s6
220 ; GFX6-NEXT: s_mov_b32 s5, s6
221 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
222 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
223 ; GFX6-NEXT: .LBB0_1: ; %atomicrmw.start
224 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
225 ; GFX6-NEXT: s_waitcnt vmcnt(0)
226 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
227 ; GFX6-NEXT: s_waitcnt expcnt(0)
228 ; GFX6-NEXT: v_sub_f32_e32 v4, v5, v2
229 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
230 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
231 ; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
232 ; GFX6-NEXT: s_waitcnt vmcnt(0)
233 ; GFX6-NEXT: buffer_wbinvl1
234 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
235 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
236 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
237 ; GFX6-NEXT: s_cbranch_execnz .LBB0_1
238 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
239 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
240 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
241 ; GFX6-NEXT: s_waitcnt expcnt(0)
242 ; GFX6-NEXT: s_setpc_b64 s[30:31]
243 %result = atomicrmw fsub ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst
244 ret float %result
245 }
247 define float @global_agent_atomic_fsub_ret_f32__offset12b_pos(ptr addrspace(1) %ptr, float %val) #0 {
248 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos:
249 ; GFX12: ; %bb.0:
250 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
251 ; GFX12-NEXT: s_wait_expcnt 0x0
252 ; GFX12-NEXT: s_wait_samplecnt 0x0
253 ; GFX12-NEXT: s_wait_bvhcnt 0x0
254 ; GFX12-NEXT: s_wait_kmcnt 0x0
255 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
256 ; GFX12-NEXT: s_mov_b32 s0, 0
257 ; GFX12-NEXT: .LBB1_1: ; %atomicrmw.start
258 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
259 ; GFX12-NEXT: s_wait_loadcnt 0x0
260 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
261 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
262 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
263 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
264 ; GFX12-NEXT: s_wait_storecnt 0x0
265 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
266 ; GFX12-NEXT: s_wait_loadcnt 0x0
267 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
268 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
269 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
270 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
271 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
272 ; GFX12-NEXT: s_cbranch_execnz .LBB1_1
273 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
274 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
275 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
276 ; GFX12-NEXT: s_setpc_b64 s[30:31]
277 ;
278 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos:
279 ; GFX940: ; %bb.0:
280 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
281 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
282 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
283 ; GFX940-NEXT: .LBB1_1: ; %atomicrmw.start
284 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
285 ; GFX940-NEXT: s_waitcnt vmcnt(0)
286 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
287 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
288 ; GFX940-NEXT: buffer_wbl2 sc1
289 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
290 ; GFX940-NEXT: s_waitcnt vmcnt(0)
291 ; GFX940-NEXT: buffer_inv sc1
292 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
293 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
294 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
295 ; GFX940-NEXT: s_cbranch_execnz .LBB1_1
296 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
297 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
298 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
299 ; GFX940-NEXT: s_setpc_b64 s[30:31]
300 ;
301 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos:
302 ; GFX11: ; %bb.0:
303 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
304 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
305 ; GFX11-NEXT: s_mov_b32 s0, 0
306 ; GFX11-NEXT: .LBB1_1: ; %atomicrmw.start
307 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
308 ; GFX11-NEXT: s_waitcnt vmcnt(0)
309 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
310 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
311 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
312 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
313 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
314 ; GFX11-NEXT: s_waitcnt vmcnt(0)
315 ; GFX11-NEXT: buffer_gl1_inv
316 ; GFX11-NEXT: buffer_gl0_inv
317 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
318 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
319 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
320 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
321 ; GFX11-NEXT: s_cbranch_execnz .LBB1_1
322 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
323 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
324 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
325 ; GFX11-NEXT: s_setpc_b64 s[30:31]
326 ;
327 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos:
328 ; GFX10: ; %bb.0:
329 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
330 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
331 ; GFX10-NEXT: s_mov_b32 s4, 0
332 ; GFX10-NEXT: .LBB1_1: ; %atomicrmw.start
333 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
334 ; GFX10-NEXT: s_waitcnt vmcnt(0)
335 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
336 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
337 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
338 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
339 ; GFX10-NEXT: s_waitcnt vmcnt(0)
340 ; GFX10-NEXT: buffer_gl1_inv
341 ; GFX10-NEXT: buffer_gl0_inv
342 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
343 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
344 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
345 ; GFX10-NEXT: s_cbranch_execnz .LBB1_1
346 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
347 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
348 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
349 ; GFX10-NEXT: s_setpc_b64 s[30:31]
350 ;
351 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos:
352 ; GFX90A: ; %bb.0:
353 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
354 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
355 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
356 ; GFX90A-NEXT: .LBB1_1: ; %atomicrmw.start
357 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
358 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
359 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
360 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
361 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
362 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
363 ; GFX90A-NEXT: buffer_wbinvl1
364 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
365 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
366 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
367 ; GFX90A-NEXT: s_cbranch_execnz .LBB1_1
368 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
369 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
370 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
371 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
372 ;
373 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos:
374 ; GFX908: ; %bb.0:
375 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
376 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
377 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
378 ; GFX908-NEXT: .LBB1_1: ; %atomicrmw.start
379 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
380 ; GFX908-NEXT: s_waitcnt vmcnt(0)
381 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
382 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
383 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
384 ; GFX908-NEXT: s_waitcnt vmcnt(0)
385 ; GFX908-NEXT: buffer_wbinvl1
386 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
387 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
388 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
389 ; GFX908-NEXT: s_cbranch_execnz .LBB1_1
390 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
391 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
392 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
393 ; GFX908-NEXT: s_setpc_b64 s[30:31]
394 ;
395 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos:
396 ; GFX8: ; %bb.0:
397 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
398 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
399 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
400 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
401 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
402 ; GFX8-NEXT: .LBB1_1: ; %atomicrmw.start
403 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
404 ; GFX8-NEXT: s_waitcnt vmcnt(0)
405 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
406 ; GFX8-NEXT: v_sub_f32_e32 v0, v1, v2
407 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
408 ; GFX8-NEXT: s_waitcnt vmcnt(0)
409 ; GFX8-NEXT: buffer_wbinvl1
410 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
411 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
412 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
413 ; GFX8-NEXT: s_cbranch_execnz .LBB1_1
414 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
415 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
416 ; GFX8-NEXT: s_setpc_b64 s[30:31]
417 ;
418 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos:
419 ; GFX7: ; %bb.0:
420 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
421 ; GFX7-NEXT: s_mov_b32 s6, 0
422 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
423 ; GFX7-NEXT: s_mov_b32 s4, s6
424 ; GFX7-NEXT: s_mov_b32 s5, s6
425 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
426 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
427 ; GFX7-NEXT: .LBB1_1: ; %atomicrmw.start
428 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
429 ; GFX7-NEXT: s_waitcnt vmcnt(0)
430 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
431 ; GFX7-NEXT: v_sub_f32_e32 v4, v5, v2
432 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
433 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
434 ; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
435 ; GFX7-NEXT: s_waitcnt vmcnt(0)
436 ; GFX7-NEXT: buffer_wbinvl1
437 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
438 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
439 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
440 ; GFX7-NEXT: s_cbranch_execnz .LBB1_1
441 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
442 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
443 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
444 ; GFX7-NEXT: s_setpc_b64 s[30:31]
445 ;
446 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos:
447 ; GFX6: ; %bb.0:
448 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
449 ; GFX6-NEXT: s_mov_b32 s6, 0
450 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
451 ; GFX6-NEXT: s_mov_b32 s4, s6
452 ; GFX6-NEXT: s_mov_b32 s5, s6
453 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
454 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
455 ; GFX6-NEXT: .LBB1_1: ; %atomicrmw.start
456 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
457 ; GFX6-NEXT: s_waitcnt vmcnt(0)
458 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
459 ; GFX6-NEXT: s_waitcnt expcnt(0)
460 ; GFX6-NEXT: v_sub_f32_e32 v4, v5, v2
461 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
462 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
463 ; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
464 ; GFX6-NEXT: s_waitcnt vmcnt(0)
465 ; GFX6-NEXT: buffer_wbinvl1
466 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
467 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
468 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
469 ; GFX6-NEXT: s_cbranch_execnz .LBB1_1
470 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
471 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
472 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
473 ; GFX6-NEXT: s_waitcnt expcnt(0)
474 ; GFX6-NEXT: s_setpc_b64 s[30:31]
475 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
476 %result = atomicrmw fsub ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
477 ret float %result
478 }
480 define float @global_agent_atomic_fsub_ret_f32__offset12b_neg(ptr addrspace(1) %ptr, float %val) #0 {
481 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg:
482 ; GFX12: ; %bb.0:
483 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
484 ; GFX12-NEXT: s_wait_expcnt 0x0
485 ; GFX12-NEXT: s_wait_samplecnt 0x0
486 ; GFX12-NEXT: s_wait_bvhcnt 0x0
487 ; GFX12-NEXT: s_wait_kmcnt 0x0
488 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
489 ; GFX12-NEXT: s_mov_b32 s0, 0
490 ; GFX12-NEXT: .LBB2_1: ; %atomicrmw.start
491 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
492 ; GFX12-NEXT: s_wait_loadcnt 0x0
493 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
494 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
495 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
496 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
497 ; GFX12-NEXT: s_wait_storecnt 0x0
498 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
499 ; GFX12-NEXT: s_wait_loadcnt 0x0
500 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
501 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
502 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
503 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
504 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
505 ; GFX12-NEXT: s_cbranch_execnz .LBB2_1
506 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
507 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
508 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
509 ; GFX12-NEXT: s_setpc_b64 s[30:31]
510 ;
511 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg:
512 ; GFX940: ; %bb.0:
513 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
514 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
515 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
516 ; GFX940-NEXT: .LBB2_1: ; %atomicrmw.start
517 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
518 ; GFX940-NEXT: s_waitcnt vmcnt(0)
519 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
520 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
521 ; GFX940-NEXT: buffer_wbl2 sc1
522 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
523 ; GFX940-NEXT: s_waitcnt vmcnt(0)
524 ; GFX940-NEXT: buffer_inv sc1
525 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
526 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
527 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
528 ; GFX940-NEXT: s_cbranch_execnz .LBB2_1
529 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
530 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
531 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
532 ; GFX940-NEXT: s_setpc_b64 s[30:31]
533 ;
534 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg:
535 ; GFX11: ; %bb.0:
536 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
537 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
538 ; GFX11-NEXT: s_mov_b32 s0, 0
539 ; GFX11-NEXT: .LBB2_1: ; %atomicrmw.start
540 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
541 ; GFX11-NEXT: s_waitcnt vmcnt(0)
542 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
543 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
544 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
545 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
546 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
547 ; GFX11-NEXT: s_waitcnt vmcnt(0)
548 ; GFX11-NEXT: buffer_gl1_inv
549 ; GFX11-NEXT: buffer_gl0_inv
550 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
551 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
552 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
553 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
554 ; GFX11-NEXT: s_cbranch_execnz .LBB2_1
555 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
556 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
557 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
558 ; GFX11-NEXT: s_setpc_b64 s[30:31]
559 ;
560 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg:
561 ; GFX10: ; %bb.0:
562 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
563 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
564 ; GFX10-NEXT: s_mov_b32 s4, 0
565 ; GFX10-NEXT: .LBB2_1: ; %atomicrmw.start
566 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
567 ; GFX10-NEXT: s_waitcnt vmcnt(0)
568 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
569 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
570 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
571 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
572 ; GFX10-NEXT: s_waitcnt vmcnt(0)
573 ; GFX10-NEXT: buffer_gl1_inv
574 ; GFX10-NEXT: buffer_gl0_inv
575 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
576 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
577 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
578 ; GFX10-NEXT: s_cbranch_execnz .LBB2_1
579 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
580 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
581 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
582 ; GFX10-NEXT: s_setpc_b64 s[30:31]
583 ;
584 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg:
585 ; GFX90A: ; %bb.0:
586 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
587 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
588 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
589 ; GFX90A-NEXT: .LBB2_1: ; %atomicrmw.start
590 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
591 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
592 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
593 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
594 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
595 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
596 ; GFX90A-NEXT: buffer_wbinvl1
597 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
598 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
599 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
600 ; GFX90A-NEXT: s_cbranch_execnz .LBB2_1
601 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
602 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
603 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
604 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
605 ;
606 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg:
607 ; GFX908: ; %bb.0:
608 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
609 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
610 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
611 ; GFX908-NEXT: .LBB2_1: ; %atomicrmw.start
612 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
613 ; GFX908-NEXT: s_waitcnt vmcnt(0)
614 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
615 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
616 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
617 ; GFX908-NEXT: s_waitcnt vmcnt(0)
618 ; GFX908-NEXT: buffer_wbinvl1
619 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
620 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
621 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
622 ; GFX908-NEXT: s_cbranch_execnz .LBB2_1
623 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
624 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
625 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
626 ; GFX908-NEXT: s_setpc_b64 s[30:31]
627 ;
628 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg:
629 ; GFX8: ; %bb.0:
630 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
631 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
632 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
633 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
634 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
635 ; GFX8-NEXT: .LBB2_1: ; %atomicrmw.start
636 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
637 ; GFX8-NEXT: s_waitcnt vmcnt(0)
638 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
639 ; GFX8-NEXT: v_sub_f32_e32 v0, v1, v2
640 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
641 ; GFX8-NEXT: s_waitcnt vmcnt(0)
642 ; GFX8-NEXT: buffer_wbinvl1
643 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
644 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
645 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
646 ; GFX8-NEXT: s_cbranch_execnz .LBB2_1
647 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
648 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
649 ; GFX8-NEXT: s_setpc_b64 s[30:31]
650 ;
651 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg:
652 ; GFX7: ; %bb.0:
653 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
654 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
655 ; GFX7-NEXT: v_mov_b32_e32 v4, v1
656 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
657 ; GFX7-NEXT: s_mov_b32 s5, -1
658 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
659 ; GFX7-NEXT: s_mov_b32 s6, 0
660 ; GFX7-NEXT: buffer_load_dword v0, v[3:4], s[4:7], 0 addr64
661 ; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0xfffff800, v3
662 ; GFX7-NEXT: v_addc_u32_e32 v4, vcc, -1, v4, vcc
663 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
664 ; GFX7-NEXT: s_mov_b32 s4, s6
665 ; GFX7-NEXT: s_mov_b32 s5, s6
666 ; GFX7-NEXT: .LBB2_1: ; %atomicrmw.start
667 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
668 ; GFX7-NEXT: s_waitcnt vmcnt(0)
669 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
670 ; GFX7-NEXT: v_sub_f32_e32 v5, v6, v2
671 ; GFX7-NEXT: v_mov_b32_e32 v0, v5
672 ; GFX7-NEXT: v_mov_b32_e32 v1, v6
673 ; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v[3:4], s[4:7], 0 addr64 glc
674 ; GFX7-NEXT: s_waitcnt vmcnt(0)
675 ; GFX7-NEXT: buffer_wbinvl1
676 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
677 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
678 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
679 ; GFX7-NEXT: s_cbranch_execnz .LBB2_1
680 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
681 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
682 ; GFX7-NEXT: s_setpc_b64 s[30:31]
683 ;
684 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg:
685 ; GFX6: ; %bb.0:
686 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
687 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
688 ; GFX6-NEXT: v_mov_b32_e32 v4, v1
689 ; GFX6-NEXT: v_mov_b32_e32 v3, v0
690 ; GFX6-NEXT: s_mov_b32 s5, -1
691 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
692 ; GFX6-NEXT: s_mov_b32 s6, 0
693 ; GFX6-NEXT: buffer_load_dword v0, v[3:4], s[4:7], 0 addr64
694 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, 0xfffff800, v3
695 ; GFX6-NEXT: v_addc_u32_e32 v4, vcc, -1, v4, vcc
696 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
697 ; GFX6-NEXT: s_mov_b32 s4, s6
698 ; GFX6-NEXT: s_mov_b32 s5, s6
699 ; GFX6-NEXT: .LBB2_1: ; %atomicrmw.start
700 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
701 ; GFX6-NEXT: s_waitcnt vmcnt(0)
702 ; GFX6-NEXT: v_mov_b32_e32 v6, v0
703 ; GFX6-NEXT: v_sub_f32_e32 v5, v6, v2
704 ; GFX6-NEXT: s_waitcnt expcnt(0)
705 ; GFX6-NEXT: v_mov_b32_e32 v0, v5
706 ; GFX6-NEXT: v_mov_b32_e32 v1, v6
707 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v[3:4], s[4:7], 0 addr64 glc
708 ; GFX6-NEXT: s_waitcnt vmcnt(0)
709 ; GFX6-NEXT: buffer_wbinvl1
710 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
711 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
712 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
713 ; GFX6-NEXT: s_cbranch_execnz .LBB2_1
714 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
715 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
716 ; GFX6-NEXT: s_waitcnt expcnt(0)
717 ; GFX6-NEXT: s_setpc_b64 s[30:31]
718 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
719 %result = atomicrmw fsub ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
720 ret float %result
721 }
723 define void @global_agent_atomic_fsub_noret_f32(ptr addrspace(1) %ptr, float %val) #0 {
724 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f32:
725 ; GFX12: ; %bb.0:
726 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
727 ; GFX12-NEXT: s_wait_expcnt 0x0
728 ; GFX12-NEXT: s_wait_samplecnt 0x0
729 ; GFX12-NEXT: s_wait_bvhcnt 0x0
730 ; GFX12-NEXT: s_wait_kmcnt 0x0
731 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off
732 ; GFX12-NEXT: s_mov_b32 s0, 0
733 ; GFX12-NEXT: .LBB3_1: ; %atomicrmw.start
734 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
735 ; GFX12-NEXT: s_wait_loadcnt 0x0
736 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
737 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
738 ; GFX12-NEXT: s_wait_storecnt 0x0
739 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
740 ; GFX12-NEXT: s_wait_loadcnt 0x0
741 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
742 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
743 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
744 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
745 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
746 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
747 ; GFX12-NEXT: s_cbranch_execnz .LBB3_1
748 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
749 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
750 ; GFX12-NEXT: s_setpc_b64 s[30:31]
751 ;
752 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f32:
753 ; GFX940: ; %bb.0:
754 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
755 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
756 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
757 ; GFX940-NEXT: .LBB3_1: ; %atomicrmw.start
758 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
759 ; GFX940-NEXT: s_waitcnt vmcnt(0)
760 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
761 ; GFX940-NEXT: buffer_wbl2 sc1
762 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
763 ; GFX940-NEXT: s_waitcnt vmcnt(0)
764 ; GFX940-NEXT: buffer_inv sc1
765 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
766 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
767 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
768 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
769 ; GFX940-NEXT: s_cbranch_execnz .LBB3_1
770 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
771 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
772 ; GFX940-NEXT: s_setpc_b64 s[30:31]
773 ;
774 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f32:
775 ; GFX11: ; %bb.0:
776 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
777 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off
778 ; GFX11-NEXT: s_mov_b32 s0, 0
779 ; GFX11-NEXT: .LBB3_1: ; %atomicrmw.start
780 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
781 ; GFX11-NEXT: s_waitcnt vmcnt(0)
782 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
783 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
784 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
785 ; GFX11-NEXT: s_waitcnt vmcnt(0)
786 ; GFX11-NEXT: buffer_gl1_inv
787 ; GFX11-NEXT: buffer_gl0_inv
788 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
789 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
790 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
791 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
792 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
793 ; GFX11-NEXT: s_cbranch_execnz .LBB3_1
794 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
795 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
796 ; GFX11-NEXT: s_setpc_b64 s[30:31]
797 ;
798 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f32:
799 ; GFX10: ; %bb.0:
800 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
801 ; GFX10-NEXT: global_load_dword v4, v[0:1], off
802 ; GFX10-NEXT: s_mov_b32 s4, 0
803 ; GFX10-NEXT: .LBB3_1: ; %atomicrmw.start
804 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
805 ; GFX10-NEXT: s_waitcnt vmcnt(0)
806 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
807 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
808 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
809 ; GFX10-NEXT: s_waitcnt vmcnt(0)
810 ; GFX10-NEXT: buffer_gl1_inv
811 ; GFX10-NEXT: buffer_gl0_inv
812 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
813 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
814 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
815 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
816 ; GFX10-NEXT: s_cbranch_execnz .LBB3_1
817 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
818 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
819 ; GFX10-NEXT: s_setpc_b64 s[30:31]
820 ;
821 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f32:
822 ; GFX90A: ; %bb.0:
823 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
824 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
825 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
826 ; GFX90A-NEXT: .LBB3_1: ; %atomicrmw.start
827 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
828 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
829 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
830 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
831 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
832 ; GFX90A-NEXT: buffer_wbinvl1
833 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
834 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
835 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
836 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
837 ; GFX90A-NEXT: s_cbranch_execnz .LBB3_1
838 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
839 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
840 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
841 ;
842 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f32:
843 ; GFX908: ; %bb.0:
844 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
845 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
846 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
847 ; GFX908-NEXT: .LBB3_1: ; %atomicrmw.start
848 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
849 ; GFX908-NEXT: s_waitcnt vmcnt(0)
850 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
851 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
852 ; GFX908-NEXT: s_waitcnt vmcnt(0)
853 ; GFX908-NEXT: buffer_wbinvl1
854 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
855 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
856 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
857 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
858 ; GFX908-NEXT: s_cbranch_execnz .LBB3_1
859 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
860 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
861 ; GFX908-NEXT: s_setpc_b64 s[30:31]
862 ;
863 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f32:
864 ; GFX8: ; %bb.0:
865 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
866 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
867 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
868 ; GFX8-NEXT: .LBB3_1: ; %atomicrmw.start
869 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
870 ; GFX8-NEXT: s_waitcnt vmcnt(0)
871 ; GFX8-NEXT: v_sub_f32_e32 v3, v4, v2
872 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
873 ; GFX8-NEXT: s_waitcnt vmcnt(0)
874 ; GFX8-NEXT: buffer_wbinvl1
875 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
876 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
877 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
878 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
879 ; GFX8-NEXT: s_cbranch_execnz .LBB3_1
880 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
881 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
882 ; GFX8-NEXT: s_setpc_b64 s[30:31]
883 ;
884 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f32:
885 ; GFX7: ; %bb.0:
886 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
887 ; GFX7-NEXT: s_mov_b32 s6, 0
888 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
889 ; GFX7-NEXT: s_mov_b32 s4, s6
890 ; GFX7-NEXT: s_mov_b32 s5, s6
891 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
892 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
893 ; GFX7-NEXT: .LBB3_1: ; %atomicrmw.start
894 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
895 ; GFX7-NEXT: s_waitcnt vmcnt(0)
896 ; GFX7-NEXT: v_sub_f32_e32 v3, v4, v2
897 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
898 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
899 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc
900 ; GFX7-NEXT: s_waitcnt vmcnt(0)
901 ; GFX7-NEXT: buffer_wbinvl1
902 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
903 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
904 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
905 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
906 ; GFX7-NEXT: s_cbranch_execnz .LBB3_1
907 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
908 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
909 ; GFX7-NEXT: s_setpc_b64 s[30:31]
910 ;
911 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f32:
912 ; GFX6: ; %bb.0:
913 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
914 ; GFX6-NEXT: s_mov_b32 s6, 0
915 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
916 ; GFX6-NEXT: s_mov_b32 s4, s6
917 ; GFX6-NEXT: s_mov_b32 s5, s6
918 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
919 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
920 ; GFX6-NEXT: .LBB3_1: ; %atomicrmw.start
921 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
922 ; GFX6-NEXT: s_waitcnt vmcnt(0)
923 ; GFX6-NEXT: v_sub_f32_e32 v3, v4, v2
924 ; GFX6-NEXT: s_waitcnt expcnt(0)
925 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
926 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
927 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc
928 ; GFX6-NEXT: s_waitcnt vmcnt(0)
929 ; GFX6-NEXT: buffer_wbinvl1
930 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
931 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
932 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
933 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
934 ; GFX6-NEXT: s_cbranch_execnz .LBB3_1
935 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
936 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
937 ; GFX6-NEXT: s_waitcnt expcnt(0)
938 ; GFX6-NEXT: s_setpc_b64 s[30:31]
939 %unused = atomicrmw fsub ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst
940 ret void
941 }
943 define void @global_agent_atomic_fsub_noret_f32__offset12b_pos(ptr addrspace(1) %ptr, float %val) #0 {
944 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos:
945 ; GFX12: ; %bb.0:
946 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
947 ; GFX12-NEXT: s_wait_expcnt 0x0
948 ; GFX12-NEXT: s_wait_samplecnt 0x0
949 ; GFX12-NEXT: s_wait_bvhcnt 0x0
950 ; GFX12-NEXT: s_wait_kmcnt 0x0
951 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:2044
952 ; GFX12-NEXT: s_mov_b32 s0, 0
953 ; GFX12-NEXT: .LBB4_1: ; %atomicrmw.start
954 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
955 ; GFX12-NEXT: s_wait_loadcnt 0x0
956 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
957 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
958 ; GFX12-NEXT: s_wait_storecnt 0x0
959 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
960 ; GFX12-NEXT: s_wait_loadcnt 0x0
961 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
962 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
963 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
964 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
965 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
966 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
967 ; GFX12-NEXT: s_cbranch_execnz .LBB4_1
968 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
969 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
970 ; GFX12-NEXT: s_setpc_b64 s[30:31]
971 ;
972 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos:
973 ; GFX940: ; %bb.0:
974 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
975 ; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:2044
976 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
977 ; GFX940-NEXT: .LBB4_1: ; %atomicrmw.start
978 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
979 ; GFX940-NEXT: s_waitcnt vmcnt(0)
980 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
981 ; GFX940-NEXT: buffer_wbl2 sc1
982 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
983 ; GFX940-NEXT: s_waitcnt vmcnt(0)
984 ; GFX940-NEXT: buffer_inv sc1
985 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
986 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
987 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
988 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
989 ; GFX940-NEXT: s_cbranch_execnz .LBB4_1
990 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
991 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
992 ; GFX940-NEXT: s_setpc_b64 s[30:31]
993 ;
994 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos:
995 ; GFX11: ; %bb.0:
996 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
997 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:2044
998 ; GFX11-NEXT: s_mov_b32 s0, 0
999 ; GFX11-NEXT: .LBB4_1: ; %atomicrmw.start
1000 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
1001 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1002 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
1003 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1004 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
1005 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1006 ; GFX11-NEXT: buffer_gl1_inv
1007 ; GFX11-NEXT: buffer_gl0_inv
1008 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1009 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
1010 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
1011 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1012 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1013 ; GFX11-NEXT: s_cbranch_execnz .LBB4_1
1014 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
1015 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
1016 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1017 ;
1018 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos:
1019 ; GFX10: ; %bb.0:
1020 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1021 ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
1022 ; GFX10-NEXT: s_mov_b32 s4, 0
1023 ; GFX10-NEXT: .LBB4_1: ; %atomicrmw.start
1024 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
1025 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1026 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
1027 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1028 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
1029 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1030 ; GFX10-NEXT: buffer_gl1_inv
1031 ; GFX10-NEXT: buffer_gl0_inv
1032 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1033 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
1034 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
1035 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
1036 ; GFX10-NEXT: s_cbranch_execnz .LBB4_1
1037 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
1038 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
1039 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1040 ;
1041 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos:
1042 ; GFX90A: ; %bb.0:
1043 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1044 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
1045 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
1046 ; GFX90A-NEXT: .LBB4_1: ; %atomicrmw.start
1047 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
1048 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1049 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
1050 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
1051 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1052 ; GFX90A-NEXT: buffer_wbinvl1
1053 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1054 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1055 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
1056 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
1057 ; GFX90A-NEXT: s_cbranch_execnz .LBB4_1
1058 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
1059 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
1060 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1061 ;
1062 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos:
1063 ; GFX908: ; %bb.0:
1064 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1065 ; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
1066 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
1067 ; GFX908-NEXT: .LBB4_1: ; %atomicrmw.start
1068 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
1069 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1070 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
1071 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
1072 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1073 ; GFX908-NEXT: buffer_wbinvl1
1074 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1075 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1076 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
1077 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
1078 ; GFX908-NEXT: s_cbranch_execnz .LBB4_1
1079 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
1080 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
1081 ; GFX908-NEXT: s_setpc_b64 s[30:31]
1082 ;
1083 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos:
1084 ; GFX8: ; %bb.0:
1085 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1086 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
1087 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1088 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
1089 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
1090 ; GFX8-NEXT: .LBB4_1: ; %atomicrmw.start
1091 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
1092 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1093 ; GFX8-NEXT: v_sub_f32_e32 v3, v4, v2
1094 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
1095 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1096 ; GFX8-NEXT: buffer_wbinvl1
1097 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1098 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1099 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
1100 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1101 ; GFX8-NEXT: s_cbranch_execnz .LBB4_1
1102 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1103 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1104 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1105 ;
1106 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos:
1107 ; GFX7: ; %bb.0:
1108 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1109 ; GFX7-NEXT: s_mov_b32 s6, 0
1110 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
1111 ; GFX7-NEXT: s_mov_b32 s4, s6
1112 ; GFX7-NEXT: s_mov_b32 s5, s6
1113 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:2044
1114 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
1115 ; GFX7-NEXT: .LBB4_1: ; %atomicrmw.start
1116 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
1117 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1118 ; GFX7-NEXT: v_sub_f32_e32 v3, v4, v2
1119 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
1120 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
1121 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
1122 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1123 ; GFX7-NEXT: buffer_wbinvl1
1124 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
1125 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
1126 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
1127 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
1128 ; GFX7-NEXT: s_cbranch_execnz .LBB4_1
1129 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
1130 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
1131 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1132 ;
1133 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos:
1134 ; GFX6: ; %bb.0:
1135 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1136 ; GFX6-NEXT: s_mov_b32 s6, 0
1137 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
1138 ; GFX6-NEXT: s_mov_b32 s4, s6
1139 ; GFX6-NEXT: s_mov_b32 s5, s6
1140 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:2044
1141 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
1142 ; GFX6-NEXT: .LBB4_1: ; %atomicrmw.start
1143 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
1144 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1145 ; GFX6-NEXT: v_sub_f32_e32 v3, v4, v2
1146 ; GFX6-NEXT: s_waitcnt expcnt(0)
1147 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
1148 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
1149 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
1150 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1151 ; GFX6-NEXT: buffer_wbinvl1
1152 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
1153 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
1154 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
1155 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
1156 ; GFX6-NEXT: s_cbranch_execnz .LBB4_1
1157 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
1158 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
1159 ; GFX6-NEXT: s_waitcnt expcnt(0)
1160 ; GFX6-NEXT: s_setpc_b64 s[30:31]
1161 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
1162 %unused = atomicrmw fsub ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
1163 ret void
1164 }
1166 define void @global_agent_atomic_fsub_noret_f32__offset12b_neg(ptr addrspace(1) %ptr, float %val) #0 {
1167 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg:
1168 ; GFX12: ; %bb.0:
1169 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
1170 ; GFX12-NEXT: s_wait_expcnt 0x0
1171 ; GFX12-NEXT: s_wait_samplecnt 0x0
1172 ; GFX12-NEXT: s_wait_bvhcnt 0x0
1173 ; GFX12-NEXT: s_wait_kmcnt 0x0
1174 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
1175 ; GFX12-NEXT: s_mov_b32 s0, 0
1176 ; GFX12-NEXT: .LBB5_1: ; %atomicrmw.start
1177 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
1178 ; GFX12-NEXT: s_wait_loadcnt 0x0
1179 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
1180 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
1181 ; GFX12-NEXT: s_wait_storecnt 0x0
1182 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
1183 ; GFX12-NEXT: s_wait_loadcnt 0x0
1184 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
1185 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1186 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
1187 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
1188 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1189 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1190 ; GFX12-NEXT: s_cbranch_execnz .LBB5_1
1191 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
1192 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
1193 ; GFX12-NEXT: s_setpc_b64 s[30:31]
1194 ;
1195 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg:
1196 ; GFX940: ; %bb.0:
1197 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1198 ; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:-2048
1199 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
1200 ; GFX940-NEXT: .LBB5_1: ; %atomicrmw.start
1201 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
1202 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1203 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
1204 ; GFX940-NEXT: buffer_wbl2 sc1
1205 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
1206 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1207 ; GFX940-NEXT: buffer_inv sc1
1208 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1209 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
1210 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
1211 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
1212 ; GFX940-NEXT: s_cbranch_execnz .LBB5_1
1213 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
1214 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
1215 ; GFX940-NEXT: s_setpc_b64 s[30:31]
1216 ;
1217 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg:
1218 ; GFX11: ; %bb.0:
1219 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1220 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
1221 ; GFX11-NEXT: s_mov_b32 s0, 0
1222 ; GFX11-NEXT: .LBB5_1: ; %atomicrmw.start
1223 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
1224 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1225 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
1226 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1227 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
1228 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1229 ; GFX11-NEXT: buffer_gl1_inv
1230 ; GFX11-NEXT: buffer_gl0_inv
1231 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1232 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
1233 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
1234 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1235 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1236 ; GFX11-NEXT: s_cbranch_execnz .LBB5_1
1237 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
1238 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
1239 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1240 ;
1241 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg:
1242 ; GFX10: ; %bb.0:
1243 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1244 ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:-2048
1245 ; GFX10-NEXT: s_mov_b32 s4, 0
1246 ; GFX10-NEXT: .LBB5_1: ; %atomicrmw.start
1247 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
1248 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1249 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
1250 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1251 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
1252 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1253 ; GFX10-NEXT: buffer_gl1_inv
1254 ; GFX10-NEXT: buffer_gl0_inv
1255 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1256 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
1257 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
1258 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
1259 ; GFX10-NEXT: s_cbranch_execnz .LBB5_1
1260 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
1261 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
1262 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1263 ;
1264 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg:
1265 ; GFX90A: ; %bb.0:
1266 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1267 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
1268 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
1269 ; GFX90A-NEXT: .LBB5_1: ; %atomicrmw.start
1270 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
1271 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1272 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
1273 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
1274 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1275 ; GFX90A-NEXT: buffer_wbinvl1
1276 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1277 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1278 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
1279 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
1280 ; GFX90A-NEXT: s_cbranch_execnz .LBB5_1
1281 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
1282 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
1283 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1284 ;
1285 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg:
1286 ; GFX908: ; %bb.0:
1287 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1288 ; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:-2048
1289 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
1290 ; GFX908-NEXT: .LBB5_1: ; %atomicrmw.start
1291 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
1292 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1293 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
1294 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
1295 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1296 ; GFX908-NEXT: buffer_wbinvl1
1297 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1298 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1299 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
1300 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
1301 ; GFX908-NEXT: s_cbranch_execnz .LBB5_1
1302 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
1303 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
1304 ; GFX908-NEXT: s_setpc_b64 s[30:31]
1305 ;
1306 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg:
1307 ; GFX8: ; %bb.0:
1308 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1309 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
1310 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
1311 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
1312 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
1313 ; GFX8-NEXT: .LBB5_1: ; %atomicrmw.start
1314 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
1315 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1316 ; GFX8-NEXT: v_sub_f32_e32 v3, v4, v2
1317 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
1318 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1319 ; GFX8-NEXT: buffer_wbinvl1
1320 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1321 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1322 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
1323 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1324 ; GFX8-NEXT: s_cbranch_execnz .LBB5_1
1325 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1326 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1327 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1328 ;
1329 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg:
1330 ; GFX7: ; %bb.0:
1331 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1332 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
1333 ; GFX7-NEXT: s_mov_b32 s5, -1
1334 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
1335 ; GFX7-NEXT: s_mov_b32 s6, 0
1336 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
1337 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
1338 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
1339 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
1340 ; GFX7-NEXT: s_mov_b32 s4, s6
1341 ; GFX7-NEXT: s_mov_b32 s5, s6
1342 ; GFX7-NEXT: .LBB5_1: ; %atomicrmw.start
1343 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
1344 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1345 ; GFX7-NEXT: v_sub_f32_e32 v3, v4, v2
1346 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
1347 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
1348 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc
1349 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1350 ; GFX7-NEXT: buffer_wbinvl1
1351 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
1352 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
1353 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
1354 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
1355 ; GFX7-NEXT: s_cbranch_execnz .LBB5_1
1356 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
1357 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
1358 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1359 ;
1360 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg:
1361 ; GFX6: ; %bb.0:
1362 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1363 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
1364 ; GFX6-NEXT: s_mov_b32 s5, -1
1365 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
1366 ; GFX6-NEXT: s_mov_b32 s6, 0
1367 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
1368 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
1369 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
1370 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
1371 ; GFX6-NEXT: s_mov_b32 s4, s6
1372 ; GFX6-NEXT: s_mov_b32 s5, s6
1373 ; GFX6-NEXT: .LBB5_1: ; %atomicrmw.start
1374 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
1375 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1376 ; GFX6-NEXT: v_sub_f32_e32 v3, v4, v2
1377 ; GFX6-NEXT: s_waitcnt expcnt(0)
1378 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
1379 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
1380 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc
1381 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1382 ; GFX6-NEXT: buffer_wbinvl1
1383 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
1384 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
1385 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
1386 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
1387 ; GFX6-NEXT: s_cbranch_execnz .LBB5_1
1388 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
1389 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
1390 ; GFX6-NEXT: s_waitcnt expcnt(0)
1391 ; GFX6-NEXT: s_setpc_b64 s[30:31]
1392 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
1393 %unused = atomicrmw fsub ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
1394 ret void
1395 }
1397 define float @global_system_atomic_fsub_ret_f32__offset12b_pos(ptr addrspace(1) %ptr, float %val) #0 {
1398 ; GFX12-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos:
1399 ; GFX12: ; %bb.0:
1400 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
1401 ; GFX12-NEXT: s_wait_expcnt 0x0
1402 ; GFX12-NEXT: s_wait_samplecnt 0x0
1403 ; GFX12-NEXT: s_wait_bvhcnt 0x0
1404 ; GFX12-NEXT: s_wait_kmcnt 0x0
1405 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
1406 ; GFX12-NEXT: s_mov_b32 s0, 0
1407 ; GFX12-NEXT: .LBB6_1: ; %atomicrmw.start
1408 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
1409 ; GFX12-NEXT: s_wait_loadcnt 0x0
1410 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
1411 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
1412 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
1413 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
1414 ; GFX12-NEXT: s_wait_storecnt 0x0
1415 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
1416 ; GFX12-NEXT: s_wait_loadcnt 0x0
1417 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
1418 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1419 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
1420 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1421 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1422 ; GFX12-NEXT: s_cbranch_execnz .LBB6_1
1423 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
1424 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
1425 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
1426 ; GFX12-NEXT: s_setpc_b64 s[30:31]
1427 ;
1428 ; GFX940-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos:
1429 ; GFX940: ; %bb.0:
1430 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1431 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
1432 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
1433 ; GFX940-NEXT: .LBB6_1: ; %atomicrmw.start
1434 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
1435 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1436 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
1437 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
1438 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
1439 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
1440 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1441 ; GFX940-NEXT: buffer_inv sc0 sc1
1442 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1443 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
1444 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
1445 ; GFX940-NEXT: s_cbranch_execnz .LBB6_1
1446 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
1447 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
1448 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
1449 ; GFX940-NEXT: s_setpc_b64 s[30:31]
1450 ;
1451 ; GFX11-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos:
1452 ; GFX11: ; %bb.0:
1453 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1454 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
1455 ; GFX11-NEXT: s_mov_b32 s0, 0
1456 ; GFX11-NEXT: .LBB6_1: ; %atomicrmw.start
1457 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
1458 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1459 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
1460 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
1461 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
1462 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1463 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
1464 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1465 ; GFX11-NEXT: buffer_gl1_inv
1466 ; GFX11-NEXT: buffer_gl0_inv
1467 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1468 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
1469 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1470 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1471 ; GFX11-NEXT: s_cbranch_execnz .LBB6_1
1472 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
1473 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
1474 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
1475 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1476 ;
1477 ; GFX10-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos:
1478 ; GFX10: ; %bb.0:
1479 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1480 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
1481 ; GFX10-NEXT: s_mov_b32 s4, 0
1482 ; GFX10-NEXT: .LBB6_1: ; %atomicrmw.start
1483 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
1484 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1485 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
1486 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
1487 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1488 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
1489 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1490 ; GFX10-NEXT: buffer_gl1_inv
1491 ; GFX10-NEXT: buffer_gl0_inv
1492 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1493 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
1494 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
1495 ; GFX10-NEXT: s_cbranch_execnz .LBB6_1
1496 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
1497 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
1498 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
1499 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1500 ;
1501 ; GFX90A-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos:
1502 ; GFX90A: ; %bb.0:
1503 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1504 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
1505 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
1506 ; GFX90A-NEXT: .LBB6_1: ; %atomicrmw.start
1507 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
1508 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1509 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
1510 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
1511 ; GFX90A-NEXT: buffer_wbl2
1512 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
1513 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1514 ; GFX90A-NEXT: buffer_invl2
1515 ; GFX90A-NEXT: buffer_wbinvl1
1516 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1517 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1518 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
1519 ; GFX90A-NEXT: s_cbranch_execnz .LBB6_1
1520 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
1521 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
1522 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
1523 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1524 ;
1525 ; GFX908-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos:
1526 ; GFX908: ; %bb.0:
1527 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1528 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
1529 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
1530 ; GFX908-NEXT: .LBB6_1: ; %atomicrmw.start
1531 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
1532 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1533 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
1534 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
1535 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
1536 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1537 ; GFX908-NEXT: buffer_wbinvl1
1538 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1539 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1540 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
1541 ; GFX908-NEXT: s_cbranch_execnz .LBB6_1
1542 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
1543 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
1544 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
1545 ; GFX908-NEXT: s_setpc_b64 s[30:31]
1546 ;
1547 ; GFX8-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos:
1548 ; GFX8: ; %bb.0:
1549 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1550 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
1551 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
1552 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
1553 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
1554 ; GFX8-NEXT: .LBB6_1: ; %atomicrmw.start
1555 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
1556 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1557 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
1558 ; GFX8-NEXT: v_sub_f32_e32 v0, v1, v2
1559 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
1560 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1561 ; GFX8-NEXT: buffer_wbinvl1
1562 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
1563 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1564 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1565 ; GFX8-NEXT: s_cbranch_execnz .LBB6_1
1566 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1567 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1568 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1569 ;
1570 ; GFX7-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos:
1571 ; GFX7: ; %bb.0:
1572 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1573 ; GFX7-NEXT: s_mov_b32 s6, 0
1574 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
1575 ; GFX7-NEXT: s_mov_b32 s4, s6
1576 ; GFX7-NEXT: s_mov_b32 s5, s6
1577 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
1578 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
1579 ; GFX7-NEXT: .LBB6_1: ; %atomicrmw.start
1580 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
1581 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1582 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
1583 ; GFX7-NEXT: v_sub_f32_e32 v4, v5, v2
1584 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
1585 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
1586 ; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
1587 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1588 ; GFX7-NEXT: buffer_wbinvl1
1589 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1590 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
1591 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
1592 ; GFX7-NEXT: s_cbranch_execnz .LBB6_1
1593 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
1594 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
1595 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
1596 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1597 ;
1598 ; GFX6-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos:
1599 ; GFX6: ; %bb.0:
1600 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1601 ; GFX6-NEXT: s_mov_b32 s6, 0
1602 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
1603 ; GFX6-NEXT: s_mov_b32 s4, s6
1604 ; GFX6-NEXT: s_mov_b32 s5, s6
1605 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
1606 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
1607 ; GFX6-NEXT: .LBB6_1: ; %atomicrmw.start
1608 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
1609 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1610 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
1611 ; GFX6-NEXT: s_waitcnt expcnt(0)
1612 ; GFX6-NEXT: v_sub_f32_e32 v4, v5, v2
1613 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
1614 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
1615 ; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
1616 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1617 ; GFX6-NEXT: buffer_wbinvl1
1618 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1619 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
1620 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
1621 ; GFX6-NEXT: s_cbranch_execnz .LBB6_1
1622 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
1623 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
1624 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
1625 ; GFX6-NEXT: s_waitcnt expcnt(0)
1626 ; GFX6-NEXT: s_setpc_b64 s[30:31]
1627 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
1628 %result = atomicrmw fsub ptr addrspace(1) %gep, float %val seq_cst
1629 ret float %result
1630 }
1632 define void @global_system_atomic_fsub_noret_f32__offset12b_pos(ptr addrspace(1) %ptr, float %val) #0 {
1633 ; GFX12-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos:
1634 ; GFX12: ; %bb.0:
1635 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
1636 ; GFX12-NEXT: s_wait_expcnt 0x0
1637 ; GFX12-NEXT: s_wait_samplecnt 0x0
1638 ; GFX12-NEXT: s_wait_bvhcnt 0x0
1639 ; GFX12-NEXT: s_wait_kmcnt 0x0
1640 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:2044
1641 ; GFX12-NEXT: s_mov_b32 s0, 0
1642 ; GFX12-NEXT: .LBB7_1: ; %atomicrmw.start
1643 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
1644 ; GFX12-NEXT: s_wait_loadcnt 0x0
1645 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
1646 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
1647 ; GFX12-NEXT: s_wait_storecnt 0x0
1648 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
1649 ; GFX12-NEXT: s_wait_loadcnt 0x0
1650 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
1651 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1652 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
1653 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
1654 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1655 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1656 ; GFX12-NEXT: s_cbranch_execnz .LBB7_1
1657 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
1658 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
1659 ; GFX12-NEXT: s_setpc_b64 s[30:31]
1660 ;
1661 ; GFX940-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos:
1662 ; GFX940: ; %bb.0:
1663 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1664 ; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:2044
1665 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
1666 ; GFX940-NEXT: .LBB7_1: ; %atomicrmw.start
1667 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
1668 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1669 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
1670 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
1671 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
1672 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1673 ; GFX940-NEXT: buffer_inv sc0 sc1
1674 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1675 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
1676 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
1677 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
1678 ; GFX940-NEXT: s_cbranch_execnz .LBB7_1
1679 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
1680 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
1681 ; GFX940-NEXT: s_setpc_b64 s[30:31]
1682 ;
1683 ; GFX11-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos:
1684 ; GFX11: ; %bb.0:
1685 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1686 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:2044
1687 ; GFX11-NEXT: s_mov_b32 s0, 0
1688 ; GFX11-NEXT: .LBB7_1: ; %atomicrmw.start
1689 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
1690 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1691 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
1692 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1693 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
1694 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1695 ; GFX11-NEXT: buffer_gl1_inv
1696 ; GFX11-NEXT: buffer_gl0_inv
1697 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1698 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
1699 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
1700 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1701 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1702 ; GFX11-NEXT: s_cbranch_execnz .LBB7_1
1703 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
1704 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
1705 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1706 ;
1707 ; GFX10-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos:
1708 ; GFX10: ; %bb.0:
1709 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1710 ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
1711 ; GFX10-NEXT: s_mov_b32 s4, 0
1712 ; GFX10-NEXT: .LBB7_1: ; %atomicrmw.start
1713 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
1714 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1715 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
1716 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1717 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
1718 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1719 ; GFX10-NEXT: buffer_gl1_inv
1720 ; GFX10-NEXT: buffer_gl0_inv
1721 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1722 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
1723 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
1724 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
1725 ; GFX10-NEXT: s_cbranch_execnz .LBB7_1
1726 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
1727 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
1728 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1729 ;
1730 ; GFX90A-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos:
1731 ; GFX90A: ; %bb.0:
1732 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1733 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
1734 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
1735 ; GFX90A-NEXT: .LBB7_1: ; %atomicrmw.start
1736 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
1737 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1738 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
1739 ; GFX90A-NEXT: buffer_wbl2
1740 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
1741 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1742 ; GFX90A-NEXT: buffer_invl2
1743 ; GFX90A-NEXT: buffer_wbinvl1
1744 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1745 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1746 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
1747 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
1748 ; GFX90A-NEXT: s_cbranch_execnz .LBB7_1
1749 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
1750 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
1751 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1752 ;
1753 ; GFX908-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos:
1754 ; GFX908: ; %bb.0:
1755 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1756 ; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
1757 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
1758 ; GFX908-NEXT: .LBB7_1: ; %atomicrmw.start
1759 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
1760 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1761 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
1762 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
1763 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1764 ; GFX908-NEXT: buffer_wbinvl1
1765 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1766 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1767 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
1768 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
1769 ; GFX908-NEXT: s_cbranch_execnz .LBB7_1
1770 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
1771 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
1772 ; GFX908-NEXT: s_setpc_b64 s[30:31]
1773 ;
1774 ; GFX8-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos:
1775 ; GFX8: ; %bb.0:
1776 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1777 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
1778 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1779 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
1780 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
1781 ; GFX8-NEXT: .LBB7_1: ; %atomicrmw.start
1782 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
1783 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1784 ; GFX8-NEXT: v_sub_f32_e32 v3, v4, v2
1785 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
1786 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1787 ; GFX8-NEXT: buffer_wbinvl1
1788 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1789 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1790 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
1791 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1792 ; GFX8-NEXT: s_cbranch_execnz .LBB7_1
1793 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1794 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1795 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1796 ;
1797 ; GFX7-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos:
1798 ; GFX7: ; %bb.0:
1799 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1800 ; GFX7-NEXT: s_mov_b32 s6, 0
1801 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
1802 ; GFX7-NEXT: s_mov_b32 s4, s6
1803 ; GFX7-NEXT: s_mov_b32 s5, s6
1804 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:2044
1805 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
1806 ; GFX7-NEXT: .LBB7_1: ; %atomicrmw.start
1807 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
1808 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1809 ; GFX7-NEXT: v_sub_f32_e32 v3, v4, v2
1810 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
1811 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
1812 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
1813 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1814 ; GFX7-NEXT: buffer_wbinvl1
1815 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
1816 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
1817 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
1818 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
1819 ; GFX7-NEXT: s_cbranch_execnz .LBB7_1
1820 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
1821 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
1822 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1823 ;
1824 ; GFX6-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos:
1825 ; GFX6: ; %bb.0:
1826 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1827 ; GFX6-NEXT: s_mov_b32 s6, 0
1828 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
1829 ; GFX6-NEXT: s_mov_b32 s4, s6
1830 ; GFX6-NEXT: s_mov_b32 s5, s6
1831 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:2044
1832 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
1833 ; GFX6-NEXT: .LBB7_1: ; %atomicrmw.start
1834 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
1835 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1836 ; GFX6-NEXT: v_sub_f32_e32 v3, v4, v2
1837 ; GFX6-NEXT: s_waitcnt expcnt(0)
1838 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
1839 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
1840 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
1841 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1842 ; GFX6-NEXT: buffer_wbinvl1
1843 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
1844 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
1845 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
1846 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
1847 ; GFX6-NEXT: s_cbranch_execnz .LBB7_1
1848 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
1849 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
1850 ; GFX6-NEXT: s_waitcnt expcnt(0)
1851 ; GFX6-NEXT: s_setpc_b64 s[30:31]
1852 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
1853 %unused = atomicrmw fsub ptr addrspace(1) %gep, float %val seq_cst
1854 ret void
1855 }
1857 ; --------------------------------------------------------------------
1858 ; float with ftz/daz
1859 ; --------------------------------------------------------------------
1861 define float @global_agent_atomic_fsub_ret_f32__ftz(ptr addrspace(1) %ptr, float %val) #1 {
1862 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f32__ftz:
1863 ; GFX12: ; %bb.0:
1864 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
1865 ; GFX12-NEXT: s_wait_expcnt 0x0
1866 ; GFX12-NEXT: s_wait_samplecnt 0x0
1867 ; GFX12-NEXT: s_wait_bvhcnt 0x0
1868 ; GFX12-NEXT: s_wait_kmcnt 0x0
1869 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
1870 ; GFX12-NEXT: s_mov_b32 s0, 0
1871 ; GFX12-NEXT: .LBB8_1: ; %atomicrmw.start
1872 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
1873 ; GFX12-NEXT: s_wait_loadcnt 0x0
1874 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
1875 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
1876 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
1877 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
1878 ; GFX12-NEXT: s_wait_storecnt 0x0
1879 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
1880 ; GFX12-NEXT: s_wait_loadcnt 0x0
1881 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
1882 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1883 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
1884 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1885 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1886 ; GFX12-NEXT: s_cbranch_execnz .LBB8_1
1887 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
1888 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
1889 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
1890 ; GFX12-NEXT: s_setpc_b64 s[30:31]
1891 ;
1892 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f32__ftz:
1893 ; GFX940: ; %bb.0:
1894 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1895 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
1896 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
1897 ; GFX940-NEXT: .LBB8_1: ; %atomicrmw.start
1898 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
1899 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1900 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
1901 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
1902 ; GFX940-NEXT: buffer_wbl2 sc1
1903 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
1904 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1905 ; GFX940-NEXT: buffer_inv sc1
1906 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1907 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
1908 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
1909 ; GFX940-NEXT: s_cbranch_execnz .LBB8_1
1910 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
1911 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
1912 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
1913 ; GFX940-NEXT: s_setpc_b64 s[30:31]
1914 ;
1915 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f32__ftz:
1916 ; GFX11: ; %bb.0:
1917 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1918 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
1919 ; GFX11-NEXT: s_mov_b32 s0, 0
1920 ; GFX11-NEXT: .LBB8_1: ; %atomicrmw.start
1921 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
1922 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1923 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
1924 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
1925 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
1926 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1927 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
1928 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1929 ; GFX11-NEXT: buffer_gl1_inv
1930 ; GFX11-NEXT: buffer_gl0_inv
1931 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1932 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
1933 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1934 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1935 ; GFX11-NEXT: s_cbranch_execnz .LBB8_1
1936 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
1937 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
1938 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
1939 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1940 ;
1941 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f32__ftz:
1942 ; GFX10: ; %bb.0:
1943 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1944 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
1945 ; GFX10-NEXT: s_mov_b32 s4, 0
1946 ; GFX10-NEXT: .LBB8_1: ; %atomicrmw.start
1947 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
1948 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1949 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
1950 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
1951 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1952 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
1953 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1954 ; GFX10-NEXT: buffer_gl1_inv
1955 ; GFX10-NEXT: buffer_gl0_inv
1956 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1957 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
1958 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
1959 ; GFX10-NEXT: s_cbranch_execnz .LBB8_1
1960 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
1961 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
1962 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
1963 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1964 ;
1965 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f32__ftz:
1966 ; GFX90A: ; %bb.0:
1967 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1968 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
1969 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
1970 ; GFX90A-NEXT: .LBB8_1: ; %atomicrmw.start
1971 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
1972 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1973 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
1974 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
1975 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
1976 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1977 ; GFX90A-NEXT: buffer_wbinvl1
1978 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1979 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1980 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
1981 ; GFX90A-NEXT: s_cbranch_execnz .LBB8_1
1982 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
1983 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
1984 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
1985 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1986 ;
1987 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f32__ftz:
1988 ; GFX908: ; %bb.0:
1989 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1990 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
1991 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
1992 ; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start
1993 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
1994 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1995 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
1996 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
1997 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
1998 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1999 ; GFX908-NEXT: buffer_wbinvl1
2000 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
2001 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2002 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
2003 ; GFX908-NEXT: s_cbranch_execnz .LBB8_1
2004 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
2005 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
2006 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
2007 ; GFX908-NEXT: s_setpc_b64 s[30:31]
2008 ;
2009 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f32__ftz:
2010 ; GFX8: ; %bb.0:
2011 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2012 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
2013 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2014 ; GFX8-NEXT: .LBB8_1: ; %atomicrmw.start
2015 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2016 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2017 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
2018 ; GFX8-NEXT: v_sub_f32_e32 v3, v4, v2
2019 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
2020 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2021 ; GFX8-NEXT: buffer_wbinvl1
2022 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
2023 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2024 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2025 ; GFX8-NEXT: s_cbranch_execnz .LBB8_1
2026 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2027 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2028 ; GFX8-NEXT: v_mov_b32_e32 v0, v3
2029 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2030 ;
2031 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f32__ftz:
2032 ; GFX7: ; %bb.0:
2033 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2034 ; GFX7-NEXT: s_mov_b32 s6, 0
2035 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
2036 ; GFX7-NEXT: s_mov_b32 s4, s6
2037 ; GFX7-NEXT: s_mov_b32 s5, s6
2038 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
2039 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
2040 ; GFX7-NEXT: .LBB8_1: ; %atomicrmw.start
2041 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
2042 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2043 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
2044 ; GFX7-NEXT: v_sub_f32_e32 v4, v5, v2
2045 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
2046 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
2047 ; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
2048 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2049 ; GFX7-NEXT: buffer_wbinvl1
2050 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2051 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
2052 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
2053 ; GFX7-NEXT: s_cbranch_execnz .LBB8_1
2054 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
2055 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
2056 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
2057 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2058 ;
2059 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f32__ftz:
2060 ; GFX6: ; %bb.0:
2061 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2062 ; GFX6-NEXT: s_mov_b32 s6, 0
2063 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
2064 ; GFX6-NEXT: s_mov_b32 s4, s6
2065 ; GFX6-NEXT: s_mov_b32 s5, s6
2066 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
2067 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
2068 ; GFX6-NEXT: .LBB8_1: ; %atomicrmw.start
2069 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
2070 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2071 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
2072 ; GFX6-NEXT: s_waitcnt expcnt(0)
2073 ; GFX6-NEXT: v_sub_f32_e32 v4, v5, v2
2074 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
2075 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
2076 ; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
2077 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2078 ; GFX6-NEXT: buffer_wbinvl1
2079 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2080 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
2081 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
2082 ; GFX6-NEXT: s_cbranch_execnz .LBB8_1
2083 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
2084 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
2085 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
2086 ; GFX6-NEXT: s_waitcnt expcnt(0)
2087 ; GFX6-NEXT: s_setpc_b64 s[30:31]
2088 %result = atomicrmw fsub ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst
2089 ret float %result
2090 }
2092 define float @global_agent_atomic_fsub_ret_f32__offset12b_pos__ftz(ptr addrspace(1) %ptr, float %val) #1 {
2093 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos__ftz:
2094 ; GFX12: ; %bb.0:
2095 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
2096 ; GFX12-NEXT: s_wait_expcnt 0x0
2097 ; GFX12-NEXT: s_wait_samplecnt 0x0
2098 ; GFX12-NEXT: s_wait_bvhcnt 0x0
2099 ; GFX12-NEXT: s_wait_kmcnt 0x0
2100 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
2101 ; GFX12-NEXT: s_mov_b32 s0, 0
2102 ; GFX12-NEXT: .LBB9_1: ; %atomicrmw.start
2103 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
2104 ; GFX12-NEXT: s_wait_loadcnt 0x0
2105 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
2106 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
2107 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
2108 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
2109 ; GFX12-NEXT: s_wait_storecnt 0x0
2110 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
2111 ; GFX12-NEXT: s_wait_loadcnt 0x0
2112 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
2113 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2114 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
2115 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
2116 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2117 ; GFX12-NEXT: s_cbranch_execnz .LBB9_1
2118 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
2119 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
2120 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
2121 ; GFX12-NEXT: s_setpc_b64 s[30:31]
2122 ;
2123 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos__ftz:
2124 ; GFX940: ; %bb.0:
2125 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2126 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
2127 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
2128 ; GFX940-NEXT: .LBB9_1: ; %atomicrmw.start
2129 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
2130 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2131 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
2132 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
2133 ; GFX940-NEXT: buffer_wbl2 sc1
2134 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
2135 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2136 ; GFX940-NEXT: buffer_inv sc1
2137 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2138 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
2139 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
2140 ; GFX940-NEXT: s_cbranch_execnz .LBB9_1
2141 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
2142 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
2143 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
2144 ; GFX940-NEXT: s_setpc_b64 s[30:31]
2145 ;
2146 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos__ftz:
2147 ; GFX11: ; %bb.0:
2148 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2149 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
2150 ; GFX11-NEXT: s_mov_b32 s0, 0
2151 ; GFX11-NEXT: .LBB9_1: ; %atomicrmw.start
2152 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
2153 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2154 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
2155 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
2156 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
2157 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2158 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
2159 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2160 ; GFX11-NEXT: buffer_gl1_inv
2161 ; GFX11-NEXT: buffer_gl0_inv
2162 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2163 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
2164 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
2165 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2166 ; GFX11-NEXT: s_cbranch_execnz .LBB9_1
2167 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
2168 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
2169 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
2170 ; GFX11-NEXT: s_setpc_b64 s[30:31]
2171 ;
2172 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos__ftz:
2173 ; GFX10: ; %bb.0:
2174 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2175 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
2176 ; GFX10-NEXT: s_mov_b32 s4, 0
2177 ; GFX10-NEXT: .LBB9_1: ; %atomicrmw.start
2178 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
2179 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2180 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
2181 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
2182 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2183 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
2184 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2185 ; GFX10-NEXT: buffer_gl1_inv
2186 ; GFX10-NEXT: buffer_gl0_inv
2187 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2188 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
2189 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
2190 ; GFX10-NEXT: s_cbranch_execnz .LBB9_1
2191 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
2192 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
2193 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
2194 ; GFX10-NEXT: s_setpc_b64 s[30:31]
2195 ;
2196 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos__ftz:
2197 ; GFX90A: ; %bb.0:
2198 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2199 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
2200 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
2201 ; GFX90A-NEXT: .LBB9_1: ; %atomicrmw.start
2202 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
2203 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2204 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
2205 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
2206 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
2207 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2208 ; GFX90A-NEXT: buffer_wbinvl1
2209 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2210 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2211 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
2212 ; GFX90A-NEXT: s_cbranch_execnz .LBB9_1
2213 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
2214 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
2215 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
2216 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
2217 ;
2218 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos__ftz:
2219 ; GFX908: ; %bb.0:
2220 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2221 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
2222 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
2223 ; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
2224 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
2225 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2226 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
2227 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
2228 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
2229 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2230 ; GFX908-NEXT: buffer_wbinvl1
2231 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
2232 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2233 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
2234 ; GFX908-NEXT: s_cbranch_execnz .LBB9_1
2235 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
2236 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
2237 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
2238 ; GFX908-NEXT: s_setpc_b64 s[30:31]
2239 ;
2240 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos__ftz:
2241 ; GFX8: ; %bb.0:
2242 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2243 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
2244 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
2245 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
2246 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2247 ; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start
2248 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2249 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2250 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
2251 ; GFX8-NEXT: v_sub_f32_e32 v0, v1, v2
2252 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
2253 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2254 ; GFX8-NEXT: buffer_wbinvl1
2255 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
2256 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2257 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2258 ; GFX8-NEXT: s_cbranch_execnz .LBB9_1
2259 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2260 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2261 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2262 ;
2263 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos__ftz:
2264 ; GFX7: ; %bb.0:
2265 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2266 ; GFX7-NEXT: s_mov_b32 s6, 0
2267 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
2268 ; GFX7-NEXT: s_mov_b32 s4, s6
2269 ; GFX7-NEXT: s_mov_b32 s5, s6
2270 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
2271 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
2272 ; GFX7-NEXT: .LBB9_1: ; %atomicrmw.start
2273 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
2274 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2275 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
2276 ; GFX7-NEXT: v_sub_f32_e32 v4, v5, v2
2277 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
2278 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
2279 ; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
2280 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2281 ; GFX7-NEXT: buffer_wbinvl1
2282 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2283 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
2284 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
2285 ; GFX7-NEXT: s_cbranch_execnz .LBB9_1
2286 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
2287 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
2288 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
2289 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2290 ;
2291 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_pos__ftz:
2292 ; GFX6: ; %bb.0:
2293 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2294 ; GFX6-NEXT: s_mov_b32 s6, 0
2295 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
2296 ; GFX6-NEXT: s_mov_b32 s4, s6
2297 ; GFX6-NEXT: s_mov_b32 s5, s6
2298 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
2299 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
2300 ; GFX6-NEXT: .LBB9_1: ; %atomicrmw.start
2301 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
2302 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2303 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
2304 ; GFX6-NEXT: s_waitcnt expcnt(0)
2305 ; GFX6-NEXT: v_sub_f32_e32 v4, v5, v2
2306 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
2307 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
2308 ; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
2309 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2310 ; GFX6-NEXT: buffer_wbinvl1
2311 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2312 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
2313 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
2314 ; GFX6-NEXT: s_cbranch_execnz .LBB9_1
2315 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
2316 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
2317 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
2318 ; GFX6-NEXT: s_waitcnt expcnt(0)
2319 ; GFX6-NEXT: s_setpc_b64 s[30:31]
2320 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
2321 %result = atomicrmw fsub ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
2322 ret float %result
2323 }
2325 define float @global_agent_atomic_fsub_ret_f32__offset12b_neg__ftz(ptr addrspace(1) %ptr, float %val) #1 {
2326 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg__ftz:
2327 ; GFX12: ; %bb.0:
2328 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
2329 ; GFX12-NEXT: s_wait_expcnt 0x0
2330 ; GFX12-NEXT: s_wait_samplecnt 0x0
2331 ; GFX12-NEXT: s_wait_bvhcnt 0x0
2332 ; GFX12-NEXT: s_wait_kmcnt 0x0
2333 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
2334 ; GFX12-NEXT: s_mov_b32 s0, 0
2335 ; GFX12-NEXT: .LBB10_1: ; %atomicrmw.start
2336 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
2337 ; GFX12-NEXT: s_wait_loadcnt 0x0
2338 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
2339 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
2340 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
2341 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
2342 ; GFX12-NEXT: s_wait_storecnt 0x0
2343 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
2344 ; GFX12-NEXT: s_wait_loadcnt 0x0
2345 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
2346 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2347 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
2348 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
2349 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2350 ; GFX12-NEXT: s_cbranch_execnz .LBB10_1
2351 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
2352 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
2353 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
2354 ; GFX12-NEXT: s_setpc_b64 s[30:31]
2355 ;
2356 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg__ftz:
2357 ; GFX940: ; %bb.0:
2358 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2359 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
2360 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
2361 ; GFX940-NEXT: .LBB10_1: ; %atomicrmw.start
2362 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
2363 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2364 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
2365 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
2366 ; GFX940-NEXT: buffer_wbl2 sc1
2367 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
2368 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2369 ; GFX940-NEXT: buffer_inv sc1
2370 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2371 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
2372 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
2373 ; GFX940-NEXT: s_cbranch_execnz .LBB10_1
2374 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
2375 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
2376 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
2377 ; GFX940-NEXT: s_setpc_b64 s[30:31]
2378 ;
2379 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg__ftz:
2380 ; GFX11: ; %bb.0:
2381 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2382 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
2383 ; GFX11-NEXT: s_mov_b32 s0, 0
2384 ; GFX11-NEXT: .LBB10_1: ; %atomicrmw.start
2385 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
2386 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2387 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
2388 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
2389 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
2390 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2391 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
2392 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2393 ; GFX11-NEXT: buffer_gl1_inv
2394 ; GFX11-NEXT: buffer_gl0_inv
2395 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2396 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
2397 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
2398 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2399 ; GFX11-NEXT: s_cbranch_execnz .LBB10_1
2400 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
2401 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
2402 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
2403 ; GFX11-NEXT: s_setpc_b64 s[30:31]
2404 ;
2405 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg__ftz:
2406 ; GFX10: ; %bb.0:
2407 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2408 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
2409 ; GFX10-NEXT: s_mov_b32 s4, 0
2410 ; GFX10-NEXT: .LBB10_1: ; %atomicrmw.start
2411 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
2412 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2413 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
2414 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
2415 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2416 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
2417 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2418 ; GFX10-NEXT: buffer_gl1_inv
2419 ; GFX10-NEXT: buffer_gl0_inv
2420 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2421 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
2422 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
2423 ; GFX10-NEXT: s_cbranch_execnz .LBB10_1
2424 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
2425 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
2426 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
2427 ; GFX10-NEXT: s_setpc_b64 s[30:31]
2428 ;
2429 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg__ftz:
2430 ; GFX90A: ; %bb.0:
2431 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2432 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
2433 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
2434 ; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start
2435 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
2436 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2437 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
2438 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
2439 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
2440 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2441 ; GFX90A-NEXT: buffer_wbinvl1
2442 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2443 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2444 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
2445 ; GFX90A-NEXT: s_cbranch_execnz .LBB10_1
2446 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
2447 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
2448 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
2449 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
2450 ;
2451 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg__ftz:
2452 ; GFX908: ; %bb.0:
2453 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2454 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
2455 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
2456 ; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
2457 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
2458 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2459 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
2460 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
2461 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
2462 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2463 ; GFX908-NEXT: buffer_wbinvl1
2464 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
2465 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2466 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
2467 ; GFX908-NEXT: s_cbranch_execnz .LBB10_1
2468 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
2469 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
2470 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
2471 ; GFX908-NEXT: s_setpc_b64 s[30:31]
2472 ;
2473 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg__ftz:
2474 ; GFX8: ; %bb.0:
2475 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2476 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
2477 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
2478 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
2479 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2480 ; GFX8-NEXT: .LBB10_1: ; %atomicrmw.start
2481 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2482 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2483 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
2484 ; GFX8-NEXT: v_sub_f32_e32 v0, v1, v2
2485 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
2486 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2487 ; GFX8-NEXT: buffer_wbinvl1
2488 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
2489 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2490 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2491 ; GFX8-NEXT: s_cbranch_execnz .LBB10_1
2492 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2493 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2494 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2495 ;
2496 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg__ftz:
2497 ; GFX7: ; %bb.0:
2498 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2499 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
2500 ; GFX7-NEXT: v_mov_b32_e32 v4, v1
2501 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
2502 ; GFX7-NEXT: s_mov_b32 s5, -1
2503 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
2504 ; GFX7-NEXT: s_mov_b32 s6, 0
2505 ; GFX7-NEXT: buffer_load_dword v0, v[3:4], s[4:7], 0 addr64
2506 ; GFX7-NEXT: v_add_i32_e32 v3, vcc, 0xfffff800, v3
2507 ; GFX7-NEXT: v_addc_u32_e32 v4, vcc, -1, v4, vcc
2508 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
2509 ; GFX7-NEXT: s_mov_b32 s4, s6
2510 ; GFX7-NEXT: s_mov_b32 s5, s6
2511 ; GFX7-NEXT: .LBB10_1: ; %atomicrmw.start
2512 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
2513 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2514 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
2515 ; GFX7-NEXT: v_sub_f32_e32 v5, v6, v2
2516 ; GFX7-NEXT: v_mov_b32_e32 v0, v5
2517 ; GFX7-NEXT: v_mov_b32_e32 v1, v6
2518 ; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v[3:4], s[4:7], 0 addr64 glc
2519 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2520 ; GFX7-NEXT: buffer_wbinvl1
2521 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
2522 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
2523 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
2524 ; GFX7-NEXT: s_cbranch_execnz .LBB10_1
2525 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
2526 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
2527 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2528 ;
2529 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f32__offset12b_neg__ftz:
2530 ; GFX6: ; %bb.0:
2531 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2532 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
2533 ; GFX6-NEXT: v_mov_b32_e32 v4, v1
2534 ; GFX6-NEXT: v_mov_b32_e32 v3, v0
2535 ; GFX6-NEXT: s_mov_b32 s5, -1
2536 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
2537 ; GFX6-NEXT: s_mov_b32 s6, 0
2538 ; GFX6-NEXT: buffer_load_dword v0, v[3:4], s[4:7], 0 addr64
2539 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, 0xfffff800, v3
2540 ; GFX6-NEXT: v_addc_u32_e32 v4, vcc, -1, v4, vcc
2541 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
2542 ; GFX6-NEXT: s_mov_b32 s4, s6
2543 ; GFX6-NEXT: s_mov_b32 s5, s6
2544 ; GFX6-NEXT: .LBB10_1: ; %atomicrmw.start
2545 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
2546 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2547 ; GFX6-NEXT: v_mov_b32_e32 v6, v0
2548 ; GFX6-NEXT: v_sub_f32_e32 v5, v6, v2
2549 ; GFX6-NEXT: s_waitcnt expcnt(0)
2550 ; GFX6-NEXT: v_mov_b32_e32 v0, v5
2551 ; GFX6-NEXT: v_mov_b32_e32 v1, v6
2552 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v[3:4], s[4:7], 0 addr64 glc
2553 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2554 ; GFX6-NEXT: buffer_wbinvl1
2555 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
2556 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
2557 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
2558 ; GFX6-NEXT: s_cbranch_execnz .LBB10_1
2559 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
2560 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
2561 ; GFX6-NEXT: s_waitcnt expcnt(0)
2562 ; GFX6-NEXT: s_setpc_b64 s[30:31]
2563 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
2564 %result = atomicrmw fsub ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
2565 ret float %result
2566 }
2568 define void @global_agent_atomic_fsub_noret_f32__ftz(ptr addrspace(1) %ptr, float %val) #1 {
2569 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f32__ftz:
2570 ; GFX12: ; %bb.0:
2571 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
2572 ; GFX12-NEXT: s_wait_expcnt 0x0
2573 ; GFX12-NEXT: s_wait_samplecnt 0x0
2574 ; GFX12-NEXT: s_wait_bvhcnt 0x0
2575 ; GFX12-NEXT: s_wait_kmcnt 0x0
2576 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off
2577 ; GFX12-NEXT: s_mov_b32 s0, 0
2578 ; GFX12-NEXT: .LBB11_1: ; %atomicrmw.start
2579 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
2580 ; GFX12-NEXT: s_wait_loadcnt 0x0
2581 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
2582 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
2583 ; GFX12-NEXT: s_wait_storecnt 0x0
2584 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
2585 ; GFX12-NEXT: s_wait_loadcnt 0x0
2586 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
2587 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2588 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
2589 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
2590 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
2591 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2592 ; GFX12-NEXT: s_cbranch_execnz .LBB11_1
2593 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
2594 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
2595 ; GFX12-NEXT: s_setpc_b64 s[30:31]
2596 ;
2597 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f32__ftz:
2598 ; GFX940: ; %bb.0:
2599 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2600 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
2601 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
2602 ; GFX940-NEXT: .LBB11_1: ; %atomicrmw.start
2603 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
2604 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2605 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
2606 ; GFX940-NEXT: buffer_wbl2 sc1
2607 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
2608 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2609 ; GFX940-NEXT: buffer_inv sc1
2610 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2611 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
2612 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
2613 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
2614 ; GFX940-NEXT: s_cbranch_execnz .LBB11_1
2615 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
2616 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
2617 ; GFX940-NEXT: s_setpc_b64 s[30:31]
2618 ;
2619 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f32__ftz:
2620 ; GFX11: ; %bb.0:
2621 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2622 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off
2623 ; GFX11-NEXT: s_mov_b32 s0, 0
2624 ; GFX11-NEXT: .LBB11_1: ; %atomicrmw.start
2625 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
2626 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2627 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
2628 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2629 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
2630 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2631 ; GFX11-NEXT: buffer_gl1_inv
2632 ; GFX11-NEXT: buffer_gl0_inv
2633 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2634 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
2635 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
2636 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
2637 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2638 ; GFX11-NEXT: s_cbranch_execnz .LBB11_1
2639 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
2640 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
2641 ; GFX11-NEXT: s_setpc_b64 s[30:31]
2642 ;
2643 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f32__ftz:
2644 ; GFX10: ; %bb.0:
2645 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2646 ; GFX10-NEXT: global_load_dword v4, v[0:1], off
2647 ; GFX10-NEXT: s_mov_b32 s4, 0
2648 ; GFX10-NEXT: .LBB11_1: ; %atomicrmw.start
2649 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
2650 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2651 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
2652 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2653 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
2654 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2655 ; GFX10-NEXT: buffer_gl1_inv
2656 ; GFX10-NEXT: buffer_gl0_inv
2657 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2658 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
2659 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
2660 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
2661 ; GFX10-NEXT: s_cbranch_execnz .LBB11_1
2662 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
2663 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
2664 ; GFX10-NEXT: s_setpc_b64 s[30:31]
2665 ;
2666 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f32__ftz:
2667 ; GFX90A: ; %bb.0:
2668 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2669 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
2670 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
2671 ; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start
2672 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
2673 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2674 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
2675 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
2676 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2677 ; GFX90A-NEXT: buffer_wbinvl1
2678 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2679 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2680 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
2681 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
2682 ; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
2683 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
2684 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
2685 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
2686 ;
2687 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f32__ftz:
2688 ; GFX908: ; %bb.0:
2689 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2690 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
2691 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
2692 ; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
2693 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
2694 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2695 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
2696 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
2697 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2698 ; GFX908-NEXT: buffer_wbinvl1
2699 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
2700 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2701 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
2702 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
2703 ; GFX908-NEXT: s_cbranch_execnz .LBB11_1
2704 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
2705 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
2706 ; GFX908-NEXT: s_setpc_b64 s[30:31]
2707 ;
2708 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f32__ftz:
2709 ; GFX8: ; %bb.0:
2710 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2711 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
2712 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2713 ; GFX8-NEXT: .LBB11_1: ; %atomicrmw.start
2714 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2715 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2716 ; GFX8-NEXT: v_sub_f32_e32 v3, v4, v2
2717 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
2718 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2719 ; GFX8-NEXT: buffer_wbinvl1
2720 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
2721 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2722 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
2723 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2724 ; GFX8-NEXT: s_cbranch_execnz .LBB11_1
2725 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2726 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2727 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2728 ;
2729 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f32__ftz:
2730 ; GFX7: ; %bb.0:
2731 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2732 ; GFX7-NEXT: s_mov_b32 s6, 0
2733 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
2734 ; GFX7-NEXT: s_mov_b32 s4, s6
2735 ; GFX7-NEXT: s_mov_b32 s5, s6
2736 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
2737 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
2738 ; GFX7-NEXT: .LBB11_1: ; %atomicrmw.start
2739 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
2740 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2741 ; GFX7-NEXT: v_sub_f32_e32 v3, v4, v2
2742 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
2743 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
2744 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc
2745 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2746 ; GFX7-NEXT: buffer_wbinvl1
2747 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
2748 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
2749 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
2750 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
2751 ; GFX7-NEXT: s_cbranch_execnz .LBB11_1
2752 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
2753 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
2754 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2755 ;
2756 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f32__ftz:
2757 ; GFX6: ; %bb.0:
2758 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2759 ; GFX6-NEXT: s_mov_b32 s6, 0
2760 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
2761 ; GFX6-NEXT: s_mov_b32 s4, s6
2762 ; GFX6-NEXT: s_mov_b32 s5, s6
2763 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
2764 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
2765 ; GFX6-NEXT: .LBB11_1: ; %atomicrmw.start
2766 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
2767 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2768 ; GFX6-NEXT: v_sub_f32_e32 v3, v4, v2
2769 ; GFX6-NEXT: s_waitcnt expcnt(0)
2770 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
2771 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
2772 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc
2773 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2774 ; GFX6-NEXT: buffer_wbinvl1
2775 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
2776 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
2777 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
2778 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
2779 ; GFX6-NEXT: s_cbranch_execnz .LBB11_1
2780 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
2781 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
2782 ; GFX6-NEXT: s_waitcnt expcnt(0)
2783 ; GFX6-NEXT: s_setpc_b64 s[30:31]
2784 %unused = atomicrmw fsub ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst
2785 ret void
2786 }
2788 define void @global_agent_atomic_fsub_noret_f32__offset12b_pos__ftz(ptr addrspace(1) %ptr, float %val) #1 {
2789 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos__ftz:
2790 ; GFX12: ; %bb.0:
2791 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
2792 ; GFX12-NEXT: s_wait_expcnt 0x0
2793 ; GFX12-NEXT: s_wait_samplecnt 0x0
2794 ; GFX12-NEXT: s_wait_bvhcnt 0x0
2795 ; GFX12-NEXT: s_wait_kmcnt 0x0
2796 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:2044
2797 ; GFX12-NEXT: s_mov_b32 s0, 0
2798 ; GFX12-NEXT: .LBB12_1: ; %atomicrmw.start
2799 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
2800 ; GFX12-NEXT: s_wait_loadcnt 0x0
2801 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
2802 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
2803 ; GFX12-NEXT: s_wait_storecnt 0x0
2804 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
2805 ; GFX12-NEXT: s_wait_loadcnt 0x0
2806 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
2807 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2808 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
2809 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
2810 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
2811 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2812 ; GFX12-NEXT: s_cbranch_execnz .LBB12_1
2813 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
2814 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
2815 ; GFX12-NEXT: s_setpc_b64 s[30:31]
2816 ;
2817 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos__ftz:
2818 ; GFX940: ; %bb.0:
2819 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2820 ; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:2044
2821 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
2822 ; GFX940-NEXT: .LBB12_1: ; %atomicrmw.start
2823 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
2824 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2825 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
2826 ; GFX940-NEXT: buffer_wbl2 sc1
2827 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
2828 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2829 ; GFX940-NEXT: buffer_inv sc1
2830 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2831 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
2832 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
2833 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
2834 ; GFX940-NEXT: s_cbranch_execnz .LBB12_1
2835 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
2836 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
2837 ; GFX940-NEXT: s_setpc_b64 s[30:31]
2838 ;
2839 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos__ftz:
2840 ; GFX11: ; %bb.0:
2841 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2842 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:2044
2843 ; GFX11-NEXT: s_mov_b32 s0, 0
2844 ; GFX11-NEXT: .LBB12_1: ; %atomicrmw.start
2845 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
2846 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2847 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
2848 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2849 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
2850 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2851 ; GFX11-NEXT: buffer_gl1_inv
2852 ; GFX11-NEXT: buffer_gl0_inv
2853 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2854 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
2855 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
2856 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
2857 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2858 ; GFX11-NEXT: s_cbranch_execnz .LBB12_1
2859 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
2860 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
2861 ; GFX11-NEXT: s_setpc_b64 s[30:31]
2862 ;
2863 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos__ftz:
2864 ; GFX10: ; %bb.0:
2865 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2866 ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
2867 ; GFX10-NEXT: s_mov_b32 s4, 0
2868 ; GFX10-NEXT: .LBB12_1: ; %atomicrmw.start
2869 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
2870 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2871 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
2872 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2873 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
2874 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2875 ; GFX10-NEXT: buffer_gl1_inv
2876 ; GFX10-NEXT: buffer_gl0_inv
2877 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2878 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
2879 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
2880 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
2881 ; GFX10-NEXT: s_cbranch_execnz .LBB12_1
2882 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
2883 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
2884 ; GFX10-NEXT: s_setpc_b64 s[30:31]
2885 ;
2886 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos__ftz:
2887 ; GFX90A: ; %bb.0:
2888 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2889 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
2890 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
2891 ; GFX90A-NEXT: .LBB12_1: ; %atomicrmw.start
2892 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
2893 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2894 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
2895 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
2896 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2897 ; GFX90A-NEXT: buffer_wbinvl1
2898 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2899 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2900 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
2901 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
2902 ; GFX90A-NEXT: s_cbranch_execnz .LBB12_1
2903 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
2904 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
2905 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
2906 ;
2907 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos__ftz:
2908 ; GFX908: ; %bb.0:
2909 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2910 ; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
2911 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
2912 ; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start
2913 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
2914 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2915 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
2916 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
2917 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2918 ; GFX908-NEXT: buffer_wbinvl1
2919 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
2920 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2921 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
2922 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
2923 ; GFX908-NEXT: s_cbranch_execnz .LBB12_1
2924 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
2925 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
2926 ; GFX908-NEXT: s_setpc_b64 s[30:31]
2927 ;
2928 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos__ftz:
2929 ; GFX8: ; %bb.0:
2930 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2931 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
2932 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2933 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
2934 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2935 ; GFX8-NEXT: .LBB12_1: ; %atomicrmw.start
2936 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2937 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2938 ; GFX8-NEXT: v_sub_f32_e32 v3, v4, v2
2939 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
2940 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2941 ; GFX8-NEXT: buffer_wbinvl1
2942 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
2943 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2944 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
2945 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2946 ; GFX8-NEXT: s_cbranch_execnz .LBB12_1
2947 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2948 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2949 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2950 ;
2951 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos__ftz:
2952 ; GFX7: ; %bb.0:
2953 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2954 ; GFX7-NEXT: s_mov_b32 s6, 0
2955 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
2956 ; GFX7-NEXT: s_mov_b32 s4, s6
2957 ; GFX7-NEXT: s_mov_b32 s5, s6
2958 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:2044
2959 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
2960 ; GFX7-NEXT: .LBB12_1: ; %atomicrmw.start
2961 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
2962 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2963 ; GFX7-NEXT: v_sub_f32_e32 v3, v4, v2
2964 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
2965 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
2966 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
2967 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2968 ; GFX7-NEXT: buffer_wbinvl1
2969 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
2970 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
2971 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
2972 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
2973 ; GFX7-NEXT: s_cbranch_execnz .LBB12_1
2974 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
2975 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
2976 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2977 ;
2978 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_pos__ftz:
2979 ; GFX6: ; %bb.0:
2980 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2981 ; GFX6-NEXT: s_mov_b32 s6, 0
2982 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
2983 ; GFX6-NEXT: s_mov_b32 s4, s6
2984 ; GFX6-NEXT: s_mov_b32 s5, s6
2985 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:2044
2986 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
2987 ; GFX6-NEXT: .LBB12_1: ; %atomicrmw.start
2988 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
2989 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2990 ; GFX6-NEXT: v_sub_f32_e32 v3, v4, v2
2991 ; GFX6-NEXT: s_waitcnt expcnt(0)
2992 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
2993 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
2994 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
2995 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2996 ; GFX6-NEXT: buffer_wbinvl1
2997 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
2998 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
2999 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
3000 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
3001 ; GFX6-NEXT: s_cbranch_execnz .LBB12_1
3002 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
3003 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
3004 ; GFX6-NEXT: s_waitcnt expcnt(0)
3005 ; GFX6-NEXT: s_setpc_b64 s[30:31]
3006 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
3007 %unused = atomicrmw fsub ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
3008 ret void
3009 }
3011 define void @global_agent_atomic_fsub_noret_f32__offset12b_neg__ftz(ptr addrspace(1) %ptr, float %val) #1 {
3012 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg__ftz:
3013 ; GFX12: ; %bb.0:
3014 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
3015 ; GFX12-NEXT: s_wait_expcnt 0x0
3016 ; GFX12-NEXT: s_wait_samplecnt 0x0
3017 ; GFX12-NEXT: s_wait_bvhcnt 0x0
3018 ; GFX12-NEXT: s_wait_kmcnt 0x0
3019 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
3020 ; GFX12-NEXT: s_mov_b32 s0, 0
3021 ; GFX12-NEXT: .LBB13_1: ; %atomicrmw.start
3022 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
3023 ; GFX12-NEXT: s_wait_loadcnt 0x0
3024 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
3025 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
3026 ; GFX12-NEXT: s_wait_storecnt 0x0
3027 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
3028 ; GFX12-NEXT: s_wait_loadcnt 0x0
3029 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
3030 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3031 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
3032 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
3033 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3034 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3035 ; GFX12-NEXT: s_cbranch_execnz .LBB13_1
3036 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
3037 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
3038 ; GFX12-NEXT: s_setpc_b64 s[30:31]
3039 ;
3040 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg__ftz:
3041 ; GFX940: ; %bb.0:
3042 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3043 ; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:-2048
3044 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
3045 ; GFX940-NEXT: .LBB13_1: ; %atomicrmw.start
3046 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
3047 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3048 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
3049 ; GFX940-NEXT: buffer_wbl2 sc1
3050 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
3051 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3052 ; GFX940-NEXT: buffer_inv sc1
3053 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
3054 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
3055 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
3056 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
3057 ; GFX940-NEXT: s_cbranch_execnz .LBB13_1
3058 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
3059 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
3060 ; GFX940-NEXT: s_setpc_b64 s[30:31]
3061 ;
3062 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg__ftz:
3063 ; GFX11: ; %bb.0:
3064 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3065 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
3066 ; GFX11-NEXT: s_mov_b32 s0, 0
3067 ; GFX11-NEXT: .LBB13_1: ; %atomicrmw.start
3068 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
3069 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3070 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
3071 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
3072 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
3073 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3074 ; GFX11-NEXT: buffer_gl1_inv
3075 ; GFX11-NEXT: buffer_gl0_inv
3076 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3077 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
3078 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
3079 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3080 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3081 ; GFX11-NEXT: s_cbranch_execnz .LBB13_1
3082 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
3083 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
3084 ; GFX11-NEXT: s_setpc_b64 s[30:31]
3085 ;
3086 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg__ftz:
3087 ; GFX10: ; %bb.0:
3088 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3089 ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:-2048
3090 ; GFX10-NEXT: s_mov_b32 s4, 0
3091 ; GFX10-NEXT: .LBB13_1: ; %atomicrmw.start
3092 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
3093 ; GFX10-NEXT: s_waitcnt vmcnt(0)
3094 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
3095 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
3096 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
3097 ; GFX10-NEXT: s_waitcnt vmcnt(0)
3098 ; GFX10-NEXT: buffer_gl1_inv
3099 ; GFX10-NEXT: buffer_gl0_inv
3100 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3101 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
3102 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
3103 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
3104 ; GFX10-NEXT: s_cbranch_execnz .LBB13_1
3105 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
3106 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
3107 ; GFX10-NEXT: s_setpc_b64 s[30:31]
3108 ;
3109 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg__ftz:
3110 ; GFX90A: ; %bb.0:
3111 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3112 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
3113 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
3114 ; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
3115 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
3116 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3117 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
3118 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
3119 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3120 ; GFX90A-NEXT: buffer_wbinvl1
3121 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
3122 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3123 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
3124 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
3125 ; GFX90A-NEXT: s_cbranch_execnz .LBB13_1
3126 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
3127 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
3128 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
3129 ;
3130 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg__ftz:
3131 ; GFX908: ; %bb.0:
3132 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3133 ; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:-2048
3134 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
3135 ; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
3136 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
3137 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3138 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
3139 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
3140 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3141 ; GFX908-NEXT: buffer_wbinvl1
3142 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
3143 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3144 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
3145 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
3146 ; GFX908-NEXT: s_cbranch_execnz .LBB13_1
3147 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
3148 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
3149 ; GFX908-NEXT: s_setpc_b64 s[30:31]
3150 ;
3151 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg__ftz:
3152 ; GFX8: ; %bb.0:
3153 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3154 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
3155 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
3156 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
3157 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
3158 ; GFX8-NEXT: .LBB13_1: ; %atomicrmw.start
3159 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3160 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3161 ; GFX8-NEXT: v_sub_f32_e32 v3, v4, v2
3162 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
3163 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3164 ; GFX8-NEXT: buffer_wbinvl1
3165 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
3166 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3167 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
3168 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
3169 ; GFX8-NEXT: s_cbranch_execnz .LBB13_1
3170 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
3171 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
3172 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3173 ;
3174 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg__ftz:
3175 ; GFX7: ; %bb.0:
3176 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3177 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
3178 ; GFX7-NEXT: s_mov_b32 s5, -1
3179 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
3180 ; GFX7-NEXT: s_mov_b32 s6, 0
3181 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
3182 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
3183 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
3184 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
3185 ; GFX7-NEXT: s_mov_b32 s4, s6
3186 ; GFX7-NEXT: s_mov_b32 s5, s6
3187 ; GFX7-NEXT: .LBB13_1: ; %atomicrmw.start
3188 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
3189 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3190 ; GFX7-NEXT: v_sub_f32_e32 v3, v4, v2
3191 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
3192 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
3193 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc
3194 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3195 ; GFX7-NEXT: buffer_wbinvl1
3196 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
3197 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
3198 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
3199 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
3200 ; GFX7-NEXT: s_cbranch_execnz .LBB13_1
3201 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
3202 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
3203 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3204 ;
3205 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f32__offset12b_neg__ftz:
3206 ; GFX6: ; %bb.0:
3207 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3208 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
3209 ; GFX6-NEXT: s_mov_b32 s5, -1
3210 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
3211 ; GFX6-NEXT: s_mov_b32 s6, 0
3212 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
3213 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
3214 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
3215 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
3216 ; GFX6-NEXT: s_mov_b32 s4, s6
3217 ; GFX6-NEXT: s_mov_b32 s5, s6
3218 ; GFX6-NEXT: .LBB13_1: ; %atomicrmw.start
3219 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
3220 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3221 ; GFX6-NEXT: v_sub_f32_e32 v3, v4, v2
3222 ; GFX6-NEXT: s_waitcnt expcnt(0)
3223 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
3224 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
3225 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc
3226 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3227 ; GFX6-NEXT: buffer_wbinvl1
3228 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
3229 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
3230 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
3231 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
3232 ; GFX6-NEXT: s_cbranch_execnz .LBB13_1
3233 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
3234 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
3235 ; GFX6-NEXT: s_waitcnt expcnt(0)
3236 ; GFX6-NEXT: s_setpc_b64 s[30:31]
3237 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
3238 %unused = atomicrmw fsub ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst
3239 ret void
3240 }
3242 define float @global_system_atomic_fsub_ret_f32__offset12b_pos__ftz(ptr addrspace(1) %ptr, float %val) #1 {
3243 ; GFX12-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos__ftz:
3244 ; GFX12: ; %bb.0:
3245 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
3246 ; GFX12-NEXT: s_wait_expcnt 0x0
3247 ; GFX12-NEXT: s_wait_samplecnt 0x0
3248 ; GFX12-NEXT: s_wait_bvhcnt 0x0
3249 ; GFX12-NEXT: s_wait_kmcnt 0x0
3250 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
3251 ; GFX12-NEXT: s_mov_b32 s0, 0
3252 ; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
3253 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
3254 ; GFX12-NEXT: s_wait_loadcnt 0x0
3255 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
3256 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
3257 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
3258 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
3259 ; GFX12-NEXT: s_wait_storecnt 0x0
3260 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
3261 ; GFX12-NEXT: s_wait_loadcnt 0x0
3262 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
3263 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3264 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
3265 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3266 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3267 ; GFX12-NEXT: s_cbranch_execnz .LBB14_1
3268 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
3269 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
3270 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
3271 ; GFX12-NEXT: s_setpc_b64 s[30:31]
3272 ;
3273 ; GFX940-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos__ftz:
3274 ; GFX940: ; %bb.0:
3275 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3276 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
3277 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
3278 ; GFX940-NEXT: .LBB14_1: ; %atomicrmw.start
3279 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
3280 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3281 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
3282 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
3283 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
3284 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
3285 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3286 ; GFX940-NEXT: buffer_inv sc0 sc1
3287 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
3288 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
3289 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
3290 ; GFX940-NEXT: s_cbranch_execnz .LBB14_1
3291 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
3292 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
3293 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
3294 ; GFX940-NEXT: s_setpc_b64 s[30:31]
3295 ;
3296 ; GFX11-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos__ftz:
3297 ; GFX11: ; %bb.0:
3298 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3299 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
3300 ; GFX11-NEXT: s_mov_b32 s0, 0
3301 ; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
3302 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
3303 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3304 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
3305 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
3306 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
3307 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
3308 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
3309 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3310 ; GFX11-NEXT: buffer_gl1_inv
3311 ; GFX11-NEXT: buffer_gl0_inv
3312 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3313 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
3314 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3315 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3316 ; GFX11-NEXT: s_cbranch_execnz .LBB14_1
3317 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
3318 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
3319 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
3320 ; GFX11-NEXT: s_setpc_b64 s[30:31]
3321 ;
3322 ; GFX10-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos__ftz:
3323 ; GFX10: ; %bb.0:
3324 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3325 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
3326 ; GFX10-NEXT: s_mov_b32 s4, 0
3327 ; GFX10-NEXT: .LBB14_1: ; %atomicrmw.start
3328 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
3329 ; GFX10-NEXT: s_waitcnt vmcnt(0)
3330 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
3331 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
3332 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
3333 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
3334 ; GFX10-NEXT: s_waitcnt vmcnt(0)
3335 ; GFX10-NEXT: buffer_gl1_inv
3336 ; GFX10-NEXT: buffer_gl0_inv
3337 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3338 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
3339 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
3340 ; GFX10-NEXT: s_cbranch_execnz .LBB14_1
3341 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
3342 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
3343 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
3344 ; GFX10-NEXT: s_setpc_b64 s[30:31]
3345 ;
3346 ; GFX90A-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos__ftz:
3347 ; GFX90A: ; %bb.0:
3348 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3349 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
3350 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
3351 ; GFX90A-NEXT: .LBB14_1: ; %atomicrmw.start
3352 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
3353 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3354 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
3355 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
3356 ; GFX90A-NEXT: buffer_wbl2
3357 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
3358 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3359 ; GFX90A-NEXT: buffer_invl2
3360 ; GFX90A-NEXT: buffer_wbinvl1
3361 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
3362 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3363 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
3364 ; GFX90A-NEXT: s_cbranch_execnz .LBB14_1
3365 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
3366 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
3367 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
3368 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
3369 ;
3370 ; GFX908-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos__ftz:
3371 ; GFX908: ; %bb.0:
3372 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3373 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
3374 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
3375 ; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
3376 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
3377 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3378 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
3379 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
3380 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
3381 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3382 ; GFX908-NEXT: buffer_wbinvl1
3383 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
3384 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3385 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
3386 ; GFX908-NEXT: s_cbranch_execnz .LBB14_1
3387 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
3388 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
3389 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
3390 ; GFX908-NEXT: s_setpc_b64 s[30:31]
3391 ;
3392 ; GFX8-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos__ftz:
3393 ; GFX8: ; %bb.0:
3394 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3395 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
3396 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
3397 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
3398 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
3399 ; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
3400 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3401 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3402 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
3403 ; GFX8-NEXT: v_sub_f32_e32 v0, v1, v2
3404 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
3405 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3406 ; GFX8-NEXT: buffer_wbinvl1
3407 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
3408 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3409 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
3410 ; GFX8-NEXT: s_cbranch_execnz .LBB14_1
3411 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
3412 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
3413 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3414 ;
3415 ; GFX7-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos__ftz:
3416 ; GFX7: ; %bb.0:
3417 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3418 ; GFX7-NEXT: s_mov_b32 s6, 0
3419 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
3420 ; GFX7-NEXT: s_mov_b32 s4, s6
3421 ; GFX7-NEXT: s_mov_b32 s5, s6
3422 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
3423 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
3424 ; GFX7-NEXT: .LBB14_1: ; %atomicrmw.start
3425 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
3426 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3427 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
3428 ; GFX7-NEXT: v_sub_f32_e32 v4, v5, v2
3429 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
3430 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
3431 ; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
3432 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3433 ; GFX7-NEXT: buffer_wbinvl1
3434 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
3435 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
3436 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
3437 ; GFX7-NEXT: s_cbranch_execnz .LBB14_1
3438 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
3439 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
3440 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
3441 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3442 ;
3443 ; GFX6-LABEL: global_system_atomic_fsub_ret_f32__offset12b_pos__ftz:
3444 ; GFX6: ; %bb.0:
3445 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3446 ; GFX6-NEXT: s_mov_b32 s6, 0
3447 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
3448 ; GFX6-NEXT: s_mov_b32 s4, s6
3449 ; GFX6-NEXT: s_mov_b32 s5, s6
3450 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2044
3451 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
3452 ; GFX6-NEXT: .LBB14_1: ; %atomicrmw.start
3453 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
3454 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3455 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
3456 ; GFX6-NEXT: s_waitcnt expcnt(0)
3457 ; GFX6-NEXT: v_sub_f32_e32 v4, v5, v2
3458 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
3459 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
3460 ; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:2044 glc
3461 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3462 ; GFX6-NEXT: buffer_wbinvl1
3463 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
3464 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
3465 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
3466 ; GFX6-NEXT: s_cbranch_execnz .LBB14_1
3467 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
3468 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
3469 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
3470 ; GFX6-NEXT: s_waitcnt expcnt(0)
3471 ; GFX6-NEXT: s_setpc_b64 s[30:31]
3472 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
3473 %result = atomicrmw fsub ptr addrspace(1) %gep, float %val seq_cst
3474 ret float %result
3475 }
3477 define void @global_system_atomic_fsub_noret_f32__offset12b_pos__ftz(ptr addrspace(1) %ptr, float %val) #1 {
3478 ; GFX12-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos__ftz:
3479 ; GFX12: ; %bb.0:
3480 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
3481 ; GFX12-NEXT: s_wait_expcnt 0x0
3482 ; GFX12-NEXT: s_wait_samplecnt 0x0
3483 ; GFX12-NEXT: s_wait_bvhcnt 0x0
3484 ; GFX12-NEXT: s_wait_kmcnt 0x0
3485 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:2044
3486 ; GFX12-NEXT: s_mov_b32 s0, 0
3487 ; GFX12-NEXT: .LBB15_1: ; %atomicrmw.start
3488 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
3489 ; GFX12-NEXT: s_wait_loadcnt 0x0
3490 ; GFX12-NEXT: v_sub_f32_e32 v3, v4, v2
3491 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
3492 ; GFX12-NEXT: s_wait_storecnt 0x0
3493 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
3494 ; GFX12-NEXT: s_wait_loadcnt 0x0
3495 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
3496 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3497 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
3498 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
3499 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3500 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3501 ; GFX12-NEXT: s_cbranch_execnz .LBB15_1
3502 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
3503 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
3504 ; GFX12-NEXT: s_setpc_b64 s[30:31]
3505 ;
3506 ; GFX940-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos__ftz:
3507 ; GFX940: ; %bb.0:
3508 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3509 ; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:2044
3510 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
3511 ; GFX940-NEXT: .LBB15_1: ; %atomicrmw.start
3512 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
3513 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3514 ; GFX940-NEXT: v_sub_f32_e32 v4, v5, v2
3515 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
3516 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
3517 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3518 ; GFX940-NEXT: buffer_inv sc0 sc1
3519 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
3520 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
3521 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
3522 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
3523 ; GFX940-NEXT: s_cbranch_execnz .LBB15_1
3524 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
3525 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
3526 ; GFX940-NEXT: s_setpc_b64 s[30:31]
3527 ;
3528 ; GFX11-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos__ftz:
3529 ; GFX11: ; %bb.0:
3530 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3531 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:2044
3532 ; GFX11-NEXT: s_mov_b32 s0, 0
3533 ; GFX11-NEXT: .LBB15_1: ; %atomicrmw.start
3534 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
3535 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3536 ; GFX11-NEXT: v_sub_f32_e32 v3, v4, v2
3537 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
3538 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
3539 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3540 ; GFX11-NEXT: buffer_gl1_inv
3541 ; GFX11-NEXT: buffer_gl0_inv
3542 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3543 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
3544 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
3545 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3546 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3547 ; GFX11-NEXT: s_cbranch_execnz .LBB15_1
3548 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
3549 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
3550 ; GFX11-NEXT: s_setpc_b64 s[30:31]
3551 ;
3552 ; GFX10-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos__ftz:
3553 ; GFX10: ; %bb.0:
3554 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3555 ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
3556 ; GFX10-NEXT: s_mov_b32 s4, 0
3557 ; GFX10-NEXT: .LBB15_1: ; %atomicrmw.start
3558 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
3559 ; GFX10-NEXT: s_waitcnt vmcnt(0)
3560 ; GFX10-NEXT: v_sub_f32_e32 v3, v4, v2
3561 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
3562 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
3563 ; GFX10-NEXT: s_waitcnt vmcnt(0)
3564 ; GFX10-NEXT: buffer_gl1_inv
3565 ; GFX10-NEXT: buffer_gl0_inv
3566 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3567 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
3568 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
3569 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
3570 ; GFX10-NEXT: s_cbranch_execnz .LBB15_1
3571 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
3572 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
3573 ; GFX10-NEXT: s_setpc_b64 s[30:31]
3574 ;
3575 ; GFX90A-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos__ftz:
3576 ; GFX90A: ; %bb.0:
3577 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3578 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
3579 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
3580 ; GFX90A-NEXT: .LBB15_1: ; %atomicrmw.start
3581 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
3582 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3583 ; GFX90A-NEXT: v_sub_f32_e32 v4, v5, v2
3584 ; GFX90A-NEXT: buffer_wbl2
3585 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
3586 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3587 ; GFX90A-NEXT: buffer_invl2
3588 ; GFX90A-NEXT: buffer_wbinvl1
3589 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
3590 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3591 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
3592 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
3593 ; GFX90A-NEXT: s_cbranch_execnz .LBB15_1
3594 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
3595 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
3596 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
3597 ;
3598 ; GFX908-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos__ftz:
3599 ; GFX908: ; %bb.0:
3600 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3601 ; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
3602 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
3603 ; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start
3604 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
3605 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3606 ; GFX908-NEXT: v_sub_f32_e32 v3, v4, v2
3607 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
3608 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3609 ; GFX908-NEXT: buffer_wbinvl1
3610 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
3611 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3612 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
3613 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
3614 ; GFX908-NEXT: s_cbranch_execnz .LBB15_1
3615 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
3616 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
3617 ; GFX908-NEXT: s_setpc_b64 s[30:31]
3618 ;
3619 ; GFX8-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos__ftz:
3620 ; GFX8: ; %bb.0:
3621 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3622 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
3623 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3624 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
3625 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
3626 ; GFX8-NEXT: .LBB15_1: ; %atomicrmw.start
3627 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3628 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3629 ; GFX8-NEXT: v_sub_f32_e32 v3, v4, v2
3630 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
3631 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3632 ; GFX8-NEXT: buffer_wbinvl1
3633 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
3634 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3635 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
3636 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
3637 ; GFX8-NEXT: s_cbranch_execnz .LBB15_1
3638 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
3639 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
3640 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3641 ;
3642 ; GFX7-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos__ftz:
3643 ; GFX7: ; %bb.0:
3644 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3645 ; GFX7-NEXT: s_mov_b32 s6, 0
3646 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
3647 ; GFX7-NEXT: s_mov_b32 s4, s6
3648 ; GFX7-NEXT: s_mov_b32 s5, s6
3649 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:2044
3650 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
3651 ; GFX7-NEXT: .LBB15_1: ; %atomicrmw.start
3652 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
3653 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3654 ; GFX7-NEXT: v_sub_f32_e32 v3, v4, v2
3655 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
3656 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
3657 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
3658 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3659 ; GFX7-NEXT: buffer_wbinvl1
3660 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
3661 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
3662 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
3663 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
3664 ; GFX7-NEXT: s_cbranch_execnz .LBB15_1
3665 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
3666 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
3667 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3668 ;
3669 ; GFX6-LABEL: global_system_atomic_fsub_noret_f32__offset12b_pos__ftz:
3670 ; GFX6: ; %bb.0:
3671 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3672 ; GFX6-NEXT: s_mov_b32 s6, 0
3673 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
3674 ; GFX6-NEXT: s_mov_b32 s4, s6
3675 ; GFX6-NEXT: s_mov_b32 s5, s6
3676 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:2044
3677 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
3678 ; GFX6-NEXT: .LBB15_1: ; %atomicrmw.start
3679 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
3680 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3681 ; GFX6-NEXT: v_sub_f32_e32 v3, v4, v2
3682 ; GFX6-NEXT: s_waitcnt expcnt(0)
3683 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
3684 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
3685 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2044 glc
3686 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3687 ; GFX6-NEXT: buffer_wbinvl1
3688 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
3689 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
3690 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
3691 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
3692 ; GFX6-NEXT: s_cbranch_execnz .LBB15_1
3693 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
3694 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
3695 ; GFX6-NEXT: s_waitcnt expcnt(0)
3696 ; GFX6-NEXT: s_setpc_b64 s[30:31]
3697 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
3698 %unused = atomicrmw fsub ptr addrspace(1) %gep, float %val seq_cst
3699 ret void
3700 }
3702 ; --------------------------------------------------------------------
3703 ; double
3704 ; --------------------------------------------------------------------
3706 define double @global_agent_atomic_fsub_ret_f64(ptr addrspace(1) %ptr, double %val) #0 {
3707 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f64:
3708 ; GFX12: ; %bb.0:
3709 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
3710 ; GFX12-NEXT: s_wait_expcnt 0x0
3711 ; GFX12-NEXT: s_wait_samplecnt 0x0
3712 ; GFX12-NEXT: s_wait_bvhcnt 0x0
3713 ; GFX12-NEXT: s_wait_kmcnt 0x0
3714 ; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
3715 ; GFX12-NEXT: s_mov_b32 s0, 0
3716 ; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start
3717 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
3718 ; GFX12-NEXT: s_wait_loadcnt 0x0
3719 ; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
3720 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
3721 ; GFX12-NEXT: v_add_f64_e64 v[4:5], v[6:7], -v[2:3]
3722 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
3723 ; GFX12-NEXT: s_wait_storecnt 0x0
3724 ; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
3725 ; GFX12-NEXT: s_wait_loadcnt 0x0
3726 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
3727 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
3728 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
3729 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3730 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3731 ; GFX12-NEXT: s_cbranch_execnz .LBB16_1
3732 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
3733 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
3734 ; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
3735 ; GFX12-NEXT: s_setpc_b64 s[30:31]
3736 ;
3737 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f64:
3738 ; GFX940: ; %bb.0:
3739 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3740 ; GFX940-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
3741 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
3742 ; GFX940-NEXT: .LBB16_1: ; %atomicrmw.start
3743 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
3744 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3745 ; GFX940-NEXT: v_mov_b64_e32 v[6:7], v[4:5]
3746 ; GFX940-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
3747 ; GFX940-NEXT: buffer_wbl2 sc1
3748 ; GFX940-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off sc0
3749 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3750 ; GFX940-NEXT: buffer_inv sc1
3751 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
3752 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
3753 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
3754 ; GFX940-NEXT: s_cbranch_execnz .LBB16_1
3755 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
3756 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
3757 ; GFX940-NEXT: v_mov_b32_e32 v0, v4
3758 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
3759 ; GFX940-NEXT: s_setpc_b64 s[30:31]
3760 ;
3761 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f64:
3762 ; GFX11: ; %bb.0:
3763 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3764 ; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
3765 ; GFX11-NEXT: s_mov_b32 s0, 0
3766 ; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start
3767 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
3768 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3769 ; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
3770 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
3771 ; GFX11-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
3772 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
3773 ; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
3774 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3775 ; GFX11-NEXT: buffer_gl1_inv
3776 ; GFX11-NEXT: buffer_gl0_inv
3777 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
3778 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
3779 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3780 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3781 ; GFX11-NEXT: s_cbranch_execnz .LBB16_1
3782 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
3783 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
3784 ; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
3785 ; GFX11-NEXT: s_setpc_b64 s[30:31]
3786 ;
3787 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f64:
3788 ; GFX10: ; %bb.0:
3789 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3790 ; GFX10-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
3791 ; GFX10-NEXT: s_mov_b32 s4, 0
3792 ; GFX10-NEXT: .LBB16_1: ; %atomicrmw.start
3793 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
3794 ; GFX10-NEXT: s_waitcnt vmcnt(0)
3795 ; GFX10-NEXT: v_mov_b32_e32 v7, v5
3796 ; GFX10-NEXT: v_mov_b32_e32 v6, v4
3797 ; GFX10-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
3798 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
3799 ; GFX10-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
3800 ; GFX10-NEXT: s_waitcnt vmcnt(0)
3801 ; GFX10-NEXT: buffer_gl1_inv
3802 ; GFX10-NEXT: buffer_gl0_inv
3803 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
3804 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
3805 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
3806 ; GFX10-NEXT: s_cbranch_execnz .LBB16_1
3807 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
3808 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
3809 ; GFX10-NEXT: v_mov_b32_e32 v0, v4
3810 ; GFX10-NEXT: v_mov_b32_e32 v1, v5
3811 ; GFX10-NEXT: s_setpc_b64 s[30:31]
3812 ;
3813 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f64:
3814 ; GFX90A: ; %bb.0:
3815 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3816 ; GFX90A-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
3817 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
3818 ; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
3819 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
3820 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3821 ; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1]
3822 ; GFX90A-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
3823 ; GFX90A-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
3824 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3825 ; GFX90A-NEXT: buffer_wbinvl1
3826 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
3827 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3828 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
3829 ; GFX90A-NEXT: s_cbranch_execnz .LBB16_1
3830 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
3831 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
3832 ; GFX90A-NEXT: v_mov_b32_e32 v0, v4
3833 ; GFX90A-NEXT: v_mov_b32_e32 v1, v5
3834 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
3835 ;
3836 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f64:
3837 ; GFX908: ; %bb.0:
3838 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3839 ; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
3840 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
3841 ; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
3842 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
3843 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3844 ; GFX908-NEXT: v_mov_b32_e32 v7, v5
3845 ; GFX908-NEXT: v_mov_b32_e32 v6, v4
3846 ; GFX908-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
3847 ; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
3848 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3849 ; GFX908-NEXT: buffer_wbinvl1
3850 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
3851 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3852 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
3853 ; GFX908-NEXT: s_cbranch_execnz .LBB16_1
3854 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
3855 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
3856 ; GFX908-NEXT: v_mov_b32_e32 v0, v4
3857 ; GFX908-NEXT: v_mov_b32_e32 v1, v5
3858 ; GFX908-NEXT: s_setpc_b64 s[30:31]
3859 ;
3860 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f64:
3861 ; GFX8: ; %bb.0:
3862 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3863 ; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
3864 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
3865 ; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start
3866 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3867 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3868 ; GFX8-NEXT: v_mov_b32_e32 v7, v5
3869 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
3870 ; GFX8-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
3871 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
3872 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3873 ; GFX8-NEXT: buffer_wbinvl1
3874 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
3875 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3876 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
3877 ; GFX8-NEXT: s_cbranch_execnz .LBB16_1
3878 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
3879 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
3880 ; GFX8-NEXT: v_mov_b32_e32 v0, v4
3881 ; GFX8-NEXT: v_mov_b32_e32 v1, v5
3882 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3883 ;
3884 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f64:
3885 ; GFX7: ; %bb.0:
3886 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3887 ; GFX7-NEXT: s_mov_b32 s6, 0
3888 ; GFX7-NEXT: v_mov_b32_e32 v7, v1
3889 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
3890 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
3891 ; GFX7-NEXT: s_mov_b32 s4, s6
3892 ; GFX7-NEXT: s_mov_b32 s5, s6
3893 ; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64
3894 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
3895 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
3896 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
3897 ; GFX7-NEXT: .LBB16_1: ; %atomicrmw.start
3898 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
3899 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3900 ; GFX7-NEXT: v_mov_b32_e32 v11, v1
3901 ; GFX7-NEXT: v_mov_b32_e32 v10, v0
3902 ; GFX7-NEXT: v_add_f64 v[8:9], v[10:11], -v[4:5]
3903 ; GFX7-NEXT: v_mov_b32_e32 v0, v8
3904 ; GFX7-NEXT: v_mov_b32_e32 v1, v9
3905 ; GFX7-NEXT: v_mov_b32_e32 v2, v10
3906 ; GFX7-NEXT: v_mov_b32_e32 v3, v11
3907 ; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 glc
3908 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3909 ; GFX7-NEXT: buffer_wbinvl1
3910 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
3911 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
3912 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
3913 ; GFX7-NEXT: s_cbranch_execnz .LBB16_1
3914 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
3915 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
3916 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3917 ;
3918 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f64:
3919 ; GFX6: ; %bb.0:
3920 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3921 ; GFX6-NEXT: s_mov_b32 s6, 0
3922 ; GFX6-NEXT: v_mov_b32_e32 v7, v1
3923 ; GFX6-NEXT: v_mov_b32_e32 v6, v0
3924 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
3925 ; GFX6-NEXT: s_mov_b32 s4, s6
3926 ; GFX6-NEXT: s_mov_b32 s5, s6
3927 ; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64
3928 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
3929 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
3930 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
3931 ; GFX6-NEXT: .LBB16_1: ; %atomicrmw.start
3932 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
3933 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3934 ; GFX6-NEXT: v_mov_b32_e32 v11, v1
3935 ; GFX6-NEXT: v_mov_b32_e32 v10, v0
3936 ; GFX6-NEXT: v_add_f64 v[8:9], v[10:11], -v[4:5]
3937 ; GFX6-NEXT: s_waitcnt expcnt(0)
3938 ; GFX6-NEXT: v_mov_b32_e32 v0, v8
3939 ; GFX6-NEXT: v_mov_b32_e32 v1, v9
3940 ; GFX6-NEXT: v_mov_b32_e32 v2, v10
3941 ; GFX6-NEXT: v_mov_b32_e32 v3, v11
3942 ; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 glc
3943 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3944 ; GFX6-NEXT: buffer_wbinvl1
3945 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
3946 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
3947 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
3948 ; GFX6-NEXT: s_cbranch_execnz .LBB16_1
3949 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
3950 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
3951 ; GFX6-NEXT: s_waitcnt expcnt(0)
3952 ; GFX6-NEXT: s_setpc_b64 s[30:31]
3953 %result = atomicrmw fsub ptr addrspace(1) %ptr, double %val syncscope("agent") seq_cst
3954 ret double %result
3955 }
3957 define double @global_agent_atomic_fsub_ret_f64__offset12b_pos(ptr addrspace(1) %ptr, double %val) #0 {
3958 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_pos:
3959 ; GFX12: ; %bb.0:
3960 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
3961 ; GFX12-NEXT: s_wait_expcnt 0x0
3962 ; GFX12-NEXT: s_wait_samplecnt 0x0
3963 ; GFX12-NEXT: s_wait_bvhcnt 0x0
3964 ; GFX12-NEXT: s_wait_kmcnt 0x0
3965 ; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
3966 ; GFX12-NEXT: s_mov_b32 s0, 0
3967 ; GFX12-NEXT: .LBB17_1: ; %atomicrmw.start
3968 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
3969 ; GFX12-NEXT: s_wait_loadcnt 0x0
3970 ; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
3971 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
3972 ; GFX12-NEXT: v_add_f64_e64 v[4:5], v[6:7], -v[2:3]
3973 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
3974 ; GFX12-NEXT: s_wait_storecnt 0x0
3975 ; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
3976 ; GFX12-NEXT: s_wait_loadcnt 0x0
3977 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
3978 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
3979 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
3980 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3981 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3982 ; GFX12-NEXT: s_cbranch_execnz .LBB17_1
3983 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
3984 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
3985 ; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
3986 ; GFX12-NEXT: s_setpc_b64 s[30:31]
3987 ;
3988 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_pos:
3989 ; GFX940: ; %bb.0:
3990 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3991 ; GFX940-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:2040
3992 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
3993 ; GFX940-NEXT: .LBB17_1: ; %atomicrmw.start
3994 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
3995 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3996 ; GFX940-NEXT: v_mov_b64_e32 v[6:7], v[4:5]
3997 ; GFX940-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
3998 ; GFX940-NEXT: buffer_wbl2 sc1
3999 ; GFX940-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 sc0
4000 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4001 ; GFX940-NEXT: buffer_inv sc1
4002 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4003 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4004 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
4005 ; GFX940-NEXT: s_cbranch_execnz .LBB17_1
4006 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
4007 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
4008 ; GFX940-NEXT: v_mov_b32_e32 v0, v4
4009 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
4010 ; GFX940-NEXT: s_setpc_b64 s[30:31]
4011 ;
4012 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_pos:
4013 ; GFX11: ; %bb.0:
4014 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4015 ; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
4016 ; GFX11-NEXT: s_mov_b32 s0, 0
4017 ; GFX11-NEXT: .LBB17_1: ; %atomicrmw.start
4018 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
4019 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4020 ; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
4021 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
4022 ; GFX11-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4023 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
4024 ; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 glc
4025 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4026 ; GFX11-NEXT: buffer_gl1_inv
4027 ; GFX11-NEXT: buffer_gl0_inv
4028 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4029 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
4030 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4031 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4032 ; GFX11-NEXT: s_cbranch_execnz .LBB17_1
4033 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
4034 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
4035 ; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
4036 ; GFX11-NEXT: s_setpc_b64 s[30:31]
4037 ;
4038 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_pos:
4039 ; GFX10: ; %bb.0:
4040 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4041 ; GFX10-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:2040
4042 ; GFX10-NEXT: s_mov_b32 s4, 0
4043 ; GFX10-NEXT: .LBB17_1: ; %atomicrmw.start
4044 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
4045 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4046 ; GFX10-NEXT: v_mov_b32_e32 v7, v5
4047 ; GFX10-NEXT: v_mov_b32_e32 v6, v4
4048 ; GFX10-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4049 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
4050 ; GFX10-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 glc
4051 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4052 ; GFX10-NEXT: buffer_gl1_inv
4053 ; GFX10-NEXT: buffer_gl0_inv
4054 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4055 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
4056 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
4057 ; GFX10-NEXT: s_cbranch_execnz .LBB17_1
4058 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
4059 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
4060 ; GFX10-NEXT: v_mov_b32_e32 v0, v4
4061 ; GFX10-NEXT: v_mov_b32_e32 v1, v5
4062 ; GFX10-NEXT: s_setpc_b64 s[30:31]
4063 ;
4064 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_pos:
4065 ; GFX90A: ; %bb.0:
4066 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4067 ; GFX90A-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:2040
4068 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
4069 ; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
4070 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
4071 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4072 ; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1]
4073 ; GFX90A-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4074 ; GFX90A-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 glc
4075 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4076 ; GFX90A-NEXT: buffer_wbinvl1
4077 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4078 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4079 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
4080 ; GFX90A-NEXT: s_cbranch_execnz .LBB17_1
4081 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
4082 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
4083 ; GFX90A-NEXT: v_mov_b32_e32 v0, v4
4084 ; GFX90A-NEXT: v_mov_b32_e32 v1, v5
4085 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
4086 ;
4087 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_pos:
4088 ; GFX908: ; %bb.0:
4089 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4090 ; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:2040
4091 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
4092 ; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
4093 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
4094 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4095 ; GFX908-NEXT: v_mov_b32_e32 v7, v5
4096 ; GFX908-NEXT: v_mov_b32_e32 v6, v4
4097 ; GFX908-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4098 ; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 glc
4099 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4100 ; GFX908-NEXT: buffer_wbinvl1
4101 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4102 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4103 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
4104 ; GFX908-NEXT: s_cbranch_execnz .LBB17_1
4105 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
4106 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
4107 ; GFX908-NEXT: v_mov_b32_e32 v0, v4
4108 ; GFX908-NEXT: v_mov_b32_e32 v1, v5
4109 ; GFX908-NEXT: s_setpc_b64 s[30:31]
4110 ;
4111 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_pos:
4112 ; GFX8: ; %bb.0:
4113 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4114 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7f8, v0
4115 ; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
4116 ; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
4117 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
4118 ; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
4119 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4120 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4121 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
4122 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
4123 ; GFX8-NEXT: v_add_f64 v[6:7], v[8:9], -v[2:3]
4124 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
4125 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4126 ; GFX8-NEXT: buffer_wbinvl1
4127 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
4128 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4129 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
4130 ; GFX8-NEXT: s_cbranch_execnz .LBB17_1
4131 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4132 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
4133 ; GFX8-NEXT: s_setpc_b64 s[30:31]
4134 ;
4135 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_pos:
4136 ; GFX7: ; %bb.0:
4137 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4138 ; GFX7-NEXT: s_mov_b32 s6, 0
4139 ; GFX7-NEXT: v_mov_b32_e32 v7, v1
4140 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
4141 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
4142 ; GFX7-NEXT: s_mov_b32 s4, s6
4143 ; GFX7-NEXT: s_mov_b32 s5, s6
4144 ; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64 offset:2040
4145 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
4146 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
4147 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
4148 ; GFX7-NEXT: .LBB17_1: ; %atomicrmw.start
4149 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4150 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4151 ; GFX7-NEXT: v_mov_b32_e32 v11, v1
4152 ; GFX7-NEXT: v_mov_b32_e32 v10, v0
4153 ; GFX7-NEXT: v_add_f64 v[8:9], v[10:11], -v[4:5]
4154 ; GFX7-NEXT: v_mov_b32_e32 v0, v8
4155 ; GFX7-NEXT: v_mov_b32_e32 v1, v9
4156 ; GFX7-NEXT: v_mov_b32_e32 v2, v10
4157 ; GFX7-NEXT: v_mov_b32_e32 v3, v11
4158 ; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 offset:2040 glc
4159 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4160 ; GFX7-NEXT: buffer_wbinvl1
4161 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
4162 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
4163 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
4164 ; GFX7-NEXT: s_cbranch_execnz .LBB17_1
4165 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4166 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
4167 ; GFX7-NEXT: s_setpc_b64 s[30:31]
4168 ;
4169 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_pos:
4170 ; GFX6: ; %bb.0:
4171 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4172 ; GFX6-NEXT: s_mov_b32 s6, 0
4173 ; GFX6-NEXT: v_mov_b32_e32 v7, v1
4174 ; GFX6-NEXT: v_mov_b32_e32 v6, v0
4175 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
4176 ; GFX6-NEXT: s_mov_b32 s4, s6
4177 ; GFX6-NEXT: s_mov_b32 s5, s6
4178 ; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64 offset:2040
4179 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
4180 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
4181 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
4182 ; GFX6-NEXT: .LBB17_1: ; %atomicrmw.start
4183 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
4184 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4185 ; GFX6-NEXT: v_mov_b32_e32 v11, v1
4186 ; GFX6-NEXT: v_mov_b32_e32 v10, v0
4187 ; GFX6-NEXT: v_add_f64 v[8:9], v[10:11], -v[4:5]
4188 ; GFX6-NEXT: s_waitcnt expcnt(0)
4189 ; GFX6-NEXT: v_mov_b32_e32 v0, v8
4190 ; GFX6-NEXT: v_mov_b32_e32 v1, v9
4191 ; GFX6-NEXT: v_mov_b32_e32 v2, v10
4192 ; GFX6-NEXT: v_mov_b32_e32 v3, v11
4193 ; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 offset:2040 glc
4194 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4195 ; GFX6-NEXT: buffer_wbinvl1
4196 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
4197 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
4198 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
4199 ; GFX6-NEXT: s_cbranch_execnz .LBB17_1
4200 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
4201 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
4202 ; GFX6-NEXT: s_waitcnt expcnt(0)
4203 ; GFX6-NEXT: s_setpc_b64 s[30:31]
4204 %gep = getelementptr double, ptr addrspace(1) %ptr, i64 255
4205 %result = atomicrmw fsub ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst
4206 ret double %result
4207 }
4209 define double @global_agent_atomic_fsub_ret_f64__offset12b_neg(ptr addrspace(1) %ptr, double %val) #0 {
4210 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_neg:
4211 ; GFX12: ; %bb.0:
4212 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
4213 ; GFX12-NEXT: s_wait_expcnt 0x0
4214 ; GFX12-NEXT: s_wait_samplecnt 0x0
4215 ; GFX12-NEXT: s_wait_bvhcnt 0x0
4216 ; GFX12-NEXT: s_wait_kmcnt 0x0
4217 ; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
4218 ; GFX12-NEXT: s_mov_b32 s0, 0
4219 ; GFX12-NEXT: .LBB18_1: ; %atomicrmw.start
4220 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
4221 ; GFX12-NEXT: s_wait_loadcnt 0x0
4222 ; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
4223 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
4224 ; GFX12-NEXT: v_add_f64_e64 v[4:5], v[6:7], -v[2:3]
4225 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
4226 ; GFX12-NEXT: s_wait_storecnt 0x0
4227 ; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
4228 ; GFX12-NEXT: s_wait_loadcnt 0x0
4229 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
4230 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4231 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
4232 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4233 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4234 ; GFX12-NEXT: s_cbranch_execnz .LBB18_1
4235 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
4236 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
4237 ; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
4238 ; GFX12-NEXT: s_setpc_b64 s[30:31]
4239 ;
4240 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_neg:
4241 ; GFX940: ; %bb.0:
4242 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4243 ; GFX940-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:-2048
4244 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
4245 ; GFX940-NEXT: .LBB18_1: ; %atomicrmw.start
4246 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
4247 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4248 ; GFX940-NEXT: v_mov_b64_e32 v[6:7], v[4:5]
4249 ; GFX940-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4250 ; GFX940-NEXT: buffer_wbl2 sc1
4251 ; GFX940-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 sc0
4252 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4253 ; GFX940-NEXT: buffer_inv sc1
4254 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4255 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4256 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
4257 ; GFX940-NEXT: s_cbranch_execnz .LBB18_1
4258 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
4259 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
4260 ; GFX940-NEXT: v_mov_b32_e32 v0, v4
4261 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
4262 ; GFX940-NEXT: s_setpc_b64 s[30:31]
4263 ;
4264 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_neg:
4265 ; GFX11: ; %bb.0:
4266 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4267 ; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
4268 ; GFX11-NEXT: s_mov_b32 s0, 0
4269 ; GFX11-NEXT: .LBB18_1: ; %atomicrmw.start
4270 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
4271 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4272 ; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
4273 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
4274 ; GFX11-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4275 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
4276 ; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
4277 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4278 ; GFX11-NEXT: buffer_gl1_inv
4279 ; GFX11-NEXT: buffer_gl0_inv
4280 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4281 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
4282 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4283 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4284 ; GFX11-NEXT: s_cbranch_execnz .LBB18_1
4285 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
4286 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
4287 ; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
4288 ; GFX11-NEXT: s_setpc_b64 s[30:31]
4289 ;
4290 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_neg:
4291 ; GFX10: ; %bb.0:
4292 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4293 ; GFX10-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:-2048
4294 ; GFX10-NEXT: s_mov_b32 s4, 0
4295 ; GFX10-NEXT: .LBB18_1: ; %atomicrmw.start
4296 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
4297 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4298 ; GFX10-NEXT: v_mov_b32_e32 v7, v5
4299 ; GFX10-NEXT: v_mov_b32_e32 v6, v4
4300 ; GFX10-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4301 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
4302 ; GFX10-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
4303 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4304 ; GFX10-NEXT: buffer_gl1_inv
4305 ; GFX10-NEXT: buffer_gl0_inv
4306 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4307 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
4308 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
4309 ; GFX10-NEXT: s_cbranch_execnz .LBB18_1
4310 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
4311 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
4312 ; GFX10-NEXT: v_mov_b32_e32 v0, v4
4313 ; GFX10-NEXT: v_mov_b32_e32 v1, v5
4314 ; GFX10-NEXT: s_setpc_b64 s[30:31]
4315 ;
4316 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_neg:
4317 ; GFX90A: ; %bb.0:
4318 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4319 ; GFX90A-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:-2048
4320 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
4321 ; GFX90A-NEXT: .LBB18_1: ; %atomicrmw.start
4322 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
4323 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4324 ; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1]
4325 ; GFX90A-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4326 ; GFX90A-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
4327 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4328 ; GFX90A-NEXT: buffer_wbinvl1
4329 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4330 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4331 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
4332 ; GFX90A-NEXT: s_cbranch_execnz .LBB18_1
4333 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
4334 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
4335 ; GFX90A-NEXT: v_mov_b32_e32 v0, v4
4336 ; GFX90A-NEXT: v_mov_b32_e32 v1, v5
4337 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
4338 ;
4339 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_neg:
4340 ; GFX908: ; %bb.0:
4341 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4342 ; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:-2048
4343 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
4344 ; GFX908-NEXT: .LBB18_1: ; %atomicrmw.start
4345 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
4346 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4347 ; GFX908-NEXT: v_mov_b32_e32 v7, v5
4348 ; GFX908-NEXT: v_mov_b32_e32 v6, v4
4349 ; GFX908-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4350 ; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
4351 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4352 ; GFX908-NEXT: buffer_wbinvl1
4353 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4354 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4355 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
4356 ; GFX908-NEXT: s_cbranch_execnz .LBB18_1
4357 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
4358 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
4359 ; GFX908-NEXT: v_mov_b32_e32 v0, v4
4360 ; GFX908-NEXT: v_mov_b32_e32 v1, v5
4361 ; GFX908-NEXT: s_setpc_b64 s[30:31]
4362 ;
4363 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_neg:
4364 ; GFX8: ; %bb.0:
4365 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4366 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
4367 ; GFX8-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
4368 ; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
4369 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
4370 ; GFX8-NEXT: .LBB18_1: ; %atomicrmw.start
4371 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4372 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4373 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
4374 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
4375 ; GFX8-NEXT: v_add_f64 v[6:7], v[8:9], -v[2:3]
4376 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
4377 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4378 ; GFX8-NEXT: buffer_wbinvl1
4379 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
4380 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4381 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
4382 ; GFX8-NEXT: s_cbranch_execnz .LBB18_1
4383 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4384 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
4385 ; GFX8-NEXT: s_setpc_b64 s[30:31]
4386 ;
4387 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_neg:
4388 ; GFX7: ; %bb.0:
4389 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4390 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
4391 ; GFX7-NEXT: v_mov_b32_e32 v7, v1
4392 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
4393 ; GFX7-NEXT: s_mov_b32 s5, -1
4394 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
4395 ; GFX7-NEXT: s_mov_b32 s6, 0
4396 ; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64
4397 ; GFX7-NEXT: v_add_i32_e32 v6, vcc, 0xfffff800, v6
4398 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
4399 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
4400 ; GFX7-NEXT: v_addc_u32_e32 v7, vcc, -1, v7, vcc
4401 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
4402 ; GFX7-NEXT: s_mov_b32 s4, s6
4403 ; GFX7-NEXT: s_mov_b32 s5, s6
4404 ; GFX7-NEXT: .LBB18_1: ; %atomicrmw.start
4405 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4406 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4407 ; GFX7-NEXT: v_mov_b32_e32 v11, v1
4408 ; GFX7-NEXT: v_mov_b32_e32 v10, v0
4409 ; GFX7-NEXT: v_add_f64 v[8:9], v[10:11], -v[4:5]
4410 ; GFX7-NEXT: v_mov_b32_e32 v0, v8
4411 ; GFX7-NEXT: v_mov_b32_e32 v1, v9
4412 ; GFX7-NEXT: v_mov_b32_e32 v2, v10
4413 ; GFX7-NEXT: v_mov_b32_e32 v3, v11
4414 ; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 glc
4415 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4416 ; GFX7-NEXT: buffer_wbinvl1
4417 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
4418 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
4419 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
4420 ; GFX7-NEXT: s_cbranch_execnz .LBB18_1
4421 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4422 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
4423 ; GFX7-NEXT: s_setpc_b64 s[30:31]
4424 ;
4425 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f64__offset12b_neg:
4426 ; GFX6: ; %bb.0:
4427 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4428 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
4429 ; GFX6-NEXT: v_mov_b32_e32 v7, v1
4430 ; GFX6-NEXT: v_mov_b32_e32 v6, v0
4431 ; GFX6-NEXT: s_mov_b32 s5, -1
4432 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
4433 ; GFX6-NEXT: s_mov_b32 s6, 0
4434 ; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64
4435 ; GFX6-NEXT: v_add_i32_e32 v6, vcc, 0xfffff800, v6
4436 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
4437 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
4438 ; GFX6-NEXT: v_addc_u32_e32 v7, vcc, -1, v7, vcc
4439 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
4440 ; GFX6-NEXT: s_mov_b32 s4, s6
4441 ; GFX6-NEXT: s_mov_b32 s5, s6
4442 ; GFX6-NEXT: .LBB18_1: ; %atomicrmw.start
4443 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
4444 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4445 ; GFX6-NEXT: v_mov_b32_e32 v11, v1
4446 ; GFX6-NEXT: v_mov_b32_e32 v10, v0
4447 ; GFX6-NEXT: v_add_f64 v[8:9], v[10:11], -v[4:5]
4448 ; GFX6-NEXT: s_waitcnt expcnt(0)
4449 ; GFX6-NEXT: v_mov_b32_e32 v0, v8
4450 ; GFX6-NEXT: v_mov_b32_e32 v1, v9
4451 ; GFX6-NEXT: v_mov_b32_e32 v2, v10
4452 ; GFX6-NEXT: v_mov_b32_e32 v3, v11
4453 ; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 glc
4454 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4455 ; GFX6-NEXT: buffer_wbinvl1
4456 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
4457 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
4458 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
4459 ; GFX6-NEXT: s_cbranch_execnz .LBB18_1
4460 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
4461 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
4462 ; GFX6-NEXT: s_waitcnt expcnt(0)
4463 ; GFX6-NEXT: s_setpc_b64 s[30:31]
4464 %gep = getelementptr double, ptr addrspace(1) %ptr, i64 -256
4465 %result = atomicrmw fsub ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst
4466 ret double %result
4467 }
4469 define void @global_agent_atomic_fsub_noret_f64(ptr addrspace(1) %ptr, double %val) #0 {
4470 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f64:
4471 ; GFX12: ; %bb.0:
4472 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
4473 ; GFX12-NEXT: s_wait_expcnt 0x0
4474 ; GFX12-NEXT: s_wait_samplecnt 0x0
4475 ; GFX12-NEXT: s_wait_bvhcnt 0x0
4476 ; GFX12-NEXT: s_wait_kmcnt 0x0
4477 ; GFX12-NEXT: global_load_b64 v[6:7], v[0:1], off
4478 ; GFX12-NEXT: s_mov_b32 s0, 0
4479 ; GFX12-NEXT: .LBB19_1: ; %atomicrmw.start
4480 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
4481 ; GFX12-NEXT: s_wait_loadcnt 0x0
4482 ; GFX12-NEXT: v_add_f64_e64 v[4:5], v[6:7], -v[2:3]
4483 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
4484 ; GFX12-NEXT: s_wait_storecnt 0x0
4485 ; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
4486 ; GFX12-NEXT: s_wait_loadcnt 0x0
4487 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
4488 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4489 ; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
4490 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
4491 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4492 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4493 ; GFX12-NEXT: s_cbranch_execnz .LBB19_1
4494 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
4495 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
4496 ; GFX12-NEXT: s_setpc_b64 s[30:31]
4497 ;
4498 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f64:
4499 ; GFX940: ; %bb.0:
4500 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4501 ; GFX940-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
4502 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
4503 ; GFX940-NEXT: .LBB19_1: ; %atomicrmw.start
4504 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
4505 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4506 ; GFX940-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4507 ; GFX940-NEXT: buffer_wbl2 sc1
4508 ; GFX940-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off sc0
4509 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4510 ; GFX940-NEXT: buffer_inv sc1
4511 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4512 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4513 ; GFX940-NEXT: v_mov_b64_e32 v[6:7], v[4:5]
4514 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
4515 ; GFX940-NEXT: s_cbranch_execnz .LBB19_1
4516 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
4517 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
4518 ; GFX940-NEXT: s_setpc_b64 s[30:31]
4519 ;
4520 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f64:
4521 ; GFX11: ; %bb.0:
4522 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4523 ; GFX11-NEXT: global_load_b64 v[6:7], v[0:1], off
4524 ; GFX11-NEXT: s_mov_b32 s0, 0
4525 ; GFX11-NEXT: .LBB19_1: ; %atomicrmw.start
4526 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
4527 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4528 ; GFX11-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4529 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
4530 ; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
4531 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4532 ; GFX11-NEXT: buffer_gl1_inv
4533 ; GFX11-NEXT: buffer_gl0_inv
4534 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4535 ; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
4536 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
4537 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4538 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4539 ; GFX11-NEXT: s_cbranch_execnz .LBB19_1
4540 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
4541 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
4542 ; GFX11-NEXT: s_setpc_b64 s[30:31]
4543 ;
4544 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f64:
4545 ; GFX10: ; %bb.0:
4546 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4547 ; GFX10-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
4548 ; GFX10-NEXT: s_mov_b32 s4, 0
4549 ; GFX10-NEXT: .LBB19_1: ; %atomicrmw.start
4550 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
4551 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4552 ; GFX10-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4553 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
4554 ; GFX10-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
4555 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4556 ; GFX10-NEXT: buffer_gl1_inv
4557 ; GFX10-NEXT: buffer_gl0_inv
4558 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4559 ; GFX10-NEXT: v_mov_b32_e32 v7, v5
4560 ; GFX10-NEXT: v_mov_b32_e32 v6, v4
4561 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
4562 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
4563 ; GFX10-NEXT: s_cbranch_execnz .LBB19_1
4564 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
4565 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
4566 ; GFX10-NEXT: s_setpc_b64 s[30:31]
4567 ;
4568 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f64:
4569 ; GFX90A: ; %bb.0:
4570 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4571 ; GFX90A-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
4572 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
4573 ; GFX90A-NEXT: .LBB19_1: ; %atomicrmw.start
4574 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
4575 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4576 ; GFX90A-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4577 ; GFX90A-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
4578 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4579 ; GFX90A-NEXT: buffer_wbinvl1
4580 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4581 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4582 ; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1]
4583 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
4584 ; GFX90A-NEXT: s_cbranch_execnz .LBB19_1
4585 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
4586 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
4587 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
4588 ;
4589 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f64:
4590 ; GFX908: ; %bb.0:
4591 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4592 ; GFX908-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
4593 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
4594 ; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
4595 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
4596 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4597 ; GFX908-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4598 ; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
4599 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4600 ; GFX908-NEXT: buffer_wbinvl1
4601 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4602 ; GFX908-NEXT: v_mov_b32_e32 v7, v5
4603 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4604 ; GFX908-NEXT: v_mov_b32_e32 v6, v4
4605 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
4606 ; GFX908-NEXT: s_cbranch_execnz .LBB19_1
4607 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
4608 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
4609 ; GFX908-NEXT: s_setpc_b64 s[30:31]
4610 ;
4611 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f64:
4612 ; GFX8: ; %bb.0:
4613 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4614 ; GFX8-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
4615 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
4616 ; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
4617 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4618 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4619 ; GFX8-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4620 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
4621 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4622 ; GFX8-NEXT: buffer_wbinvl1
4623 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4624 ; GFX8-NEXT: v_mov_b32_e32 v7, v5
4625 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4626 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
4627 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
4628 ; GFX8-NEXT: s_cbranch_execnz .LBB19_1
4629 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4630 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
4631 ; GFX8-NEXT: s_setpc_b64 s[30:31]
4632 ;
4633 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f64:
4634 ; GFX7: ; %bb.0:
4635 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4636 ; GFX7-NEXT: s_mov_b32 s6, 0
4637 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
4638 ; GFX7-NEXT: s_mov_b32 s4, s6
4639 ; GFX7-NEXT: s_mov_b32 s5, s6
4640 ; GFX7-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64
4641 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
4642 ; GFX7-NEXT: .LBB19_1: ; %atomicrmw.start
4643 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4644 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4645 ; GFX7-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4646 ; GFX7-NEXT: v_mov_b32_e32 v11, v7
4647 ; GFX7-NEXT: v_mov_b32_e32 v10, v6
4648 ; GFX7-NEXT: v_mov_b32_e32 v9, v5
4649 ; GFX7-NEXT: v_mov_b32_e32 v8, v4
4650 ; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 glc
4651 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4652 ; GFX7-NEXT: buffer_wbinvl1
4653 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
4654 ; GFX7-NEXT: v_mov_b32_e32 v6, v8
4655 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
4656 ; GFX7-NEXT: v_mov_b32_e32 v7, v9
4657 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
4658 ; GFX7-NEXT: s_cbranch_execnz .LBB19_1
4659 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4660 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
4661 ; GFX7-NEXT: s_setpc_b64 s[30:31]
4662 ;
4663 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f64:
4664 ; GFX6: ; %bb.0:
4665 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4666 ; GFX6-NEXT: s_mov_b32 s6, 0
4667 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
4668 ; GFX6-NEXT: s_mov_b32 s4, s6
4669 ; GFX6-NEXT: s_mov_b32 s5, s6
4670 ; GFX6-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64
4671 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
4672 ; GFX6-NEXT: .LBB19_1: ; %atomicrmw.start
4673 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
4674 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4675 ; GFX6-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4676 ; GFX6-NEXT: s_waitcnt expcnt(0)
4677 ; GFX6-NEXT: v_mov_b32_e32 v11, v7
4678 ; GFX6-NEXT: v_mov_b32_e32 v10, v6
4679 ; GFX6-NEXT: v_mov_b32_e32 v9, v5
4680 ; GFX6-NEXT: v_mov_b32_e32 v8, v4
4681 ; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 glc
4682 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4683 ; GFX6-NEXT: buffer_wbinvl1
4684 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
4685 ; GFX6-NEXT: v_mov_b32_e32 v6, v8
4686 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
4687 ; GFX6-NEXT: v_mov_b32_e32 v7, v9
4688 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
4689 ; GFX6-NEXT: s_cbranch_execnz .LBB19_1
4690 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
4691 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
4692 ; GFX6-NEXT: s_waitcnt expcnt(0)
4693 ; GFX6-NEXT: s_setpc_b64 s[30:31]
4694 %unused = atomicrmw fsub ptr addrspace(1) %ptr, double %val syncscope("agent") seq_cst
4695 ret void
4696 }
4698 define void @global_agent_atomic_fsub_noret_f64__offset12b_pos(ptr addrspace(1) %ptr, double %val) #0 {
4699 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_pos:
4700 ; GFX12: ; %bb.0:
4701 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
4702 ; GFX12-NEXT: s_wait_expcnt 0x0
4703 ; GFX12-NEXT: s_wait_samplecnt 0x0
4704 ; GFX12-NEXT: s_wait_bvhcnt 0x0
4705 ; GFX12-NEXT: s_wait_kmcnt 0x0
4706 ; GFX12-NEXT: global_load_b64 v[6:7], v[0:1], off offset:2040
4707 ; GFX12-NEXT: s_mov_b32 s0, 0
4708 ; GFX12-NEXT: .LBB20_1: ; %atomicrmw.start
4709 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
4710 ; GFX12-NEXT: s_wait_loadcnt 0x0
4711 ; GFX12-NEXT: v_add_f64_e64 v[4:5], v[6:7], -v[2:3]
4712 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
4713 ; GFX12-NEXT: s_wait_storecnt 0x0
4714 ; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
4715 ; GFX12-NEXT: s_wait_loadcnt 0x0
4716 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
4717 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4718 ; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
4719 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
4720 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4721 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4722 ; GFX12-NEXT: s_cbranch_execnz .LBB20_1
4723 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
4724 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
4725 ; GFX12-NEXT: s_setpc_b64 s[30:31]
4726 ;
4727 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_pos:
4728 ; GFX940: ; %bb.0:
4729 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4730 ; GFX940-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:2040
4731 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
4732 ; GFX940-NEXT: .LBB20_1: ; %atomicrmw.start
4733 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
4734 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4735 ; GFX940-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4736 ; GFX940-NEXT: buffer_wbl2 sc1
4737 ; GFX940-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 sc0
4738 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4739 ; GFX940-NEXT: buffer_inv sc1
4740 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4741 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4742 ; GFX940-NEXT: v_mov_b64_e32 v[6:7], v[4:5]
4743 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
4744 ; GFX940-NEXT: s_cbranch_execnz .LBB20_1
4745 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
4746 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
4747 ; GFX940-NEXT: s_setpc_b64 s[30:31]
4748 ;
4749 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_pos:
4750 ; GFX11: ; %bb.0:
4751 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4752 ; GFX11-NEXT: global_load_b64 v[6:7], v[0:1], off offset:2040
4753 ; GFX11-NEXT: s_mov_b32 s0, 0
4754 ; GFX11-NEXT: .LBB20_1: ; %atomicrmw.start
4755 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
4756 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4757 ; GFX11-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4758 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
4759 ; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 glc
4760 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4761 ; GFX11-NEXT: buffer_gl1_inv
4762 ; GFX11-NEXT: buffer_gl0_inv
4763 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4764 ; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
4765 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
4766 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4767 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4768 ; GFX11-NEXT: s_cbranch_execnz .LBB20_1
4769 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
4770 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
4771 ; GFX11-NEXT: s_setpc_b64 s[30:31]
4772 ;
4773 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_pos:
4774 ; GFX10: ; %bb.0:
4775 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4776 ; GFX10-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:2040
4777 ; GFX10-NEXT: s_mov_b32 s4, 0
4778 ; GFX10-NEXT: .LBB20_1: ; %atomicrmw.start
4779 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
4780 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4781 ; GFX10-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4782 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
4783 ; GFX10-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 glc
4784 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4785 ; GFX10-NEXT: buffer_gl1_inv
4786 ; GFX10-NEXT: buffer_gl0_inv
4787 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4788 ; GFX10-NEXT: v_mov_b32_e32 v7, v5
4789 ; GFX10-NEXT: v_mov_b32_e32 v6, v4
4790 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
4791 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
4792 ; GFX10-NEXT: s_cbranch_execnz .LBB20_1
4793 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
4794 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
4795 ; GFX10-NEXT: s_setpc_b64 s[30:31]
4796 ;
4797 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_pos:
4798 ; GFX90A: ; %bb.0:
4799 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4800 ; GFX90A-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:2040
4801 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
4802 ; GFX90A-NEXT: .LBB20_1: ; %atomicrmw.start
4803 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
4804 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4805 ; GFX90A-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4806 ; GFX90A-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 glc
4807 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4808 ; GFX90A-NEXT: buffer_wbinvl1
4809 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4810 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4811 ; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1]
4812 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
4813 ; GFX90A-NEXT: s_cbranch_execnz .LBB20_1
4814 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
4815 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
4816 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
4817 ;
4818 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_pos:
4819 ; GFX908: ; %bb.0:
4820 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4821 ; GFX908-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:2040
4822 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
4823 ; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
4824 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
4825 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4826 ; GFX908-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4827 ; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 glc
4828 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4829 ; GFX908-NEXT: buffer_wbinvl1
4830 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4831 ; GFX908-NEXT: v_mov_b32_e32 v7, v5
4832 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4833 ; GFX908-NEXT: v_mov_b32_e32 v6, v4
4834 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
4835 ; GFX908-NEXT: s_cbranch_execnz .LBB20_1
4836 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
4837 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
4838 ; GFX908-NEXT: s_setpc_b64 s[30:31]
4839 ;
4840 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_pos:
4841 ; GFX8: ; %bb.0:
4842 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4843 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7f8, v0
4844 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
4845 ; GFX8-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
4846 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
4847 ; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
4848 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4849 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4850 ; GFX8-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4851 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
4852 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4853 ; GFX8-NEXT: buffer_wbinvl1
4854 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4855 ; GFX8-NEXT: v_mov_b32_e32 v7, v5
4856 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4857 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
4858 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
4859 ; GFX8-NEXT: s_cbranch_execnz .LBB20_1
4860 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4861 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
4862 ; GFX8-NEXT: s_setpc_b64 s[30:31]
4863 ;
4864 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_pos:
4865 ; GFX7: ; %bb.0:
4866 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4867 ; GFX7-NEXT: s_mov_b32 s6, 0
4868 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
4869 ; GFX7-NEXT: s_mov_b32 s4, s6
4870 ; GFX7-NEXT: s_mov_b32 s5, s6
4871 ; GFX7-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:2040
4872 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
4873 ; GFX7-NEXT: .LBB20_1: ; %atomicrmw.start
4874 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4875 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4876 ; GFX7-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4877 ; GFX7-NEXT: v_mov_b32_e32 v11, v7
4878 ; GFX7-NEXT: v_mov_b32_e32 v10, v6
4879 ; GFX7-NEXT: v_mov_b32_e32 v9, v5
4880 ; GFX7-NEXT: v_mov_b32_e32 v8, v4
4881 ; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:2040 glc
4882 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4883 ; GFX7-NEXT: buffer_wbinvl1
4884 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
4885 ; GFX7-NEXT: v_mov_b32_e32 v6, v8
4886 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
4887 ; GFX7-NEXT: v_mov_b32_e32 v7, v9
4888 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
4889 ; GFX7-NEXT: s_cbranch_execnz .LBB20_1
4890 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4891 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
4892 ; GFX7-NEXT: s_setpc_b64 s[30:31]
4893 ;
4894 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_pos:
4895 ; GFX6: ; %bb.0:
4896 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4897 ; GFX6-NEXT: s_mov_b32 s6, 0
4898 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
4899 ; GFX6-NEXT: s_mov_b32 s4, s6
4900 ; GFX6-NEXT: s_mov_b32 s5, s6
4901 ; GFX6-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:2040
4902 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
4903 ; GFX6-NEXT: .LBB20_1: ; %atomicrmw.start
4904 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
4905 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4906 ; GFX6-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4907 ; GFX6-NEXT: s_waitcnt expcnt(0)
4908 ; GFX6-NEXT: v_mov_b32_e32 v11, v7
4909 ; GFX6-NEXT: v_mov_b32_e32 v10, v6
4910 ; GFX6-NEXT: v_mov_b32_e32 v9, v5
4911 ; GFX6-NEXT: v_mov_b32_e32 v8, v4
4912 ; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:2040 glc
4913 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4914 ; GFX6-NEXT: buffer_wbinvl1
4915 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
4916 ; GFX6-NEXT: v_mov_b32_e32 v6, v8
4917 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
4918 ; GFX6-NEXT: v_mov_b32_e32 v7, v9
4919 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
4920 ; GFX6-NEXT: s_cbranch_execnz .LBB20_1
4921 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
4922 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
4923 ; GFX6-NEXT: s_waitcnt expcnt(0)
4924 ; GFX6-NEXT: s_setpc_b64 s[30:31]
4925 %gep = getelementptr double, ptr addrspace(1) %ptr, i64 255
4926 %unused = atomicrmw fsub ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst
4927 ret void
4928 }
4930 define void @global_agent_atomic_fsub_noret_f64__offset12b_neg(ptr addrspace(1) %ptr, double %val) #0 {
4931 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_neg:
4932 ; GFX12: ; %bb.0:
4933 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
4934 ; GFX12-NEXT: s_wait_expcnt 0x0
4935 ; GFX12-NEXT: s_wait_samplecnt 0x0
4936 ; GFX12-NEXT: s_wait_bvhcnt 0x0
4937 ; GFX12-NEXT: s_wait_kmcnt 0x0
4938 ; GFX12-NEXT: global_load_b64 v[6:7], v[0:1], off offset:-2048
4939 ; GFX12-NEXT: s_mov_b32 s0, 0
4940 ; GFX12-NEXT: .LBB21_1: ; %atomicrmw.start
4941 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
4942 ; GFX12-NEXT: s_wait_loadcnt 0x0
4943 ; GFX12-NEXT: v_add_f64_e64 v[4:5], v[6:7], -v[2:3]
4944 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
4945 ; GFX12-NEXT: s_wait_storecnt 0x0
4946 ; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
4947 ; GFX12-NEXT: s_wait_loadcnt 0x0
4948 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
4949 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4950 ; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
4951 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
4952 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4953 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4954 ; GFX12-NEXT: s_cbranch_execnz .LBB21_1
4955 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
4956 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
4957 ; GFX12-NEXT: s_setpc_b64 s[30:31]
4958 ;
4959 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_neg:
4960 ; GFX940: ; %bb.0:
4961 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4962 ; GFX940-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:-2048
4963 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
4964 ; GFX940-NEXT: .LBB21_1: ; %atomicrmw.start
4965 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
4966 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4967 ; GFX940-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4968 ; GFX940-NEXT: buffer_wbl2 sc1
4969 ; GFX940-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 sc0
4970 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4971 ; GFX940-NEXT: buffer_inv sc1
4972 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4973 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4974 ; GFX940-NEXT: v_mov_b64_e32 v[6:7], v[4:5]
4975 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
4976 ; GFX940-NEXT: s_cbranch_execnz .LBB21_1
4977 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
4978 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
4979 ; GFX940-NEXT: s_setpc_b64 s[30:31]
4980 ;
4981 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_neg:
4982 ; GFX11: ; %bb.0:
4983 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4984 ; GFX11-NEXT: global_load_b64 v[6:7], v[0:1], off offset:-2048
4985 ; GFX11-NEXT: s_mov_b32 s0, 0
4986 ; GFX11-NEXT: .LBB21_1: ; %atomicrmw.start
4987 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
4988 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4989 ; GFX11-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
4990 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
4991 ; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
4992 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4993 ; GFX11-NEXT: buffer_gl1_inv
4994 ; GFX11-NEXT: buffer_gl0_inv
4995 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4996 ; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
4997 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
4998 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4999 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5000 ; GFX11-NEXT: s_cbranch_execnz .LBB21_1
5001 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
5002 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
5003 ; GFX11-NEXT: s_setpc_b64 s[30:31]
5004 ;
5005 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_neg:
5006 ; GFX10: ; %bb.0:
5007 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5008 ; GFX10-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:-2048
5009 ; GFX10-NEXT: s_mov_b32 s4, 0
5010 ; GFX10-NEXT: .LBB21_1: ; %atomicrmw.start
5011 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
5012 ; GFX10-NEXT: s_waitcnt vmcnt(0)
5013 ; GFX10-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
5014 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
5015 ; GFX10-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
5016 ; GFX10-NEXT: s_waitcnt vmcnt(0)
5017 ; GFX10-NEXT: buffer_gl1_inv
5018 ; GFX10-NEXT: buffer_gl0_inv
5019 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
5020 ; GFX10-NEXT: v_mov_b32_e32 v7, v5
5021 ; GFX10-NEXT: v_mov_b32_e32 v6, v4
5022 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
5023 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
5024 ; GFX10-NEXT: s_cbranch_execnz .LBB21_1
5025 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
5026 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
5027 ; GFX10-NEXT: s_setpc_b64 s[30:31]
5028 ;
5029 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_neg:
5030 ; GFX90A: ; %bb.0:
5031 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5032 ; GFX90A-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:-2048
5033 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
5034 ; GFX90A-NEXT: .LBB21_1: ; %atomicrmw.start
5035 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
5036 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
5037 ; GFX90A-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
5038 ; GFX90A-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
5039 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
5040 ; GFX90A-NEXT: buffer_wbinvl1
5041 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
5042 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5043 ; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1]
5044 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
5045 ; GFX90A-NEXT: s_cbranch_execnz .LBB21_1
5046 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
5047 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
5048 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
5049 ;
5050 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_neg:
5051 ; GFX908: ; %bb.0:
5052 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5053 ; GFX908-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:-2048
5054 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
5055 ; GFX908-NEXT: .LBB21_1: ; %atomicrmw.start
5056 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
5057 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5058 ; GFX908-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
5059 ; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
5060 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5061 ; GFX908-NEXT: buffer_wbinvl1
5062 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
5063 ; GFX908-NEXT: v_mov_b32_e32 v7, v5
5064 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5065 ; GFX908-NEXT: v_mov_b32_e32 v6, v4
5066 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
5067 ; GFX908-NEXT: s_cbranch_execnz .LBB21_1
5068 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
5069 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
5070 ; GFX908-NEXT: s_setpc_b64 s[30:31]
5071 ;
5072 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_neg:
5073 ; GFX8: ; %bb.0:
5074 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5075 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
5076 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
5077 ; GFX8-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
5078 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
5079 ; GFX8-NEXT: .LBB21_1: ; %atomicrmw.start
5080 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5081 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5082 ; GFX8-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
5083 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
5084 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5085 ; GFX8-NEXT: buffer_wbinvl1
5086 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
5087 ; GFX8-NEXT: v_mov_b32_e32 v7, v5
5088 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5089 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
5090 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
5091 ; GFX8-NEXT: s_cbranch_execnz .LBB21_1
5092 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5093 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
5094 ; GFX8-NEXT: s_setpc_b64 s[30:31]
5095 ;
5096 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_neg:
5097 ; GFX7: ; %bb.0:
5098 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5099 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
5100 ; GFX7-NEXT: s_mov_b32 s5, -1
5101 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
5102 ; GFX7-NEXT: s_mov_b32 s6, 0
5103 ; GFX7-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64
5104 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
5105 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
5106 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
5107 ; GFX7-NEXT: s_mov_b32 s4, s6
5108 ; GFX7-NEXT: s_mov_b32 s5, s6
5109 ; GFX7-NEXT: .LBB21_1: ; %atomicrmw.start
5110 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5111 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5112 ; GFX7-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
5113 ; GFX7-NEXT: v_mov_b32_e32 v11, v7
5114 ; GFX7-NEXT: v_mov_b32_e32 v10, v6
5115 ; GFX7-NEXT: v_mov_b32_e32 v9, v5
5116 ; GFX7-NEXT: v_mov_b32_e32 v8, v4
5117 ; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 glc
5118 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5119 ; GFX7-NEXT: buffer_wbinvl1
5120 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
5121 ; GFX7-NEXT: v_mov_b32_e32 v6, v8
5122 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
5123 ; GFX7-NEXT: v_mov_b32_e32 v7, v9
5124 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
5125 ; GFX7-NEXT: s_cbranch_execnz .LBB21_1
5126 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5127 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
5128 ; GFX7-NEXT: s_setpc_b64 s[30:31]
5129 ;
5130 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f64__offset12b_neg:
5131 ; GFX6: ; %bb.0:
5132 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5133 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
5134 ; GFX6-NEXT: s_mov_b32 s5, -1
5135 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
5136 ; GFX6-NEXT: s_mov_b32 s6, 0
5137 ; GFX6-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64
5138 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
5139 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
5140 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
5141 ; GFX6-NEXT: s_mov_b32 s4, s6
5142 ; GFX6-NEXT: s_mov_b32 s5, s6
5143 ; GFX6-NEXT: .LBB21_1: ; %atomicrmw.start
5144 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
5145 ; GFX6-NEXT: s_waitcnt vmcnt(0)
5146 ; GFX6-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
5147 ; GFX6-NEXT: s_waitcnt expcnt(0)
5148 ; GFX6-NEXT: v_mov_b32_e32 v11, v7
5149 ; GFX6-NEXT: v_mov_b32_e32 v10, v6
5150 ; GFX6-NEXT: v_mov_b32_e32 v9, v5
5151 ; GFX6-NEXT: v_mov_b32_e32 v8, v4
5152 ; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 glc
5153 ; GFX6-NEXT: s_waitcnt vmcnt(0)
5154 ; GFX6-NEXT: buffer_wbinvl1
5155 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
5156 ; GFX6-NEXT: v_mov_b32_e32 v6, v8
5157 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
5158 ; GFX6-NEXT: v_mov_b32_e32 v7, v9
5159 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
5160 ; GFX6-NEXT: s_cbranch_execnz .LBB21_1
5161 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
5162 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
5163 ; GFX6-NEXT: s_waitcnt expcnt(0)
5164 ; GFX6-NEXT: s_setpc_b64 s[30:31]
5165 %gep = getelementptr double, ptr addrspace(1) %ptr, i64 -256
5166 %unused = atomicrmw fsub ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst
5167 ret void
5168 }
5170 ; --------------------------------------------------------------------
5171 ; half
5172 ; --------------------------------------------------------------------
5174 define half @global_agent_atomic_fsub_ret_f16(ptr addrspace(1) %ptr, half %val) #0 {
5175 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f16:
5176 ; GFX12: ; %bb.0:
5177 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
5178 ; GFX12-NEXT: s_wait_expcnt 0x0
5179 ; GFX12-NEXT: s_wait_samplecnt 0x0
5180 ; GFX12-NEXT: s_wait_bvhcnt 0x0
5181 ; GFX12-NEXT: s_wait_kmcnt 0x0
5182 ; GFX12-NEXT: v_mov_b32_e32 v3, v0
5183 ; GFX12-NEXT: s_mov_b32 s0, 0
5184 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
5185 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
5186 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
5187 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
5188 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5189 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
5190 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
5191 ; GFX12-NEXT: v_not_b32_e32 v4, v4
5192 ; GFX12-NEXT: .LBB22_1: ; %atomicrmw.start
5193 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
5194 ; GFX12-NEXT: s_wait_loadcnt 0x0
5195 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
5196 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5197 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5198 ; GFX12-NEXT: v_sub_f16_e32 v5, v5, v2
5199 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5200 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff, v5
5201 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5202 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
5203 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
5204 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
5205 ; GFX12-NEXT: s_wait_storecnt 0x0
5206 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
5207 ; GFX12-NEXT: s_wait_loadcnt 0x0
5208 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
5209 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
5210 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
5211 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
5212 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5213 ; GFX12-NEXT: s_cbranch_execnz .LBB22_1
5214 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
5215 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
5216 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5217 ; GFX12-NEXT: s_setpc_b64 s[30:31]
5218 ;
5219 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f16:
5220 ; GFX940: ; %bb.0:
5221 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5222 ; GFX940-NEXT: v_mov_b32_e32 v3, v0
5223 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v3
5224 ; GFX940-NEXT: global_load_dword v4, v[0:1], off
5225 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v3
5226 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5227 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
5228 ; GFX940-NEXT: v_lshlrev_b32_e64 v5, v3, s0
5229 ; GFX940-NEXT: v_not_b32_e32 v5, v5
5230 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
5231 ; GFX940-NEXT: .LBB22_1: ; %atomicrmw.start
5232 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
5233 ; GFX940-NEXT: s_waitcnt vmcnt(0)
5234 ; GFX940-NEXT: v_mov_b32_e32 v7, v4
5235 ; GFX940-NEXT: v_lshrrev_b32_e32 v4, v3, v7
5236 ; GFX940-NEXT: v_sub_f16_e32 v4, v4, v2
5237 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, v3, v4
5238 ; GFX940-NEXT: v_and_or_b32 v6, v7, v5, v4
5239 ; GFX940-NEXT: buffer_wbl2 sc1
5240 ; GFX940-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off sc0
5241 ; GFX940-NEXT: s_waitcnt vmcnt(0)
5242 ; GFX940-NEXT: buffer_inv sc1
5243 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
5244 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
5245 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
5246 ; GFX940-NEXT: s_cbranch_execnz .LBB22_1
5247 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
5248 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
5249 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v4
5250 ; GFX940-NEXT: s_setpc_b64 s[30:31]
5251 ;
5252 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f16:
5253 ; GFX11: ; %bb.0:
5254 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5255 ; GFX11-NEXT: v_mov_b32_e32 v3, v0
5256 ; GFX11-NEXT: s_mov_b32 s0, 0
5257 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
5258 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
5259 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
5260 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
5261 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5262 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
5263 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
5264 ; GFX11-NEXT: v_not_b32_e32 v4, v4
5265 ; GFX11-NEXT: .LBB22_1: ; %atomicrmw.start
5266 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
5267 ; GFX11-NEXT: s_waitcnt vmcnt(0)
5268 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
5269 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5270 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5271 ; GFX11-NEXT: v_sub_f16_e32 v5, v5, v2
5272 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5273 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
5274 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5275 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
5276 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
5277 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
5278 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
5279 ; GFX11-NEXT: s_waitcnt vmcnt(0)
5280 ; GFX11-NEXT: buffer_gl1_inv
5281 ; GFX11-NEXT: buffer_gl0_inv
5282 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
5283 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
5284 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
5285 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5286 ; GFX11-NEXT: s_cbranch_execnz .LBB22_1
5287 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
5288 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
5289 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5290 ; GFX11-NEXT: s_setpc_b64 s[30:31]
5291 ;
5292 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f16:
5293 ; GFX10: ; %bb.0:
5294 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5295 ; GFX10-NEXT: v_mov_b32_e32 v3, v0
5296 ; GFX10-NEXT: s_mov_b32 s4, 0
5297 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
5298 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
5299 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
5300 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5301 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
5302 ; GFX10-NEXT: v_not_b32_e32 v4, v4
5303 ; GFX10-NEXT: .LBB22_1: ; %atomicrmw.start
5304 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
5305 ; GFX10-NEXT: s_waitcnt vmcnt(0)
5306 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
5307 ; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5308 ; GFX10-NEXT: v_sub_f16_e32 v5, v5, v2
5309 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
5310 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
5311 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
5312 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
5313 ; GFX10-NEXT: s_waitcnt vmcnt(0)
5314 ; GFX10-NEXT: buffer_gl1_inv
5315 ; GFX10-NEXT: buffer_gl0_inv
5316 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
5317 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
5318 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
5319 ; GFX10-NEXT: s_cbranch_execnz .LBB22_1
5320 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
5321 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
5322 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5323 ; GFX10-NEXT: s_setpc_b64 s[30:31]
5324 ;
5325 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f16:
5326 ; GFX90A: ; %bb.0:
5327 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5328 ; GFX90A-NEXT: v_mov_b32_e32 v3, v0
5329 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
5330 ; GFX90A-NEXT: global_load_dword v4, v[0:1], off
5331 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
5332 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5333 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
5334 ; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v3, s4
5335 ; GFX90A-NEXT: v_not_b32_e32 v5, v5
5336 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
5337 ; GFX90A-NEXT: .LBB22_1: ; %atomicrmw.start
5338 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
5339 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
5340 ; GFX90A-NEXT: v_mov_b32_e32 v7, v4
5341 ; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v7
5342 ; GFX90A-NEXT: v_sub_f16_e32 v4, v4, v2
5343 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
5344 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v5, v4
5345 ; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off glc
5346 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
5347 ; GFX90A-NEXT: buffer_wbinvl1
5348 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
5349 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5350 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
5351 ; GFX90A-NEXT: s_cbranch_execnz .LBB22_1
5352 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
5353 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
5354 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v4
5355 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
5356 ;
5357 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f16:
5358 ; GFX908: ; %bb.0:
5359 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5360 ; GFX908-NEXT: v_mov_b32_e32 v3, v0
5361 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
5362 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
5363 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
5364 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5365 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
5366 ; GFX908-NEXT: v_lshlrev_b32_e64 v5, v3, s4
5367 ; GFX908-NEXT: v_not_b32_e32 v5, v5
5368 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
5369 ; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
5370 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
5371 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5372 ; GFX908-NEXT: v_mov_b32_e32 v7, v4
5373 ; GFX908-NEXT: v_lshrrev_b32_e32 v4, v3, v7
5374 ; GFX908-NEXT: v_sub_f16_e32 v4, v4, v2
5375 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, v3, v4
5376 ; GFX908-NEXT: v_and_or_b32 v6, v7, v5, v4
5377 ; GFX908-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off glc
5378 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5379 ; GFX908-NEXT: buffer_wbinvl1
5380 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
5381 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5382 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
5383 ; GFX908-NEXT: s_cbranch_execnz .LBB22_1
5384 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
5385 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
5386 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v4
5387 ; GFX908-NEXT: s_setpc_b64 s[30:31]
5388 ;
5389 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f16:
5390 ; GFX8: ; %bb.0:
5391 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5392 ; GFX8-NEXT: v_mov_b32_e32 v3, v0
5393 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
5394 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
5395 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
5396 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5397 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
5398 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
5399 ; GFX8-NEXT: v_not_b32_e32 v4, v4
5400 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
5401 ; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
5402 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5403 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5404 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
5405 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5406 ; GFX8-NEXT: v_sub_f16_e32 v5, v5, v2
5407 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
5408 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5409 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
5410 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
5411 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5412 ; GFX8-NEXT: buffer_wbinvl1
5413 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
5414 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5415 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
5416 ; GFX8-NEXT: s_cbranch_execnz .LBB22_1
5417 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5418 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
5419 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5420 ; GFX8-NEXT: s_setpc_b64 s[30:31]
5421 ;
5422 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f16:
5423 ; GFX7: ; %bb.0:
5424 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5425 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
5426 ; GFX7-NEXT: s_mov_b32 s6, 0
5427 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
5428 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
5429 ; GFX7-NEXT: s_mov_b32 s4, s6
5430 ; GFX7-NEXT: s_mov_b32 s5, s6
5431 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
5432 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
5433 ; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
5434 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 3, v3
5435 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
5436 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v2
5437 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v5
5438 ; GFX7-NEXT: v_not_b32_e32 v7, v2
5439 ; GFX7-NEXT: .LBB22_1: ; %atomicrmw.start
5440 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5441 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5442 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v5, v4
5443 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
5444 ; GFX7-NEXT: v_and_b32_e32 v3, v4, v7
5445 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v6
5446 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
5447 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v5, v2
5448 ; GFX7-NEXT: v_or_b32_e32 v3, v3, v2
5449 ; GFX7-NEXT: v_mov_b32_e32 v2, v3
5450 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
5451 ; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v[0:1], s[4:7], 0 addr64 glc
5452 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5453 ; GFX7-NEXT: buffer_wbinvl1
5454 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
5455 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
5456 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
5457 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
5458 ; GFX7-NEXT: s_cbranch_execnz .LBB22_1
5459 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5460 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
5461 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v5, v2
5462 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
5463 ; GFX7-NEXT: s_setpc_b64 s[30:31]
5464 ;
5465 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f16:
5466 ; GFX6: ; %bb.0:
5467 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5468 ; GFX6-NEXT: v_mov_b32_e32 v3, v0
5469 ; GFX6-NEXT: s_mov_b32 s6, 0
5470 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v3
5471 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
5472 ; GFX6-NEXT: s_mov_b32 s4, s6
5473 ; GFX6-NEXT: s_mov_b32 s5, s6
5474 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
5475 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
5476 ; GFX6-NEXT: v_and_b32_e32 v3, 3, v3
5477 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 3, v3
5478 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
5479 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v2
5480 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v5
5481 ; GFX6-NEXT: v_not_b32_e32 v7, v2
5482 ; GFX6-NEXT: .LBB22_1: ; %atomicrmw.start
5483 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
5484 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
5485 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v5, v4
5486 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
5487 ; GFX6-NEXT: v_and_b32_e32 v3, v4, v7
5488 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v6
5489 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
5490 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v5, v2
5491 ; GFX6-NEXT: v_or_b32_e32 v3, v3, v2
5492 ; GFX6-NEXT: v_mov_b32_e32 v2, v3
5493 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
5494 ; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v[0:1], s[4:7], 0 addr64 glc
5495 ; GFX6-NEXT: s_waitcnt vmcnt(0)
5496 ; GFX6-NEXT: buffer_wbinvl1
5497 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
5498 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
5499 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
5500 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
5501 ; GFX6-NEXT: s_cbranch_execnz .LBB22_1
5502 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
5503 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
5504 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v5, v2
5505 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
5506 ; GFX6-NEXT: s_waitcnt expcnt(0)
5507 ; GFX6-NEXT: s_setpc_b64 s[30:31]
5508 %result = atomicrmw fsub ptr addrspace(1) %ptr, half %val syncscope("agent") seq_cst
5509 ret half %result
5510 }
5512 define half @global_agent_atomic_fsub_ret_f16__offset12b_pos(ptr addrspace(1) %ptr, half %val) #0 {
5513 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos:
5514 ; GFX12: ; %bb.0:
5515 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
5516 ; GFX12-NEXT: s_wait_expcnt 0x0
5517 ; GFX12-NEXT: s_wait_samplecnt 0x0
5518 ; GFX12-NEXT: s_wait_bvhcnt 0x0
5519 ; GFX12-NEXT: s_wait_kmcnt 0x0
5520 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
5521 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
5522 ; GFX12-NEXT: s_mov_b32 s0, 0
5523 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
5524 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
5525 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
5526 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
5527 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5528 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
5529 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
5530 ; GFX12-NEXT: v_not_b32_e32 v4, v4
5531 ; GFX12-NEXT: .LBB23_1: ; %atomicrmw.start
5532 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
5533 ; GFX12-NEXT: s_wait_loadcnt 0x0
5534 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
5535 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5536 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5537 ; GFX12-NEXT: v_sub_f16_e32 v5, v5, v2
5538 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5539 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff, v5
5540 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5541 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
5542 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
5543 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
5544 ; GFX12-NEXT: s_wait_storecnt 0x0
5545 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
5546 ; GFX12-NEXT: s_wait_loadcnt 0x0
5547 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
5548 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
5549 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
5550 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
5551 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5552 ; GFX12-NEXT: s_cbranch_execnz .LBB23_1
5553 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
5554 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
5555 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5556 ; GFX12-NEXT: s_setpc_b64 s[30:31]
5557 ;
5558 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos:
5559 ; GFX940: ; %bb.0:
5560 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5561 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
5562 ; GFX940-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, s[0:1]
5563 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v6
5564 ; GFX940-NEXT: v_mov_b32_e32 v1, v7
5565 ; GFX940-NEXT: global_load_dword v4, v[0:1], off
5566 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v6
5567 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5568 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
5569 ; GFX940-NEXT: v_lshlrev_b32_e64 v5, v3, s0
5570 ; GFX940-NEXT: v_not_b32_e32 v5, v5
5571 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
5572 ; GFX940-NEXT: .LBB23_1: ; %atomicrmw.start
5573 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
5574 ; GFX940-NEXT: s_waitcnt vmcnt(0)
5575 ; GFX940-NEXT: v_mov_b32_e32 v7, v4
5576 ; GFX940-NEXT: v_lshrrev_b32_e32 v4, v3, v7
5577 ; GFX940-NEXT: v_sub_f16_e32 v4, v4, v2
5578 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, v3, v4
5579 ; GFX940-NEXT: v_and_or_b32 v6, v7, v5, v4
5580 ; GFX940-NEXT: buffer_wbl2 sc1
5581 ; GFX940-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off sc0
5582 ; GFX940-NEXT: s_waitcnt vmcnt(0)
5583 ; GFX940-NEXT: buffer_inv sc1
5584 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
5585 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
5586 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
5587 ; GFX940-NEXT: s_cbranch_execnz .LBB23_1
5588 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
5589 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
5590 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v4
5591 ; GFX940-NEXT: s_setpc_b64 s[30:31]
5592 ;
5593 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos:
5594 ; GFX11: ; %bb.0:
5595 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5596 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
5597 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
5598 ; GFX11-NEXT: s_mov_b32 s0, 0
5599 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
5600 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
5601 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
5602 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
5603 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5604 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
5605 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
5606 ; GFX11-NEXT: v_not_b32_e32 v4, v4
5607 ; GFX11-NEXT: .LBB23_1: ; %atomicrmw.start
5608 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
5609 ; GFX11-NEXT: s_waitcnt vmcnt(0)
5610 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
5611 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5612 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5613 ; GFX11-NEXT: v_sub_f16_e32 v5, v5, v2
5614 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5615 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
5616 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5617 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
5618 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
5619 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
5620 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
5621 ; GFX11-NEXT: s_waitcnt vmcnt(0)
5622 ; GFX11-NEXT: buffer_gl1_inv
5623 ; GFX11-NEXT: buffer_gl0_inv
5624 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
5625 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
5626 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
5627 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5628 ; GFX11-NEXT: s_cbranch_execnz .LBB23_1
5629 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
5630 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
5631 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5632 ; GFX11-NEXT: s_setpc_b64 s[30:31]
5633 ;
5634 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos:
5635 ; GFX10: ; %bb.0:
5636 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5637 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
5638 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
5639 ; GFX10-NEXT: s_mov_b32 s4, 0
5640 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
5641 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
5642 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
5643 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5644 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
5645 ; GFX10-NEXT: v_not_b32_e32 v4, v4
5646 ; GFX10-NEXT: .LBB23_1: ; %atomicrmw.start
5647 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
5648 ; GFX10-NEXT: s_waitcnt vmcnt(0)
5649 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
5650 ; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5651 ; GFX10-NEXT: v_sub_f16_e32 v5, v5, v2
5652 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
5653 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
5654 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
5655 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
5656 ; GFX10-NEXT: s_waitcnt vmcnt(0)
5657 ; GFX10-NEXT: buffer_gl1_inv
5658 ; GFX10-NEXT: buffer_gl0_inv
5659 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
5660 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
5661 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
5662 ; GFX10-NEXT: s_cbranch_execnz .LBB23_1
5663 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
5664 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
5665 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5666 ; GFX10-NEXT: s_setpc_b64 s[30:31]
5667 ;
5668 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos:
5669 ; GFX90A: ; %bb.0:
5670 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5671 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
5672 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
5673 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
5674 ; GFX90A-NEXT: global_load_dword v4, v[0:1], off
5675 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
5676 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5677 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
5678 ; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v3, s4
5679 ; GFX90A-NEXT: v_not_b32_e32 v5, v5
5680 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
5681 ; GFX90A-NEXT: .LBB23_1: ; %atomicrmw.start
5682 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
5683 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
5684 ; GFX90A-NEXT: v_mov_b32_e32 v7, v4
5685 ; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v7
5686 ; GFX90A-NEXT: v_sub_f16_e32 v4, v4, v2
5687 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
5688 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v5, v4
5689 ; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off glc
5690 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
5691 ; GFX90A-NEXT: buffer_wbinvl1
5692 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
5693 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5694 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
5695 ; GFX90A-NEXT: s_cbranch_execnz .LBB23_1
5696 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
5697 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
5698 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v4
5699 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
5700 ;
5701 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos:
5702 ; GFX908: ; %bb.0:
5703 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5704 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
5705 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
5706 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
5707 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
5708 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
5709 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5710 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
5711 ; GFX908-NEXT: v_lshlrev_b32_e64 v5, v3, s4
5712 ; GFX908-NEXT: v_not_b32_e32 v5, v5
5713 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
5714 ; GFX908-NEXT: .LBB23_1: ; %atomicrmw.start
5715 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
5716 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5717 ; GFX908-NEXT: v_mov_b32_e32 v7, v4
5718 ; GFX908-NEXT: v_lshrrev_b32_e32 v4, v3, v7
5719 ; GFX908-NEXT: v_sub_f16_e32 v4, v4, v2
5720 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, v3, v4
5721 ; GFX908-NEXT: v_and_or_b32 v6, v7, v5, v4
5722 ; GFX908-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off glc
5723 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5724 ; GFX908-NEXT: buffer_wbinvl1
5725 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
5726 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5727 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
5728 ; GFX908-NEXT: s_cbranch_execnz .LBB23_1
5729 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
5730 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
5731 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v4
5732 ; GFX908-NEXT: s_setpc_b64 s[30:31]
5733 ;
5734 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos:
5735 ; GFX8: ; %bb.0:
5736 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5737 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
5738 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
5739 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
5740 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
5741 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
5742 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5743 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
5744 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
5745 ; GFX8-NEXT: v_not_b32_e32 v4, v4
5746 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
5747 ; GFX8-NEXT: .LBB23_1: ; %atomicrmw.start
5748 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5749 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5750 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
5751 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5752 ; GFX8-NEXT: v_sub_f16_e32 v5, v5, v2
5753 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
5754 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5755 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
5756 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
5757 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5758 ; GFX8-NEXT: buffer_wbinvl1
5759 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
5760 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5761 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
5762 ; GFX8-NEXT: s_cbranch_execnz .LBB23_1
5763 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5764 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
5765 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5766 ; GFX8-NEXT: s_setpc_b64 s[30:31]
5767 ;
5768 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos:
5769 ; GFX7: ; %bb.0:
5770 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5771 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
5772 ; GFX7-NEXT: s_mov_b32 s6, 0
5773 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
5774 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
5775 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
5776 ; GFX7-NEXT: s_mov_b32 s4, s6
5777 ; GFX7-NEXT: s_mov_b32 s5, s6
5778 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
5779 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
5780 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
5781 ; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
5782 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
5783 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
5784 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
5785 ; GFX7-NEXT: v_not_b32_e32 v8, v2
5786 ; GFX7-NEXT: .LBB23_1: ; %atomicrmw.start
5787 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5788 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5789 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
5790 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
5791 ; GFX7-NEXT: v_and_b32_e32 v4, v3, v8
5792 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v7
5793 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
5794 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v6, v2
5795 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
5796 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
5797 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
5798 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
5799 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5800 ; GFX7-NEXT: buffer_wbinvl1
5801 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
5802 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
5803 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
5804 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
5805 ; GFX7-NEXT: s_cbranch_execnz .LBB23_1
5806 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5807 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
5808 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
5809 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
5810 ; GFX7-NEXT: s_setpc_b64 s[30:31]
5811 ;
5812 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos:
5813 ; GFX6: ; %bb.0:
5814 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5815 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
5816 ; GFX6-NEXT: s_mov_b32 s6, 0
5817 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
5818 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
5819 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
5820 ; GFX6-NEXT: s_mov_b32 s4, s6
5821 ; GFX6-NEXT: s_mov_b32 s5, s6
5822 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
5823 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
5824 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
5825 ; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
5826 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
5827 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
5828 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
5829 ; GFX6-NEXT: v_not_b32_e32 v8, v2
5830 ; GFX6-NEXT: .LBB23_1: ; %atomicrmw.start
5831 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
5832 ; GFX6-NEXT: s_waitcnt vmcnt(0)
5833 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
5834 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
5835 ; GFX6-NEXT: s_waitcnt expcnt(0)
5836 ; GFX6-NEXT: v_and_b32_e32 v4, v3, v8
5837 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v7
5838 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
5839 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v6, v2
5840 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
5841 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
5842 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
5843 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
5844 ; GFX6-NEXT: s_waitcnt vmcnt(0)
5845 ; GFX6-NEXT: buffer_wbinvl1
5846 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
5847 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
5848 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
5849 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
5850 ; GFX6-NEXT: s_cbranch_execnz .LBB23_1
5851 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
5852 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
5853 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
5854 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
5855 ; GFX6-NEXT: s_waitcnt expcnt(0)
5856 ; GFX6-NEXT: s_setpc_b64 s[30:31]
5857 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
5858 %result = atomicrmw fsub ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst
5859 ret half %result
5860 }
5862 define half @global_agent_atomic_fsub_ret_f16__offset12b_neg(ptr addrspace(1) %ptr, half %val) #0 {
5863 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_neg:
5864 ; GFX12: ; %bb.0:
5865 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
5866 ; GFX12-NEXT: s_wait_expcnt 0x0
5867 ; GFX12-NEXT: s_wait_samplecnt 0x0
5868 ; GFX12-NEXT: s_wait_bvhcnt 0x0
5869 ; GFX12-NEXT: s_wait_kmcnt 0x0
5870 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
5871 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
5872 ; GFX12-NEXT: s_mov_b32 s0, 0
5873 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
5874 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
5875 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
5876 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
5877 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5878 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
5879 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
5880 ; GFX12-NEXT: v_not_b32_e32 v4, v4
5881 ; GFX12-NEXT: .LBB24_1: ; %atomicrmw.start
5882 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
5883 ; GFX12-NEXT: s_wait_loadcnt 0x0
5884 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
5885 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5886 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5887 ; GFX12-NEXT: v_sub_f16_e32 v5, v5, v2
5888 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5889 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff, v5
5890 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5891 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
5892 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
5893 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
5894 ; GFX12-NEXT: s_wait_storecnt 0x0
5895 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
5896 ; GFX12-NEXT: s_wait_loadcnt 0x0
5897 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
5898 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
5899 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
5900 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
5901 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5902 ; GFX12-NEXT: s_cbranch_execnz .LBB24_1
5903 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
5904 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
5905 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5906 ; GFX12-NEXT: s_setpc_b64 s[30:31]
5907 ;
5908 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_neg:
5909 ; GFX940: ; %bb.0:
5910 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5911 ; GFX940-NEXT: s_movk_i32 s0, 0xf800
5912 ; GFX940-NEXT: s_mov_b32 s1, -1
5913 ; GFX940-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, s[0:1]
5914 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v6
5915 ; GFX940-NEXT: v_mov_b32_e32 v1, v7
5916 ; GFX940-NEXT: global_load_dword v4, v[0:1], off
5917 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v6
5918 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5919 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
5920 ; GFX940-NEXT: v_lshlrev_b32_e64 v5, v3, s0
5921 ; GFX940-NEXT: v_not_b32_e32 v5, v5
5922 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
5923 ; GFX940-NEXT: .LBB24_1: ; %atomicrmw.start
5924 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
5925 ; GFX940-NEXT: s_waitcnt vmcnt(0)
5926 ; GFX940-NEXT: v_mov_b32_e32 v7, v4
5927 ; GFX940-NEXT: v_lshrrev_b32_e32 v4, v3, v7
5928 ; GFX940-NEXT: v_sub_f16_e32 v4, v4, v2
5929 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, v3, v4
5930 ; GFX940-NEXT: v_and_or_b32 v6, v7, v5, v4
5931 ; GFX940-NEXT: buffer_wbl2 sc1
5932 ; GFX940-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off sc0
5933 ; GFX940-NEXT: s_waitcnt vmcnt(0)
5934 ; GFX940-NEXT: buffer_inv sc1
5935 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
5936 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
5937 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
5938 ; GFX940-NEXT: s_cbranch_execnz .LBB24_1
5939 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
5940 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
5941 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v4
5942 ; GFX940-NEXT: s_setpc_b64 s[30:31]
5943 ;
5944 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_neg:
5945 ; GFX11: ; %bb.0:
5946 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5947 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
5948 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
5949 ; GFX11-NEXT: s_mov_b32 s0, 0
5950 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
5951 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
5952 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
5953 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
5954 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5955 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
5956 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
5957 ; GFX11-NEXT: v_not_b32_e32 v4, v4
5958 ; GFX11-NEXT: .LBB24_1: ; %atomicrmw.start
5959 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
5960 ; GFX11-NEXT: s_waitcnt vmcnt(0)
5961 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
5962 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5963 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5964 ; GFX11-NEXT: v_sub_f16_e32 v5, v5, v2
5965 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5966 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
5967 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5968 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
5969 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
5970 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
5971 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
5972 ; GFX11-NEXT: s_waitcnt vmcnt(0)
5973 ; GFX11-NEXT: buffer_gl1_inv
5974 ; GFX11-NEXT: buffer_gl0_inv
5975 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
5976 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
5977 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
5978 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5979 ; GFX11-NEXT: s_cbranch_execnz .LBB24_1
5980 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
5981 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
5982 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5983 ; GFX11-NEXT: s_setpc_b64 s[30:31]
5984 ;
5985 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_neg:
5986 ; GFX10: ; %bb.0:
5987 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5988 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
5989 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
5990 ; GFX10-NEXT: s_mov_b32 s4, 0
5991 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
5992 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
5993 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
5994 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5995 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
5996 ; GFX10-NEXT: v_not_b32_e32 v4, v4
5997 ; GFX10-NEXT: .LBB24_1: ; %atomicrmw.start
5998 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
5999 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6000 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
6001 ; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
6002 ; GFX10-NEXT: v_sub_f16_e32 v5, v5, v2
6003 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
6004 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
6005 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
6006 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
6007 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6008 ; GFX10-NEXT: buffer_gl1_inv
6009 ; GFX10-NEXT: buffer_gl0_inv
6010 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
6011 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
6012 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
6013 ; GFX10-NEXT: s_cbranch_execnz .LBB24_1
6014 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
6015 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
6016 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
6017 ; GFX10-NEXT: s_setpc_b64 s[30:31]
6018 ;
6019 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_neg:
6020 ; GFX90A: ; %bb.0:
6021 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6022 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
6023 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
6024 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
6025 ; GFX90A-NEXT: global_load_dword v4, v[0:1], off
6026 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
6027 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
6028 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
6029 ; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v3, s4
6030 ; GFX90A-NEXT: v_not_b32_e32 v5, v5
6031 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
6032 ; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
6033 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
6034 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6035 ; GFX90A-NEXT: v_mov_b32_e32 v7, v4
6036 ; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v7
6037 ; GFX90A-NEXT: v_sub_f16_e32 v4, v4, v2
6038 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
6039 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v5, v4
6040 ; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off glc
6041 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6042 ; GFX90A-NEXT: buffer_wbinvl1
6043 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
6044 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6045 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
6046 ; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
6047 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
6048 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
6049 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v4
6050 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
6051 ;
6052 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_neg:
6053 ; GFX908: ; %bb.0:
6054 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6055 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
6056 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
6057 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
6058 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
6059 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
6060 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
6061 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
6062 ; GFX908-NEXT: v_lshlrev_b32_e64 v5, v3, s4
6063 ; GFX908-NEXT: v_not_b32_e32 v5, v5
6064 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
6065 ; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start
6066 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
6067 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6068 ; GFX908-NEXT: v_mov_b32_e32 v7, v4
6069 ; GFX908-NEXT: v_lshrrev_b32_e32 v4, v3, v7
6070 ; GFX908-NEXT: v_sub_f16_e32 v4, v4, v2
6071 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, v3, v4
6072 ; GFX908-NEXT: v_and_or_b32 v6, v7, v5, v4
6073 ; GFX908-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off glc
6074 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6075 ; GFX908-NEXT: buffer_wbinvl1
6076 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
6077 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6078 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
6079 ; GFX908-NEXT: s_cbranch_execnz .LBB24_1
6080 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
6081 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
6082 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v4
6083 ; GFX908-NEXT: s_setpc_b64 s[30:31]
6084 ;
6085 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_neg:
6086 ; GFX8: ; %bb.0:
6087 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6088 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
6089 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
6090 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
6091 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
6092 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
6093 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
6094 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
6095 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
6096 ; GFX8-NEXT: v_not_b32_e32 v4, v4
6097 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
6098 ; GFX8-NEXT: .LBB24_1: ; %atomicrmw.start
6099 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6100 ; GFX8-NEXT: s_waitcnt vmcnt(0)
6101 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
6102 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
6103 ; GFX8-NEXT: v_sub_f16_e32 v5, v5, v2
6104 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
6105 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
6106 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
6107 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
6108 ; GFX8-NEXT: s_waitcnt vmcnt(0)
6109 ; GFX8-NEXT: buffer_wbinvl1
6110 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
6111 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6112 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
6113 ; GFX8-NEXT: s_cbranch_execnz .LBB24_1
6114 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6115 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
6116 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
6117 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6118 ;
6119 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_neg:
6120 ; GFX7: ; %bb.0:
6121 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6122 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
6123 ; GFX7-NEXT: s_mov_b32 s6, 0
6124 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
6125 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
6126 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
6127 ; GFX7-NEXT: s_mov_b32 s4, s6
6128 ; GFX7-NEXT: s_mov_b32 s5, s6
6129 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
6130 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
6131 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
6132 ; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
6133 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
6134 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
6135 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
6136 ; GFX7-NEXT: v_not_b32_e32 v8, v2
6137 ; GFX7-NEXT: .LBB24_1: ; %atomicrmw.start
6138 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
6139 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6140 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
6141 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
6142 ; GFX7-NEXT: v_and_b32_e32 v4, v3, v8
6143 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v7
6144 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
6145 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v6, v2
6146 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
6147 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
6148 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
6149 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
6150 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6151 ; GFX7-NEXT: buffer_wbinvl1
6152 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
6153 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
6154 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
6155 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
6156 ; GFX7-NEXT: s_cbranch_execnz .LBB24_1
6157 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6158 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
6159 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
6160 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
6161 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6162 ;
6163 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_neg:
6164 ; GFX6: ; %bb.0:
6165 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6166 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
6167 ; GFX6-NEXT: s_mov_b32 s6, 0
6168 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
6169 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
6170 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
6171 ; GFX6-NEXT: s_mov_b32 s4, s6
6172 ; GFX6-NEXT: s_mov_b32 s5, s6
6173 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
6174 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
6175 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
6176 ; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
6177 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
6178 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
6179 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
6180 ; GFX6-NEXT: v_not_b32_e32 v8, v2
6181 ; GFX6-NEXT: .LBB24_1: ; %atomicrmw.start
6182 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
6183 ; GFX6-NEXT: s_waitcnt vmcnt(0)
6184 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
6185 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
6186 ; GFX6-NEXT: s_waitcnt expcnt(0)
6187 ; GFX6-NEXT: v_and_b32_e32 v4, v3, v8
6188 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v7
6189 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
6190 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v6, v2
6191 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
6192 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
6193 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
6194 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
6195 ; GFX6-NEXT: s_waitcnt vmcnt(0)
6196 ; GFX6-NEXT: buffer_wbinvl1
6197 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
6198 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
6199 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
6200 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
6201 ; GFX6-NEXT: s_cbranch_execnz .LBB24_1
6202 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
6203 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
6204 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
6205 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
6206 ; GFX6-NEXT: s_waitcnt expcnt(0)
6207 ; GFX6-NEXT: s_setpc_b64 s[30:31]
6208 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 -1024
6209 %result = atomicrmw fsub ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst
6210 ret half %result
6211 }
6213 define void @global_agent_atomic_fsub_noret_f16(ptr addrspace(1) %ptr, half %val) #0 {
6214 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f16:
6215 ; GFX12: ; %bb.0:
6216 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
6217 ; GFX12-NEXT: s_wait_expcnt 0x0
6218 ; GFX12-NEXT: s_wait_samplecnt 0x0
6219 ; GFX12-NEXT: s_wait_bvhcnt 0x0
6220 ; GFX12-NEXT: s_wait_kmcnt 0x0
6221 ; GFX12-NEXT: v_mov_b32_e32 v3, v0
6222 ; GFX12-NEXT: s_mov_b32 s0, 0
6223 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
6224 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
6225 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
6226 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off
6227 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6228 ; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
6229 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
6230 ; GFX12-NEXT: v_not_b32_e32 v6, v3
6231 ; GFX12-NEXT: .LBB25_1: ; %atomicrmw.start
6232 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
6233 ; GFX12-NEXT: s_wait_loadcnt 0x0
6234 ; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
6235 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6236 ; GFX12-NEXT: v_sub_f16_e32 v3, v3, v2
6237 ; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
6238 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6239 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
6240 ; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
6241 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
6242 ; GFX12-NEXT: s_wait_storecnt 0x0
6243 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
6244 ; GFX12-NEXT: s_wait_loadcnt 0x0
6245 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
6246 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
6247 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
6248 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
6249 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
6250 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
6251 ; GFX12-NEXT: s_cbranch_execnz .LBB25_1
6252 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
6253 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
6254 ; GFX12-NEXT: s_setpc_b64 s[30:31]
6255 ;
6256 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f16:
6257 ; GFX940: ; %bb.0:
6258 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6259 ; GFX940-NEXT: v_mov_b32_e32 v3, v0
6260 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v3
6261 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
6262 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v3
6263 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
6264 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
6265 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
6266 ; GFX940-NEXT: v_not_b32_e32 v6, v4
6267 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
6268 ; GFX940-NEXT: .LBB25_1: ; %atomicrmw.start
6269 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
6270 ; GFX940-NEXT: s_waitcnt vmcnt(0)
6271 ; GFX940-NEXT: v_lshrrev_b32_e32 v4, v3, v5
6272 ; GFX940-NEXT: v_sub_f16_e32 v4, v4, v2
6273 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, v3, v4
6274 ; GFX940-NEXT: v_and_or_b32 v4, v5, v6, v4
6275 ; GFX940-NEXT: buffer_wbl2 sc1
6276 ; GFX940-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0
6277 ; GFX940-NEXT: s_waitcnt vmcnt(0)
6278 ; GFX940-NEXT: buffer_inv sc1
6279 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
6280 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
6281 ; GFX940-NEXT: v_mov_b32_e32 v5, v4
6282 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
6283 ; GFX940-NEXT: s_cbranch_execnz .LBB25_1
6284 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
6285 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
6286 ; GFX940-NEXT: s_setpc_b64 s[30:31]
6287 ;
6288 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f16:
6289 ; GFX11: ; %bb.0:
6290 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6291 ; GFX11-NEXT: v_mov_b32_e32 v3, v0
6292 ; GFX11-NEXT: s_mov_b32 s0, 0
6293 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
6294 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
6295 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
6296 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off
6297 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6298 ; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
6299 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
6300 ; GFX11-NEXT: v_not_b32_e32 v6, v3
6301 ; GFX11-NEXT: .LBB25_1: ; %atomicrmw.start
6302 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
6303 ; GFX11-NEXT: s_waitcnt vmcnt(0)
6304 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
6305 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6306 ; GFX11-NEXT: v_sub_f16_e32 v3, v3, v2
6307 ; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
6308 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6309 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
6310 ; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
6311 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
6312 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
6313 ; GFX11-NEXT: s_waitcnt vmcnt(0)
6314 ; GFX11-NEXT: buffer_gl1_inv
6315 ; GFX11-NEXT: buffer_gl0_inv
6316 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
6317 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
6318 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
6319 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
6320 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
6321 ; GFX11-NEXT: s_cbranch_execnz .LBB25_1
6322 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
6323 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
6324 ; GFX11-NEXT: s_setpc_b64 s[30:31]
6325 ;
6326 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f16:
6327 ; GFX10: ; %bb.0:
6328 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6329 ; GFX10-NEXT: v_mov_b32_e32 v3, v0
6330 ; GFX10-NEXT: s_mov_b32 s4, 0
6331 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
6332 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
6333 ; GFX10-NEXT: global_load_dword v4, v[0:1], off
6334 ; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6335 ; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
6336 ; GFX10-NEXT: v_not_b32_e32 v6, v3
6337 ; GFX10-NEXT: .LBB25_1: ; %atomicrmw.start
6338 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
6339 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6340 ; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
6341 ; GFX10-NEXT: v_sub_f16_e32 v3, v3, v2
6342 ; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
6343 ; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
6344 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
6345 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
6346 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6347 ; GFX10-NEXT: buffer_gl1_inv
6348 ; GFX10-NEXT: buffer_gl0_inv
6349 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
6350 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
6351 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
6352 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
6353 ; GFX10-NEXT: s_cbranch_execnz .LBB25_1
6354 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
6355 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
6356 ; GFX10-NEXT: s_setpc_b64 s[30:31]
6357 ;
6358 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f16:
6359 ; GFX90A: ; %bb.0:
6360 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6361 ; GFX90A-NEXT: v_mov_b32_e32 v3, v0
6362 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
6363 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
6364 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
6365 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
6366 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
6367 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
6368 ; GFX90A-NEXT: v_not_b32_e32 v6, v4
6369 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
6370 ; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
6371 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
6372 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6373 ; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
6374 ; GFX90A-NEXT: v_sub_f16_e32 v4, v4, v2
6375 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
6376 ; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
6377 ; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
6378 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6379 ; GFX90A-NEXT: buffer_wbinvl1
6380 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
6381 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6382 ; GFX90A-NEXT: v_mov_b32_e32 v5, v4
6383 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
6384 ; GFX90A-NEXT: s_cbranch_execnz .LBB25_1
6385 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
6386 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
6387 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
6388 ;
6389 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f16:
6390 ; GFX908: ; %bb.0:
6391 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6392 ; GFX908-NEXT: v_mov_b32_e32 v3, v0
6393 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
6394 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
6395 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
6396 ; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6397 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
6398 ; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
6399 ; GFX908-NEXT: v_not_b32_e32 v6, v3
6400 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
6401 ; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start
6402 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
6403 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6404 ; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
6405 ; GFX908-NEXT: v_sub_f16_e32 v3, v3, v2
6406 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
6407 ; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
6408 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
6409 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6410 ; GFX908-NEXT: buffer_wbinvl1
6411 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
6412 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6413 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
6414 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
6415 ; GFX908-NEXT: s_cbranch_execnz .LBB25_1
6416 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
6417 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
6418 ; GFX908-NEXT: s_setpc_b64 s[30:31]
6419 ;
6420 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f16:
6421 ; GFX8: ; %bb.0:
6422 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6423 ; GFX8-NEXT: v_mov_b32_e32 v3, v0
6424 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
6425 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
6426 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
6427 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6428 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
6429 ; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
6430 ; GFX8-NEXT: v_not_b32_e32 v6, v3
6431 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
6432 ; GFX8-NEXT: .LBB25_1: ; %atomicrmw.start
6433 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6434 ; GFX8-NEXT: s_waitcnt vmcnt(0)
6435 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
6436 ; GFX8-NEXT: v_sub_f16_e32 v3, v3, v2
6437 ; GFX8-NEXT: v_and_b32_e32 v7, v4, v6
6438 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
6439 ; GFX8-NEXT: v_or_b32_e32 v3, v7, v3
6440 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
6441 ; GFX8-NEXT: s_waitcnt vmcnt(0)
6442 ; GFX8-NEXT: buffer_wbinvl1
6443 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
6444 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6445 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
6446 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
6447 ; GFX8-NEXT: s_cbranch_execnz .LBB25_1
6448 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6449 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
6450 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6451 ;
6452 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f16:
6453 ; GFX7: ; %bb.0:
6454 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6455 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
6456 ; GFX7-NEXT: s_mov_b32 s6, 0
6457 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
6458 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
6459 ; GFX7-NEXT: s_mov_b32 s4, s6
6460 ; GFX7-NEXT: s_mov_b32 s5, s6
6461 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
6462 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v2
6463 ; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
6464 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
6465 ; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
6466 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
6467 ; GFX7-NEXT: v_not_b32_e32 v6, v3
6468 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
6469 ; GFX7-NEXT: .LBB25_1: ; %atomicrmw.start
6470 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
6471 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6472 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, v2, v4
6473 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
6474 ; GFX7-NEXT: v_and_b32_e32 v7, v4, v6
6475 ; GFX7-NEXT: v_sub_f32_e32 v3, v3, v5
6476 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
6477 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, v2, v3
6478 ; GFX7-NEXT: v_or_b32_e32 v3, v7, v3
6479 ; GFX7-NEXT: v_mov_b32_e32 v8, v4
6480 ; GFX7-NEXT: v_mov_b32_e32 v7, v3
6481 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
6482 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6483 ; GFX7-NEXT: buffer_wbinvl1
6484 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v4
6485 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
6486 ; GFX7-NEXT: v_mov_b32_e32 v4, v7
6487 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
6488 ; GFX7-NEXT: s_cbranch_execnz .LBB25_1
6489 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6490 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
6491 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6492 ;
6493 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f16:
6494 ; GFX6: ; %bb.0:
6495 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6496 ; GFX6-NEXT: v_mov_b32_e32 v3, v0
6497 ; GFX6-NEXT: s_mov_b32 s6, 0
6498 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v3
6499 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
6500 ; GFX6-NEXT: s_mov_b32 s4, s6
6501 ; GFX6-NEXT: s_mov_b32 s5, s6
6502 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
6503 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v2
6504 ; GFX6-NEXT: v_and_b32_e32 v2, 3, v3
6505 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 3, v2
6506 ; GFX6-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
6507 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
6508 ; GFX6-NEXT: v_not_b32_e32 v6, v3
6509 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
6510 ; GFX6-NEXT: .LBB25_1: ; %atomicrmw.start
6511 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
6512 ; GFX6-NEXT: s_waitcnt vmcnt(0)
6513 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, v2, v4
6514 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
6515 ; GFX6-NEXT: s_waitcnt expcnt(0)
6516 ; GFX6-NEXT: v_and_b32_e32 v7, v4, v6
6517 ; GFX6-NEXT: v_sub_f32_e32 v3, v3, v5
6518 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
6519 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, v2, v3
6520 ; GFX6-NEXT: v_or_b32_e32 v3, v7, v3
6521 ; GFX6-NEXT: v_mov_b32_e32 v8, v4
6522 ; GFX6-NEXT: v_mov_b32_e32 v7, v3
6523 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
6524 ; GFX6-NEXT: s_waitcnt vmcnt(0)
6525 ; GFX6-NEXT: buffer_wbinvl1
6526 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v4
6527 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
6528 ; GFX6-NEXT: v_mov_b32_e32 v4, v7
6529 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
6530 ; GFX6-NEXT: s_cbranch_execnz .LBB25_1
6531 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
6532 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
6533 ; GFX6-NEXT: s_waitcnt expcnt(0)
6534 ; GFX6-NEXT: s_setpc_b64 s[30:31]
6535 %unused = atomicrmw fsub ptr addrspace(1) %ptr, half %val syncscope("agent") seq_cst
6536 ret void
6537 }
6539 define void @global_agent_atomic_fsub_noret_f16__offset12b_pos(ptr addrspace(1) %ptr, half %val) #0 {
6540 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_pos:
6541 ; GFX12: ; %bb.0:
6542 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
6543 ; GFX12-NEXT: s_wait_expcnt 0x0
6544 ; GFX12-NEXT: s_wait_samplecnt 0x0
6545 ; GFX12-NEXT: s_wait_bvhcnt 0x0
6546 ; GFX12-NEXT: s_wait_kmcnt 0x0
6547 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
6548 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
6549 ; GFX12-NEXT: s_mov_b32 s0, 0
6550 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
6551 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
6552 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
6553 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off
6554 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6555 ; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
6556 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
6557 ; GFX12-NEXT: v_not_b32_e32 v6, v3
6558 ; GFX12-NEXT: .LBB26_1: ; %atomicrmw.start
6559 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
6560 ; GFX12-NEXT: s_wait_loadcnt 0x0
6561 ; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
6562 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6563 ; GFX12-NEXT: v_sub_f16_e32 v3, v3, v2
6564 ; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
6565 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6566 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
6567 ; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
6568 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
6569 ; GFX12-NEXT: s_wait_storecnt 0x0
6570 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
6571 ; GFX12-NEXT: s_wait_loadcnt 0x0
6572 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
6573 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
6574 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
6575 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
6576 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
6577 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
6578 ; GFX12-NEXT: s_cbranch_execnz .LBB26_1
6579 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
6580 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
6581 ; GFX12-NEXT: s_setpc_b64 s[30:31]
6582 ;
6583 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_pos:
6584 ; GFX940: ; %bb.0:
6585 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6586 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
6587 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
6588 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
6589 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
6590 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
6591 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v4
6592 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
6593 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
6594 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
6595 ; GFX940-NEXT: v_not_b32_e32 v6, v4
6596 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
6597 ; GFX940-NEXT: .LBB26_1: ; %atomicrmw.start
6598 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
6599 ; GFX940-NEXT: s_waitcnt vmcnt(0)
6600 ; GFX940-NEXT: v_lshrrev_b32_e32 v4, v3, v5
6601 ; GFX940-NEXT: v_sub_f16_e32 v4, v4, v2
6602 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, v3, v4
6603 ; GFX940-NEXT: v_and_or_b32 v4, v5, v6, v4
6604 ; GFX940-NEXT: buffer_wbl2 sc1
6605 ; GFX940-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0
6606 ; GFX940-NEXT: s_waitcnt vmcnt(0)
6607 ; GFX940-NEXT: buffer_inv sc1
6608 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
6609 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
6610 ; GFX940-NEXT: v_mov_b32_e32 v5, v4
6611 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
6612 ; GFX940-NEXT: s_cbranch_execnz .LBB26_1
6613 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
6614 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
6615 ; GFX940-NEXT: s_setpc_b64 s[30:31]
6616 ;
6617 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_pos:
6618 ; GFX11: ; %bb.0:
6619 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6620 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
6621 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
6622 ; GFX11-NEXT: s_mov_b32 s0, 0
6623 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
6624 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
6625 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
6626 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off
6627 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6628 ; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
6629 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
6630 ; GFX11-NEXT: v_not_b32_e32 v6, v3
6631 ; GFX11-NEXT: .LBB26_1: ; %atomicrmw.start
6632 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
6633 ; GFX11-NEXT: s_waitcnt vmcnt(0)
6634 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
6635 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6636 ; GFX11-NEXT: v_sub_f16_e32 v3, v3, v2
6637 ; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
6638 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6639 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
6640 ; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
6641 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
6642 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
6643 ; GFX11-NEXT: s_waitcnt vmcnt(0)
6644 ; GFX11-NEXT: buffer_gl1_inv
6645 ; GFX11-NEXT: buffer_gl0_inv
6646 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
6647 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
6648 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
6649 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
6650 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
6651 ; GFX11-NEXT: s_cbranch_execnz .LBB26_1
6652 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
6653 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
6654 ; GFX11-NEXT: s_setpc_b64 s[30:31]
6655 ;
6656 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_pos:
6657 ; GFX10: ; %bb.0:
6658 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6659 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
6660 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
6661 ; GFX10-NEXT: s_mov_b32 s4, 0
6662 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
6663 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
6664 ; GFX10-NEXT: global_load_dword v4, v[0:1], off
6665 ; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6666 ; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
6667 ; GFX10-NEXT: v_not_b32_e32 v6, v3
6668 ; GFX10-NEXT: .LBB26_1: ; %atomicrmw.start
6669 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
6670 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6671 ; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
6672 ; GFX10-NEXT: v_sub_f16_e32 v3, v3, v2
6673 ; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
6674 ; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
6675 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
6676 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
6677 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6678 ; GFX10-NEXT: buffer_gl1_inv
6679 ; GFX10-NEXT: buffer_gl0_inv
6680 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
6681 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
6682 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
6683 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
6684 ; GFX10-NEXT: s_cbranch_execnz .LBB26_1
6685 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
6686 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
6687 ; GFX10-NEXT: s_setpc_b64 s[30:31]
6688 ;
6689 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_pos:
6690 ; GFX90A: ; %bb.0:
6691 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6692 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
6693 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
6694 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
6695 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
6696 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
6697 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
6698 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
6699 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
6700 ; GFX90A-NEXT: v_not_b32_e32 v6, v4
6701 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
6702 ; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
6703 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
6704 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6705 ; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
6706 ; GFX90A-NEXT: v_sub_f16_e32 v4, v4, v2
6707 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
6708 ; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
6709 ; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
6710 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6711 ; GFX90A-NEXT: buffer_wbinvl1
6712 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
6713 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6714 ; GFX90A-NEXT: v_mov_b32_e32 v5, v4
6715 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
6716 ; GFX90A-NEXT: s_cbranch_execnz .LBB26_1
6717 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
6718 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
6719 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
6720 ;
6721 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_pos:
6722 ; GFX908: ; %bb.0:
6723 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6724 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
6725 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
6726 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
6727 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
6728 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
6729 ; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6730 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
6731 ; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
6732 ; GFX908-NEXT: v_not_b32_e32 v6, v3
6733 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
6734 ; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
6735 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
6736 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6737 ; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
6738 ; GFX908-NEXT: v_sub_f16_e32 v3, v3, v2
6739 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
6740 ; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
6741 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
6742 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6743 ; GFX908-NEXT: buffer_wbinvl1
6744 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
6745 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6746 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
6747 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
6748 ; GFX908-NEXT: s_cbranch_execnz .LBB26_1
6749 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
6750 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
6751 ; GFX908-NEXT: s_setpc_b64 s[30:31]
6752 ;
6753 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_pos:
6754 ; GFX8: ; %bb.0:
6755 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6756 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
6757 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
6758 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
6759 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
6760 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
6761 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6762 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
6763 ; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
6764 ; GFX8-NEXT: v_not_b32_e32 v6, v3
6765 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
6766 ; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start
6767 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6768 ; GFX8-NEXT: s_waitcnt vmcnt(0)
6769 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
6770 ; GFX8-NEXT: v_sub_f16_e32 v3, v3, v2
6771 ; GFX8-NEXT: v_and_b32_e32 v7, v4, v6
6772 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
6773 ; GFX8-NEXT: v_or_b32_e32 v3, v7, v3
6774 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
6775 ; GFX8-NEXT: s_waitcnt vmcnt(0)
6776 ; GFX8-NEXT: buffer_wbinvl1
6777 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
6778 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6779 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
6780 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
6781 ; GFX8-NEXT: s_cbranch_execnz .LBB26_1
6782 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6783 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
6784 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6785 ;
6786 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_pos:
6787 ; GFX7: ; %bb.0:
6788 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6789 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
6790 ; GFX7-NEXT: s_mov_b32 s6, 0
6791 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
6792 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
6793 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
6794 ; GFX7-NEXT: s_mov_b32 s4, s6
6795 ; GFX7-NEXT: s_mov_b32 s5, s6
6796 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
6797 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
6798 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
6799 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
6800 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
6801 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
6802 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
6803 ; GFX7-NEXT: v_not_b32_e32 v6, v2
6804 ; GFX7-NEXT: .LBB26_1: ; %atomicrmw.start
6805 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
6806 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6807 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
6808 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
6809 ; GFX7-NEXT: v_and_b32_e32 v7, v3, v6
6810 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v5
6811 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
6812 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v4, v2
6813 ; GFX7-NEXT: v_or_b32_e32 v2, v7, v2
6814 ; GFX7-NEXT: v_mov_b32_e32 v8, v3
6815 ; GFX7-NEXT: v_mov_b32_e32 v7, v2
6816 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
6817 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6818 ; GFX7-NEXT: buffer_wbinvl1
6819 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
6820 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
6821 ; GFX7-NEXT: v_mov_b32_e32 v3, v7
6822 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
6823 ; GFX7-NEXT: s_cbranch_execnz .LBB26_1
6824 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6825 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
6826 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6827 ;
6828 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_pos:
6829 ; GFX6: ; %bb.0:
6830 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6831 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
6832 ; GFX6-NEXT: s_mov_b32 s6, 0
6833 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
6834 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
6835 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
6836 ; GFX6-NEXT: s_mov_b32 s4, s6
6837 ; GFX6-NEXT: s_mov_b32 s5, s6
6838 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
6839 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
6840 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
6841 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
6842 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
6843 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
6844 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
6845 ; GFX6-NEXT: v_not_b32_e32 v6, v2
6846 ; GFX6-NEXT: .LBB26_1: ; %atomicrmw.start
6847 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
6848 ; GFX6-NEXT: s_waitcnt vmcnt(0)
6849 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
6850 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
6851 ; GFX6-NEXT: s_waitcnt expcnt(0)
6852 ; GFX6-NEXT: v_and_b32_e32 v7, v3, v6
6853 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v5
6854 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
6855 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v4, v2
6856 ; GFX6-NEXT: v_or_b32_e32 v2, v7, v2
6857 ; GFX6-NEXT: v_mov_b32_e32 v8, v3
6858 ; GFX6-NEXT: v_mov_b32_e32 v7, v2
6859 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
6860 ; GFX6-NEXT: s_waitcnt vmcnt(0)
6861 ; GFX6-NEXT: buffer_wbinvl1
6862 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
6863 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
6864 ; GFX6-NEXT: v_mov_b32_e32 v3, v7
6865 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
6866 ; GFX6-NEXT: s_cbranch_execnz .LBB26_1
6867 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
6868 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
6869 ; GFX6-NEXT: s_waitcnt expcnt(0)
6870 ; GFX6-NEXT: s_setpc_b64 s[30:31]
6871 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
6872 %unused = atomicrmw fsub ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst
6873 ret void
6874 }
6876 define void @global_agent_atomic_fsub_noret_f16__offset12b_neg(ptr addrspace(1) %ptr, half %val) #0 {
6877 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_neg:
6878 ; GFX12: ; %bb.0:
6879 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
6880 ; GFX12-NEXT: s_wait_expcnt 0x0
6881 ; GFX12-NEXT: s_wait_samplecnt 0x0
6882 ; GFX12-NEXT: s_wait_bvhcnt 0x0
6883 ; GFX12-NEXT: s_wait_kmcnt 0x0
6884 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
6885 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
6886 ; GFX12-NEXT: s_mov_b32 s0, 0
6887 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
6888 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
6889 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
6890 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off
6891 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6892 ; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
6893 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
6894 ; GFX12-NEXT: v_not_b32_e32 v6, v3
6895 ; GFX12-NEXT: .LBB27_1: ; %atomicrmw.start
6896 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
6897 ; GFX12-NEXT: s_wait_loadcnt 0x0
6898 ; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
6899 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6900 ; GFX12-NEXT: v_sub_f16_e32 v3, v3, v2
6901 ; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
6902 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6903 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
6904 ; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
6905 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
6906 ; GFX12-NEXT: s_wait_storecnt 0x0
6907 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
6908 ; GFX12-NEXT: s_wait_loadcnt 0x0
6909 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
6910 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
6911 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
6912 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
6913 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
6914 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
6915 ; GFX12-NEXT: s_cbranch_execnz .LBB27_1
6916 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
6917 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
6918 ; GFX12-NEXT: s_setpc_b64 s[30:31]
6919 ;
6920 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_neg:
6921 ; GFX940: ; %bb.0:
6922 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6923 ; GFX940-NEXT: s_movk_i32 s0, 0xf800
6924 ; GFX940-NEXT: s_mov_b32 s1, -1
6925 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
6926 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
6927 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
6928 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
6929 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v4
6930 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
6931 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
6932 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
6933 ; GFX940-NEXT: v_not_b32_e32 v6, v4
6934 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
6935 ; GFX940-NEXT: .LBB27_1: ; %atomicrmw.start
6936 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
6937 ; GFX940-NEXT: s_waitcnt vmcnt(0)
6938 ; GFX940-NEXT: v_lshrrev_b32_e32 v4, v3, v5
6939 ; GFX940-NEXT: v_sub_f16_e32 v4, v4, v2
6940 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, v3, v4
6941 ; GFX940-NEXT: v_and_or_b32 v4, v5, v6, v4
6942 ; GFX940-NEXT: buffer_wbl2 sc1
6943 ; GFX940-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0
6944 ; GFX940-NEXT: s_waitcnt vmcnt(0)
6945 ; GFX940-NEXT: buffer_inv sc1
6946 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
6947 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
6948 ; GFX940-NEXT: v_mov_b32_e32 v5, v4
6949 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
6950 ; GFX940-NEXT: s_cbranch_execnz .LBB27_1
6951 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
6952 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
6953 ; GFX940-NEXT: s_setpc_b64 s[30:31]
6954 ;
6955 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_neg:
6956 ; GFX11: ; %bb.0:
6957 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6958 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
6959 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
6960 ; GFX11-NEXT: s_mov_b32 s0, 0
6961 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
6962 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
6963 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
6964 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off
6965 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
6966 ; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
6967 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
6968 ; GFX11-NEXT: v_not_b32_e32 v6, v3
6969 ; GFX11-NEXT: .LBB27_1: ; %atomicrmw.start
6970 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
6971 ; GFX11-NEXT: s_waitcnt vmcnt(0)
6972 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
6973 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6974 ; GFX11-NEXT: v_sub_f16_e32 v3, v3, v2
6975 ; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
6976 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6977 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
6978 ; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
6979 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
6980 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
6981 ; GFX11-NEXT: s_waitcnt vmcnt(0)
6982 ; GFX11-NEXT: buffer_gl1_inv
6983 ; GFX11-NEXT: buffer_gl0_inv
6984 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
6985 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
6986 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
6987 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
6988 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
6989 ; GFX11-NEXT: s_cbranch_execnz .LBB27_1
6990 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
6991 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
6992 ; GFX11-NEXT: s_setpc_b64 s[30:31]
6993 ;
6994 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_neg:
6995 ; GFX10: ; %bb.0:
6996 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6997 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
6998 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
6999 ; GFX10-NEXT: s_mov_b32 s4, 0
7000 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
7001 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
7002 ; GFX10-NEXT: global_load_dword v4, v[0:1], off
7003 ; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
7004 ; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
7005 ; GFX10-NEXT: v_not_b32_e32 v6, v3
7006 ; GFX10-NEXT: .LBB27_1: ; %atomicrmw.start
7007 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
7008 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7009 ; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
7010 ; GFX10-NEXT: v_sub_f16_e32 v3, v3, v2
7011 ; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
7012 ; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
7013 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
7014 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
7015 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7016 ; GFX10-NEXT: buffer_gl1_inv
7017 ; GFX10-NEXT: buffer_gl0_inv
7018 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
7019 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
7020 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
7021 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
7022 ; GFX10-NEXT: s_cbranch_execnz .LBB27_1
7023 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
7024 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
7025 ; GFX10-NEXT: s_setpc_b64 s[30:31]
7026 ;
7027 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_neg:
7028 ; GFX90A: ; %bb.0:
7029 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7030 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
7031 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
7032 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
7033 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
7034 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
7035 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7036 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
7037 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
7038 ; GFX90A-NEXT: v_not_b32_e32 v6, v4
7039 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
7040 ; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
7041 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
7042 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7043 ; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
7044 ; GFX90A-NEXT: v_sub_f16_e32 v4, v4, v2
7045 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
7046 ; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
7047 ; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
7048 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7049 ; GFX90A-NEXT: buffer_wbinvl1
7050 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
7051 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7052 ; GFX90A-NEXT: v_mov_b32_e32 v5, v4
7053 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
7054 ; GFX90A-NEXT: s_cbranch_execnz .LBB27_1
7055 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
7056 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
7057 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
7058 ;
7059 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_neg:
7060 ; GFX908: ; %bb.0:
7061 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7062 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
7063 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
7064 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
7065 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
7066 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
7067 ; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
7068 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
7069 ; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
7070 ; GFX908-NEXT: v_not_b32_e32 v6, v3
7071 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
7072 ; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
7073 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
7074 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7075 ; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
7076 ; GFX908-NEXT: v_sub_f16_e32 v3, v3, v2
7077 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
7078 ; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
7079 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
7080 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7081 ; GFX908-NEXT: buffer_wbinvl1
7082 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
7083 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7084 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
7085 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
7086 ; GFX908-NEXT: s_cbranch_execnz .LBB27_1
7087 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
7088 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
7089 ; GFX908-NEXT: s_setpc_b64 s[30:31]
7090 ;
7091 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_neg:
7092 ; GFX8: ; %bb.0:
7093 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7094 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
7095 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
7096 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
7097 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
7098 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
7099 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
7100 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
7101 ; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
7102 ; GFX8-NEXT: v_not_b32_e32 v6, v3
7103 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
7104 ; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start
7105 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7106 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7107 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
7108 ; GFX8-NEXT: v_sub_f16_e32 v3, v3, v2
7109 ; GFX8-NEXT: v_and_b32_e32 v7, v4, v6
7110 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
7111 ; GFX8-NEXT: v_or_b32_e32 v3, v7, v3
7112 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
7113 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7114 ; GFX8-NEXT: buffer_wbinvl1
7115 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
7116 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7117 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
7118 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
7119 ; GFX8-NEXT: s_cbranch_execnz .LBB27_1
7120 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7121 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
7122 ; GFX8-NEXT: s_setpc_b64 s[30:31]
7123 ;
7124 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_neg:
7125 ; GFX7: ; %bb.0:
7126 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7127 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
7128 ; GFX7-NEXT: s_mov_b32 s6, 0
7129 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
7130 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
7131 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
7132 ; GFX7-NEXT: s_mov_b32 s4, s6
7133 ; GFX7-NEXT: s_mov_b32 s5, s6
7134 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
7135 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
7136 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
7137 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
7138 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
7139 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
7140 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
7141 ; GFX7-NEXT: v_not_b32_e32 v6, v2
7142 ; GFX7-NEXT: .LBB27_1: ; %atomicrmw.start
7143 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7144 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7145 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
7146 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
7147 ; GFX7-NEXT: v_and_b32_e32 v7, v3, v6
7148 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v5
7149 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
7150 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v4, v2
7151 ; GFX7-NEXT: v_or_b32_e32 v2, v7, v2
7152 ; GFX7-NEXT: v_mov_b32_e32 v8, v3
7153 ; GFX7-NEXT: v_mov_b32_e32 v7, v2
7154 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
7155 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7156 ; GFX7-NEXT: buffer_wbinvl1
7157 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
7158 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
7159 ; GFX7-NEXT: v_mov_b32_e32 v3, v7
7160 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
7161 ; GFX7-NEXT: s_cbranch_execnz .LBB27_1
7162 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
7163 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
7164 ; GFX7-NEXT: s_setpc_b64 s[30:31]
7165 ;
7166 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f16__offset12b_neg:
7167 ; GFX6: ; %bb.0:
7168 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7169 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
7170 ; GFX6-NEXT: s_mov_b32 s6, 0
7171 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
7172 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
7173 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
7174 ; GFX6-NEXT: s_mov_b32 s4, s6
7175 ; GFX6-NEXT: s_mov_b32 s5, s6
7176 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
7177 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
7178 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
7179 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
7180 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
7181 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
7182 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
7183 ; GFX6-NEXT: v_not_b32_e32 v6, v2
7184 ; GFX6-NEXT: .LBB27_1: ; %atomicrmw.start
7185 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
7186 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7187 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
7188 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
7189 ; GFX6-NEXT: s_waitcnt expcnt(0)
7190 ; GFX6-NEXT: v_and_b32_e32 v7, v3, v6
7191 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v5
7192 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
7193 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v4, v2
7194 ; GFX6-NEXT: v_or_b32_e32 v2, v7, v2
7195 ; GFX6-NEXT: v_mov_b32_e32 v8, v3
7196 ; GFX6-NEXT: v_mov_b32_e32 v7, v2
7197 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
7198 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7199 ; GFX6-NEXT: buffer_wbinvl1
7200 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
7201 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
7202 ; GFX6-NEXT: v_mov_b32_e32 v3, v7
7203 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
7204 ; GFX6-NEXT: s_cbranch_execnz .LBB27_1
7205 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
7206 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
7207 ; GFX6-NEXT: s_waitcnt expcnt(0)
7208 ; GFX6-NEXT: s_setpc_b64 s[30:31]
7209 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 -1024
7210 %unused = atomicrmw fsub ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst
7211 ret void
7212 }
7214 define half @global_agent_atomic_fsub_ret_f16__offset12b_pos__align4(ptr addrspace(1) %ptr, half %val) #0 {
7215 ; GFX12-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos__align4:
7216 ; GFX12: ; %bb.0:
7217 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
7218 ; GFX12-NEXT: s_wait_expcnt 0x0
7219 ; GFX12-NEXT: s_wait_samplecnt 0x0
7220 ; GFX12-NEXT: s_wait_bvhcnt 0x0
7221 ; GFX12-NEXT: s_wait_kmcnt 0x0
7222 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2046
7223 ; GFX12-NEXT: s_mov_b32 s0, 0
7224 ; GFX12-NEXT: .LBB28_1: ; %atomicrmw.start
7225 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
7226 ; GFX12-NEXT: s_wait_loadcnt 0x0
7227 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
7228 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7229 ; GFX12-NEXT: v_sub_f16_e32 v3, v4, v2
7230 ; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
7231 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
7232 ; GFX12-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
7233 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
7234 ; GFX12-NEXT: s_wait_storecnt 0x0
7235 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
7236 ; GFX12-NEXT: s_wait_loadcnt 0x0
7237 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
7238 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
7239 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
7240 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
7241 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
7242 ; GFX12-NEXT: s_cbranch_execnz .LBB28_1
7243 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
7244 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
7245 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
7246 ; GFX12-NEXT: s_setpc_b64 s[30:31]
7247 ;
7248 ; GFX940-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos__align4:
7249 ; GFX940: ; %bb.0:
7250 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7251 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2046
7252 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
7253 ; GFX940-NEXT: s_mov_b32 s2, 0xffff0000
7254 ; GFX940-NEXT: .LBB28_1: ; %atomicrmw.start
7255 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
7256 ; GFX940-NEXT: s_waitcnt vmcnt(0)
7257 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
7258 ; GFX940-NEXT: v_sub_f16_e32 v3, v5, v2
7259 ; GFX940-NEXT: v_and_or_b32 v4, v5, s2, v3
7260 ; GFX940-NEXT: buffer_wbl2 sc1
7261 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 sc0
7262 ; GFX940-NEXT: s_waitcnt vmcnt(0)
7263 ; GFX940-NEXT: buffer_inv sc1
7264 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
7265 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
7266 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
7267 ; GFX940-NEXT: s_cbranch_execnz .LBB28_1
7268 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
7269 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
7270 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
7271 ; GFX940-NEXT: s_setpc_b64 s[30:31]
7272 ;
7273 ; GFX11-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos__align4:
7274 ; GFX11: ; %bb.0:
7275 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7276 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2046
7277 ; GFX11-NEXT: s_mov_b32 s0, 0
7278 ; GFX11-NEXT: .LBB28_1: ; %atomicrmw.start
7279 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
7280 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7281 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
7282 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7283 ; GFX11-NEXT: v_sub_f16_e32 v3, v4, v2
7284 ; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
7285 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
7286 ; GFX11-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
7287 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
7288 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
7289 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7290 ; GFX11-NEXT: buffer_gl1_inv
7291 ; GFX11-NEXT: buffer_gl0_inv
7292 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
7293 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
7294 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
7295 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
7296 ; GFX11-NEXT: s_cbranch_execnz .LBB28_1
7297 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
7298 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
7299 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
7300 ; GFX11-NEXT: s_setpc_b64 s[30:31]
7301 ;
7302 ; GFX10-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos__align4:
7303 ; GFX10: ; %bb.0:
7304 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7305 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2046
7306 ; GFX10-NEXT: s_mov_b32 s4, 0
7307 ; GFX10-NEXT: .LBB28_1: ; %atomicrmw.start
7308 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
7309 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7310 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
7311 ; GFX10-NEXT: v_sub_f16_e32 v3, v4, v2
7312 ; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
7313 ; GFX10-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
7314 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
7315 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2046 glc
7316 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7317 ; GFX10-NEXT: buffer_gl1_inv
7318 ; GFX10-NEXT: buffer_gl0_inv
7319 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
7320 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
7321 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
7322 ; GFX10-NEXT: s_cbranch_execnz .LBB28_1
7323 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
7324 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
7325 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
7326 ; GFX10-NEXT: s_setpc_b64 s[30:31]
7327 ;
7328 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos__align4:
7329 ; GFX90A: ; %bb.0:
7330 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7331 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2046
7332 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
7333 ; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
7334 ; GFX90A-NEXT: .LBB28_1: ; %atomicrmw.start
7335 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
7336 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7337 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
7338 ; GFX90A-NEXT: v_sub_f16_e32 v3, v5, v2
7339 ; GFX90A-NEXT: v_and_or_b32 v4, v5, s6, v3
7340 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 glc
7341 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7342 ; GFX90A-NEXT: buffer_wbinvl1
7343 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
7344 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7345 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
7346 ; GFX90A-NEXT: s_cbranch_execnz .LBB28_1
7347 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
7348 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
7349 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
7350 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
7351 ;
7352 ; GFX908-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos__align4:
7353 ; GFX908: ; %bb.0:
7354 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7355 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2046
7356 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
7357 ; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
7358 ; GFX908-NEXT: .LBB28_1: ; %atomicrmw.start
7359 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
7360 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7361 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
7362 ; GFX908-NEXT: v_sub_f16_e32 v3, v4, v2
7363 ; GFX908-NEXT: v_and_or_b32 v3, v4, s6, v3
7364 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2046 glc
7365 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7366 ; GFX908-NEXT: buffer_wbinvl1
7367 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
7368 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7369 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
7370 ; GFX908-NEXT: s_cbranch_execnz .LBB28_1
7371 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
7372 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
7373 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
7374 ; GFX908-NEXT: s_setpc_b64 s[30:31]
7375 ;
7376 ; GFX8-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos__align4:
7377 ; GFX8: ; %bb.0:
7378 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7379 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
7380 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
7381 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
7382 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
7383 ; GFX8-NEXT: .LBB28_1: ; %atomicrmw.start
7384 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7385 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7386 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
7387 ; GFX8-NEXT: v_sub_f16_e32 v0, v1, v2
7388 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
7389 ; GFX8-NEXT: v_or_b32_e32 v0, v5, v0
7390 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
7391 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7392 ; GFX8-NEXT: buffer_wbinvl1
7393 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
7394 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7395 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
7396 ; GFX8-NEXT: s_cbranch_execnz .LBB28_1
7397 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7398 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
7399 ; GFX8-NEXT: s_setpc_b64 s[30:31]
7400 ;
7401 ; GFX7-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos__align4:
7402 ; GFX7: ; %bb.0:
7403 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7404 ; GFX7-NEXT: s_mov_b32 s6, 0
7405 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
7406 ; GFX7-NEXT: s_mov_b32 s4, s6
7407 ; GFX7-NEXT: s_mov_b32 s5, s6
7408 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
7409 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
7410 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
7411 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v2
7412 ; GFX7-NEXT: .LBB28_1: ; %atomicrmw.start
7413 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7414 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7415 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
7416 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
7417 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v6
7418 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
7419 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
7420 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
7421 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
7422 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 offset:2046 glc
7423 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7424 ; GFX7-NEXT: buffer_wbinvl1
7425 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
7426 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
7427 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
7428 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
7429 ; GFX7-NEXT: s_cbranch_execnz .LBB28_1
7430 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
7431 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
7432 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v4
7433 ; GFX7-NEXT: s_setpc_b64 s[30:31]
7434 ;
7435 ; GFX6-LABEL: global_agent_atomic_fsub_ret_f16__offset12b_pos__align4:
7436 ; GFX6: ; %bb.0:
7437 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7438 ; GFX6-NEXT: s_mov_b32 s6, 0
7439 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
7440 ; GFX6-NEXT: s_mov_b32 s4, s6
7441 ; GFX6-NEXT: s_mov_b32 s5, s6
7442 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
7443 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
7444 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
7445 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v2
7446 ; GFX6-NEXT: .LBB28_1: ; %atomicrmw.start
7447 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
7448 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7449 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v3
7450 ; GFX6-NEXT: s_waitcnt expcnt(0)
7451 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
7452 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v6
7453 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
7454 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
7455 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
7456 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
7457 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 offset:2046 glc
7458 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7459 ; GFX6-NEXT: buffer_wbinvl1
7460 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
7461 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
7462 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
7463 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
7464 ; GFX6-NEXT: s_cbranch_execnz .LBB28_1
7465 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
7466 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
7467 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v4
7468 ; GFX6-NEXT: s_waitcnt expcnt(0)
7469 ; GFX6-NEXT: s_setpc_b64 s[30:31]
7470 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
7471 %result = atomicrmw fsub ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, align 4
7472 ret half %result
7473 }
7475 define void @global_agent_atomic_fsub_noret_f16__offset12b__align4_pos(ptr addrspace(1) %ptr, half %val) #0 {
7476 ; GFX12-LABEL: global_agent_atomic_fsub_noret_f16__offset12b__align4_pos:
7477 ; GFX12: ; %bb.0:
7478 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
7479 ; GFX12-NEXT: s_wait_expcnt 0x0
7480 ; GFX12-NEXT: s_wait_samplecnt 0x0
7481 ; GFX12-NEXT: s_wait_bvhcnt 0x0
7482 ; GFX12-NEXT: s_wait_kmcnt 0x0
7483 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:2046
7484 ; GFX12-NEXT: s_mov_b32 s0, 0
7485 ; GFX12-NEXT: .LBB29_1: ; %atomicrmw.start
7486 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
7487 ; GFX12-NEXT: s_wait_loadcnt 0x0
7488 ; GFX12-NEXT: v_sub_f16_e32 v3, v4, v2
7489 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7490 ; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
7491 ; GFX12-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
7492 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
7493 ; GFX12-NEXT: s_wait_storecnt 0x0
7494 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
7495 ; GFX12-NEXT: s_wait_loadcnt 0x0
7496 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
7497 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
7498 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
7499 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
7500 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
7501 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
7502 ; GFX12-NEXT: s_cbranch_execnz .LBB29_1
7503 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
7504 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
7505 ; GFX12-NEXT: s_setpc_b64 s[30:31]
7506 ;
7507 ; GFX940-LABEL: global_agent_atomic_fsub_noret_f16__offset12b__align4_pos:
7508 ; GFX940: ; %bb.0:
7509 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7510 ; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:2046
7511 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
7512 ; GFX940-NEXT: s_mov_b32 s2, 0xffff0000
7513 ; GFX940-NEXT: .LBB29_1: ; %atomicrmw.start
7514 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
7515 ; GFX940-NEXT: s_waitcnt vmcnt(0)
7516 ; GFX940-NEXT: v_sub_f16_e32 v3, v5, v2
7517 ; GFX940-NEXT: v_and_or_b32 v4, v5, s2, v3
7518 ; GFX940-NEXT: buffer_wbl2 sc1
7519 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 sc0
7520 ; GFX940-NEXT: s_waitcnt vmcnt(0)
7521 ; GFX940-NEXT: buffer_inv sc1
7522 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
7523 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
7524 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
7525 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
7526 ; GFX940-NEXT: s_cbranch_execnz .LBB29_1
7527 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
7528 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
7529 ; GFX940-NEXT: s_setpc_b64 s[30:31]
7530 ;
7531 ; GFX11-LABEL: global_agent_atomic_fsub_noret_f16__offset12b__align4_pos:
7532 ; GFX11: ; %bb.0:
7533 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7534 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:2046
7535 ; GFX11-NEXT: s_mov_b32 s0, 0
7536 ; GFX11-NEXT: .LBB29_1: ; %atomicrmw.start
7537 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
7538 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7539 ; GFX11-NEXT: v_sub_f16_e32 v3, v4, v2
7540 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7541 ; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
7542 ; GFX11-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
7543 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
7544 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
7545 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7546 ; GFX11-NEXT: buffer_gl1_inv
7547 ; GFX11-NEXT: buffer_gl0_inv
7548 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
7549 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
7550 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
7551 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
7552 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
7553 ; GFX11-NEXT: s_cbranch_execnz .LBB29_1
7554 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
7555 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
7556 ; GFX11-NEXT: s_setpc_b64 s[30:31]
7557 ;
7558 ; GFX10-LABEL: global_agent_atomic_fsub_noret_f16__offset12b__align4_pos:
7559 ; GFX10: ; %bb.0:
7560 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7561 ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2046
7562 ; GFX10-NEXT: s_mov_b32 s4, 0
7563 ; GFX10-NEXT: .LBB29_1: ; %atomicrmw.start
7564 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
7565 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7566 ; GFX10-NEXT: v_sub_f16_e32 v3, v4, v2
7567 ; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
7568 ; GFX10-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
7569 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
7570 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2046 glc
7571 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7572 ; GFX10-NEXT: buffer_gl1_inv
7573 ; GFX10-NEXT: buffer_gl0_inv
7574 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
7575 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
7576 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
7577 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
7578 ; GFX10-NEXT: s_cbranch_execnz .LBB29_1
7579 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
7580 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
7581 ; GFX10-NEXT: s_setpc_b64 s[30:31]
7582 ;
7583 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_f16__offset12b__align4_pos:
7584 ; GFX90A: ; %bb.0:
7585 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7586 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2046
7587 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
7588 ; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
7589 ; GFX90A-NEXT: .LBB29_1: ; %atomicrmw.start
7590 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
7591 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7592 ; GFX90A-NEXT: v_sub_f16_e32 v3, v5, v2
7593 ; GFX90A-NEXT: v_and_or_b32 v4, v5, s6, v3
7594 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 glc
7595 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7596 ; GFX90A-NEXT: buffer_wbinvl1
7597 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
7598 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7599 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
7600 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
7601 ; GFX90A-NEXT: s_cbranch_execnz .LBB29_1
7602 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
7603 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
7604 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
7605 ;
7606 ; GFX908-LABEL: global_agent_atomic_fsub_noret_f16__offset12b__align4_pos:
7607 ; GFX908: ; %bb.0:
7608 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7609 ; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2046
7610 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
7611 ; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
7612 ; GFX908-NEXT: .LBB29_1: ; %atomicrmw.start
7613 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
7614 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7615 ; GFX908-NEXT: v_sub_f16_e32 v3, v4, v2
7616 ; GFX908-NEXT: v_and_or_b32 v3, v4, s6, v3
7617 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2046 glc
7618 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7619 ; GFX908-NEXT: buffer_wbinvl1
7620 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
7621 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7622 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
7623 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
7624 ; GFX908-NEXT: s_cbranch_execnz .LBB29_1
7625 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
7626 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
7627 ; GFX908-NEXT: s_setpc_b64 s[30:31]
7628 ;
7629 ; GFX8-LABEL: global_agent_atomic_fsub_noret_f16__offset12b__align4_pos:
7630 ; GFX8: ; %bb.0:
7631 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7632 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fe, v0
7633 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
7634 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
7635 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
7636 ; GFX8-NEXT: .LBB29_1: ; %atomicrmw.start
7637 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7638 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7639 ; GFX8-NEXT: v_sub_f16_e32 v3, v4, v2
7640 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
7641 ; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
7642 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
7643 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7644 ; GFX8-NEXT: buffer_wbinvl1
7645 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
7646 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7647 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
7648 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
7649 ; GFX8-NEXT: s_cbranch_execnz .LBB29_1
7650 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7651 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
7652 ; GFX8-NEXT: s_setpc_b64 s[30:31]
7653 ;
7654 ; GFX7-LABEL: global_agent_atomic_fsub_noret_f16__offset12b__align4_pos:
7655 ; GFX7: ; %bb.0:
7656 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7657 ; GFX7-NEXT: s_mov_b32 s6, 0
7658 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
7659 ; GFX7-NEXT: s_mov_b32 s4, s6
7660 ; GFX7-NEXT: s_mov_b32 s5, s6
7661 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
7662 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
7663 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
7664 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v2
7665 ; GFX7-NEXT: .LBB29_1: ; %atomicrmw.start
7666 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7667 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7668 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
7669 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
7670 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v4
7671 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
7672 ; GFX7-NEXT: v_or_b32_e32 v2, v5, v2
7673 ; GFX7-NEXT: v_mov_b32_e32 v6, v3
7674 ; GFX7-NEXT: v_mov_b32_e32 v5, v2
7675 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2046 glc
7676 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7677 ; GFX7-NEXT: buffer_wbinvl1
7678 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
7679 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
7680 ; GFX7-NEXT: v_mov_b32_e32 v3, v5
7681 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
7682 ; GFX7-NEXT: s_cbranch_execnz .LBB29_1
7683 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
7684 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
7685 ; GFX7-NEXT: s_setpc_b64 s[30:31]
7686 ;
7687 ; GFX6-LABEL: global_agent_atomic_fsub_noret_f16__offset12b__align4_pos:
7688 ; GFX6: ; %bb.0:
7689 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7690 ; GFX6-NEXT: s_mov_b32 s6, 0
7691 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
7692 ; GFX6-NEXT: s_mov_b32 s4, s6
7693 ; GFX6-NEXT: s_mov_b32 s5, s6
7694 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
7695 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
7696 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
7697 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v2
7698 ; GFX6-NEXT: .LBB29_1: ; %atomicrmw.start
7699 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
7700 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7701 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v3
7702 ; GFX6-NEXT: s_waitcnt expcnt(0)
7703 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
7704 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v4
7705 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
7706 ; GFX6-NEXT: v_or_b32_e32 v2, v5, v2
7707 ; GFX6-NEXT: v_mov_b32_e32 v6, v3
7708 ; GFX6-NEXT: v_mov_b32_e32 v5, v2
7709 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2046 glc
7710 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7711 ; GFX6-NEXT: buffer_wbinvl1
7712 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
7713 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
7714 ; GFX6-NEXT: v_mov_b32_e32 v3, v5
7715 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
7716 ; GFX6-NEXT: s_cbranch_execnz .LBB29_1
7717 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
7718 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
7719 ; GFX6-NEXT: s_waitcnt expcnt(0)
7720 ; GFX6-NEXT: s_setpc_b64 s[30:31]
7721 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
7722 %unused = atomicrmw fsub ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, align 4
7723 ret void
7724 }
7726 define half @global_system_atomic_fsub_ret_f16__offset12b_pos(ptr addrspace(1) %ptr, half %val) #0 {
7727 ; GFX12-LABEL: global_system_atomic_fsub_ret_f16__offset12b_pos:
7728 ; GFX12: ; %bb.0:
7729 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
7730 ; GFX12-NEXT: s_wait_expcnt 0x0
7731 ; GFX12-NEXT: s_wait_samplecnt 0x0
7732 ; GFX12-NEXT: s_wait_bvhcnt 0x0
7733 ; GFX12-NEXT: s_wait_kmcnt 0x0
7734 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
7735 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
7736 ; GFX12-NEXT: s_mov_b32 s0, 0
7737 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
7738 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
7739 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
7740 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
7741 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7742 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
7743 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
7744 ; GFX12-NEXT: v_not_b32_e32 v4, v4
7745 ; GFX12-NEXT: .LBB30_1: ; %atomicrmw.start
7746 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
7747 ; GFX12-NEXT: s_wait_loadcnt 0x0
7748 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
7749 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7750 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
7751 ; GFX12-NEXT: v_sub_f16_e32 v5, v5, v2
7752 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7753 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff, v5
7754 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
7755 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
7756 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
7757 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
7758 ; GFX12-NEXT: s_wait_storecnt 0x0
7759 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
7760 ; GFX12-NEXT: s_wait_loadcnt 0x0
7761 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
7762 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
7763 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
7764 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
7765 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
7766 ; GFX12-NEXT: s_cbranch_execnz .LBB30_1
7767 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
7768 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
7769 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
7770 ; GFX12-NEXT: s_setpc_b64 s[30:31]
7771 ;
7772 ; GFX940-LABEL: global_system_atomic_fsub_ret_f16__offset12b_pos:
7773 ; GFX940: ; %bb.0:
7774 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7775 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
7776 ; GFX940-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, s[0:1]
7777 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v6
7778 ; GFX940-NEXT: v_mov_b32_e32 v1, v7
7779 ; GFX940-NEXT: global_load_dword v4, v[0:1], off
7780 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v6
7781 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7782 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
7783 ; GFX940-NEXT: v_lshlrev_b32_e64 v5, v3, s0
7784 ; GFX940-NEXT: v_not_b32_e32 v5, v5
7785 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
7786 ; GFX940-NEXT: .LBB30_1: ; %atomicrmw.start
7787 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
7788 ; GFX940-NEXT: s_waitcnt vmcnt(0)
7789 ; GFX940-NEXT: v_mov_b32_e32 v7, v4
7790 ; GFX940-NEXT: v_lshrrev_b32_e32 v4, v3, v7
7791 ; GFX940-NEXT: v_sub_f16_e32 v4, v4, v2
7792 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, v3, v4
7793 ; GFX940-NEXT: v_and_or_b32 v6, v7, v5, v4
7794 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
7795 ; GFX940-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off sc0 sc1
7796 ; GFX940-NEXT: s_waitcnt vmcnt(0)
7797 ; GFX940-NEXT: buffer_inv sc0 sc1
7798 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
7799 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
7800 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
7801 ; GFX940-NEXT: s_cbranch_execnz .LBB30_1
7802 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
7803 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
7804 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v4
7805 ; GFX940-NEXT: s_setpc_b64 s[30:31]
7806 ;
7807 ; GFX11-LABEL: global_system_atomic_fsub_ret_f16__offset12b_pos:
7808 ; GFX11: ; %bb.0:
7809 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7810 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
7811 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
7812 ; GFX11-NEXT: s_mov_b32 s0, 0
7813 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
7814 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
7815 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
7816 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
7817 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7818 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
7819 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
7820 ; GFX11-NEXT: v_not_b32_e32 v4, v4
7821 ; GFX11-NEXT: .LBB30_1: ; %atomicrmw.start
7822 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
7823 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7824 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
7825 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7826 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
7827 ; GFX11-NEXT: v_sub_f16_e32 v5, v5, v2
7828 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7829 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
7830 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
7831 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
7832 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
7833 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
7834 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
7835 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7836 ; GFX11-NEXT: buffer_gl1_inv
7837 ; GFX11-NEXT: buffer_gl0_inv
7838 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
7839 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
7840 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
7841 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
7842 ; GFX11-NEXT: s_cbranch_execnz .LBB30_1
7843 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
7844 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
7845 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
7846 ; GFX11-NEXT: s_setpc_b64 s[30:31]
7847 ;
7848 ; GFX10-LABEL: global_system_atomic_fsub_ret_f16__offset12b_pos:
7849 ; GFX10: ; %bb.0:
7850 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7851 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
7852 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
7853 ; GFX10-NEXT: s_mov_b32 s4, 0
7854 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
7855 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
7856 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
7857 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7858 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
7859 ; GFX10-NEXT: v_not_b32_e32 v4, v4
7860 ; GFX10-NEXT: .LBB30_1: ; %atomicrmw.start
7861 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
7862 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7863 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
7864 ; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
7865 ; GFX10-NEXT: v_sub_f16_e32 v5, v5, v2
7866 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
7867 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
7868 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
7869 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
7870 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7871 ; GFX10-NEXT: buffer_gl1_inv
7872 ; GFX10-NEXT: buffer_gl0_inv
7873 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
7874 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
7875 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
7876 ; GFX10-NEXT: s_cbranch_execnz .LBB30_1
7877 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
7878 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
7879 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
7880 ; GFX10-NEXT: s_setpc_b64 s[30:31]
7881 ;
7882 ; GFX90A-LABEL: global_system_atomic_fsub_ret_f16__offset12b_pos:
7883 ; GFX90A: ; %bb.0:
7884 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7885 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
7886 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
7887 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
7888 ; GFX90A-NEXT: global_load_dword v4, v[0:1], off
7889 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
7890 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7891 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
7892 ; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v3, s4
7893 ; GFX90A-NEXT: v_not_b32_e32 v5, v5
7894 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
7895 ; GFX90A-NEXT: .LBB30_1: ; %atomicrmw.start
7896 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
7897 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7898 ; GFX90A-NEXT: v_mov_b32_e32 v7, v4
7899 ; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v7
7900 ; GFX90A-NEXT: v_sub_f16_e32 v4, v4, v2
7901 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
7902 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v5, v4
7903 ; GFX90A-NEXT: buffer_wbl2
7904 ; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off glc
7905 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7906 ; GFX90A-NEXT: buffer_invl2
7907 ; GFX90A-NEXT: buffer_wbinvl1
7908 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
7909 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7910 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
7911 ; GFX90A-NEXT: s_cbranch_execnz .LBB30_1
7912 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
7913 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
7914 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v4
7915 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
7916 ;
7917 ; GFX908-LABEL: global_system_atomic_fsub_ret_f16__offset12b_pos:
7918 ; GFX908: ; %bb.0:
7919 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7920 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
7921 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
7922 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
7923 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
7924 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
7925 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7926 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
7927 ; GFX908-NEXT: v_lshlrev_b32_e64 v5, v3, s4
7928 ; GFX908-NEXT: v_not_b32_e32 v5, v5
7929 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
7930 ; GFX908-NEXT: .LBB30_1: ; %atomicrmw.start
7931 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
7932 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7933 ; GFX908-NEXT: v_mov_b32_e32 v7, v4
7934 ; GFX908-NEXT: v_lshrrev_b32_e32 v4, v3, v7
7935 ; GFX908-NEXT: v_sub_f16_e32 v4, v4, v2
7936 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, v3, v4
7937 ; GFX908-NEXT: v_and_or_b32 v6, v7, v5, v4
7938 ; GFX908-NEXT: global_atomic_cmpswap v4, v[0:1], v[6:7], off glc
7939 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7940 ; GFX908-NEXT: buffer_wbinvl1
7941 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
7942 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7943 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
7944 ; GFX908-NEXT: s_cbranch_execnz .LBB30_1
7945 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
7946 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
7947 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v4
7948 ; GFX908-NEXT: s_setpc_b64 s[30:31]
7949 ;
7950 ; GFX8-LABEL: global_system_atomic_fsub_ret_f16__offset12b_pos:
7951 ; GFX8: ; %bb.0:
7952 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7953 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
7954 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
7955 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
7956 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
7957 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
7958 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7959 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
7960 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
7961 ; GFX8-NEXT: v_not_b32_e32 v4, v4
7962 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
7963 ; GFX8-NEXT: .LBB30_1: ; %atomicrmw.start
7964 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7965 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7966 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
7967 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
7968 ; GFX8-NEXT: v_sub_f16_e32 v5, v5, v2
7969 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
7970 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
7971 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
7972 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
7973 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7974 ; GFX8-NEXT: buffer_wbinvl1
7975 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
7976 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7977 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
7978 ; GFX8-NEXT: s_cbranch_execnz .LBB30_1
7979 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7980 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
7981 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
7982 ; GFX8-NEXT: s_setpc_b64 s[30:31]
7983 ;
7984 ; GFX7-LABEL: global_system_atomic_fsub_ret_f16__offset12b_pos:
7985 ; GFX7: ; %bb.0:
7986 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7987 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
7988 ; GFX7-NEXT: s_mov_b32 s6, 0
7989 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
7990 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
7991 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
7992 ; GFX7-NEXT: s_mov_b32 s4, s6
7993 ; GFX7-NEXT: s_mov_b32 s5, s6
7994 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
7995 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
7996 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
7997 ; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
7998 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
7999 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
8000 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
8001 ; GFX7-NEXT: v_not_b32_e32 v8, v2
8002 ; GFX7-NEXT: .LBB30_1: ; %atomicrmw.start
8003 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
8004 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8005 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
8006 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
8007 ; GFX7-NEXT: v_and_b32_e32 v4, v3, v8
8008 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v7
8009 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
8010 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v6, v2
8011 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
8012 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
8013 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
8014 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
8015 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8016 ; GFX7-NEXT: buffer_wbinvl1
8017 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
8018 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
8019 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
8020 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
8021 ; GFX7-NEXT: s_cbranch_execnz .LBB30_1
8022 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
8023 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
8024 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
8025 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
8026 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8027 ;
8028 ; GFX6-LABEL: global_system_atomic_fsub_ret_f16__offset12b_pos:
8029 ; GFX6: ; %bb.0:
8030 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8031 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
8032 ; GFX6-NEXT: s_mov_b32 s6, 0
8033 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8034 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
8035 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
8036 ; GFX6-NEXT: s_mov_b32 s4, s6
8037 ; GFX6-NEXT: s_mov_b32 s5, s6
8038 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
8039 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
8040 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
8041 ; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
8042 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
8043 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
8044 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
8045 ; GFX6-NEXT: v_not_b32_e32 v8, v2
8046 ; GFX6-NEXT: .LBB30_1: ; %atomicrmw.start
8047 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
8048 ; GFX6-NEXT: s_waitcnt vmcnt(0)
8049 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
8050 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
8051 ; GFX6-NEXT: s_waitcnt expcnt(0)
8052 ; GFX6-NEXT: v_and_b32_e32 v4, v3, v8
8053 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v7
8054 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
8055 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v6, v2
8056 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
8057 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
8058 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
8059 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
8060 ; GFX6-NEXT: s_waitcnt vmcnt(0)
8061 ; GFX6-NEXT: buffer_wbinvl1
8062 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
8063 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
8064 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
8065 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
8066 ; GFX6-NEXT: s_cbranch_execnz .LBB30_1
8067 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
8068 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
8069 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
8070 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
8071 ; GFX6-NEXT: s_waitcnt expcnt(0)
8072 ; GFX6-NEXT: s_setpc_b64 s[30:31]
8073 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
8074 %result = atomicrmw fsub ptr addrspace(1) %gep, half %val seq_cst
8075 ret half %result
8076 }
8078 define void @global_system_atomic_fsub_noret_f16__offset12b_pos(ptr addrspace(1) %ptr, half %val) #0 {
8079 ; GFX12-LABEL: global_system_atomic_fsub_noret_f16__offset12b_pos:
8080 ; GFX12: ; %bb.0:
8081 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
8082 ; GFX12-NEXT: s_wait_expcnt 0x0
8083 ; GFX12-NEXT: s_wait_samplecnt 0x0
8084 ; GFX12-NEXT: s_wait_bvhcnt 0x0
8085 ; GFX12-NEXT: s_wait_kmcnt 0x0
8086 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
8087 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
8088 ; GFX12-NEXT: s_mov_b32 s0, 0
8089 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
8090 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
8091 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
8092 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off
8093 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
8094 ; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
8095 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
8096 ; GFX12-NEXT: v_not_b32_e32 v6, v3
8097 ; GFX12-NEXT: .LBB31_1: ; %atomicrmw.start
8098 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
8099 ; GFX12-NEXT: s_wait_loadcnt 0x0
8100 ; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
8101 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8102 ; GFX12-NEXT: v_sub_f16_e32 v3, v3, v2
8103 ; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
8104 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8105 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
8106 ; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
8107 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
8108 ; GFX12-NEXT: s_wait_storecnt 0x0
8109 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
8110 ; GFX12-NEXT: s_wait_loadcnt 0x0
8111 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
8112 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
8113 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
8114 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
8115 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
8116 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
8117 ; GFX12-NEXT: s_cbranch_execnz .LBB31_1
8118 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
8119 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
8120 ; GFX12-NEXT: s_setpc_b64 s[30:31]
8121 ;
8122 ; GFX940-LABEL: global_system_atomic_fsub_noret_f16__offset12b_pos:
8123 ; GFX940: ; %bb.0:
8124 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8125 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
8126 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
8127 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
8128 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
8129 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
8130 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v4
8131 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8132 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
8133 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
8134 ; GFX940-NEXT: v_not_b32_e32 v6, v4
8135 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
8136 ; GFX940-NEXT: .LBB31_1: ; %atomicrmw.start
8137 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
8138 ; GFX940-NEXT: s_waitcnt vmcnt(0)
8139 ; GFX940-NEXT: v_lshrrev_b32_e32 v4, v3, v5
8140 ; GFX940-NEXT: v_sub_f16_e32 v4, v4, v2
8141 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, v3, v4
8142 ; GFX940-NEXT: v_and_or_b32 v4, v5, v6, v4
8143 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
8144 ; GFX940-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0 sc1
8145 ; GFX940-NEXT: s_waitcnt vmcnt(0)
8146 ; GFX940-NEXT: buffer_inv sc0 sc1
8147 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
8148 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
8149 ; GFX940-NEXT: v_mov_b32_e32 v5, v4
8150 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
8151 ; GFX940-NEXT: s_cbranch_execnz .LBB31_1
8152 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
8153 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
8154 ; GFX940-NEXT: s_setpc_b64 s[30:31]
8155 ;
8156 ; GFX11-LABEL: global_system_atomic_fsub_noret_f16__offset12b_pos:
8157 ; GFX11: ; %bb.0:
8158 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8159 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
8160 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
8161 ; GFX11-NEXT: s_mov_b32 s0, 0
8162 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
8163 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
8164 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
8165 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off
8166 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
8167 ; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
8168 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
8169 ; GFX11-NEXT: v_not_b32_e32 v6, v3
8170 ; GFX11-NEXT: .LBB31_1: ; %atomicrmw.start
8171 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
8172 ; GFX11-NEXT: s_waitcnt vmcnt(0)
8173 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
8174 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8175 ; GFX11-NEXT: v_sub_f16_e32 v3, v3, v2
8176 ; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
8177 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8178 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
8179 ; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
8180 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
8181 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
8182 ; GFX11-NEXT: s_waitcnt vmcnt(0)
8183 ; GFX11-NEXT: buffer_gl1_inv
8184 ; GFX11-NEXT: buffer_gl0_inv
8185 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
8186 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
8187 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
8188 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
8189 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
8190 ; GFX11-NEXT: s_cbranch_execnz .LBB31_1
8191 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
8192 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
8193 ; GFX11-NEXT: s_setpc_b64 s[30:31]
8194 ;
8195 ; GFX10-LABEL: global_system_atomic_fsub_noret_f16__offset12b_pos:
8196 ; GFX10: ; %bb.0:
8197 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8198 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
8199 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
8200 ; GFX10-NEXT: s_mov_b32 s4, 0
8201 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
8202 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
8203 ; GFX10-NEXT: global_load_dword v4, v[0:1], off
8204 ; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
8205 ; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
8206 ; GFX10-NEXT: v_not_b32_e32 v6, v3
8207 ; GFX10-NEXT: .LBB31_1: ; %atomicrmw.start
8208 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
8209 ; GFX10-NEXT: s_waitcnt vmcnt(0)
8210 ; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
8211 ; GFX10-NEXT: v_sub_f16_e32 v3, v3, v2
8212 ; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
8213 ; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
8214 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
8215 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
8216 ; GFX10-NEXT: s_waitcnt vmcnt(0)
8217 ; GFX10-NEXT: buffer_gl1_inv
8218 ; GFX10-NEXT: buffer_gl0_inv
8219 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
8220 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
8221 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
8222 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
8223 ; GFX10-NEXT: s_cbranch_execnz .LBB31_1
8224 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
8225 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
8226 ; GFX10-NEXT: s_setpc_b64 s[30:31]
8227 ;
8228 ; GFX90A-LABEL: global_system_atomic_fsub_noret_f16__offset12b_pos:
8229 ; GFX90A: ; %bb.0:
8230 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8231 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
8232 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
8233 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
8234 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
8235 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
8236 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8237 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
8238 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
8239 ; GFX90A-NEXT: v_not_b32_e32 v6, v4
8240 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
8241 ; GFX90A-NEXT: .LBB31_1: ; %atomicrmw.start
8242 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
8243 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
8244 ; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
8245 ; GFX90A-NEXT: v_sub_f16_e32 v4, v4, v2
8246 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
8247 ; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
8248 ; GFX90A-NEXT: buffer_wbl2
8249 ; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
8250 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
8251 ; GFX90A-NEXT: buffer_invl2
8252 ; GFX90A-NEXT: buffer_wbinvl1
8253 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
8254 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8255 ; GFX90A-NEXT: v_mov_b32_e32 v5, v4
8256 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
8257 ; GFX90A-NEXT: s_cbranch_execnz .LBB31_1
8258 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
8259 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
8260 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
8261 ;
8262 ; GFX908-LABEL: global_system_atomic_fsub_noret_f16__offset12b_pos:
8263 ; GFX908: ; %bb.0:
8264 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8265 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
8266 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
8267 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
8268 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
8269 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
8270 ; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
8271 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
8272 ; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
8273 ; GFX908-NEXT: v_not_b32_e32 v6, v3
8274 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
8275 ; GFX908-NEXT: .LBB31_1: ; %atomicrmw.start
8276 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
8277 ; GFX908-NEXT: s_waitcnt vmcnt(0)
8278 ; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
8279 ; GFX908-NEXT: v_sub_f16_e32 v3, v3, v2
8280 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
8281 ; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
8282 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
8283 ; GFX908-NEXT: s_waitcnt vmcnt(0)
8284 ; GFX908-NEXT: buffer_wbinvl1
8285 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
8286 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8287 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
8288 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
8289 ; GFX908-NEXT: s_cbranch_execnz .LBB31_1
8290 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
8291 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
8292 ; GFX908-NEXT: s_setpc_b64 s[30:31]
8293 ;
8294 ; GFX8-LABEL: global_system_atomic_fsub_noret_f16__offset12b_pos:
8295 ; GFX8: ; %bb.0:
8296 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8297 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
8298 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8299 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
8300 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
8301 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
8302 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
8303 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
8304 ; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
8305 ; GFX8-NEXT: v_not_b32_e32 v6, v3
8306 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
8307 ; GFX8-NEXT: .LBB31_1: ; %atomicrmw.start
8308 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
8309 ; GFX8-NEXT: s_waitcnt vmcnt(0)
8310 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
8311 ; GFX8-NEXT: v_sub_f16_e32 v3, v3, v2
8312 ; GFX8-NEXT: v_and_b32_e32 v7, v4, v6
8313 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
8314 ; GFX8-NEXT: v_or_b32_e32 v3, v7, v3
8315 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
8316 ; GFX8-NEXT: s_waitcnt vmcnt(0)
8317 ; GFX8-NEXT: buffer_wbinvl1
8318 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
8319 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8320 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
8321 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
8322 ; GFX8-NEXT: s_cbranch_execnz .LBB31_1
8323 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
8324 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
8325 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8326 ;
8327 ; GFX7-LABEL: global_system_atomic_fsub_noret_f16__offset12b_pos:
8328 ; GFX7: ; %bb.0:
8329 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8330 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
8331 ; GFX7-NEXT: s_mov_b32 s6, 0
8332 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8333 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
8334 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
8335 ; GFX7-NEXT: s_mov_b32 s4, s6
8336 ; GFX7-NEXT: s_mov_b32 s5, s6
8337 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
8338 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
8339 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
8340 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
8341 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
8342 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
8343 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
8344 ; GFX7-NEXT: v_not_b32_e32 v6, v2
8345 ; GFX7-NEXT: .LBB31_1: ; %atomicrmw.start
8346 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
8347 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8348 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
8349 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
8350 ; GFX7-NEXT: v_and_b32_e32 v7, v3, v6
8351 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v5
8352 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
8353 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v4, v2
8354 ; GFX7-NEXT: v_or_b32_e32 v2, v7, v2
8355 ; GFX7-NEXT: v_mov_b32_e32 v8, v3
8356 ; GFX7-NEXT: v_mov_b32_e32 v7, v2
8357 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
8358 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8359 ; GFX7-NEXT: buffer_wbinvl1
8360 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
8361 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
8362 ; GFX7-NEXT: v_mov_b32_e32 v3, v7
8363 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
8364 ; GFX7-NEXT: s_cbranch_execnz .LBB31_1
8365 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
8366 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
8367 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8368 ;
8369 ; GFX6-LABEL: global_system_atomic_fsub_noret_f16__offset12b_pos:
8370 ; GFX6: ; %bb.0:
8371 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8372 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
8373 ; GFX6-NEXT: s_mov_b32 s6, 0
8374 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8375 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
8376 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
8377 ; GFX6-NEXT: s_mov_b32 s4, s6
8378 ; GFX6-NEXT: s_mov_b32 s5, s6
8379 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
8380 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
8381 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
8382 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
8383 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
8384 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
8385 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
8386 ; GFX6-NEXT: v_not_b32_e32 v6, v2
8387 ; GFX6-NEXT: .LBB31_1: ; %atomicrmw.start
8388 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
8389 ; GFX6-NEXT: s_waitcnt vmcnt(0)
8390 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
8391 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
8392 ; GFX6-NEXT: s_waitcnt expcnt(0)
8393 ; GFX6-NEXT: v_and_b32_e32 v7, v3, v6
8394 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v5
8395 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
8396 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v4, v2
8397 ; GFX6-NEXT: v_or_b32_e32 v2, v7, v2
8398 ; GFX6-NEXT: v_mov_b32_e32 v8, v3
8399 ; GFX6-NEXT: v_mov_b32_e32 v7, v2
8400 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
8401 ; GFX6-NEXT: s_waitcnt vmcnt(0)
8402 ; GFX6-NEXT: buffer_wbinvl1
8403 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
8404 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
8405 ; GFX6-NEXT: v_mov_b32_e32 v3, v7
8406 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
8407 ; GFX6-NEXT: s_cbranch_execnz .LBB31_1
8408 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
8409 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
8410 ; GFX6-NEXT: s_waitcnt expcnt(0)
8411 ; GFX6-NEXT: s_setpc_b64 s[30:31]
8412 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
8413 %unused = atomicrmw fsub ptr addrspace(1) %gep, half %val seq_cst
8414 ret void
8415 }
8417 ; --------------------------------------------------------------------
8418 ; bfloat
8419 ; --------------------------------------------------------------------
8421 define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %val) #0 {
8422 ; GFX12-LABEL: global_agent_atomic_fsub_ret_bf16:
8423 ; GFX12: ; %bb.0:
8424 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
8425 ; GFX12-NEXT: s_wait_expcnt 0x0
8426 ; GFX12-NEXT: s_wait_samplecnt 0x0
8427 ; GFX12-NEXT: s_wait_bvhcnt 0x0
8428 ; GFX12-NEXT: s_wait_kmcnt 0x0
8429 ; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
8430 ; GFX12-NEXT: s_mov_b32 s0, 0
8431 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
8432 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
8433 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
8434 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
8435 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8436 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
8437 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
8438 ; GFX12-NEXT: v_not_b32_e32 v4, v4
8439 ; GFX12-NEXT: .LBB32_1: ; %atomicrmw.start
8440 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
8441 ; GFX12-NEXT: s_wait_loadcnt 0x0
8442 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
8443 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8444 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
8445 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
8446 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8447 ; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
8448 ; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
8449 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
8450 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
8451 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
8452 ; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
8453 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
8454 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8455 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
8456 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
8457 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
8458 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
8459 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
8460 ; GFX12-NEXT: s_wait_storecnt 0x0
8461 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
8462 ; GFX12-NEXT: s_wait_loadcnt 0x0
8463 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
8464 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
8465 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
8466 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
8467 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
8468 ; GFX12-NEXT: s_cbranch_execnz .LBB32_1
8469 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
8470 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
8471 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8472 ; GFX12-NEXT: s_setpc_b64 s[30:31]
8473 ;
8474 ; GFX940-LABEL: global_agent_atomic_fsub_ret_bf16:
8475 ; GFX940: ; %bb.0:
8476 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8477 ; GFX940-NEXT: v_mov_b32_e32 v3, v0
8478 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v3
8479 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
8480 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v3
8481 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8482 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
8483 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
8484 ; GFX940-NEXT: v_not_b32_e32 v4, v4
8485 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
8486 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8487 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
8488 ; GFX940-NEXT: .LBB32_1: ; %atomicrmw.start
8489 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
8490 ; GFX940-NEXT: s_waitcnt vmcnt(0)
8491 ; GFX940-NEXT: v_mov_b32_e32 v7, v5
8492 ; GFX940-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8493 ; GFX940-NEXT: s_nop 0
8494 ; GFX940-NEXT: v_sub_f32_e32 v5, v5, v2
8495 ; GFX940-NEXT: v_bfe_u32 v6, v5, 16, 1
8496 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v5
8497 ; GFX940-NEXT: v_add3_u32 v6, v6, v5, s2
8498 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
8499 ; GFX940-NEXT: s_nop 1
8500 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
8501 ; GFX940-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8502 ; GFX940-NEXT: v_and_or_b32 v6, v7, v4, v5
8503 ; GFX940-NEXT: buffer_wbl2 sc1
8504 ; GFX940-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off sc0
8505 ; GFX940-NEXT: s_waitcnt vmcnt(0)
8506 ; GFX940-NEXT: buffer_inv sc1
8507 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
8508 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
8509 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
8510 ; GFX940-NEXT: s_cbranch_execnz .LBB32_1
8511 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
8512 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
8513 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8514 ; GFX940-NEXT: s_setpc_b64 s[30:31]
8515 ;
8516 ; GFX11-LABEL: global_agent_atomic_fsub_ret_bf16:
8517 ; GFX11: ; %bb.0:
8518 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8519 ; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
8520 ; GFX11-NEXT: s_mov_b32 s0, 0
8521 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
8522 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
8523 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
8524 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
8525 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8526 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
8527 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
8528 ; GFX11-NEXT: v_not_b32_e32 v4, v4
8529 ; GFX11-NEXT: .p2align 6
8530 ; GFX11-NEXT: .LBB32_1: ; %atomicrmw.start
8531 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
8532 ; GFX11-NEXT: s_waitcnt vmcnt(0)
8533 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
8534 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8535 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
8536 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
8537 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8538 ; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
8539 ; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
8540 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
8541 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
8542 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
8543 ; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
8544 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
8545 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8546 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
8547 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
8548 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
8549 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
8550 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
8551 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
8552 ; GFX11-NEXT: s_waitcnt vmcnt(0)
8553 ; GFX11-NEXT: buffer_gl1_inv
8554 ; GFX11-NEXT: buffer_gl0_inv
8555 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
8556 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
8557 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
8558 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
8559 ; GFX11-NEXT: s_cbranch_execnz .LBB32_1
8560 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
8561 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
8562 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8563 ; GFX11-NEXT: s_setpc_b64 s[30:31]
8564 ;
8565 ; GFX10-LABEL: global_agent_atomic_fsub_ret_bf16:
8566 ; GFX10: ; %bb.0:
8567 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8568 ; GFX10-NEXT: v_mov_b32_e32 v3, v0
8569 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8570 ; GFX10-NEXT: s_mov_b32 s4, 0
8571 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
8572 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
8573 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
8574 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8575 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
8576 ; GFX10-NEXT: v_not_b32_e32 v4, v4
8577 ; GFX10-NEXT: .LBB32_1: ; %atomicrmw.start
8578 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
8579 ; GFX10-NEXT: s_waitcnt vmcnt(0)
8580 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
8581 ; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8582 ; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
8583 ; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
8584 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v5
8585 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
8586 ; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
8587 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
8588 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8589 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
8590 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
8591 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
8592 ; GFX10-NEXT: s_waitcnt vmcnt(0)
8593 ; GFX10-NEXT: buffer_gl1_inv
8594 ; GFX10-NEXT: buffer_gl0_inv
8595 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
8596 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
8597 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
8598 ; GFX10-NEXT: s_cbranch_execnz .LBB32_1
8599 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
8600 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
8601 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8602 ; GFX10-NEXT: s_setpc_b64 s[30:31]
8603 ;
8604 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_bf16:
8605 ; GFX90A: ; %bb.0:
8606 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8607 ; GFX90A-NEXT: v_mov_b32_e32 v3, v0
8608 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
8609 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
8610 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
8611 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8612 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
8613 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
8614 ; GFX90A-NEXT: v_not_b32_e32 v4, v4
8615 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
8616 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8617 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
8618 ; GFX90A-NEXT: .LBB32_1: ; %atomicrmw.start
8619 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
8620 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
8621 ; GFX90A-NEXT: v_mov_b32_e32 v7, v5
8622 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8623 ; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v2
8624 ; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
8625 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v5
8626 ; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
8627 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
8628 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
8629 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8630 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v4, v5
8631 ; GFX90A-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off glc
8632 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
8633 ; GFX90A-NEXT: buffer_wbinvl1
8634 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
8635 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8636 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
8637 ; GFX90A-NEXT: s_cbranch_execnz .LBB32_1
8638 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
8639 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
8640 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8641 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
8642 ;
8643 ; GFX908-LABEL: global_agent_atomic_fsub_ret_bf16:
8644 ; GFX908: ; %bb.0:
8645 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8646 ; GFX908-NEXT: v_mov_b32_e32 v3, v0
8647 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
8648 ; GFX908-NEXT: global_load_dword v5, v[0:1], off
8649 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
8650 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8651 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
8652 ; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
8653 ; GFX908-NEXT: v_not_b32_e32 v4, v4
8654 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
8655 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8656 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
8657 ; GFX908-NEXT: .LBB32_1: ; %atomicrmw.start
8658 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
8659 ; GFX908-NEXT: s_waitcnt vmcnt(0)
8660 ; GFX908-NEXT: v_mov_b32_e32 v6, v5
8661 ; GFX908-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8662 ; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
8663 ; GFX908-NEXT: v_bfe_u32 v7, v5, 16, 1
8664 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v5
8665 ; GFX908-NEXT: v_add3_u32 v7, v7, v5, s6
8666 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
8667 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc
8668 ; GFX908-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8669 ; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
8670 ; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
8671 ; GFX908-NEXT: s_waitcnt vmcnt(0)
8672 ; GFX908-NEXT: buffer_wbinvl1
8673 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
8674 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8675 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
8676 ; GFX908-NEXT: s_cbranch_execnz .LBB32_1
8677 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
8678 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
8679 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8680 ; GFX908-NEXT: s_setpc_b64 s[30:31]
8681 ;
8682 ; GFX8-LABEL: global_agent_atomic_fsub_ret_bf16:
8683 ; GFX8: ; %bb.0:
8684 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8685 ; GFX8-NEXT: v_mov_b32_e32 v3, v0
8686 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
8687 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
8688 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
8689 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8690 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
8691 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
8692 ; GFX8-NEXT: v_not_b32_e32 v4, v4
8693 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
8694 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8695 ; GFX8-NEXT: .LBB32_1: ; %atomicrmw.start
8696 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
8697 ; GFX8-NEXT: s_waitcnt vmcnt(0)
8698 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
8699 ; GFX8-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8700 ; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
8701 ; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
8702 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
8703 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
8704 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
8705 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
8706 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
8707 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
8708 ; GFX8-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8709 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
8710 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
8711 ; GFX8-NEXT: s_waitcnt vmcnt(0)
8712 ; GFX8-NEXT: buffer_wbinvl1
8713 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
8714 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8715 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
8716 ; GFX8-NEXT: s_cbranch_execnz .LBB32_1
8717 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
8718 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
8719 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8720 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8721 ;
8722 ; GFX7-LABEL: global_agent_atomic_fsub_ret_bf16:
8723 ; GFX7: ; %bb.0:
8724 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8725 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
8726 ; GFX7-NEXT: s_mov_b32 s6, 0
8727 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
8728 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
8729 ; GFX7-NEXT: s_mov_b32 s4, s6
8730 ; GFX7-NEXT: s_mov_b32 s5, s6
8731 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
8732 ; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
8733 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 3, v3
8734 ; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
8735 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
8736 ; GFX7-NEXT: v_not_b32_e32 v6, v3
8737 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
8738 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
8739 ; GFX7-NEXT: .LBB32_1: ; %atomicrmw.start
8740 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
8741 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8742 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v5, v4
8743 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8744 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v7
8745 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
8746 ; GFX7-NEXT: v_and_b32_e32 v3, v4, v6
8747 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v5, v2
8748 ; GFX7-NEXT: v_or_b32_e32 v3, v3, v2
8749 ; GFX7-NEXT: v_mov_b32_e32 v2, v3
8750 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
8751 ; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v[0:1], s[4:7], 0 addr64 glc
8752 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8753 ; GFX7-NEXT: buffer_wbinvl1
8754 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
8755 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
8756 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
8757 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
8758 ; GFX7-NEXT: s_cbranch_execnz .LBB32_1
8759 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
8760 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
8761 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v5, v2
8762 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
8763 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8764 ;
8765 ; GFX6-LABEL: global_agent_atomic_fsub_ret_bf16:
8766 ; GFX6: ; %bb.0:
8767 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8768 ; GFX6-NEXT: v_mov_b32_e32 v3, v0
8769 ; GFX6-NEXT: s_mov_b32 s6, 0
8770 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v3
8771 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
8772 ; GFX6-NEXT: s_mov_b32 s4, s6
8773 ; GFX6-NEXT: s_mov_b32 s5, s6
8774 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
8775 ; GFX6-NEXT: v_and_b32_e32 v3, 3, v3
8776 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 3, v3
8777 ; GFX6-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
8778 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
8779 ; GFX6-NEXT: v_not_b32_e32 v6, v3
8780 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
8781 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
8782 ; GFX6-NEXT: .LBB32_1: ; %atomicrmw.start
8783 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
8784 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
8785 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v5, v4
8786 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8787 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v7
8788 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
8789 ; GFX6-NEXT: v_and_b32_e32 v3, v4, v6
8790 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v5, v2
8791 ; GFX6-NEXT: v_or_b32_e32 v3, v3, v2
8792 ; GFX6-NEXT: v_mov_b32_e32 v2, v3
8793 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
8794 ; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v[0:1], s[4:7], 0 addr64 glc
8795 ; GFX6-NEXT: s_waitcnt vmcnt(0)
8796 ; GFX6-NEXT: buffer_wbinvl1
8797 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
8798 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
8799 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
8800 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
8801 ; GFX6-NEXT: s_cbranch_execnz .LBB32_1
8802 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
8803 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
8804 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v5, v2
8805 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
8806 ; GFX6-NEXT: s_waitcnt expcnt(0)
8807 ; GFX6-NEXT: s_setpc_b64 s[30:31]
8808 %result = atomicrmw fsub ptr addrspace(1) %ptr, bfloat %val syncscope("agent") seq_cst
8809 ret bfloat %result
8810 }
8812 define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
8813 ; GFX12-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
8814 ; GFX12: ; %bb.0:
8815 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
8816 ; GFX12-NEXT: s_wait_expcnt 0x0
8817 ; GFX12-NEXT: s_wait_samplecnt 0x0
8818 ; GFX12-NEXT: s_wait_bvhcnt 0x0
8819 ; GFX12-NEXT: s_wait_kmcnt 0x0
8820 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
8821 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
8822 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8823 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
8824 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
8825 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
8826 ; GFX12-NEXT: s_mov_b32 s0, 0
8827 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
8828 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8829 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
8830 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
8831 ; GFX12-NEXT: v_not_b32_e32 v4, v4
8832 ; GFX12-NEXT: .LBB33_1: ; %atomicrmw.start
8833 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
8834 ; GFX12-NEXT: s_wait_loadcnt 0x0
8835 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
8836 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8837 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
8838 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
8839 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8840 ; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
8841 ; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
8842 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
8843 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
8844 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
8845 ; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
8846 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
8847 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8848 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
8849 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
8850 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
8851 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
8852 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
8853 ; GFX12-NEXT: s_wait_storecnt 0x0
8854 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
8855 ; GFX12-NEXT: s_wait_loadcnt 0x0
8856 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
8857 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
8858 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
8859 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
8860 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
8861 ; GFX12-NEXT: s_cbranch_execnz .LBB33_1
8862 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
8863 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
8864 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8865 ; GFX12-NEXT: s_setpc_b64 s[30:31]
8866 ;
8867 ; GFX940-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
8868 ; GFX940: ; %bb.0:
8869 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8870 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
8871 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
8872 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
8873 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
8874 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
8875 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v4
8876 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8877 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
8878 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
8879 ; GFX940-NEXT: v_not_b32_e32 v4, v4
8880 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
8881 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8882 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
8883 ; GFX940-NEXT: .LBB33_1: ; %atomicrmw.start
8884 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
8885 ; GFX940-NEXT: s_waitcnt vmcnt(0)
8886 ; GFX940-NEXT: v_mov_b32_e32 v7, v5
8887 ; GFX940-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8888 ; GFX940-NEXT: s_nop 0
8889 ; GFX940-NEXT: v_sub_f32_e32 v5, v5, v2
8890 ; GFX940-NEXT: v_bfe_u32 v6, v5, 16, 1
8891 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v5
8892 ; GFX940-NEXT: v_add3_u32 v6, v6, v5, s2
8893 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
8894 ; GFX940-NEXT: s_nop 1
8895 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
8896 ; GFX940-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8897 ; GFX940-NEXT: v_and_or_b32 v6, v7, v4, v5
8898 ; GFX940-NEXT: buffer_wbl2 sc1
8899 ; GFX940-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off sc0
8900 ; GFX940-NEXT: s_waitcnt vmcnt(0)
8901 ; GFX940-NEXT: buffer_inv sc1
8902 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
8903 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
8904 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
8905 ; GFX940-NEXT: s_cbranch_execnz .LBB33_1
8906 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
8907 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
8908 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8909 ; GFX940-NEXT: s_setpc_b64 s[30:31]
8910 ;
8911 ; GFX11-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
8912 ; GFX11: ; %bb.0:
8913 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8914 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
8915 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
8916 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8917 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
8918 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
8919 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
8920 ; GFX11-NEXT: s_mov_b32 s0, 0
8921 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
8922 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8923 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
8924 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
8925 ; GFX11-NEXT: v_not_b32_e32 v4, v4
8926 ; GFX11-NEXT: .p2align 6
8927 ; GFX11-NEXT: .LBB33_1: ; %atomicrmw.start
8928 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
8929 ; GFX11-NEXT: s_waitcnt vmcnt(0)
8930 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
8931 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8932 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
8933 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
8934 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8935 ; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
8936 ; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
8937 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
8938 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
8939 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
8940 ; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
8941 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
8942 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8943 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
8944 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
8945 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
8946 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
8947 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
8948 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
8949 ; GFX11-NEXT: s_waitcnt vmcnt(0)
8950 ; GFX11-NEXT: buffer_gl1_inv
8951 ; GFX11-NEXT: buffer_gl0_inv
8952 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
8953 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
8954 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
8955 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
8956 ; GFX11-NEXT: s_cbranch_execnz .LBB33_1
8957 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
8958 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
8959 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8960 ; GFX11-NEXT: s_setpc_b64 s[30:31]
8961 ;
8962 ; GFX10-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
8963 ; GFX10: ; %bb.0:
8964 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8965 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
8966 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
8967 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8968 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
8969 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
8970 ; GFX10-NEXT: s_mov_b32 s4, 0
8971 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
8972 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8973 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
8974 ; GFX10-NEXT: v_not_b32_e32 v4, v4
8975 ; GFX10-NEXT: .LBB33_1: ; %atomicrmw.start
8976 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
8977 ; GFX10-NEXT: s_waitcnt vmcnt(0)
8978 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
8979 ; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8980 ; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
8981 ; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
8982 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v5
8983 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
8984 ; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
8985 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
8986 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8987 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
8988 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
8989 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
8990 ; GFX10-NEXT: s_waitcnt vmcnt(0)
8991 ; GFX10-NEXT: buffer_gl1_inv
8992 ; GFX10-NEXT: buffer_gl0_inv
8993 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
8994 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
8995 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
8996 ; GFX10-NEXT: s_cbranch_execnz .LBB33_1
8997 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
8998 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
8999 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
9000 ; GFX10-NEXT: s_setpc_b64 s[30:31]
9001 ;
9002 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
9003 ; GFX90A: ; %bb.0:
9004 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9005 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
9006 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
9007 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
9008 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
9009 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
9010 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9011 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
9012 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
9013 ; GFX90A-NEXT: v_not_b32_e32 v4, v4
9014 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
9015 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9016 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
9017 ; GFX90A-NEXT: .LBB33_1: ; %atomicrmw.start
9018 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
9019 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
9020 ; GFX90A-NEXT: v_mov_b32_e32 v7, v5
9021 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9022 ; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v2
9023 ; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
9024 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v5
9025 ; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
9026 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
9027 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
9028 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9029 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v4, v5
9030 ; GFX90A-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off glc
9031 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
9032 ; GFX90A-NEXT: buffer_wbinvl1
9033 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
9034 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9035 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
9036 ; GFX90A-NEXT: s_cbranch_execnz .LBB33_1
9037 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
9038 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
9039 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
9040 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
9041 ;
9042 ; GFX908-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
9043 ; GFX908: ; %bb.0:
9044 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9045 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
9046 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
9047 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
9048 ; GFX908-NEXT: global_load_dword v5, v[0:1], off
9049 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
9050 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9051 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
9052 ; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
9053 ; GFX908-NEXT: v_not_b32_e32 v4, v4
9054 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
9055 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9056 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
9057 ; GFX908-NEXT: .LBB33_1: ; %atomicrmw.start
9058 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
9059 ; GFX908-NEXT: s_waitcnt vmcnt(0)
9060 ; GFX908-NEXT: v_mov_b32_e32 v6, v5
9061 ; GFX908-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9062 ; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
9063 ; GFX908-NEXT: v_bfe_u32 v7, v5, 16, 1
9064 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v5
9065 ; GFX908-NEXT: v_add3_u32 v7, v7, v5, s6
9066 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
9067 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc
9068 ; GFX908-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9069 ; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
9070 ; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
9071 ; GFX908-NEXT: s_waitcnt vmcnt(0)
9072 ; GFX908-NEXT: buffer_wbinvl1
9073 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
9074 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9075 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
9076 ; GFX908-NEXT: s_cbranch_execnz .LBB33_1
9077 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
9078 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
9079 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
9080 ; GFX908-NEXT: s_setpc_b64 s[30:31]
9081 ;
9082 ; GFX8-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
9083 ; GFX8: ; %bb.0:
9084 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9085 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
9086 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
9087 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
9088 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
9089 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
9090 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9091 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
9092 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
9093 ; GFX8-NEXT: v_not_b32_e32 v4, v4
9094 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
9095 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9096 ; GFX8-NEXT: .LBB33_1: ; %atomicrmw.start
9097 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
9098 ; GFX8-NEXT: s_waitcnt vmcnt(0)
9099 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
9100 ; GFX8-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9101 ; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
9102 ; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
9103 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
9104 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
9105 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
9106 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
9107 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
9108 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
9109 ; GFX8-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9110 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
9111 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
9112 ; GFX8-NEXT: s_waitcnt vmcnt(0)
9113 ; GFX8-NEXT: buffer_wbinvl1
9114 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
9115 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9116 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
9117 ; GFX8-NEXT: s_cbranch_execnz .LBB33_1
9118 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
9119 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
9120 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
9121 ; GFX8-NEXT: s_setpc_b64 s[30:31]
9122 ;
9123 ; GFX7-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
9124 ; GFX7: ; %bb.0:
9125 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9126 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
9127 ; GFX7-NEXT: s_mov_b32 s6, 0
9128 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
9129 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
9130 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
9131 ; GFX7-NEXT: s_mov_b32 s4, s6
9132 ; GFX7-NEXT: s_mov_b32 s5, s6
9133 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
9134 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
9135 ; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
9136 ; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
9137 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
9138 ; GFX7-NEXT: v_not_b32_e32 v7, v4
9139 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
9140 ; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
9141 ; GFX7-NEXT: .LBB33_1: ; %atomicrmw.start
9142 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
9143 ; GFX7-NEXT: s_waitcnt vmcnt(0)
9144 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
9145 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9146 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v8
9147 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
9148 ; GFX7-NEXT: v_and_b32_e32 v4, v3, v7
9149 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v6, v2
9150 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
9151 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
9152 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
9153 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
9154 ; GFX7-NEXT: s_waitcnt vmcnt(0)
9155 ; GFX7-NEXT: buffer_wbinvl1
9156 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
9157 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
9158 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
9159 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
9160 ; GFX7-NEXT: s_cbranch_execnz .LBB33_1
9161 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
9162 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
9163 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
9164 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
9165 ; GFX7-NEXT: s_setpc_b64 s[30:31]
9166 ;
9167 ; GFX6-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
9168 ; GFX6: ; %bb.0:
9169 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9170 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
9171 ; GFX6-NEXT: s_mov_b32 s6, 0
9172 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
9173 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
9174 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
9175 ; GFX6-NEXT: s_mov_b32 s4, s6
9176 ; GFX6-NEXT: s_mov_b32 s5, s6
9177 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
9178 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
9179 ; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
9180 ; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
9181 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
9182 ; GFX6-NEXT: v_not_b32_e32 v7, v4
9183 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
9184 ; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
9185 ; GFX6-NEXT: .LBB33_1: ; %atomicrmw.start
9186 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
9187 ; GFX6-NEXT: s_waitcnt vmcnt(0)
9188 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
9189 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9190 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v8
9191 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
9192 ; GFX6-NEXT: s_waitcnt expcnt(0)
9193 ; GFX6-NEXT: v_and_b32_e32 v4, v3, v7
9194 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v6, v2
9195 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
9196 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
9197 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
9198 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
9199 ; GFX6-NEXT: s_waitcnt vmcnt(0)
9200 ; GFX6-NEXT: buffer_wbinvl1
9201 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
9202 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
9203 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
9204 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
9205 ; GFX6-NEXT: s_cbranch_execnz .LBB33_1
9206 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
9207 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
9208 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
9209 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
9210 ; GFX6-NEXT: s_waitcnt expcnt(0)
9211 ; GFX6-NEXT: s_setpc_b64 s[30:31]
9212 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
9213 %result = atomicrmw fsub ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst
9214 ret bfloat %result
9215 }
9217 define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1) %ptr, bfloat %val) #0 {
9218 ; GFX12-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
9219 ; GFX12: ; %bb.0:
9220 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
9221 ; GFX12-NEXT: s_wait_expcnt 0x0
9222 ; GFX12-NEXT: s_wait_samplecnt 0x0
9223 ; GFX12-NEXT: s_wait_bvhcnt 0x0
9224 ; GFX12-NEXT: s_wait_kmcnt 0x0
9225 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
9226 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
9227 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9228 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
9229 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
9230 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
9231 ; GFX12-NEXT: s_mov_b32 s0, 0
9232 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
9233 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9234 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
9235 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
9236 ; GFX12-NEXT: v_not_b32_e32 v4, v4
9237 ; GFX12-NEXT: .LBB34_1: ; %atomicrmw.start
9238 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
9239 ; GFX12-NEXT: s_wait_loadcnt 0x0
9240 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
9241 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9242 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
9243 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
9244 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9245 ; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
9246 ; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
9247 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
9248 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
9249 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
9250 ; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
9251 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
9252 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9253 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
9254 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
9255 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
9256 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
9257 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
9258 ; GFX12-NEXT: s_wait_storecnt 0x0
9259 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
9260 ; GFX12-NEXT: s_wait_loadcnt 0x0
9261 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
9262 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
9263 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
9264 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
9265 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
9266 ; GFX12-NEXT: s_cbranch_execnz .LBB34_1
9267 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
9268 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
9269 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
9270 ; GFX12-NEXT: s_setpc_b64 s[30:31]
9271 ;
9272 ; GFX940-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
9273 ; GFX940: ; %bb.0:
9274 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9275 ; GFX940-NEXT: s_movk_i32 s0, 0xf800
9276 ; GFX940-NEXT: s_mov_b32 s1, -1
9277 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
9278 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
9279 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
9280 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
9281 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v4
9282 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9283 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
9284 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
9285 ; GFX940-NEXT: v_not_b32_e32 v4, v4
9286 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
9287 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9288 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
9289 ; GFX940-NEXT: .LBB34_1: ; %atomicrmw.start
9290 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
9291 ; GFX940-NEXT: s_waitcnt vmcnt(0)
9292 ; GFX940-NEXT: v_mov_b32_e32 v7, v5
9293 ; GFX940-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9294 ; GFX940-NEXT: s_nop 0
9295 ; GFX940-NEXT: v_sub_f32_e32 v5, v5, v2
9296 ; GFX940-NEXT: v_bfe_u32 v6, v5, 16, 1
9297 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v5
9298 ; GFX940-NEXT: v_add3_u32 v6, v6, v5, s2
9299 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
9300 ; GFX940-NEXT: s_nop 1
9301 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
9302 ; GFX940-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9303 ; GFX940-NEXT: v_and_or_b32 v6, v7, v4, v5
9304 ; GFX940-NEXT: buffer_wbl2 sc1
9305 ; GFX940-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off sc0
9306 ; GFX940-NEXT: s_waitcnt vmcnt(0)
9307 ; GFX940-NEXT: buffer_inv sc1
9308 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
9309 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
9310 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
9311 ; GFX940-NEXT: s_cbranch_execnz .LBB34_1
9312 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
9313 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
9314 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
9315 ; GFX940-NEXT: s_setpc_b64 s[30:31]
9316 ;
9317 ; GFX11-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
9318 ; GFX11: ; %bb.0:
9319 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9320 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
9321 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
9322 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9323 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
9324 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
9325 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
9326 ; GFX11-NEXT: s_mov_b32 s0, 0
9327 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
9328 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9329 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
9330 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
9331 ; GFX11-NEXT: v_not_b32_e32 v4, v4
9332 ; GFX11-NEXT: .p2align 6
9333 ; GFX11-NEXT: .LBB34_1: ; %atomicrmw.start
9334 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
9335 ; GFX11-NEXT: s_waitcnt vmcnt(0)
9336 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
9337 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9338 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
9339 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
9340 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9341 ; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
9342 ; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
9343 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
9344 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
9345 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
9346 ; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
9347 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
9348 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9349 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
9350 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
9351 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
9352 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
9353 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
9354 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
9355 ; GFX11-NEXT: s_waitcnt vmcnt(0)
9356 ; GFX11-NEXT: buffer_gl1_inv
9357 ; GFX11-NEXT: buffer_gl0_inv
9358 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
9359 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
9360 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
9361 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
9362 ; GFX11-NEXT: s_cbranch_execnz .LBB34_1
9363 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
9364 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
9365 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
9366 ; GFX11-NEXT: s_setpc_b64 s[30:31]
9367 ;
9368 ; GFX10-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
9369 ; GFX10: ; %bb.0:
9370 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9371 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
9372 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
9373 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9374 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
9375 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
9376 ; GFX10-NEXT: s_mov_b32 s4, 0
9377 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
9378 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9379 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
9380 ; GFX10-NEXT: v_not_b32_e32 v4, v4
9381 ; GFX10-NEXT: .LBB34_1: ; %atomicrmw.start
9382 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
9383 ; GFX10-NEXT: s_waitcnt vmcnt(0)
9384 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
9385 ; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9386 ; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
9387 ; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
9388 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v5
9389 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
9390 ; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
9391 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
9392 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9393 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
9394 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
9395 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
9396 ; GFX10-NEXT: s_waitcnt vmcnt(0)
9397 ; GFX10-NEXT: buffer_gl1_inv
9398 ; GFX10-NEXT: buffer_gl0_inv
9399 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
9400 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
9401 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
9402 ; GFX10-NEXT: s_cbranch_execnz .LBB34_1
9403 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
9404 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
9405 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
9406 ; GFX10-NEXT: s_setpc_b64 s[30:31]
9407 ;
9408 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
9409 ; GFX90A: ; %bb.0:
9410 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9411 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
9412 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
9413 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
9414 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
9415 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
9416 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9417 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
9418 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
9419 ; GFX90A-NEXT: v_not_b32_e32 v4, v4
9420 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
9421 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9422 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
9423 ; GFX90A-NEXT: .LBB34_1: ; %atomicrmw.start
9424 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
9425 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
9426 ; GFX90A-NEXT: v_mov_b32_e32 v7, v5
9427 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9428 ; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v2
9429 ; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
9430 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v5
9431 ; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
9432 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
9433 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
9434 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9435 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v4, v5
9436 ; GFX90A-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off glc
9437 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
9438 ; GFX90A-NEXT: buffer_wbinvl1
9439 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
9440 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9441 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
9442 ; GFX90A-NEXT: s_cbranch_execnz .LBB34_1
9443 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
9444 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
9445 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
9446 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
9447 ;
9448 ; GFX908-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
9449 ; GFX908: ; %bb.0:
9450 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9451 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
9452 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
9453 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
9454 ; GFX908-NEXT: global_load_dword v5, v[0:1], off
9455 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
9456 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9457 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
9458 ; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
9459 ; GFX908-NEXT: v_not_b32_e32 v4, v4
9460 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
9461 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9462 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
9463 ; GFX908-NEXT: .LBB34_1: ; %atomicrmw.start
9464 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
9465 ; GFX908-NEXT: s_waitcnt vmcnt(0)
9466 ; GFX908-NEXT: v_mov_b32_e32 v6, v5
9467 ; GFX908-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9468 ; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
9469 ; GFX908-NEXT: v_bfe_u32 v7, v5, 16, 1
9470 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v5
9471 ; GFX908-NEXT: v_add3_u32 v7, v7, v5, s6
9472 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
9473 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc
9474 ; GFX908-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9475 ; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
9476 ; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
9477 ; GFX908-NEXT: s_waitcnt vmcnt(0)
9478 ; GFX908-NEXT: buffer_wbinvl1
9479 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
9480 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9481 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
9482 ; GFX908-NEXT: s_cbranch_execnz .LBB34_1
9483 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
9484 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
9485 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
9486 ; GFX908-NEXT: s_setpc_b64 s[30:31]
9487 ;
9488 ; GFX8-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
9489 ; GFX8: ; %bb.0:
9490 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9491 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
9492 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
9493 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
9494 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
9495 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
9496 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9497 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
9498 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
9499 ; GFX8-NEXT: v_not_b32_e32 v4, v4
9500 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
9501 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9502 ; GFX8-NEXT: .LBB34_1: ; %atomicrmw.start
9503 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
9504 ; GFX8-NEXT: s_waitcnt vmcnt(0)
9505 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
9506 ; GFX8-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9507 ; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
9508 ; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
9509 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
9510 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
9511 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
9512 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
9513 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
9514 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
9515 ; GFX8-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9516 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
9517 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
9518 ; GFX8-NEXT: s_waitcnt vmcnt(0)
9519 ; GFX8-NEXT: buffer_wbinvl1
9520 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
9521 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9522 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
9523 ; GFX8-NEXT: s_cbranch_execnz .LBB34_1
9524 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
9525 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
9526 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
9527 ; GFX8-NEXT: s_setpc_b64 s[30:31]
9528 ;
9529 ; GFX7-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
9530 ; GFX7: ; %bb.0:
9531 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9532 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
9533 ; GFX7-NEXT: s_mov_b32 s6, 0
9534 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
9535 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
9536 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
9537 ; GFX7-NEXT: s_mov_b32 s4, s6
9538 ; GFX7-NEXT: s_mov_b32 s5, s6
9539 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
9540 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
9541 ; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
9542 ; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
9543 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
9544 ; GFX7-NEXT: v_not_b32_e32 v7, v4
9545 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
9546 ; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
9547 ; GFX7-NEXT: .LBB34_1: ; %atomicrmw.start
9548 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
9549 ; GFX7-NEXT: s_waitcnt vmcnt(0)
9550 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
9551 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9552 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v8
9553 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
9554 ; GFX7-NEXT: v_and_b32_e32 v4, v3, v7
9555 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v6, v2
9556 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
9557 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
9558 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
9559 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
9560 ; GFX7-NEXT: s_waitcnt vmcnt(0)
9561 ; GFX7-NEXT: buffer_wbinvl1
9562 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
9563 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
9564 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
9565 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
9566 ; GFX7-NEXT: s_cbranch_execnz .LBB34_1
9567 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
9568 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
9569 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
9570 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
9571 ; GFX7-NEXT: s_setpc_b64 s[30:31]
9572 ;
9573 ; GFX6-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
9574 ; GFX6: ; %bb.0:
9575 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9576 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
9577 ; GFX6-NEXT: s_mov_b32 s6, 0
9578 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
9579 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
9580 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
9581 ; GFX6-NEXT: s_mov_b32 s4, s6
9582 ; GFX6-NEXT: s_mov_b32 s5, s6
9583 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
9584 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
9585 ; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
9586 ; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
9587 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
9588 ; GFX6-NEXT: v_not_b32_e32 v7, v4
9589 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
9590 ; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
9591 ; GFX6-NEXT: .LBB34_1: ; %atomicrmw.start
9592 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
9593 ; GFX6-NEXT: s_waitcnt vmcnt(0)
9594 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
9595 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9596 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v8
9597 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
9598 ; GFX6-NEXT: s_waitcnt expcnt(0)
9599 ; GFX6-NEXT: v_and_b32_e32 v4, v3, v7
9600 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v6, v2
9601 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
9602 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
9603 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
9604 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
9605 ; GFX6-NEXT: s_waitcnt vmcnt(0)
9606 ; GFX6-NEXT: buffer_wbinvl1
9607 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
9608 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
9609 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
9610 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
9611 ; GFX6-NEXT: s_cbranch_execnz .LBB34_1
9612 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
9613 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
9614 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
9615 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
9616 ; GFX6-NEXT: s_waitcnt expcnt(0)
9617 ; GFX6-NEXT: s_setpc_b64 s[30:31]
9618 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 -1024
9619 %result = atomicrmw fsub ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst
9620 ret bfloat %result
9621 }
9623 define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat %val) #0 {
9624 ; GFX12-LABEL: global_agent_atomic_fsub_noret_bf16:
9625 ; GFX12: ; %bb.0:
9626 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
9627 ; GFX12-NEXT: s_wait_expcnt 0x0
9628 ; GFX12-NEXT: s_wait_samplecnt 0x0
9629 ; GFX12-NEXT: s_wait_bvhcnt 0x0
9630 ; GFX12-NEXT: s_wait_kmcnt 0x0
9631 ; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
9632 ; GFX12-NEXT: s_mov_b32 s0, 0
9633 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
9634 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
9635 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
9636 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off
9637 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
9638 ; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
9639 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
9640 ; GFX12-NEXT: v_not_b32_e32 v6, v3
9641 ; GFX12-NEXT: .LBB35_1: ; %atomicrmw.start
9642 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
9643 ; GFX12-NEXT: s_wait_loadcnt 0x0
9644 ; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
9645 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9646 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
9647 ; GFX12-NEXT: v_sub_f32_e32 v3, v3, v2
9648 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
9649 ; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
9650 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
9651 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
9652 ; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
9653 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9654 ; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
9655 ; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
9656 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9657 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
9658 ; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
9659 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
9660 ; GFX12-NEXT: s_wait_storecnt 0x0
9661 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
9662 ; GFX12-NEXT: s_wait_loadcnt 0x0
9663 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
9664 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
9665 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
9666 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
9667 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
9668 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
9669 ; GFX12-NEXT: s_cbranch_execnz .LBB35_1
9670 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
9671 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
9672 ; GFX12-NEXT: s_setpc_b64 s[30:31]
9673 ;
9674 ; GFX940-LABEL: global_agent_atomic_fsub_noret_bf16:
9675 ; GFX940: ; %bb.0:
9676 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9677 ; GFX940-NEXT: v_mov_b32_e32 v3, v0
9678 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v3
9679 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
9680 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v3
9681 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9682 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
9683 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
9684 ; GFX940-NEXT: v_not_b32_e32 v6, v4
9685 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
9686 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9687 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
9688 ; GFX940-NEXT: .LBB35_1: ; %atomicrmw.start
9689 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
9690 ; GFX940-NEXT: s_waitcnt vmcnt(0)
9691 ; GFX940-NEXT: v_lshrrev_b32_sdwa v4, v3, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9692 ; GFX940-NEXT: s_nop 0
9693 ; GFX940-NEXT: v_sub_f32_e32 v4, v4, v2
9694 ; GFX940-NEXT: v_bfe_u32 v7, v4, 16, 1
9695 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v4
9696 ; GFX940-NEXT: v_add3_u32 v7, v7, v4, s2
9697 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
9698 ; GFX940-NEXT: s_nop 1
9699 ; GFX940-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
9700 ; GFX940-NEXT: v_lshlrev_b32_sdwa v4, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9701 ; GFX940-NEXT: v_and_or_b32 v4, v5, v6, v4
9702 ; GFX940-NEXT: buffer_wbl2 sc1
9703 ; GFX940-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0
9704 ; GFX940-NEXT: s_waitcnt vmcnt(0)
9705 ; GFX940-NEXT: buffer_inv sc1
9706 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
9707 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
9708 ; GFX940-NEXT: v_mov_b32_e32 v5, v4
9709 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
9710 ; GFX940-NEXT: s_cbranch_execnz .LBB35_1
9711 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
9712 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
9713 ; GFX940-NEXT: s_setpc_b64 s[30:31]
9714 ;
9715 ; GFX11-LABEL: global_agent_atomic_fsub_noret_bf16:
9716 ; GFX11: ; %bb.0:
9717 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9718 ; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
9719 ; GFX11-NEXT: s_mov_b32 s0, 0
9720 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
9721 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
9722 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
9723 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off
9724 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
9725 ; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
9726 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
9727 ; GFX11-NEXT: v_not_b32_e32 v6, v3
9728 ; GFX11-NEXT: .p2align 6
9729 ; GFX11-NEXT: .LBB35_1: ; %atomicrmw.start
9730 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
9731 ; GFX11-NEXT: s_waitcnt vmcnt(0)
9732 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
9733 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9734 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
9735 ; GFX11-NEXT: v_sub_f32_e32 v3, v3, v2
9736 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
9737 ; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
9738 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
9739 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
9740 ; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
9741 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9742 ; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
9743 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
9744 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9745 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
9746 ; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
9747 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
9748 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
9749 ; GFX11-NEXT: s_waitcnt vmcnt(0)
9750 ; GFX11-NEXT: buffer_gl1_inv
9751 ; GFX11-NEXT: buffer_gl0_inv
9752 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
9753 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
9754 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
9755 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
9756 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
9757 ; GFX11-NEXT: s_cbranch_execnz .LBB35_1
9758 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
9759 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
9760 ; GFX11-NEXT: s_setpc_b64 s[30:31]
9761 ;
9762 ; GFX10-LABEL: global_agent_atomic_fsub_noret_bf16:
9763 ; GFX10: ; %bb.0:
9764 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9765 ; GFX10-NEXT: v_mov_b32_e32 v3, v0
9766 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9767 ; GFX10-NEXT: s_mov_b32 s4, 0
9768 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
9769 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
9770 ; GFX10-NEXT: global_load_dword v4, v[0:1], off
9771 ; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
9772 ; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
9773 ; GFX10-NEXT: v_not_b32_e32 v6, v3
9774 ; GFX10-NEXT: .LBB35_1: ; %atomicrmw.start
9775 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
9776 ; GFX10-NEXT: s_waitcnt vmcnt(0)
9777 ; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9778 ; GFX10-NEXT: v_sub_f32_e32 v3, v3, v2
9779 ; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
9780 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
9781 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
9782 ; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
9783 ; GFX10-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
9784 ; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9785 ; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
9786 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
9787 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
9788 ; GFX10-NEXT: s_waitcnt vmcnt(0)
9789 ; GFX10-NEXT: buffer_gl1_inv
9790 ; GFX10-NEXT: buffer_gl0_inv
9791 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
9792 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
9793 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
9794 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
9795 ; GFX10-NEXT: s_cbranch_execnz .LBB35_1
9796 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
9797 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
9798 ; GFX10-NEXT: s_setpc_b64 s[30:31]
9799 ;
9800 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_bf16:
9801 ; GFX90A: ; %bb.0:
9802 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9803 ; GFX90A-NEXT: v_mov_b32_e32 v3, v0
9804 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
9805 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
9806 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
9807 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9808 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
9809 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
9810 ; GFX90A-NEXT: v_not_b32_e32 v6, v4
9811 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
9812 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9813 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
9814 ; GFX90A-NEXT: .LBB35_1: ; %atomicrmw.start
9815 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
9816 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
9817 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v4, v3, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9818 ; GFX90A-NEXT: v_sub_f32_e32 v4, v4, v2
9819 ; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
9820 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
9821 ; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s6
9822 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
9823 ; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
9824 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v4, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9825 ; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
9826 ; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
9827 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
9828 ; GFX90A-NEXT: buffer_wbinvl1
9829 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
9830 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9831 ; GFX90A-NEXT: v_mov_b32_e32 v5, v4
9832 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
9833 ; GFX90A-NEXT: s_cbranch_execnz .LBB35_1
9834 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
9835 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
9836 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
9837 ;
9838 ; GFX908-LABEL: global_agent_atomic_fsub_noret_bf16:
9839 ; GFX908: ; %bb.0:
9840 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9841 ; GFX908-NEXT: v_mov_b32_e32 v3, v0
9842 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
9843 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
9844 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
9845 ; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
9846 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
9847 ; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
9848 ; GFX908-NEXT: v_not_b32_e32 v6, v3
9849 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
9850 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9851 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
9852 ; GFX908-NEXT: .LBB35_1: ; %atomicrmw.start
9853 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
9854 ; GFX908-NEXT: s_waitcnt vmcnt(0)
9855 ; GFX908-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9856 ; GFX908-NEXT: v_sub_f32_e32 v3, v3, v2
9857 ; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
9858 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
9859 ; GFX908-NEXT: v_add3_u32 v7, v7, v3, s6
9860 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
9861 ; GFX908-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
9862 ; GFX908-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9863 ; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
9864 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
9865 ; GFX908-NEXT: s_waitcnt vmcnt(0)
9866 ; GFX908-NEXT: buffer_wbinvl1
9867 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
9868 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9869 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
9870 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
9871 ; GFX908-NEXT: s_cbranch_execnz .LBB35_1
9872 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
9873 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
9874 ; GFX908-NEXT: s_setpc_b64 s[30:31]
9875 ;
9876 ; GFX8-LABEL: global_agent_atomic_fsub_noret_bf16:
9877 ; GFX8: ; %bb.0:
9878 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9879 ; GFX8-NEXT: v_mov_b32_e32 v3, v0
9880 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
9881 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
9882 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
9883 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
9884 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
9885 ; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
9886 ; GFX8-NEXT: v_not_b32_e32 v6, v3
9887 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
9888 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9889 ; GFX8-NEXT: .LBB35_1: ; %atomicrmw.start
9890 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
9891 ; GFX8-NEXT: s_waitcnt vmcnt(0)
9892 ; GFX8-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9893 ; GFX8-NEXT: v_sub_f32_e32 v3, v3, v2
9894 ; GFX8-NEXT: v_bfe_u32 v8, v3, 16, 1
9895 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v3
9896 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
9897 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v3
9898 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
9899 ; GFX8-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc
9900 ; GFX8-NEXT: v_and_b32_e32 v7, v4, v6
9901 ; GFX8-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9902 ; GFX8-NEXT: v_or_b32_e32 v3, v7, v3
9903 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
9904 ; GFX8-NEXT: s_waitcnt vmcnt(0)
9905 ; GFX8-NEXT: buffer_wbinvl1
9906 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
9907 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9908 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
9909 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
9910 ; GFX8-NEXT: s_cbranch_execnz .LBB35_1
9911 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
9912 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
9913 ; GFX8-NEXT: s_setpc_b64 s[30:31]
9914 ;
9915 ; GFX7-LABEL: global_agent_atomic_fsub_noret_bf16:
9916 ; GFX7: ; %bb.0:
9917 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9918 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
9919 ; GFX7-NEXT: s_mov_b32 s6, 0
9920 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
9921 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
9922 ; GFX7-NEXT: s_mov_b32 s4, s6
9923 ; GFX7-NEXT: s_mov_b32 s5, s6
9924 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
9925 ; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
9926 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 3, v3
9927 ; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
9928 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
9929 ; GFX7-NEXT: v_not_b32_e32 v6, v3
9930 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
9931 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
9932 ; GFX7-NEXT: .LBB35_1: ; %atomicrmw.start
9933 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
9934 ; GFX7-NEXT: s_waitcnt vmcnt(0)
9935 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, v5, v4
9936 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
9937 ; GFX7-NEXT: v_sub_f32_e32 v3, v3, v2
9938 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
9939 ; GFX7-NEXT: v_and_b32_e32 v7, v4, v6
9940 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, v5, v3
9941 ; GFX7-NEXT: v_or_b32_e32 v3, v7, v3
9942 ; GFX7-NEXT: v_mov_b32_e32 v8, v4
9943 ; GFX7-NEXT: v_mov_b32_e32 v7, v3
9944 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
9945 ; GFX7-NEXT: s_waitcnt vmcnt(0)
9946 ; GFX7-NEXT: buffer_wbinvl1
9947 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v4
9948 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
9949 ; GFX7-NEXT: v_mov_b32_e32 v4, v7
9950 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
9951 ; GFX7-NEXT: s_cbranch_execnz .LBB35_1
9952 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
9953 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
9954 ; GFX7-NEXT: s_setpc_b64 s[30:31]
9955 ;
9956 ; GFX6-LABEL: global_agent_atomic_fsub_noret_bf16:
9957 ; GFX6: ; %bb.0:
9958 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9959 ; GFX6-NEXT: v_mov_b32_e32 v3, v0
9960 ; GFX6-NEXT: s_mov_b32 s6, 0
9961 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v3
9962 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
9963 ; GFX6-NEXT: s_mov_b32 s4, s6
9964 ; GFX6-NEXT: s_mov_b32 s5, s6
9965 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
9966 ; GFX6-NEXT: v_and_b32_e32 v3, 3, v3
9967 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 3, v3
9968 ; GFX6-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
9969 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
9970 ; GFX6-NEXT: v_not_b32_e32 v6, v3
9971 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
9972 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
9973 ; GFX6-NEXT: .LBB35_1: ; %atomicrmw.start
9974 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
9975 ; GFX6-NEXT: s_waitcnt vmcnt(0)
9976 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, v5, v4
9977 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
9978 ; GFX6-NEXT: v_sub_f32_e32 v3, v3, v2
9979 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
9980 ; GFX6-NEXT: s_waitcnt expcnt(0)
9981 ; GFX6-NEXT: v_and_b32_e32 v7, v4, v6
9982 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, v5, v3
9983 ; GFX6-NEXT: v_or_b32_e32 v3, v7, v3
9984 ; GFX6-NEXT: v_mov_b32_e32 v8, v4
9985 ; GFX6-NEXT: v_mov_b32_e32 v7, v3
9986 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
9987 ; GFX6-NEXT: s_waitcnt vmcnt(0)
9988 ; GFX6-NEXT: buffer_wbinvl1
9989 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v4
9990 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
9991 ; GFX6-NEXT: v_mov_b32_e32 v4, v7
9992 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
9993 ; GFX6-NEXT: s_cbranch_execnz .LBB35_1
9994 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
9995 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
9996 ; GFX6-NEXT: s_waitcnt expcnt(0)
9997 ; GFX6-NEXT: s_setpc_b64 s[30:31]
9998 %unused = atomicrmw fsub ptr addrspace(1) %ptr, bfloat %val syncscope("agent") seq_cst
9999 ret void
10000 }
10002 define void @global_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
10003 ; GFX12-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_pos:
10004 ; GFX12: ; %bb.0:
10005 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
10006 ; GFX12-NEXT: s_wait_expcnt 0x0
10007 ; GFX12-NEXT: s_wait_samplecnt 0x0
10008 ; GFX12-NEXT: s_wait_bvhcnt 0x0
10009 ; GFX12-NEXT: s_wait_kmcnt 0x0
10010 ; GFX12-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
10011 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
10012 ; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10013 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
10014 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v4
10015 ; GFX12-NEXT: v_and_b32_e32 v4, 3, v4
10016 ; GFX12-NEXT: s_mov_b32 s0, 0
10017 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
10018 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10019 ; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
10020 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
10021 ; GFX12-NEXT: v_not_b32_e32 v5, v5
10022 ; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
10023 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
10024 ; GFX12-NEXT: s_wait_loadcnt 0x0
10025 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
10026 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10027 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10028 ; GFX12-NEXT: v_sub_f32_e32 v2, v2, v6
10029 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
10030 ; GFX12-NEXT: v_bfe_u32 v7, v2, 16, 1
10031 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v2
10032 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
10033 ; GFX12-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
10034 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10035 ; GFX12-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
10036 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10037 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10038 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, v4, v2
10039 ; GFX12-NEXT: v_and_or_b32 v2, v3, v5, v2
10040 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
10041 ; GFX12-NEXT: s_wait_storecnt 0x0
10042 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
10043 ; GFX12-NEXT: s_wait_loadcnt 0x0
10044 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
10045 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
10046 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
10047 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
10048 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
10049 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
10050 ; GFX12-NEXT: s_cbranch_execnz .LBB36_1
10051 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
10052 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
10053 ; GFX12-NEXT: s_setpc_b64 s[30:31]
10054 ;
10055 ; GFX940-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_pos:
10056 ; GFX940: ; %bb.0:
10057 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10058 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
10059 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
10060 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
10061 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
10062 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
10063 ; GFX940-NEXT: v_and_b32_e32 v4, 3, v4
10064 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10065 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
10066 ; GFX940-NEXT: v_lshlrev_b32_e64 v5, v4, s0
10067 ; GFX940-NEXT: v_not_b32_e32 v5, v5
10068 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
10069 ; GFX940-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10070 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
10071 ; GFX940-NEXT: .LBB36_1: ; %atomicrmw.start
10072 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
10073 ; GFX940-NEXT: s_waitcnt vmcnt(0)
10074 ; GFX940-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10075 ; GFX940-NEXT: s_nop 0
10076 ; GFX940-NEXT: v_sub_f32_e32 v2, v2, v6
10077 ; GFX940-NEXT: v_bfe_u32 v7, v2, 16, 1
10078 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v2
10079 ; GFX940-NEXT: v_add3_u32 v7, v7, v2, s2
10080 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
10081 ; GFX940-NEXT: s_nop 1
10082 ; GFX940-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
10083 ; GFX940-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10084 ; GFX940-NEXT: v_and_or_b32 v2, v3, v5, v2
10085 ; GFX940-NEXT: buffer_wbl2 sc1
10086 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
10087 ; GFX940-NEXT: s_waitcnt vmcnt(0)
10088 ; GFX940-NEXT: buffer_inv sc1
10089 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
10090 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
10091 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
10092 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
10093 ; GFX940-NEXT: s_cbranch_execnz .LBB36_1
10094 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
10095 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
10096 ; GFX940-NEXT: s_setpc_b64 s[30:31]
10097 ;
10098 ; GFX11-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_pos:
10099 ; GFX11: ; %bb.0:
10100 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10101 ; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
10102 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
10103 ; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10104 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
10105 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v4
10106 ; GFX11-NEXT: v_and_b32_e32 v4, 3, v4
10107 ; GFX11-NEXT: s_mov_b32 s0, 0
10108 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
10109 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10110 ; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
10111 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
10112 ; GFX11-NEXT: v_not_b32_e32 v5, v5
10113 ; GFX11-NEXT: .p2align 6
10114 ; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
10115 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
10116 ; GFX11-NEXT: s_waitcnt vmcnt(0)
10117 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
10118 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10119 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10120 ; GFX11-NEXT: v_sub_f32_e32 v2, v2, v6
10121 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
10122 ; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
10123 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v2
10124 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
10125 ; GFX11-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
10126 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10127 ; GFX11-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
10128 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10129 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10130 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, v4, v2
10131 ; GFX11-NEXT: v_and_or_b32 v2, v3, v5, v2
10132 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
10133 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
10134 ; GFX11-NEXT: s_waitcnt vmcnt(0)
10135 ; GFX11-NEXT: buffer_gl1_inv
10136 ; GFX11-NEXT: buffer_gl0_inv
10137 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
10138 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
10139 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
10140 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
10141 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
10142 ; GFX11-NEXT: s_cbranch_execnz .LBB36_1
10143 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
10144 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
10145 ; GFX11-NEXT: s_setpc_b64 s[30:31]
10146 ;
10147 ; GFX10-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_pos:
10148 ; GFX10: ; %bb.0:
10149 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10150 ; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
10151 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
10152 ; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10153 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
10154 ; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
10155 ; GFX10-NEXT: s_mov_b32 s4, 0
10156 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
10157 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10158 ; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
10159 ; GFX10-NEXT: v_not_b32_e32 v5, v5
10160 ; GFX10-NEXT: .LBB36_1: ; %atomicrmw.start
10161 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
10162 ; GFX10-NEXT: s_waitcnt vmcnt(0)
10163 ; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10164 ; GFX10-NEXT: v_sub_f32_e32 v2, v2, v6
10165 ; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
10166 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v2
10167 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
10168 ; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
10169 ; GFX10-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
10170 ; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10171 ; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
10172 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
10173 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
10174 ; GFX10-NEXT: s_waitcnt vmcnt(0)
10175 ; GFX10-NEXT: buffer_gl1_inv
10176 ; GFX10-NEXT: buffer_gl0_inv
10177 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
10178 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
10179 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
10180 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
10181 ; GFX10-NEXT: s_cbranch_execnz .LBB36_1
10182 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
10183 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
10184 ; GFX10-NEXT: s_setpc_b64 s[30:31]
10185 ;
10186 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_pos:
10187 ; GFX90A: ; %bb.0:
10188 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10189 ; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
10190 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
10191 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
10192 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
10193 ; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
10194 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10195 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
10196 ; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
10197 ; GFX90A-NEXT: v_not_b32_e32 v5, v5
10198 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
10199 ; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10200 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
10201 ; GFX90A-NEXT: .LBB36_1: ; %atomicrmw.start
10202 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
10203 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
10204 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10205 ; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v6
10206 ; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
10207 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
10208 ; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s6
10209 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
10210 ; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
10211 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10212 ; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
10213 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
10214 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
10215 ; GFX90A-NEXT: buffer_wbinvl1
10216 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
10217 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
10218 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
10219 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
10220 ; GFX90A-NEXT: s_cbranch_execnz .LBB36_1
10221 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
10222 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
10223 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
10224 ;
10225 ; GFX908-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_pos:
10226 ; GFX908: ; %bb.0:
10227 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10228 ; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
10229 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
10230 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
10231 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
10232 ; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
10233 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10234 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
10235 ; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
10236 ; GFX908-NEXT: v_not_b32_e32 v5, v5
10237 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
10238 ; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10239 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
10240 ; GFX908-NEXT: .LBB36_1: ; %atomicrmw.start
10241 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
10242 ; GFX908-NEXT: s_waitcnt vmcnt(0)
10243 ; GFX908-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10244 ; GFX908-NEXT: v_sub_f32_e32 v2, v2, v6
10245 ; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
10246 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
10247 ; GFX908-NEXT: v_add3_u32 v7, v7, v2, s6
10248 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
10249 ; GFX908-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
10250 ; GFX908-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10251 ; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
10252 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
10253 ; GFX908-NEXT: s_waitcnt vmcnt(0)
10254 ; GFX908-NEXT: buffer_wbinvl1
10255 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
10256 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
10257 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
10258 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
10259 ; GFX908-NEXT: s_cbranch_execnz .LBB36_1
10260 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
10261 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
10262 ; GFX908-NEXT: s_setpc_b64 s[30:31]
10263 ;
10264 ; GFX8-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_pos:
10265 ; GFX8: ; %bb.0:
10266 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10267 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
10268 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
10269 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
10270 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
10271 ; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
10272 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10273 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
10274 ; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
10275 ; GFX8-NEXT: v_not_b32_e32 v5, v5
10276 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
10277 ; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10278 ; GFX8-NEXT: .LBB36_1: ; %atomicrmw.start
10279 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
10280 ; GFX8-NEXT: s_waitcnt vmcnt(0)
10281 ; GFX8-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10282 ; GFX8-NEXT: v_sub_f32_e32 v2, v2, v6
10283 ; GFX8-NEXT: v_bfe_u32 v8, v2, 16, 1
10284 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v2
10285 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
10286 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v2
10287 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
10288 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v8, v9, vcc
10289 ; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
10290 ; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10291 ; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
10292 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10293 ; GFX8-NEXT: s_waitcnt vmcnt(0)
10294 ; GFX8-NEXT: buffer_wbinvl1
10295 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
10296 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
10297 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
10298 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
10299 ; GFX8-NEXT: s_cbranch_execnz .LBB36_1
10300 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
10301 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
10302 ; GFX8-NEXT: s_setpc_b64 s[30:31]
10303 ;
10304 ; GFX7-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_pos:
10305 ; GFX7: ; %bb.0:
10306 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10307 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
10308 ; GFX7-NEXT: s_mov_b32 s6, 0
10309 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
10310 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
10311 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
10312 ; GFX7-NEXT: s_mov_b32 s4, s6
10313 ; GFX7-NEXT: s_mov_b32 s5, s6
10314 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
10315 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
10316 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10317 ; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
10318 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
10319 ; GFX7-NEXT: v_not_b32_e32 v5, v5
10320 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
10321 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
10322 ; GFX7-NEXT: .LBB36_1: ; %atomicrmw.start
10323 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
10324 ; GFX7-NEXT: s_waitcnt vmcnt(0)
10325 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
10326 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10327 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v6
10328 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10329 ; GFX7-NEXT: v_and_b32_e32 v7, v3, v5
10330 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v4, v2
10331 ; GFX7-NEXT: v_or_b32_e32 v2, v7, v2
10332 ; GFX7-NEXT: v_mov_b32_e32 v8, v3
10333 ; GFX7-NEXT: v_mov_b32_e32 v7, v2
10334 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
10335 ; GFX7-NEXT: s_waitcnt vmcnt(0)
10336 ; GFX7-NEXT: buffer_wbinvl1
10337 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
10338 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
10339 ; GFX7-NEXT: v_mov_b32_e32 v3, v7
10340 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
10341 ; GFX7-NEXT: s_cbranch_execnz .LBB36_1
10342 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
10343 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
10344 ; GFX7-NEXT: s_setpc_b64 s[30:31]
10345 ;
10346 ; GFX6-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_pos:
10347 ; GFX6: ; %bb.0:
10348 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10349 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
10350 ; GFX6-NEXT: s_mov_b32 s6, 0
10351 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
10352 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
10353 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
10354 ; GFX6-NEXT: s_mov_b32 s4, s6
10355 ; GFX6-NEXT: s_mov_b32 s5, s6
10356 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
10357 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
10358 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10359 ; GFX6-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
10360 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
10361 ; GFX6-NEXT: v_not_b32_e32 v5, v5
10362 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
10363 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
10364 ; GFX6-NEXT: .LBB36_1: ; %atomicrmw.start
10365 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
10366 ; GFX6-NEXT: s_waitcnt vmcnt(0)
10367 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
10368 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10369 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v6
10370 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10371 ; GFX6-NEXT: s_waitcnt expcnt(0)
10372 ; GFX6-NEXT: v_and_b32_e32 v7, v3, v5
10373 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v4, v2
10374 ; GFX6-NEXT: v_or_b32_e32 v2, v7, v2
10375 ; GFX6-NEXT: v_mov_b32_e32 v8, v3
10376 ; GFX6-NEXT: v_mov_b32_e32 v7, v2
10377 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
10378 ; GFX6-NEXT: s_waitcnt vmcnt(0)
10379 ; GFX6-NEXT: buffer_wbinvl1
10380 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
10381 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
10382 ; GFX6-NEXT: v_mov_b32_e32 v3, v7
10383 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
10384 ; GFX6-NEXT: s_cbranch_execnz .LBB36_1
10385 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
10386 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
10387 ; GFX6-NEXT: s_waitcnt expcnt(0)
10388 ; GFX6-NEXT: s_setpc_b64 s[30:31]
10389 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
10390 %unused = atomicrmw fsub ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst
10391 ret void
10392 }
10394 define void @global_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr addrspace(1) %ptr, bfloat %val) #0 {
10395 ; GFX12-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_neg:
10396 ; GFX12: ; %bb.0:
10397 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
10398 ; GFX12-NEXT: s_wait_expcnt 0x0
10399 ; GFX12-NEXT: s_wait_samplecnt 0x0
10400 ; GFX12-NEXT: s_wait_bvhcnt 0x0
10401 ; GFX12-NEXT: s_wait_kmcnt 0x0
10402 ; GFX12-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
10403 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
10404 ; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10405 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
10406 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v4
10407 ; GFX12-NEXT: v_and_b32_e32 v4, 3, v4
10408 ; GFX12-NEXT: s_mov_b32 s0, 0
10409 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
10410 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10411 ; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
10412 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
10413 ; GFX12-NEXT: v_not_b32_e32 v5, v5
10414 ; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
10415 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
10416 ; GFX12-NEXT: s_wait_loadcnt 0x0
10417 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
10418 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10419 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10420 ; GFX12-NEXT: v_sub_f32_e32 v2, v2, v6
10421 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
10422 ; GFX12-NEXT: v_bfe_u32 v7, v2, 16, 1
10423 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v2
10424 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
10425 ; GFX12-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
10426 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10427 ; GFX12-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
10428 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10429 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10430 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, v4, v2
10431 ; GFX12-NEXT: v_and_or_b32 v2, v3, v5, v2
10432 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
10433 ; GFX12-NEXT: s_wait_storecnt 0x0
10434 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
10435 ; GFX12-NEXT: s_wait_loadcnt 0x0
10436 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
10437 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
10438 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
10439 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
10440 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
10441 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
10442 ; GFX12-NEXT: s_cbranch_execnz .LBB37_1
10443 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
10444 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
10445 ; GFX12-NEXT: s_setpc_b64 s[30:31]
10446 ;
10447 ; GFX940-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_neg:
10448 ; GFX940: ; %bb.0:
10449 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10450 ; GFX940-NEXT: s_movk_i32 s0, 0xf800
10451 ; GFX940-NEXT: s_mov_b32 s1, -1
10452 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
10453 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
10454 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
10455 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
10456 ; GFX940-NEXT: v_and_b32_e32 v4, 3, v4
10457 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10458 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
10459 ; GFX940-NEXT: v_lshlrev_b32_e64 v5, v4, s0
10460 ; GFX940-NEXT: v_not_b32_e32 v5, v5
10461 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
10462 ; GFX940-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10463 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
10464 ; GFX940-NEXT: .LBB37_1: ; %atomicrmw.start
10465 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
10466 ; GFX940-NEXT: s_waitcnt vmcnt(0)
10467 ; GFX940-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10468 ; GFX940-NEXT: s_nop 0
10469 ; GFX940-NEXT: v_sub_f32_e32 v2, v2, v6
10470 ; GFX940-NEXT: v_bfe_u32 v7, v2, 16, 1
10471 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v2
10472 ; GFX940-NEXT: v_add3_u32 v7, v7, v2, s2
10473 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
10474 ; GFX940-NEXT: s_nop 1
10475 ; GFX940-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
10476 ; GFX940-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10477 ; GFX940-NEXT: v_and_or_b32 v2, v3, v5, v2
10478 ; GFX940-NEXT: buffer_wbl2 sc1
10479 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
10480 ; GFX940-NEXT: s_waitcnt vmcnt(0)
10481 ; GFX940-NEXT: buffer_inv sc1
10482 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
10483 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
10484 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
10485 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
10486 ; GFX940-NEXT: s_cbranch_execnz .LBB37_1
10487 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
10488 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
10489 ; GFX940-NEXT: s_setpc_b64 s[30:31]
10490 ;
10491 ; GFX11-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_neg:
10492 ; GFX11: ; %bb.0:
10493 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10494 ; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
10495 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
10496 ; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10497 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
10498 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v4
10499 ; GFX11-NEXT: v_and_b32_e32 v4, 3, v4
10500 ; GFX11-NEXT: s_mov_b32 s0, 0
10501 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
10502 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10503 ; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
10504 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
10505 ; GFX11-NEXT: v_not_b32_e32 v5, v5
10506 ; GFX11-NEXT: .p2align 6
10507 ; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
10508 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
10509 ; GFX11-NEXT: s_waitcnt vmcnt(0)
10510 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
10511 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10512 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10513 ; GFX11-NEXT: v_sub_f32_e32 v2, v2, v6
10514 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
10515 ; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
10516 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v2
10517 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
10518 ; GFX11-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
10519 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10520 ; GFX11-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
10521 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10522 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10523 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, v4, v2
10524 ; GFX11-NEXT: v_and_or_b32 v2, v3, v5, v2
10525 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
10526 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
10527 ; GFX11-NEXT: s_waitcnt vmcnt(0)
10528 ; GFX11-NEXT: buffer_gl1_inv
10529 ; GFX11-NEXT: buffer_gl0_inv
10530 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
10531 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
10532 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
10533 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
10534 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
10535 ; GFX11-NEXT: s_cbranch_execnz .LBB37_1
10536 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
10537 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
10538 ; GFX11-NEXT: s_setpc_b64 s[30:31]
10539 ;
10540 ; GFX10-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_neg:
10541 ; GFX10: ; %bb.0:
10542 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10543 ; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
10544 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
10545 ; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10546 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
10547 ; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
10548 ; GFX10-NEXT: s_mov_b32 s4, 0
10549 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
10550 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10551 ; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
10552 ; GFX10-NEXT: v_not_b32_e32 v5, v5
10553 ; GFX10-NEXT: .LBB37_1: ; %atomicrmw.start
10554 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
10555 ; GFX10-NEXT: s_waitcnt vmcnt(0)
10556 ; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10557 ; GFX10-NEXT: v_sub_f32_e32 v2, v2, v6
10558 ; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
10559 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v2
10560 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
10561 ; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
10562 ; GFX10-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
10563 ; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10564 ; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
10565 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
10566 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
10567 ; GFX10-NEXT: s_waitcnt vmcnt(0)
10568 ; GFX10-NEXT: buffer_gl1_inv
10569 ; GFX10-NEXT: buffer_gl0_inv
10570 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
10571 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
10572 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
10573 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
10574 ; GFX10-NEXT: s_cbranch_execnz .LBB37_1
10575 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
10576 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
10577 ; GFX10-NEXT: s_setpc_b64 s[30:31]
10578 ;
10579 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_neg:
10580 ; GFX90A: ; %bb.0:
10581 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10582 ; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
10583 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
10584 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
10585 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
10586 ; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
10587 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10588 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
10589 ; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
10590 ; GFX90A-NEXT: v_not_b32_e32 v5, v5
10591 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
10592 ; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10593 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
10594 ; GFX90A-NEXT: .LBB37_1: ; %atomicrmw.start
10595 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
10596 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
10597 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10598 ; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v6
10599 ; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
10600 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
10601 ; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s6
10602 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
10603 ; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
10604 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10605 ; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
10606 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
10607 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
10608 ; GFX90A-NEXT: buffer_wbinvl1
10609 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
10610 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
10611 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
10612 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
10613 ; GFX90A-NEXT: s_cbranch_execnz .LBB37_1
10614 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
10615 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
10616 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
10617 ;
10618 ; GFX908-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_neg:
10619 ; GFX908: ; %bb.0:
10620 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10621 ; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
10622 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
10623 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
10624 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
10625 ; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
10626 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10627 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
10628 ; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
10629 ; GFX908-NEXT: v_not_b32_e32 v5, v5
10630 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
10631 ; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10632 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
10633 ; GFX908-NEXT: .LBB37_1: ; %atomicrmw.start
10634 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
10635 ; GFX908-NEXT: s_waitcnt vmcnt(0)
10636 ; GFX908-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10637 ; GFX908-NEXT: v_sub_f32_e32 v2, v2, v6
10638 ; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
10639 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
10640 ; GFX908-NEXT: v_add3_u32 v7, v7, v2, s6
10641 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
10642 ; GFX908-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
10643 ; GFX908-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10644 ; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
10645 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
10646 ; GFX908-NEXT: s_waitcnt vmcnt(0)
10647 ; GFX908-NEXT: buffer_wbinvl1
10648 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
10649 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
10650 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
10651 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
10652 ; GFX908-NEXT: s_cbranch_execnz .LBB37_1
10653 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
10654 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
10655 ; GFX908-NEXT: s_setpc_b64 s[30:31]
10656 ;
10657 ; GFX8-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_neg:
10658 ; GFX8: ; %bb.0:
10659 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10660 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
10661 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
10662 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
10663 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
10664 ; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
10665 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10666 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
10667 ; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
10668 ; GFX8-NEXT: v_not_b32_e32 v5, v5
10669 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
10670 ; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10671 ; GFX8-NEXT: .LBB37_1: ; %atomicrmw.start
10672 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
10673 ; GFX8-NEXT: s_waitcnt vmcnt(0)
10674 ; GFX8-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10675 ; GFX8-NEXT: v_sub_f32_e32 v2, v2, v6
10676 ; GFX8-NEXT: v_bfe_u32 v8, v2, 16, 1
10677 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v2
10678 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
10679 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v2
10680 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
10681 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v8, v9, vcc
10682 ; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
10683 ; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10684 ; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
10685 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10686 ; GFX8-NEXT: s_waitcnt vmcnt(0)
10687 ; GFX8-NEXT: buffer_wbinvl1
10688 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
10689 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
10690 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
10691 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
10692 ; GFX8-NEXT: s_cbranch_execnz .LBB37_1
10693 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
10694 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
10695 ; GFX8-NEXT: s_setpc_b64 s[30:31]
10696 ;
10697 ; GFX7-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_neg:
10698 ; GFX7: ; %bb.0:
10699 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10700 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
10701 ; GFX7-NEXT: s_mov_b32 s6, 0
10702 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
10703 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
10704 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
10705 ; GFX7-NEXT: s_mov_b32 s4, s6
10706 ; GFX7-NEXT: s_mov_b32 s5, s6
10707 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
10708 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
10709 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10710 ; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
10711 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
10712 ; GFX7-NEXT: v_not_b32_e32 v5, v5
10713 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
10714 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
10715 ; GFX7-NEXT: .LBB37_1: ; %atomicrmw.start
10716 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
10717 ; GFX7-NEXT: s_waitcnt vmcnt(0)
10718 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
10719 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10720 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v6
10721 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10722 ; GFX7-NEXT: v_and_b32_e32 v7, v3, v5
10723 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v4, v2
10724 ; GFX7-NEXT: v_or_b32_e32 v2, v7, v2
10725 ; GFX7-NEXT: v_mov_b32_e32 v8, v3
10726 ; GFX7-NEXT: v_mov_b32_e32 v7, v2
10727 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
10728 ; GFX7-NEXT: s_waitcnt vmcnt(0)
10729 ; GFX7-NEXT: buffer_wbinvl1
10730 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
10731 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
10732 ; GFX7-NEXT: v_mov_b32_e32 v3, v7
10733 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
10734 ; GFX7-NEXT: s_cbranch_execnz .LBB37_1
10735 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
10736 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
10737 ; GFX7-NEXT: s_setpc_b64 s[30:31]
10738 ;
10739 ; GFX6-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_neg:
10740 ; GFX6: ; %bb.0:
10741 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10742 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
10743 ; GFX6-NEXT: s_mov_b32 s6, 0
10744 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
10745 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
10746 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
10747 ; GFX6-NEXT: s_mov_b32 s4, s6
10748 ; GFX6-NEXT: s_mov_b32 s5, s6
10749 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
10750 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
10751 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10752 ; GFX6-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
10753 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
10754 ; GFX6-NEXT: v_not_b32_e32 v5, v5
10755 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
10756 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
10757 ; GFX6-NEXT: .LBB37_1: ; %atomicrmw.start
10758 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
10759 ; GFX6-NEXT: s_waitcnt vmcnt(0)
10760 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
10761 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10762 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v6
10763 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10764 ; GFX6-NEXT: s_waitcnt expcnt(0)
10765 ; GFX6-NEXT: v_and_b32_e32 v7, v3, v5
10766 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v4, v2
10767 ; GFX6-NEXT: v_or_b32_e32 v2, v7, v2
10768 ; GFX6-NEXT: v_mov_b32_e32 v8, v3
10769 ; GFX6-NEXT: v_mov_b32_e32 v7, v2
10770 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
10771 ; GFX6-NEXT: s_waitcnt vmcnt(0)
10772 ; GFX6-NEXT: buffer_wbinvl1
10773 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
10774 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
10775 ; GFX6-NEXT: v_mov_b32_e32 v3, v7
10776 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
10777 ; GFX6-NEXT: s_cbranch_execnz .LBB37_1
10778 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
10779 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
10780 ; GFX6-NEXT: s_waitcnt expcnt(0)
10781 ; GFX6-NEXT: s_setpc_b64 s[30:31]
10782 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 -1024
10783 %unused = atomicrmw fsub ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst
10784 ret void
10785 }
10787 define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr addrspace(1) %ptr, bfloat %val) #0 {
10788 ; GFX12-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
10789 ; GFX12: ; %bb.0:
10790 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
10791 ; GFX12-NEXT: s_wait_expcnt 0x0
10792 ; GFX12-NEXT: s_wait_samplecnt 0x0
10793 ; GFX12-NEXT: s_wait_bvhcnt 0x0
10794 ; GFX12-NEXT: s_wait_kmcnt 0x0
10795 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2046
10796 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10797 ; GFX12-NEXT: s_mov_b32 s0, 0
10798 ; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
10799 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
10800 ; GFX12-NEXT: s_wait_loadcnt 0x0
10801 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
10802 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10803 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v4
10804 ; GFX12-NEXT: v_sub_f32_e32 v3, v3, v2
10805 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
10806 ; GFX12-NEXT: v_bfe_u32 v5, v3, 16, 1
10807 ; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v3
10808 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
10809 ; GFX12-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
10810 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10811 ; GFX12-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
10812 ; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
10813 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
10814 ; GFX12-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
10815 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
10816 ; GFX12-NEXT: s_wait_storecnt 0x0
10817 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
10818 ; GFX12-NEXT: s_wait_loadcnt 0x0
10819 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
10820 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
10821 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
10822 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
10823 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
10824 ; GFX12-NEXT: s_cbranch_execnz .LBB38_1
10825 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
10826 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
10827 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
10828 ; GFX12-NEXT: s_setpc_b64 s[30:31]
10829 ;
10830 ; GFX940-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
10831 ; GFX940: ; %bb.0:
10832 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10833 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2046
10834 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
10835 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10836 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
10837 ; GFX940-NEXT: s_mov_b32 s3, 0xffff0000
10838 ; GFX940-NEXT: .LBB38_1: ; %atomicrmw.start
10839 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
10840 ; GFX940-NEXT: s_waitcnt vmcnt(0)
10841 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
10842 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 16, v5
10843 ; GFX940-NEXT: v_sub_f32_e32 v3, v3, v2
10844 ; GFX940-NEXT: v_bfe_u32 v4, v3, 16, 1
10845 ; GFX940-NEXT: v_or_b32_e32 v6, 0x400000, v3
10846 ; GFX940-NEXT: v_add3_u32 v4, v4, v3, s2
10847 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
10848 ; GFX940-NEXT: s_nop 1
10849 ; GFX940-NEXT: v_cndmask_b32_e32 v3, v4, v6, vcc
10850 ; GFX940-NEXT: v_lshrrev_b32_e32 v3, 16, v3
10851 ; GFX940-NEXT: v_and_or_b32 v4, v5, s3, v3
10852 ; GFX940-NEXT: buffer_wbl2 sc1
10853 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 sc0
10854 ; GFX940-NEXT: s_waitcnt vmcnt(0)
10855 ; GFX940-NEXT: buffer_inv sc1
10856 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
10857 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
10858 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
10859 ; GFX940-NEXT: s_cbranch_execnz .LBB38_1
10860 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
10861 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
10862 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
10863 ; GFX940-NEXT: s_setpc_b64 s[30:31]
10864 ;
10865 ; GFX11-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
10866 ; GFX11: ; %bb.0:
10867 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10868 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2046
10869 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10870 ; GFX11-NEXT: s_mov_b32 s0, 0
10871 ; GFX11-NEXT: .p2align 6
10872 ; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
10873 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
10874 ; GFX11-NEXT: s_waitcnt vmcnt(0)
10875 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
10876 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10877 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v4
10878 ; GFX11-NEXT: v_sub_f32_e32 v3, v3, v2
10879 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
10880 ; GFX11-NEXT: v_bfe_u32 v5, v3, 16, 1
10881 ; GFX11-NEXT: v_or_b32_e32 v6, 0x400000, v3
10882 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
10883 ; GFX11-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
10884 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10885 ; GFX11-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
10886 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
10887 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
10888 ; GFX11-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
10889 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
10890 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
10891 ; GFX11-NEXT: s_waitcnt vmcnt(0)
10892 ; GFX11-NEXT: buffer_gl1_inv
10893 ; GFX11-NEXT: buffer_gl0_inv
10894 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
10895 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
10896 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
10897 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
10898 ; GFX11-NEXT: s_cbranch_execnz .LBB38_1
10899 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
10900 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
10901 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
10902 ; GFX11-NEXT: s_setpc_b64 s[30:31]
10903 ;
10904 ; GFX10-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
10905 ; GFX10: ; %bb.0:
10906 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10907 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2046
10908 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10909 ; GFX10-NEXT: s_mov_b32 s4, 0
10910 ; GFX10-NEXT: .LBB38_1: ; %atomicrmw.start
10911 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
10912 ; GFX10-NEXT: s_waitcnt vmcnt(0)
10913 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
10914 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v4
10915 ; GFX10-NEXT: v_sub_f32_e32 v3, v3, v2
10916 ; GFX10-NEXT: v_bfe_u32 v5, v3, 16, 1
10917 ; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v3
10918 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
10919 ; GFX10-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
10920 ; GFX10-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
10921 ; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v3
10922 ; GFX10-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
10923 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
10924 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2046 glc
10925 ; GFX10-NEXT: s_waitcnt vmcnt(0)
10926 ; GFX10-NEXT: buffer_gl1_inv
10927 ; GFX10-NEXT: buffer_gl0_inv
10928 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
10929 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
10930 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
10931 ; GFX10-NEXT: s_cbranch_execnz .LBB38_1
10932 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
10933 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
10934 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
10935 ; GFX10-NEXT: s_setpc_b64 s[30:31]
10936 ;
10937 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
10938 ; GFX90A: ; %bb.0:
10939 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10940 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2046
10941 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
10942 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10943 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
10944 ; GFX90A-NEXT: s_mov_b32 s7, 0xffff0000
10945 ; GFX90A-NEXT: .LBB38_1: ; %atomicrmw.start
10946 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
10947 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
10948 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
10949 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v5
10950 ; GFX90A-NEXT: v_sub_f32_e32 v3, v3, v2
10951 ; GFX90A-NEXT: v_bfe_u32 v4, v3, 16, 1
10952 ; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v3
10953 ; GFX90A-NEXT: v_add3_u32 v4, v4, v3, s6
10954 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
10955 ; GFX90A-NEXT: v_cndmask_b32_e32 v3, v4, v6, vcc
10956 ; GFX90A-NEXT: v_lshrrev_b32_e32 v3, 16, v3
10957 ; GFX90A-NEXT: v_and_or_b32 v4, v5, s7, v3
10958 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 glc
10959 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
10960 ; GFX90A-NEXT: buffer_wbinvl1
10961 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
10962 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
10963 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
10964 ; GFX90A-NEXT: s_cbranch_execnz .LBB38_1
10965 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
10966 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
10967 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
10968 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
10969 ;
10970 ; GFX908-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
10971 ; GFX908: ; %bb.0:
10972 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10973 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2046
10974 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
10975 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10976 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
10977 ; GFX908-NEXT: s_mov_b32 s7, 0xffff0000
10978 ; GFX908-NEXT: .LBB38_1: ; %atomicrmw.start
10979 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
10980 ; GFX908-NEXT: s_waitcnt vmcnt(0)
10981 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
10982 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v4
10983 ; GFX908-NEXT: v_sub_f32_e32 v3, v3, v2
10984 ; GFX908-NEXT: v_bfe_u32 v5, v3, 16, 1
10985 ; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v3
10986 ; GFX908-NEXT: v_add3_u32 v5, v5, v3, s6
10987 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
10988 ; GFX908-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
10989 ; GFX908-NEXT: v_lshrrev_b32_e32 v3, 16, v3
10990 ; GFX908-NEXT: v_and_or_b32 v3, v4, s7, v3
10991 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2046 glc
10992 ; GFX908-NEXT: s_waitcnt vmcnt(0)
10993 ; GFX908-NEXT: buffer_wbinvl1
10994 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
10995 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
10996 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
10997 ; GFX908-NEXT: s_cbranch_execnz .LBB38_1
10998 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
10999 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
11000 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
11001 ; GFX908-NEXT: s_setpc_b64 s[30:31]
11002 ;
11003 ; GFX8-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
11004 ; GFX8: ; %bb.0:
11005 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11006 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
11007 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
11008 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
11009 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
11010 ; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
11011 ; GFX8-NEXT: .LBB38_1: ; %atomicrmw.start
11012 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
11013 ; GFX8-NEXT: s_waitcnt vmcnt(0)
11014 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
11015 ; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
11016 ; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1
11017 ; GFX8-NEXT: v_bfe_u32 v5, v0, 16, 1
11018 ; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v0
11019 ; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
11020 ; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v0
11021 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
11022 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
11023 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v5, v7, vcc
11024 ; GFX8-NEXT: v_or_b32_sdwa v5, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11025 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
11026 ; GFX8-NEXT: s_waitcnt vmcnt(0)
11027 ; GFX8-NEXT: buffer_wbinvl1
11028 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
11029 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11030 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
11031 ; GFX8-NEXT: s_cbranch_execnz .LBB38_1
11032 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
11033 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
11034 ; GFX8-NEXT: s_setpc_b64 s[30:31]
11035 ;
11036 ; GFX7-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
11037 ; GFX7: ; %bb.0:
11038 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11039 ; GFX7-NEXT: s_mov_b32 s6, 0
11040 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
11041 ; GFX7-NEXT: s_mov_b32 s4, s6
11042 ; GFX7-NEXT: s_mov_b32 s5, s6
11043 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
11044 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
11045 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
11046 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
11047 ; GFX7-NEXT: .LBB38_1: ; %atomicrmw.start
11048 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
11049 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11050 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
11051 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v6
11052 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
11053 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11054 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
11055 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
11056 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
11057 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 offset:2046 glc
11058 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11059 ; GFX7-NEXT: buffer_wbinvl1
11060 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
11061 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
11062 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
11063 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
11064 ; GFX7-NEXT: s_cbranch_execnz .LBB38_1
11065 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
11066 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
11067 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v4
11068 ; GFX7-NEXT: s_setpc_b64 s[30:31]
11069 ;
11070 ; GFX6-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
11071 ; GFX6: ; %bb.0:
11072 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11073 ; GFX6-NEXT: s_mov_b32 s6, 0
11074 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
11075 ; GFX6-NEXT: s_mov_b32 s4, s6
11076 ; GFX6-NEXT: s_mov_b32 s5, s6
11077 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
11078 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
11079 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
11080 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
11081 ; GFX6-NEXT: .LBB38_1: ; %atomicrmw.start
11082 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
11083 ; GFX6-NEXT: s_waitcnt vmcnt(0)
11084 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
11085 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v6
11086 ; GFX6-NEXT: s_waitcnt expcnt(0)
11087 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
11088 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11089 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
11090 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
11091 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
11092 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 offset:2046 glc
11093 ; GFX6-NEXT: s_waitcnt vmcnt(0)
11094 ; GFX6-NEXT: buffer_wbinvl1
11095 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
11096 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
11097 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
11098 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
11099 ; GFX6-NEXT: s_cbranch_execnz .LBB38_1
11100 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
11101 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
11102 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v4
11103 ; GFX6-NEXT: s_waitcnt expcnt(0)
11104 ; GFX6-NEXT: s_setpc_b64 s[30:31]
11105 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
11106 %result = atomicrmw fsub ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, align 4
11107 ret bfloat %result
11108 }
11110 define void @global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
11111 ; GFX12-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos:
11112 ; GFX12: ; %bb.0:
11113 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
11114 ; GFX12-NEXT: s_wait_expcnt 0x0
11115 ; GFX12-NEXT: s_wait_samplecnt 0x0
11116 ; GFX12-NEXT: s_wait_bvhcnt 0x0
11117 ; GFX12-NEXT: s_wait_kmcnt 0x0
11118 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2046
11119 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
11120 ; GFX12-NEXT: s_mov_b32 s0, 0
11121 ; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
11122 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
11123 ; GFX12-NEXT: s_wait_loadcnt 0x0
11124 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
11125 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11126 ; GFX12-NEXT: v_sub_f32_e32 v2, v2, v4
11127 ; GFX12-NEXT: v_bfe_u32 v5, v2, 16, 1
11128 ; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v2
11129 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
11130 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
11131 ; GFX12-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
11132 ; GFX12-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
11133 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11134 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11135 ; GFX12-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
11136 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
11137 ; GFX12-NEXT: s_wait_storecnt 0x0
11138 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
11139 ; GFX12-NEXT: s_wait_loadcnt 0x0
11140 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
11141 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
11142 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
11143 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
11144 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
11145 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
11146 ; GFX12-NEXT: s_cbranch_execnz .LBB39_1
11147 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
11148 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
11149 ; GFX12-NEXT: s_setpc_b64 s[30:31]
11150 ;
11151 ; GFX940-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos:
11152 ; GFX940: ; %bb.0:
11153 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11154 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2046
11155 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
11156 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
11157 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
11158 ; GFX940-NEXT: s_mov_b32 s3, 0xffff0000
11159 ; GFX940-NEXT: .LBB39_1: ; %atomicrmw.start
11160 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
11161 ; GFX940-NEXT: s_waitcnt vmcnt(0)
11162 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
11163 ; GFX940-NEXT: v_sub_f32_e32 v2, v2, v4
11164 ; GFX940-NEXT: v_bfe_u32 v5, v2, 16, 1
11165 ; GFX940-NEXT: v_or_b32_e32 v6, 0x400000, v2
11166 ; GFX940-NEXT: v_add3_u32 v5, v5, v2, s2
11167 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
11168 ; GFX940-NEXT: s_nop 1
11169 ; GFX940-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
11170 ; GFX940-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11171 ; GFX940-NEXT: v_and_or_b32 v2, v3, s3, v2
11172 ; GFX940-NEXT: buffer_wbl2 sc1
11173 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 sc0
11174 ; GFX940-NEXT: s_waitcnt vmcnt(0)
11175 ; GFX940-NEXT: buffer_inv sc1
11176 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
11177 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
11178 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
11179 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
11180 ; GFX940-NEXT: s_cbranch_execnz .LBB39_1
11181 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
11182 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
11183 ; GFX940-NEXT: s_setpc_b64 s[30:31]
11184 ;
11185 ; GFX11-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos:
11186 ; GFX11: ; %bb.0:
11187 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11188 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2046
11189 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
11190 ; GFX11-NEXT: s_mov_b32 s0, 0
11191 ; GFX11-NEXT: .p2align 6
11192 ; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
11193 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
11194 ; GFX11-NEXT: s_waitcnt vmcnt(0)
11195 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
11196 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11197 ; GFX11-NEXT: v_sub_f32_e32 v2, v2, v4
11198 ; GFX11-NEXT: v_bfe_u32 v5, v2, 16, 1
11199 ; GFX11-NEXT: v_or_b32_e32 v6, 0x400000, v2
11200 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
11201 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
11202 ; GFX11-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
11203 ; GFX11-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
11204 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11205 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11206 ; GFX11-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
11207 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
11208 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 glc
11209 ; GFX11-NEXT: s_waitcnt vmcnt(0)
11210 ; GFX11-NEXT: buffer_gl1_inv
11211 ; GFX11-NEXT: buffer_gl0_inv
11212 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
11213 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
11214 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
11215 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
11216 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
11217 ; GFX11-NEXT: s_cbranch_execnz .LBB39_1
11218 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
11219 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
11220 ; GFX11-NEXT: s_setpc_b64 s[30:31]
11221 ;
11222 ; GFX10-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos:
11223 ; GFX10: ; %bb.0:
11224 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11225 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2046
11226 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
11227 ; GFX10-NEXT: s_mov_b32 s4, 0
11228 ; GFX10-NEXT: .LBB39_1: ; %atomicrmw.start
11229 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
11230 ; GFX10-NEXT: s_waitcnt vmcnt(0)
11231 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
11232 ; GFX10-NEXT: v_sub_f32_e32 v2, v2, v4
11233 ; GFX10-NEXT: v_bfe_u32 v5, v2, 16, 1
11234 ; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v2
11235 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
11236 ; GFX10-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
11237 ; GFX10-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
11238 ; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11239 ; GFX10-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
11240 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
11241 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 glc
11242 ; GFX10-NEXT: s_waitcnt vmcnt(0)
11243 ; GFX10-NEXT: buffer_gl1_inv
11244 ; GFX10-NEXT: buffer_gl0_inv
11245 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
11246 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
11247 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
11248 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
11249 ; GFX10-NEXT: s_cbranch_execnz .LBB39_1
11250 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
11251 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
11252 ; GFX10-NEXT: s_setpc_b64 s[30:31]
11253 ;
11254 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos:
11255 ; GFX90A: ; %bb.0:
11256 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11257 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2046
11258 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
11259 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
11260 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
11261 ; GFX90A-NEXT: s_mov_b32 s7, 0xffff0000
11262 ; GFX90A-NEXT: .LBB39_1: ; %atomicrmw.start
11263 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
11264 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
11265 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
11266 ; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
11267 ; GFX90A-NEXT: v_bfe_u32 v5, v2, 16, 1
11268 ; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v2
11269 ; GFX90A-NEXT: v_add3_u32 v5, v5, v2, s6
11270 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
11271 ; GFX90A-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
11272 ; GFX90A-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11273 ; GFX90A-NEXT: v_and_or_b32 v2, v3, s7, v2
11274 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 glc
11275 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
11276 ; GFX90A-NEXT: buffer_wbinvl1
11277 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
11278 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11279 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
11280 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
11281 ; GFX90A-NEXT: s_cbranch_execnz .LBB39_1
11282 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
11283 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
11284 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
11285 ;
11286 ; GFX908-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos:
11287 ; GFX908: ; %bb.0:
11288 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11289 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2046
11290 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
11291 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
11292 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
11293 ; GFX908-NEXT: s_mov_b32 s7, 0xffff0000
11294 ; GFX908-NEXT: .LBB39_1: ; %atomicrmw.start
11295 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
11296 ; GFX908-NEXT: s_waitcnt vmcnt(0)
11297 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
11298 ; GFX908-NEXT: v_sub_f32_e32 v2, v2, v4
11299 ; GFX908-NEXT: v_bfe_u32 v5, v2, 16, 1
11300 ; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v2
11301 ; GFX908-NEXT: v_add3_u32 v5, v5, v2, s6
11302 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
11303 ; GFX908-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
11304 ; GFX908-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11305 ; GFX908-NEXT: v_and_or_b32 v2, v3, s7, v2
11306 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 glc
11307 ; GFX908-NEXT: s_waitcnt vmcnt(0)
11308 ; GFX908-NEXT: buffer_wbinvl1
11309 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
11310 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11311 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
11312 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
11313 ; GFX908-NEXT: s_cbranch_execnz .LBB39_1
11314 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
11315 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
11316 ; GFX908-NEXT: s_setpc_b64 s[30:31]
11317 ;
11318 ; GFX8-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos:
11319 ; GFX8: ; %bb.0:
11320 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11321 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fe, v0
11322 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
11323 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
11324 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
11325 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
11326 ; GFX8-NEXT: .LBB39_1: ; %atomicrmw.start
11327 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
11328 ; GFX8-NEXT: s_waitcnt vmcnt(0)
11329 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
11330 ; GFX8-NEXT: v_sub_f32_e32 v2, v2, v4
11331 ; GFX8-NEXT: v_bfe_u32 v6, v2, 16, 1
11332 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v2
11333 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
11334 ; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v2
11335 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
11336 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
11337 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
11338 ; GFX8-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11339 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11340 ; GFX8-NEXT: s_waitcnt vmcnt(0)
11341 ; GFX8-NEXT: buffer_wbinvl1
11342 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
11343 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11344 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
11345 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
11346 ; GFX8-NEXT: s_cbranch_execnz .LBB39_1
11347 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
11348 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
11349 ; GFX8-NEXT: s_setpc_b64 s[30:31]
11350 ;
11351 ; GFX7-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos:
11352 ; GFX7: ; %bb.0:
11353 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11354 ; GFX7-NEXT: s_mov_b32 s6, 0
11355 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
11356 ; GFX7-NEXT: s_mov_b32 s4, s6
11357 ; GFX7-NEXT: s_mov_b32 s5, s6
11358 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
11359 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
11360 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
11361 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
11362 ; GFX7-NEXT: .LBB39_1: ; %atomicrmw.start
11363 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
11364 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11365 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
11366 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v4
11367 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
11368 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11369 ; GFX7-NEXT: v_or_b32_e32 v2, v5, v2
11370 ; GFX7-NEXT: v_mov_b32_e32 v6, v3
11371 ; GFX7-NEXT: v_mov_b32_e32 v5, v2
11372 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2046 glc
11373 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11374 ; GFX7-NEXT: buffer_wbinvl1
11375 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
11376 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
11377 ; GFX7-NEXT: v_mov_b32_e32 v3, v5
11378 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
11379 ; GFX7-NEXT: s_cbranch_execnz .LBB39_1
11380 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
11381 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
11382 ; GFX7-NEXT: s_setpc_b64 s[30:31]
11383 ;
11384 ; GFX6-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos:
11385 ; GFX6: ; %bb.0:
11386 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11387 ; GFX6-NEXT: s_mov_b32 s6, 0
11388 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
11389 ; GFX6-NEXT: s_mov_b32 s4, s6
11390 ; GFX6-NEXT: s_mov_b32 s5, s6
11391 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
11392 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
11393 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
11394 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
11395 ; GFX6-NEXT: .LBB39_1: ; %atomicrmw.start
11396 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
11397 ; GFX6-NEXT: s_waitcnt vmcnt(0)
11398 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
11399 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v4
11400 ; GFX6-NEXT: s_waitcnt expcnt(0)
11401 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
11402 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11403 ; GFX6-NEXT: v_or_b32_e32 v2, v5, v2
11404 ; GFX6-NEXT: v_mov_b32_e32 v6, v3
11405 ; GFX6-NEXT: v_mov_b32_e32 v5, v2
11406 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2046 glc
11407 ; GFX6-NEXT: s_waitcnt vmcnt(0)
11408 ; GFX6-NEXT: buffer_wbinvl1
11409 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
11410 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
11411 ; GFX6-NEXT: v_mov_b32_e32 v3, v5
11412 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
11413 ; GFX6-NEXT: s_cbranch_execnz .LBB39_1
11414 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
11415 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
11416 ; GFX6-NEXT: s_waitcnt expcnt(0)
11417 ; GFX6-NEXT: s_setpc_b64 s[30:31]
11418 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
11419 %unused = atomicrmw fsub ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, align 4
11420 ret void
11421 }
11423 define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
11424 ; GFX12-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
11425 ; GFX12: ; %bb.0:
11426 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
11427 ; GFX12-NEXT: s_wait_expcnt 0x0
11428 ; GFX12-NEXT: s_wait_samplecnt 0x0
11429 ; GFX12-NEXT: s_wait_bvhcnt 0x0
11430 ; GFX12-NEXT: s_wait_kmcnt 0x0
11431 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
11432 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
11433 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11434 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
11435 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
11436 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
11437 ; GFX12-NEXT: s_mov_b32 s0, 0
11438 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
11439 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
11440 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
11441 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
11442 ; GFX12-NEXT: v_not_b32_e32 v4, v4
11443 ; GFX12-NEXT: .LBB40_1: ; %atomicrmw.start
11444 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
11445 ; GFX12-NEXT: s_wait_loadcnt 0x0
11446 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
11447 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11448 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
11449 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
11450 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11451 ; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
11452 ; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
11453 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
11454 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
11455 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
11456 ; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
11457 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
11458 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11459 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
11460 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
11461 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
11462 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
11463 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
11464 ; GFX12-NEXT: s_wait_storecnt 0x0
11465 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
11466 ; GFX12-NEXT: s_wait_loadcnt 0x0
11467 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
11468 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
11469 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
11470 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
11471 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
11472 ; GFX12-NEXT: s_cbranch_execnz .LBB40_1
11473 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
11474 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
11475 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
11476 ; GFX12-NEXT: s_setpc_b64 s[30:31]
11477 ;
11478 ; GFX940-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
11479 ; GFX940: ; %bb.0:
11480 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11481 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
11482 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
11483 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
11484 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
11485 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
11486 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v4
11487 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
11488 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
11489 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
11490 ; GFX940-NEXT: v_not_b32_e32 v4, v4
11491 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
11492 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11493 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
11494 ; GFX940-NEXT: .LBB40_1: ; %atomicrmw.start
11495 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
11496 ; GFX940-NEXT: s_waitcnt vmcnt(0)
11497 ; GFX940-NEXT: v_mov_b32_e32 v7, v5
11498 ; GFX940-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
11499 ; GFX940-NEXT: s_nop 0
11500 ; GFX940-NEXT: v_sub_f32_e32 v5, v5, v2
11501 ; GFX940-NEXT: v_bfe_u32 v6, v5, 16, 1
11502 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v5
11503 ; GFX940-NEXT: v_add3_u32 v6, v6, v5, s2
11504 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
11505 ; GFX940-NEXT: s_nop 1
11506 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
11507 ; GFX940-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11508 ; GFX940-NEXT: v_and_or_b32 v6, v7, v4, v5
11509 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
11510 ; GFX940-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off sc0 sc1
11511 ; GFX940-NEXT: s_waitcnt vmcnt(0)
11512 ; GFX940-NEXT: buffer_inv sc0 sc1
11513 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
11514 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
11515 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
11516 ; GFX940-NEXT: s_cbranch_execnz .LBB40_1
11517 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
11518 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
11519 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
11520 ; GFX940-NEXT: s_setpc_b64 s[30:31]
11521 ;
11522 ; GFX11-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
11523 ; GFX11: ; %bb.0:
11524 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11525 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
11526 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
11527 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11528 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
11529 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
11530 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
11531 ; GFX11-NEXT: s_mov_b32 s0, 0
11532 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
11533 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
11534 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
11535 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
11536 ; GFX11-NEXT: v_not_b32_e32 v4, v4
11537 ; GFX11-NEXT: .p2align 6
11538 ; GFX11-NEXT: .LBB40_1: ; %atomicrmw.start
11539 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
11540 ; GFX11-NEXT: s_waitcnt vmcnt(0)
11541 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
11542 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11543 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
11544 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
11545 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11546 ; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
11547 ; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
11548 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
11549 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
11550 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
11551 ; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
11552 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
11553 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11554 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
11555 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
11556 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
11557 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
11558 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
11559 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
11560 ; GFX11-NEXT: s_waitcnt vmcnt(0)
11561 ; GFX11-NEXT: buffer_gl1_inv
11562 ; GFX11-NEXT: buffer_gl0_inv
11563 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
11564 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
11565 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
11566 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
11567 ; GFX11-NEXT: s_cbranch_execnz .LBB40_1
11568 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
11569 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
11570 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
11571 ; GFX11-NEXT: s_setpc_b64 s[30:31]
11572 ;
11573 ; GFX10-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
11574 ; GFX10: ; %bb.0:
11575 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11576 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
11577 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
11578 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11579 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
11580 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
11581 ; GFX10-NEXT: s_mov_b32 s4, 0
11582 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
11583 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
11584 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
11585 ; GFX10-NEXT: v_not_b32_e32 v4, v4
11586 ; GFX10-NEXT: .LBB40_1: ; %atomicrmw.start
11587 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
11588 ; GFX10-NEXT: s_waitcnt vmcnt(0)
11589 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
11590 ; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
11591 ; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
11592 ; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
11593 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v5
11594 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
11595 ; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
11596 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
11597 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11598 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
11599 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
11600 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
11601 ; GFX10-NEXT: s_waitcnt vmcnt(0)
11602 ; GFX10-NEXT: buffer_gl1_inv
11603 ; GFX10-NEXT: buffer_gl0_inv
11604 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
11605 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
11606 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
11607 ; GFX10-NEXT: s_cbranch_execnz .LBB40_1
11608 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
11609 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
11610 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
11611 ; GFX10-NEXT: s_setpc_b64 s[30:31]
11612 ;
11613 ; GFX90A-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
11614 ; GFX90A: ; %bb.0:
11615 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11616 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
11617 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
11618 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
11619 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
11620 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
11621 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
11622 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
11623 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
11624 ; GFX90A-NEXT: v_not_b32_e32 v4, v4
11625 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
11626 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11627 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
11628 ; GFX90A-NEXT: .LBB40_1: ; %atomicrmw.start
11629 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
11630 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
11631 ; GFX90A-NEXT: v_mov_b32_e32 v7, v5
11632 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
11633 ; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v2
11634 ; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
11635 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v5
11636 ; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
11637 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
11638 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
11639 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11640 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v4, v5
11641 ; GFX90A-NEXT: buffer_wbl2
11642 ; GFX90A-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off glc
11643 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
11644 ; GFX90A-NEXT: buffer_invl2
11645 ; GFX90A-NEXT: buffer_wbinvl1
11646 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
11647 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11648 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
11649 ; GFX90A-NEXT: s_cbranch_execnz .LBB40_1
11650 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
11651 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
11652 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
11653 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
11654 ;
11655 ; GFX908-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
11656 ; GFX908: ; %bb.0:
11657 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11658 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
11659 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
11660 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
11661 ; GFX908-NEXT: global_load_dword v5, v[0:1], off
11662 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
11663 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
11664 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
11665 ; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
11666 ; GFX908-NEXT: v_not_b32_e32 v4, v4
11667 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
11668 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11669 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
11670 ; GFX908-NEXT: .LBB40_1: ; %atomicrmw.start
11671 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
11672 ; GFX908-NEXT: s_waitcnt vmcnt(0)
11673 ; GFX908-NEXT: v_mov_b32_e32 v6, v5
11674 ; GFX908-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
11675 ; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
11676 ; GFX908-NEXT: v_bfe_u32 v7, v5, 16, 1
11677 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v5
11678 ; GFX908-NEXT: v_add3_u32 v7, v7, v5, s6
11679 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
11680 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc
11681 ; GFX908-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11682 ; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
11683 ; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
11684 ; GFX908-NEXT: s_waitcnt vmcnt(0)
11685 ; GFX908-NEXT: buffer_wbinvl1
11686 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
11687 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11688 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
11689 ; GFX908-NEXT: s_cbranch_execnz .LBB40_1
11690 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
11691 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
11692 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
11693 ; GFX908-NEXT: s_setpc_b64 s[30:31]
11694 ;
11695 ; GFX8-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
11696 ; GFX8: ; %bb.0:
11697 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11698 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
11699 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
11700 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
11701 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
11702 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
11703 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
11704 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
11705 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
11706 ; GFX8-NEXT: v_not_b32_e32 v4, v4
11707 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
11708 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11709 ; GFX8-NEXT: .LBB40_1: ; %atomicrmw.start
11710 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
11711 ; GFX8-NEXT: s_waitcnt vmcnt(0)
11712 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
11713 ; GFX8-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
11714 ; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
11715 ; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
11716 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
11717 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
11718 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
11719 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
11720 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
11721 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
11722 ; GFX8-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11723 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
11724 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
11725 ; GFX8-NEXT: s_waitcnt vmcnt(0)
11726 ; GFX8-NEXT: buffer_wbinvl1
11727 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
11728 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11729 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
11730 ; GFX8-NEXT: s_cbranch_execnz .LBB40_1
11731 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
11732 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
11733 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
11734 ; GFX8-NEXT: s_setpc_b64 s[30:31]
11735 ;
11736 ; GFX7-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
11737 ; GFX7: ; %bb.0:
11738 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11739 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
11740 ; GFX7-NEXT: s_mov_b32 s6, 0
11741 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
11742 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
11743 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
11744 ; GFX7-NEXT: s_mov_b32 s4, s6
11745 ; GFX7-NEXT: s_mov_b32 s5, s6
11746 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
11747 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
11748 ; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
11749 ; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
11750 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
11751 ; GFX7-NEXT: v_not_b32_e32 v7, v4
11752 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
11753 ; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
11754 ; GFX7-NEXT: .LBB40_1: ; %atomicrmw.start
11755 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
11756 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11757 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
11758 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11759 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v8
11760 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11761 ; GFX7-NEXT: v_and_b32_e32 v4, v3, v7
11762 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v6, v2
11763 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
11764 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
11765 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
11766 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
11767 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11768 ; GFX7-NEXT: buffer_wbinvl1
11769 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
11770 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
11771 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
11772 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
11773 ; GFX7-NEXT: s_cbranch_execnz .LBB40_1
11774 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
11775 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
11776 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
11777 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
11778 ; GFX7-NEXT: s_setpc_b64 s[30:31]
11779 ;
11780 ; GFX6-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
11781 ; GFX6: ; %bb.0:
11782 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11783 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
11784 ; GFX6-NEXT: s_mov_b32 s6, 0
11785 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
11786 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
11787 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
11788 ; GFX6-NEXT: s_mov_b32 s4, s6
11789 ; GFX6-NEXT: s_mov_b32 s5, s6
11790 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
11791 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
11792 ; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
11793 ; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
11794 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
11795 ; GFX6-NEXT: v_not_b32_e32 v7, v4
11796 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
11797 ; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
11798 ; GFX6-NEXT: .LBB40_1: ; %atomicrmw.start
11799 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
11800 ; GFX6-NEXT: s_waitcnt vmcnt(0)
11801 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
11802 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11803 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v8
11804 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11805 ; GFX6-NEXT: s_waitcnt expcnt(0)
11806 ; GFX6-NEXT: v_and_b32_e32 v4, v3, v7
11807 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v6, v2
11808 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
11809 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
11810 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
11811 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
11812 ; GFX6-NEXT: s_waitcnt vmcnt(0)
11813 ; GFX6-NEXT: buffer_wbinvl1
11814 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
11815 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
11816 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
11817 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
11818 ; GFX6-NEXT: s_cbranch_execnz .LBB40_1
11819 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
11820 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
11821 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
11822 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
11823 ; GFX6-NEXT: s_waitcnt expcnt(0)
11824 ; GFX6-NEXT: s_setpc_b64 s[30:31]
11825 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
11826 %result = atomicrmw fsub ptr addrspace(1) %gep, bfloat %val seq_cst
11827 ret bfloat %result
11828 }
11830 define void @global_system_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
11831 ; GFX12-LABEL: global_system_atomic_fsub_noret_bf16__offset12b_pos:
11832 ; GFX12: ; %bb.0:
11833 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
11834 ; GFX12-NEXT: s_wait_expcnt 0x0
11835 ; GFX12-NEXT: s_wait_samplecnt 0x0
11836 ; GFX12-NEXT: s_wait_bvhcnt 0x0
11837 ; GFX12-NEXT: s_wait_kmcnt 0x0
11838 ; GFX12-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
11839 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
11840 ; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v2
11841 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
11842 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v4
11843 ; GFX12-NEXT: v_and_b32_e32 v4, 3, v4
11844 ; GFX12-NEXT: s_mov_b32 s0, 0
11845 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
11846 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 3, v4
11847 ; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
11848 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
11849 ; GFX12-NEXT: v_not_b32_e32 v5, v5
11850 ; GFX12-NEXT: .LBB41_1: ; %atomicrmw.start
11851 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
11852 ; GFX12-NEXT: s_wait_loadcnt 0x0
11853 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
11854 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11855 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11856 ; GFX12-NEXT: v_sub_f32_e32 v2, v2, v6
11857 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
11858 ; GFX12-NEXT: v_bfe_u32 v7, v2, 16, 1
11859 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v2
11860 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
11861 ; GFX12-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
11862 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11863 ; GFX12-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
11864 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11865 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11866 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, v4, v2
11867 ; GFX12-NEXT: v_and_or_b32 v2, v3, v5, v2
11868 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
11869 ; GFX12-NEXT: s_wait_storecnt 0x0
11870 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
11871 ; GFX12-NEXT: s_wait_loadcnt 0x0
11872 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
11873 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
11874 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
11875 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
11876 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
11877 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
11878 ; GFX12-NEXT: s_cbranch_execnz .LBB41_1
11879 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
11880 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
11881 ; GFX12-NEXT: s_setpc_b64 s[30:31]
11882 ;
11883 ; GFX940-LABEL: global_system_atomic_fsub_noret_bf16__offset12b_pos:
11884 ; GFX940: ; %bb.0:
11885 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11886 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
11887 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
11888 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
11889 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
11890 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
11891 ; GFX940-NEXT: v_and_b32_e32 v4, 3, v4
11892 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 3, v4
11893 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
11894 ; GFX940-NEXT: v_lshlrev_b32_e64 v5, v4, s0
11895 ; GFX940-NEXT: v_not_b32_e32 v5, v5
11896 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
11897 ; GFX940-NEXT: v_lshlrev_b32_e32 v6, 16, v2
11898 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
11899 ; GFX940-NEXT: .LBB41_1: ; %atomicrmw.start
11900 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
11901 ; GFX940-NEXT: s_waitcnt vmcnt(0)
11902 ; GFX940-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
11903 ; GFX940-NEXT: s_nop 0
11904 ; GFX940-NEXT: v_sub_f32_e32 v2, v2, v6
11905 ; GFX940-NEXT: v_bfe_u32 v7, v2, 16, 1
11906 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v2
11907 ; GFX940-NEXT: v_add3_u32 v7, v7, v2, s2
11908 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
11909 ; GFX940-NEXT: s_nop 1
11910 ; GFX940-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
11911 ; GFX940-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11912 ; GFX940-NEXT: v_and_or_b32 v2, v3, v5, v2
11913 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
11914 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0 sc1
11915 ; GFX940-NEXT: s_waitcnt vmcnt(0)
11916 ; GFX940-NEXT: buffer_inv sc0 sc1
11917 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
11918 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
11919 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
11920 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
11921 ; GFX940-NEXT: s_cbranch_execnz .LBB41_1
11922 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
11923 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
11924 ; GFX940-NEXT: s_setpc_b64 s[30:31]
11925 ;
11926 ; GFX11-LABEL: global_system_atomic_fsub_noret_bf16__offset12b_pos:
11927 ; GFX11: ; %bb.0:
11928 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11929 ; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
11930 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
11931 ; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v2
11932 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
11933 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v4
11934 ; GFX11-NEXT: v_and_b32_e32 v4, 3, v4
11935 ; GFX11-NEXT: s_mov_b32 s0, 0
11936 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
11937 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v4
11938 ; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
11939 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
11940 ; GFX11-NEXT: v_not_b32_e32 v5, v5
11941 ; GFX11-NEXT: .p2align 6
11942 ; GFX11-NEXT: .LBB41_1: ; %atomicrmw.start
11943 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
11944 ; GFX11-NEXT: s_waitcnt vmcnt(0)
11945 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
11946 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11947 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11948 ; GFX11-NEXT: v_sub_f32_e32 v2, v2, v6
11949 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
11950 ; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
11951 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v2
11952 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
11953 ; GFX11-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
11954 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11955 ; GFX11-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
11956 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11957 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11958 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, v4, v2
11959 ; GFX11-NEXT: v_and_or_b32 v2, v3, v5, v2
11960 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
11961 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
11962 ; GFX11-NEXT: s_waitcnt vmcnt(0)
11963 ; GFX11-NEXT: buffer_gl1_inv
11964 ; GFX11-NEXT: buffer_gl0_inv
11965 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
11966 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
11967 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
11968 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
11969 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
11970 ; GFX11-NEXT: s_cbranch_execnz .LBB41_1
11971 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
11972 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
11973 ; GFX11-NEXT: s_setpc_b64 s[30:31]
11974 ;
11975 ; GFX10-LABEL: global_system_atomic_fsub_noret_bf16__offset12b_pos:
11976 ; GFX10: ; %bb.0:
11977 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11978 ; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
11979 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
11980 ; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v2
11981 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
11982 ; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
11983 ; GFX10-NEXT: s_mov_b32 s4, 0
11984 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
11985 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
11986 ; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
11987 ; GFX10-NEXT: v_not_b32_e32 v5, v5
11988 ; GFX10-NEXT: .LBB41_1: ; %atomicrmw.start
11989 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
11990 ; GFX10-NEXT: s_waitcnt vmcnt(0)
11991 ; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
11992 ; GFX10-NEXT: v_sub_f32_e32 v2, v2, v6
11993 ; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
11994 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v2
11995 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
11996 ; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
11997 ; GFX10-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
11998 ; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11999 ; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
12000 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
12001 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
12002 ; GFX10-NEXT: s_waitcnt vmcnt(0)
12003 ; GFX10-NEXT: buffer_gl1_inv
12004 ; GFX10-NEXT: buffer_gl0_inv
12005 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
12006 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
12007 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
12008 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
12009 ; GFX10-NEXT: s_cbranch_execnz .LBB41_1
12010 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
12011 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
12012 ; GFX10-NEXT: s_setpc_b64 s[30:31]
12013 ;
12014 ; GFX90A-LABEL: global_system_atomic_fsub_noret_bf16__offset12b_pos:
12015 ; GFX90A: ; %bb.0:
12016 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12017 ; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
12018 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
12019 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
12020 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
12021 ; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
12022 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
12023 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
12024 ; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
12025 ; GFX90A-NEXT: v_not_b32_e32 v5, v5
12026 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
12027 ; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v2
12028 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
12029 ; GFX90A-NEXT: .LBB41_1: ; %atomicrmw.start
12030 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
12031 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12032 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
12033 ; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v6
12034 ; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
12035 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
12036 ; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s6
12037 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
12038 ; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
12039 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
12040 ; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
12041 ; GFX90A-NEXT: buffer_wbl2
12042 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
12043 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12044 ; GFX90A-NEXT: buffer_invl2
12045 ; GFX90A-NEXT: buffer_wbinvl1
12046 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
12047 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12048 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
12049 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
12050 ; GFX90A-NEXT: s_cbranch_execnz .LBB41_1
12051 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
12052 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
12053 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
12054 ;
12055 ; GFX908-LABEL: global_system_atomic_fsub_noret_bf16__offset12b_pos:
12056 ; GFX908: ; %bb.0:
12057 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12058 ; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
12059 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
12060 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
12061 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
12062 ; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
12063 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
12064 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
12065 ; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
12066 ; GFX908-NEXT: v_not_b32_e32 v5, v5
12067 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
12068 ; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v2
12069 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
12070 ; GFX908-NEXT: .LBB41_1: ; %atomicrmw.start
12071 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
12072 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12073 ; GFX908-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
12074 ; GFX908-NEXT: v_sub_f32_e32 v2, v2, v6
12075 ; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
12076 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
12077 ; GFX908-NEXT: v_add3_u32 v7, v7, v2, s6
12078 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
12079 ; GFX908-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
12080 ; GFX908-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
12081 ; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
12082 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
12083 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12084 ; GFX908-NEXT: buffer_wbinvl1
12085 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
12086 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12087 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
12088 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
12089 ; GFX908-NEXT: s_cbranch_execnz .LBB41_1
12090 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
12091 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
12092 ; GFX908-NEXT: s_setpc_b64 s[30:31]
12093 ;
12094 ; GFX8-LABEL: global_system_atomic_fsub_noret_bf16__offset12b_pos:
12095 ; GFX8: ; %bb.0:
12096 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12097 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
12098 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
12099 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
12100 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
12101 ; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
12102 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
12103 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
12104 ; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
12105 ; GFX8-NEXT: v_not_b32_e32 v5, v5
12106 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
12107 ; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v2
12108 ; GFX8-NEXT: .LBB41_1: ; %atomicrmw.start
12109 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
12110 ; GFX8-NEXT: s_waitcnt vmcnt(0)
12111 ; GFX8-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
12112 ; GFX8-NEXT: v_sub_f32_e32 v2, v2, v6
12113 ; GFX8-NEXT: v_bfe_u32 v8, v2, 16, 1
12114 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v2
12115 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
12116 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v2
12117 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
12118 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v8, v9, vcc
12119 ; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
12120 ; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
12121 ; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
12122 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12123 ; GFX8-NEXT: s_waitcnt vmcnt(0)
12124 ; GFX8-NEXT: buffer_wbinvl1
12125 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
12126 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12127 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
12128 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
12129 ; GFX8-NEXT: s_cbranch_execnz .LBB41_1
12130 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
12131 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
12132 ; GFX8-NEXT: s_setpc_b64 s[30:31]
12133 ;
12134 ; GFX7-LABEL: global_system_atomic_fsub_noret_bf16__offset12b_pos:
12135 ; GFX7: ; %bb.0:
12136 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12137 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
12138 ; GFX7-NEXT: s_mov_b32 s6, 0
12139 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
12140 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
12141 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
12142 ; GFX7-NEXT: s_mov_b32 s4, s6
12143 ; GFX7-NEXT: s_mov_b32 s5, s6
12144 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
12145 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
12146 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
12147 ; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
12148 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
12149 ; GFX7-NEXT: v_not_b32_e32 v5, v5
12150 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
12151 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
12152 ; GFX7-NEXT: .LBB41_1: ; %atomicrmw.start
12153 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
12154 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12155 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
12156 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
12157 ; GFX7-NEXT: v_sub_f32_e32 v2, v2, v6
12158 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
12159 ; GFX7-NEXT: v_and_b32_e32 v7, v3, v5
12160 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v4, v2
12161 ; GFX7-NEXT: v_or_b32_e32 v2, v7, v2
12162 ; GFX7-NEXT: v_mov_b32_e32 v8, v3
12163 ; GFX7-NEXT: v_mov_b32_e32 v7, v2
12164 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
12165 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12166 ; GFX7-NEXT: buffer_wbinvl1
12167 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
12168 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
12169 ; GFX7-NEXT: v_mov_b32_e32 v3, v7
12170 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
12171 ; GFX7-NEXT: s_cbranch_execnz .LBB41_1
12172 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
12173 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
12174 ; GFX7-NEXT: s_setpc_b64 s[30:31]
12175 ;
12176 ; GFX6-LABEL: global_system_atomic_fsub_noret_bf16__offset12b_pos:
12177 ; GFX6: ; %bb.0:
12178 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12179 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
12180 ; GFX6-NEXT: s_mov_b32 s6, 0
12181 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
12182 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
12183 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
12184 ; GFX6-NEXT: s_mov_b32 s4, s6
12185 ; GFX6-NEXT: s_mov_b32 s5, s6
12186 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
12187 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
12188 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
12189 ; GFX6-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
12190 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
12191 ; GFX6-NEXT: v_not_b32_e32 v5, v5
12192 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
12193 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
12194 ; GFX6-NEXT: .LBB41_1: ; %atomicrmw.start
12195 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
12196 ; GFX6-NEXT: s_waitcnt vmcnt(0)
12197 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
12198 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
12199 ; GFX6-NEXT: v_sub_f32_e32 v2, v2, v6
12200 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
12201 ; GFX6-NEXT: s_waitcnt expcnt(0)
12202 ; GFX6-NEXT: v_and_b32_e32 v7, v3, v5
12203 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v4, v2
12204 ; GFX6-NEXT: v_or_b32_e32 v2, v7, v2
12205 ; GFX6-NEXT: v_mov_b32_e32 v8, v3
12206 ; GFX6-NEXT: v_mov_b32_e32 v7, v2
12207 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
12208 ; GFX6-NEXT: s_waitcnt vmcnt(0)
12209 ; GFX6-NEXT: buffer_wbinvl1
12210 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
12211 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
12212 ; GFX6-NEXT: v_mov_b32_e32 v3, v7
12213 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
12214 ; GFX6-NEXT: s_cbranch_execnz .LBB41_1
12215 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
12216 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
12217 ; GFX6-NEXT: s_waitcnt expcnt(0)
12218 ; GFX6-NEXT: s_setpc_b64 s[30:31]
12219 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
12220 %unused = atomicrmw fsub ptr addrspace(1) %gep, bfloat %val seq_cst
12221 ret void
12222 }
12224 ; --------------------------------------------------------------------
12225 ; <2 x half>
12226 ; --------------------------------------------------------------------
12228 define <2 x half> @global_agent_atomic_fsub_ret_v2f16(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
12229 ; GFX12-LABEL: global_agent_atomic_fsub_ret_v2f16:
12230 ; GFX12: ; %bb.0:
12231 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
12232 ; GFX12-NEXT: s_wait_expcnt 0x0
12233 ; GFX12-NEXT: s_wait_samplecnt 0x0
12234 ; GFX12-NEXT: s_wait_bvhcnt 0x0
12235 ; GFX12-NEXT: s_wait_kmcnt 0x0
12236 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
12237 ; GFX12-NEXT: s_mov_b32 s0, 0
12238 ; GFX12-NEXT: .LBB42_1: ; %atomicrmw.start
12239 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
12240 ; GFX12-NEXT: s_wait_loadcnt 0x0
12241 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
12242 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
12243 ; GFX12-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12244 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
12245 ; GFX12-NEXT: s_wait_storecnt 0x0
12246 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
12247 ; GFX12-NEXT: s_wait_loadcnt 0x0
12248 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
12249 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
12250 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
12251 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
12252 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
12253 ; GFX12-NEXT: s_cbranch_execnz .LBB42_1
12254 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
12255 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
12256 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
12257 ; GFX12-NEXT: s_setpc_b64 s[30:31]
12258 ;
12259 ; GFX940-LABEL: global_agent_atomic_fsub_ret_v2f16:
12260 ; GFX940: ; %bb.0:
12261 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12262 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
12263 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
12264 ; GFX940-NEXT: .LBB42_1: ; %atomicrmw.start
12265 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
12266 ; GFX940-NEXT: s_waitcnt vmcnt(0)
12267 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
12268 ; GFX940-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
12269 ; GFX940-NEXT: buffer_wbl2 sc1
12270 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
12271 ; GFX940-NEXT: s_waitcnt vmcnt(0)
12272 ; GFX940-NEXT: buffer_inv sc1
12273 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
12274 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
12275 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
12276 ; GFX940-NEXT: s_cbranch_execnz .LBB42_1
12277 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
12278 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
12279 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
12280 ; GFX940-NEXT: s_setpc_b64 s[30:31]
12281 ;
12282 ; GFX11-LABEL: global_agent_atomic_fsub_ret_v2f16:
12283 ; GFX11: ; %bb.0:
12284 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12285 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
12286 ; GFX11-NEXT: s_mov_b32 s0, 0
12287 ; GFX11-NEXT: .LBB42_1: ; %atomicrmw.start
12288 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
12289 ; GFX11-NEXT: s_waitcnt vmcnt(0)
12290 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
12291 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
12292 ; GFX11-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12293 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
12294 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
12295 ; GFX11-NEXT: s_waitcnt vmcnt(0)
12296 ; GFX11-NEXT: buffer_gl1_inv
12297 ; GFX11-NEXT: buffer_gl0_inv
12298 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
12299 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
12300 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
12301 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
12302 ; GFX11-NEXT: s_cbranch_execnz .LBB42_1
12303 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
12304 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
12305 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
12306 ; GFX11-NEXT: s_setpc_b64 s[30:31]
12307 ;
12308 ; GFX10-LABEL: global_agent_atomic_fsub_ret_v2f16:
12309 ; GFX10: ; %bb.0:
12310 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12311 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
12312 ; GFX10-NEXT: s_mov_b32 s4, 0
12313 ; GFX10-NEXT: .LBB42_1: ; %atomicrmw.start
12314 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
12315 ; GFX10-NEXT: s_waitcnt vmcnt(0)
12316 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
12317 ; GFX10-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12318 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
12319 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
12320 ; GFX10-NEXT: s_waitcnt vmcnt(0)
12321 ; GFX10-NEXT: buffer_gl1_inv
12322 ; GFX10-NEXT: buffer_gl0_inv
12323 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
12324 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
12325 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
12326 ; GFX10-NEXT: s_cbranch_execnz .LBB42_1
12327 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
12328 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
12329 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
12330 ; GFX10-NEXT: s_setpc_b64 s[30:31]
12331 ;
12332 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_v2f16:
12333 ; GFX90A: ; %bb.0:
12334 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12335 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
12336 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
12337 ; GFX90A-NEXT: .LBB42_1: ; %atomicrmw.start
12338 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
12339 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12340 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
12341 ; GFX90A-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
12342 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
12343 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12344 ; GFX90A-NEXT: buffer_wbinvl1
12345 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
12346 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12347 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
12348 ; GFX90A-NEXT: s_cbranch_execnz .LBB42_1
12349 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
12350 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
12351 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
12352 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
12353 ;
12354 ; GFX908-LABEL: global_agent_atomic_fsub_ret_v2f16:
12355 ; GFX908: ; %bb.0:
12356 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12357 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
12358 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
12359 ; GFX908-NEXT: .LBB42_1: ; %atomicrmw.start
12360 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
12361 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12362 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
12363 ; GFX908-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12364 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
12365 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12366 ; GFX908-NEXT: buffer_wbinvl1
12367 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
12368 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12369 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
12370 ; GFX908-NEXT: s_cbranch_execnz .LBB42_1
12371 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
12372 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
12373 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
12374 ; GFX908-NEXT: s_setpc_b64 s[30:31]
12375 ;
12376 ; GFX8-LABEL: global_agent_atomic_fsub_ret_v2f16:
12377 ; GFX8: ; %bb.0:
12378 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12379 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
12380 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
12381 ; GFX8-NEXT: .LBB42_1: ; %atomicrmw.start
12382 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
12383 ; GFX8-NEXT: s_waitcnt vmcnt(0)
12384 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
12385 ; GFX8-NEXT: v_sub_f16_sdwa v3, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
12386 ; GFX8-NEXT: v_sub_f16_e32 v5, v4, v2
12387 ; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
12388 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
12389 ; GFX8-NEXT: s_waitcnt vmcnt(0)
12390 ; GFX8-NEXT: buffer_wbinvl1
12391 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
12392 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12393 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
12394 ; GFX8-NEXT: s_cbranch_execnz .LBB42_1
12395 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
12396 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
12397 ; GFX8-NEXT: v_mov_b32_e32 v0, v3
12398 ; GFX8-NEXT: s_setpc_b64 s[30:31]
12399 ;
12400 ; GFX7-LABEL: global_agent_atomic_fsub_ret_v2f16:
12401 ; GFX7: ; %bb.0:
12402 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12403 ; GFX7-NEXT: s_mov_b32 s6, 0
12404 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
12405 ; GFX7-NEXT: s_mov_b32 s4, s6
12406 ; GFX7-NEXT: s_mov_b32 s5, s6
12407 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
12408 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
12409 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v2
12410 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
12411 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v3
12412 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12413 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
12414 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v5
12415 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
12416 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v6
12417 ; GFX7-NEXT: .LBB42_1: ; %atomicrmw.start
12418 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
12419 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
12420 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
12421 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v3
12422 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
12423 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
12424 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v4
12425 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v5
12426 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6
12427 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v7
12428 ; GFX7-NEXT: v_or_b32_e32 v7, v2, v3
12429 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
12430 ; GFX7-NEXT: v_or_b32_e32 v6, v8, v2
12431 ; GFX7-NEXT: v_mov_b32_e32 v9, v7
12432 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
12433 ; GFX7-NEXT: buffer_atomic_cmpswap v[8:9], v[0:1], s[4:7], 0 addr64 glc
12434 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12435 ; GFX7-NEXT: buffer_wbinvl1
12436 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
12437 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v8
12438 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
12439 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
12440 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
12441 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
12442 ; GFX7-NEXT: s_cbranch_execnz .LBB42_1
12443 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
12444 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
12445 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
12446 ; GFX7-NEXT: v_mov_b32_e32 v1, v3
12447 ; GFX7-NEXT: s_setpc_b64 s[30:31]
12448 ;
12449 ; GFX6-LABEL: global_agent_atomic_fsub_ret_v2f16:
12450 ; GFX6: ; %bb.0:
12451 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12452 ; GFX6-NEXT: s_mov_b32 s6, 0
12453 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
12454 ; GFX6-NEXT: s_mov_b32 s4, s6
12455 ; GFX6-NEXT: s_mov_b32 s5, s6
12456 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
12457 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
12458 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v2
12459 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
12460 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v3
12461 ; GFX6-NEXT: s_waitcnt vmcnt(0)
12462 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
12463 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v5
12464 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
12465 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v6
12466 ; GFX6-NEXT: .LBB42_1: ; %atomicrmw.start
12467 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
12468 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
12469 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
12470 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v3
12471 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
12472 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
12473 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v4
12474 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v5
12475 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
12476 ; GFX6-NEXT: s_waitcnt expcnt(0)
12477 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v7
12478 ; GFX6-NEXT: v_or_b32_e32 v7, v2, v3
12479 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v6
12480 ; GFX6-NEXT: v_or_b32_e32 v6, v8, v2
12481 ; GFX6-NEXT: v_mov_b32_e32 v9, v7
12482 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
12483 ; GFX6-NEXT: buffer_atomic_cmpswap v[8:9], v[0:1], s[4:7], 0 addr64 glc
12484 ; GFX6-NEXT: s_waitcnt vmcnt(0)
12485 ; GFX6-NEXT: buffer_wbinvl1
12486 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
12487 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v8
12488 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
12489 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
12490 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
12491 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
12492 ; GFX6-NEXT: s_cbranch_execnz .LBB42_1
12493 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
12494 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
12495 ; GFX6-NEXT: v_mov_b32_e32 v0, v2
12496 ; GFX6-NEXT: v_mov_b32_e32 v1, v3
12497 ; GFX6-NEXT: s_waitcnt expcnt(0)
12498 ; GFX6-NEXT: s_setpc_b64 s[30:31]
12499 %result = atomicrmw fsub ptr addrspace(1) %ptr, <2 x half> %val syncscope("agent") seq_cst
12500 ret <2 x half> %result
12501 }
12503 define <2 x half> @global_agent_atomic_fsub_ret_v2f16__offset12b_pos(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
12504 ; GFX12-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_pos:
12505 ; GFX12: ; %bb.0:
12506 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
12507 ; GFX12-NEXT: s_wait_expcnt 0x0
12508 ; GFX12-NEXT: s_wait_samplecnt 0x0
12509 ; GFX12-NEXT: s_wait_bvhcnt 0x0
12510 ; GFX12-NEXT: s_wait_kmcnt 0x0
12511 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
12512 ; GFX12-NEXT: s_mov_b32 s0, 0
12513 ; GFX12-NEXT: .LBB43_1: ; %atomicrmw.start
12514 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
12515 ; GFX12-NEXT: s_wait_loadcnt 0x0
12516 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
12517 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
12518 ; GFX12-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12519 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
12520 ; GFX12-NEXT: s_wait_storecnt 0x0
12521 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
12522 ; GFX12-NEXT: s_wait_loadcnt 0x0
12523 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
12524 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
12525 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
12526 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
12527 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
12528 ; GFX12-NEXT: s_cbranch_execnz .LBB43_1
12529 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
12530 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
12531 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
12532 ; GFX12-NEXT: s_setpc_b64 s[30:31]
12533 ;
12534 ; GFX940-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_pos:
12535 ; GFX940: ; %bb.0:
12536 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12537 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
12538 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
12539 ; GFX940-NEXT: .LBB43_1: ; %atomicrmw.start
12540 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
12541 ; GFX940-NEXT: s_waitcnt vmcnt(0)
12542 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
12543 ; GFX940-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
12544 ; GFX940-NEXT: buffer_wbl2 sc1
12545 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
12546 ; GFX940-NEXT: s_waitcnt vmcnt(0)
12547 ; GFX940-NEXT: buffer_inv sc1
12548 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
12549 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
12550 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
12551 ; GFX940-NEXT: s_cbranch_execnz .LBB43_1
12552 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
12553 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
12554 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
12555 ; GFX940-NEXT: s_setpc_b64 s[30:31]
12556 ;
12557 ; GFX11-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_pos:
12558 ; GFX11: ; %bb.0:
12559 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12560 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
12561 ; GFX11-NEXT: s_mov_b32 s0, 0
12562 ; GFX11-NEXT: .LBB43_1: ; %atomicrmw.start
12563 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
12564 ; GFX11-NEXT: s_waitcnt vmcnt(0)
12565 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
12566 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
12567 ; GFX11-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12568 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
12569 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
12570 ; GFX11-NEXT: s_waitcnt vmcnt(0)
12571 ; GFX11-NEXT: buffer_gl1_inv
12572 ; GFX11-NEXT: buffer_gl0_inv
12573 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
12574 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
12575 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
12576 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
12577 ; GFX11-NEXT: s_cbranch_execnz .LBB43_1
12578 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
12579 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
12580 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
12581 ; GFX11-NEXT: s_setpc_b64 s[30:31]
12582 ;
12583 ; GFX10-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_pos:
12584 ; GFX10: ; %bb.0:
12585 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12586 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
12587 ; GFX10-NEXT: s_mov_b32 s4, 0
12588 ; GFX10-NEXT: .LBB43_1: ; %atomicrmw.start
12589 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
12590 ; GFX10-NEXT: s_waitcnt vmcnt(0)
12591 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
12592 ; GFX10-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12593 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
12594 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
12595 ; GFX10-NEXT: s_waitcnt vmcnt(0)
12596 ; GFX10-NEXT: buffer_gl1_inv
12597 ; GFX10-NEXT: buffer_gl0_inv
12598 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
12599 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
12600 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
12601 ; GFX10-NEXT: s_cbranch_execnz .LBB43_1
12602 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
12603 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
12604 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
12605 ; GFX10-NEXT: s_setpc_b64 s[30:31]
12606 ;
12607 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_pos:
12608 ; GFX90A: ; %bb.0:
12609 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12610 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
12611 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
12612 ; GFX90A-NEXT: .LBB43_1: ; %atomicrmw.start
12613 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
12614 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12615 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
12616 ; GFX90A-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
12617 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
12618 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12619 ; GFX90A-NEXT: buffer_wbinvl1
12620 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
12621 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12622 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
12623 ; GFX90A-NEXT: s_cbranch_execnz .LBB43_1
12624 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
12625 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
12626 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
12627 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
12628 ;
12629 ; GFX908-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_pos:
12630 ; GFX908: ; %bb.0:
12631 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12632 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
12633 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
12634 ; GFX908-NEXT: .LBB43_1: ; %atomicrmw.start
12635 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
12636 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12637 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
12638 ; GFX908-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12639 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
12640 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12641 ; GFX908-NEXT: buffer_wbinvl1
12642 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
12643 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12644 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
12645 ; GFX908-NEXT: s_cbranch_execnz .LBB43_1
12646 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
12647 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
12648 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
12649 ; GFX908-NEXT: s_setpc_b64 s[30:31]
12650 ;
12651 ; GFX8-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_pos:
12652 ; GFX8: ; %bb.0:
12653 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12654 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
12655 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
12656 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
12657 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
12658 ; GFX8-NEXT: .LBB43_1: ; %atomicrmw.start
12659 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
12660 ; GFX8-NEXT: s_waitcnt vmcnt(0)
12661 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
12662 ; GFX8-NEXT: v_sub_f16_sdwa v0, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
12663 ; GFX8-NEXT: v_sub_f16_e32 v5, v1, v2
12664 ; GFX8-NEXT: v_or_b32_e32 v0, v5, v0
12665 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
12666 ; GFX8-NEXT: s_waitcnt vmcnt(0)
12667 ; GFX8-NEXT: buffer_wbinvl1
12668 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
12669 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12670 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
12671 ; GFX8-NEXT: s_cbranch_execnz .LBB43_1
12672 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
12673 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
12674 ; GFX8-NEXT: s_setpc_b64 s[30:31]
12675 ;
12676 ; GFX7-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_pos:
12677 ; GFX7: ; %bb.0:
12678 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12679 ; GFX7-NEXT: s_mov_b32 s6, 0
12680 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
12681 ; GFX7-NEXT: s_mov_b32 s4, s6
12682 ; GFX7-NEXT: s_mov_b32 s5, s6
12683 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
12684 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
12685 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v2
12686 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
12687 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v3
12688 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12689 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
12690 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v5
12691 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
12692 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v6
12693 ; GFX7-NEXT: .LBB43_1: ; %atomicrmw.start
12694 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
12695 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
12696 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
12697 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v3
12698 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
12699 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
12700 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v4
12701 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v5
12702 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6
12703 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v7
12704 ; GFX7-NEXT: v_or_b32_e32 v7, v2, v3
12705 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
12706 ; GFX7-NEXT: v_or_b32_e32 v6, v8, v2
12707 ; GFX7-NEXT: v_mov_b32_e32 v9, v7
12708 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
12709 ; GFX7-NEXT: buffer_atomic_cmpswap v[8:9], v[0:1], s[4:7], 0 addr64 offset:2044 glc
12710 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12711 ; GFX7-NEXT: buffer_wbinvl1
12712 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
12713 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v8
12714 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
12715 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
12716 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
12717 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
12718 ; GFX7-NEXT: s_cbranch_execnz .LBB43_1
12719 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
12720 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
12721 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
12722 ; GFX7-NEXT: v_mov_b32_e32 v1, v3
12723 ; GFX7-NEXT: s_setpc_b64 s[30:31]
12724 ;
12725 ; GFX6-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_pos:
12726 ; GFX6: ; %bb.0:
12727 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12728 ; GFX6-NEXT: s_mov_b32 s6, 0
12729 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
12730 ; GFX6-NEXT: s_mov_b32 s4, s6
12731 ; GFX6-NEXT: s_mov_b32 s5, s6
12732 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
12733 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
12734 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v2
12735 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
12736 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v3
12737 ; GFX6-NEXT: s_waitcnt vmcnt(0)
12738 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
12739 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v5
12740 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
12741 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v6
12742 ; GFX6-NEXT: .LBB43_1: ; %atomicrmw.start
12743 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
12744 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
12745 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
12746 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v3
12747 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
12748 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
12749 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v4
12750 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v5
12751 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
12752 ; GFX6-NEXT: s_waitcnt expcnt(0)
12753 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v7
12754 ; GFX6-NEXT: v_or_b32_e32 v7, v2, v3
12755 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v6
12756 ; GFX6-NEXT: v_or_b32_e32 v6, v8, v2
12757 ; GFX6-NEXT: v_mov_b32_e32 v9, v7
12758 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
12759 ; GFX6-NEXT: buffer_atomic_cmpswap v[8:9], v[0:1], s[4:7], 0 addr64 offset:2044 glc
12760 ; GFX6-NEXT: s_waitcnt vmcnt(0)
12761 ; GFX6-NEXT: buffer_wbinvl1
12762 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
12763 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v8
12764 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
12765 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
12766 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
12767 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
12768 ; GFX6-NEXT: s_cbranch_execnz .LBB43_1
12769 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
12770 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
12771 ; GFX6-NEXT: v_mov_b32_e32 v0, v2
12772 ; GFX6-NEXT: v_mov_b32_e32 v1, v3
12773 ; GFX6-NEXT: s_waitcnt expcnt(0)
12774 ; GFX6-NEXT: s_setpc_b64 s[30:31]
12775 %gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 511
12776 %result = atomicrmw fsub ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst
12777 ret <2 x half> %result
12778 }
12780 define <2 x half> @global_agent_atomic_fsub_ret_v2f16__offset12b_neg(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
12781 ; GFX12-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_neg:
12782 ; GFX12: ; %bb.0:
12783 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
12784 ; GFX12-NEXT: s_wait_expcnt 0x0
12785 ; GFX12-NEXT: s_wait_samplecnt 0x0
12786 ; GFX12-NEXT: s_wait_bvhcnt 0x0
12787 ; GFX12-NEXT: s_wait_kmcnt 0x0
12788 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
12789 ; GFX12-NEXT: s_mov_b32 s0, 0
12790 ; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
12791 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
12792 ; GFX12-NEXT: s_wait_loadcnt 0x0
12793 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
12794 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
12795 ; GFX12-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12796 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
12797 ; GFX12-NEXT: s_wait_storecnt 0x0
12798 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
12799 ; GFX12-NEXT: s_wait_loadcnt 0x0
12800 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
12801 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
12802 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
12803 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
12804 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
12805 ; GFX12-NEXT: s_cbranch_execnz .LBB44_1
12806 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
12807 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
12808 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
12809 ; GFX12-NEXT: s_setpc_b64 s[30:31]
12810 ;
12811 ; GFX940-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_neg:
12812 ; GFX940: ; %bb.0:
12813 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12814 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
12815 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
12816 ; GFX940-NEXT: .LBB44_1: ; %atomicrmw.start
12817 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
12818 ; GFX940-NEXT: s_waitcnt vmcnt(0)
12819 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
12820 ; GFX940-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
12821 ; GFX940-NEXT: buffer_wbl2 sc1
12822 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
12823 ; GFX940-NEXT: s_waitcnt vmcnt(0)
12824 ; GFX940-NEXT: buffer_inv sc1
12825 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
12826 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
12827 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
12828 ; GFX940-NEXT: s_cbranch_execnz .LBB44_1
12829 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
12830 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
12831 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
12832 ; GFX940-NEXT: s_setpc_b64 s[30:31]
12833 ;
12834 ; GFX11-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_neg:
12835 ; GFX11: ; %bb.0:
12836 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12837 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
12838 ; GFX11-NEXT: s_mov_b32 s0, 0
12839 ; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
12840 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
12841 ; GFX11-NEXT: s_waitcnt vmcnt(0)
12842 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
12843 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
12844 ; GFX11-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12845 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
12846 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
12847 ; GFX11-NEXT: s_waitcnt vmcnt(0)
12848 ; GFX11-NEXT: buffer_gl1_inv
12849 ; GFX11-NEXT: buffer_gl0_inv
12850 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
12851 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
12852 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
12853 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
12854 ; GFX11-NEXT: s_cbranch_execnz .LBB44_1
12855 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
12856 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
12857 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
12858 ; GFX11-NEXT: s_setpc_b64 s[30:31]
12859 ;
12860 ; GFX10-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_neg:
12861 ; GFX10: ; %bb.0:
12862 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12863 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
12864 ; GFX10-NEXT: s_mov_b32 s4, 0
12865 ; GFX10-NEXT: .LBB44_1: ; %atomicrmw.start
12866 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
12867 ; GFX10-NEXT: s_waitcnt vmcnt(0)
12868 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
12869 ; GFX10-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12870 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
12871 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
12872 ; GFX10-NEXT: s_waitcnt vmcnt(0)
12873 ; GFX10-NEXT: buffer_gl1_inv
12874 ; GFX10-NEXT: buffer_gl0_inv
12875 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
12876 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
12877 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
12878 ; GFX10-NEXT: s_cbranch_execnz .LBB44_1
12879 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
12880 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
12881 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
12882 ; GFX10-NEXT: s_setpc_b64 s[30:31]
12883 ;
12884 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_neg:
12885 ; GFX90A: ; %bb.0:
12886 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12887 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
12888 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
12889 ; GFX90A-NEXT: .LBB44_1: ; %atomicrmw.start
12890 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
12891 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12892 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
12893 ; GFX90A-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
12894 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
12895 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12896 ; GFX90A-NEXT: buffer_wbinvl1
12897 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
12898 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12899 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
12900 ; GFX90A-NEXT: s_cbranch_execnz .LBB44_1
12901 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
12902 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
12903 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
12904 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
12905 ;
12906 ; GFX908-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_neg:
12907 ; GFX908: ; %bb.0:
12908 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12909 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
12910 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
12911 ; GFX908-NEXT: .LBB44_1: ; %atomicrmw.start
12912 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
12913 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12914 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
12915 ; GFX908-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
12916 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
12917 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12918 ; GFX908-NEXT: buffer_wbinvl1
12919 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
12920 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12921 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
12922 ; GFX908-NEXT: s_cbranch_execnz .LBB44_1
12923 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
12924 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
12925 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
12926 ; GFX908-NEXT: s_setpc_b64 s[30:31]
12927 ;
12928 ; GFX8-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_neg:
12929 ; GFX8: ; %bb.0:
12930 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12931 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
12932 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
12933 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
12934 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
12935 ; GFX8-NEXT: .LBB44_1: ; %atomicrmw.start
12936 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
12937 ; GFX8-NEXT: s_waitcnt vmcnt(0)
12938 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
12939 ; GFX8-NEXT: v_sub_f16_sdwa v0, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
12940 ; GFX8-NEXT: v_sub_f16_e32 v5, v1, v2
12941 ; GFX8-NEXT: v_or_b32_e32 v0, v5, v0
12942 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
12943 ; GFX8-NEXT: s_waitcnt vmcnt(0)
12944 ; GFX8-NEXT: buffer_wbinvl1
12945 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
12946 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12947 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
12948 ; GFX8-NEXT: s_cbranch_execnz .LBB44_1
12949 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
12950 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
12951 ; GFX8-NEXT: s_setpc_b64 s[30:31]
12952 ;
12953 ; GFX7-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_neg:
12954 ; GFX7: ; %bb.0:
12955 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12956 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
12957 ; GFX7-NEXT: s_mov_b32 s5, -1
12958 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
12959 ; GFX7-NEXT: s_mov_b32 s6, 0
12960 ; GFX7-NEXT: buffer_load_dword v6, v[0:1], s[4:7], 0 addr64
12961 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
12962 ; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v3
12963 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v2
12964 ; GFX7-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
12965 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v0
12966 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
12967 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
12968 ; GFX7-NEXT: s_mov_b32 s4, s6
12969 ; GFX7-NEXT: s_mov_b32 s5, s6
12970 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12971 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v6
12972 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v6
12973 ; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
12974 ; GFX7-NEXT: .LBB44_1: ; %atomicrmw.start
12975 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
12976 ; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
12977 ; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
12978 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v1
12979 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v0
12980 ; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
12981 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
12982 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
12983 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6
12984 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v7
12985 ; GFX7-NEXT: v_or_b32_e32 v7, v0, v1
12986 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v6
12987 ; GFX7-NEXT: v_or_b32_e32 v6, v8, v0
12988 ; GFX7-NEXT: v_mov_b32_e32 v9, v7
12989 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
12990 ; GFX7-NEXT: buffer_atomic_cmpswap v[8:9], v[4:5], s[4:7], 0 addr64 glc
12991 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12992 ; GFX7-NEXT: buffer_wbinvl1
12993 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v8
12994 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v8
12995 ; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
12996 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
12997 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
12998 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
12999 ; GFX7-NEXT: s_cbranch_execnz .LBB44_1
13000 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
13001 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
13002 ; GFX7-NEXT: s_setpc_b64 s[30:31]
13003 ;
13004 ; GFX6-LABEL: global_agent_atomic_fsub_ret_v2f16__offset12b_neg:
13005 ; GFX6: ; %bb.0:
13006 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13007 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
13008 ; GFX6-NEXT: s_mov_b32 s5, -1
13009 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
13010 ; GFX6-NEXT: s_mov_b32 s6, 0
13011 ; GFX6-NEXT: buffer_load_dword v6, v[0:1], s[4:7], 0 addr64
13012 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
13013 ; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v3
13014 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v2
13015 ; GFX6-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
13016 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v0
13017 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
13018 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
13019 ; GFX6-NEXT: s_mov_b32 s4, s6
13020 ; GFX6-NEXT: s_mov_b32 s5, s6
13021 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13022 ; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v6
13023 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v6
13024 ; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
13025 ; GFX6-NEXT: .LBB44_1: ; %atomicrmw.start
13026 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
13027 ; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
13028 ; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
13029 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v1
13030 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v0
13031 ; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
13032 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
13033 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
13034 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
13035 ; GFX6-NEXT: s_waitcnt expcnt(0)
13036 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v7
13037 ; GFX6-NEXT: v_or_b32_e32 v7, v0, v1
13038 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v6
13039 ; GFX6-NEXT: v_or_b32_e32 v6, v8, v0
13040 ; GFX6-NEXT: v_mov_b32_e32 v9, v7
13041 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
13042 ; GFX6-NEXT: buffer_atomic_cmpswap v[8:9], v[4:5], s[4:7], 0 addr64 glc
13043 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13044 ; GFX6-NEXT: buffer_wbinvl1
13045 ; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v8
13046 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v8
13047 ; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
13048 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
13049 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
13050 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
13051 ; GFX6-NEXT: s_cbranch_execnz .LBB44_1
13052 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
13053 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
13054 ; GFX6-NEXT: s_waitcnt expcnt(0)
13055 ; GFX6-NEXT: s_setpc_b64 s[30:31]
13056 %gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 -512
13057 %result = atomicrmw fsub ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst
13058 ret <2 x half> %result
13059 }
13061 define void @global_agent_atomic_fsub_noret_v2f16(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
13062 ; GFX12-LABEL: global_agent_atomic_fsub_noret_v2f16:
13063 ; GFX12: ; %bb.0:
13064 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
13065 ; GFX12-NEXT: s_wait_expcnt 0x0
13066 ; GFX12-NEXT: s_wait_samplecnt 0x0
13067 ; GFX12-NEXT: s_wait_bvhcnt 0x0
13068 ; GFX12-NEXT: s_wait_kmcnt 0x0
13069 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off
13070 ; GFX12-NEXT: s_mov_b32 s0, 0
13071 ; GFX12-NEXT: .LBB45_1: ; %atomicrmw.start
13072 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
13073 ; GFX12-NEXT: s_wait_loadcnt 0x0
13074 ; GFX12-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13075 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
13076 ; GFX12-NEXT: s_wait_storecnt 0x0
13077 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
13078 ; GFX12-NEXT: s_wait_loadcnt 0x0
13079 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
13080 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13081 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
13082 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
13083 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
13084 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
13085 ; GFX12-NEXT: s_cbranch_execnz .LBB45_1
13086 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
13087 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
13088 ; GFX12-NEXT: s_setpc_b64 s[30:31]
13089 ;
13090 ; GFX940-LABEL: global_agent_atomic_fsub_noret_v2f16:
13091 ; GFX940: ; %bb.0:
13092 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13093 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
13094 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
13095 ; GFX940-NEXT: .LBB45_1: ; %atomicrmw.start
13096 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
13097 ; GFX940-NEXT: s_waitcnt vmcnt(0)
13098 ; GFX940-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
13099 ; GFX940-NEXT: buffer_wbl2 sc1
13100 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
13101 ; GFX940-NEXT: s_waitcnt vmcnt(0)
13102 ; GFX940-NEXT: buffer_inv sc1
13103 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
13104 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
13105 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
13106 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
13107 ; GFX940-NEXT: s_cbranch_execnz .LBB45_1
13108 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
13109 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
13110 ; GFX940-NEXT: s_setpc_b64 s[30:31]
13111 ;
13112 ; GFX11-LABEL: global_agent_atomic_fsub_noret_v2f16:
13113 ; GFX11: ; %bb.0:
13114 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13115 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off
13116 ; GFX11-NEXT: s_mov_b32 s0, 0
13117 ; GFX11-NEXT: .LBB45_1: ; %atomicrmw.start
13118 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
13119 ; GFX11-NEXT: s_waitcnt vmcnt(0)
13120 ; GFX11-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13121 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
13122 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
13123 ; GFX11-NEXT: s_waitcnt vmcnt(0)
13124 ; GFX11-NEXT: buffer_gl1_inv
13125 ; GFX11-NEXT: buffer_gl0_inv
13126 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13127 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
13128 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
13129 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
13130 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
13131 ; GFX11-NEXT: s_cbranch_execnz .LBB45_1
13132 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
13133 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
13134 ; GFX11-NEXT: s_setpc_b64 s[30:31]
13135 ;
13136 ; GFX10-LABEL: global_agent_atomic_fsub_noret_v2f16:
13137 ; GFX10: ; %bb.0:
13138 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13139 ; GFX10-NEXT: global_load_dword v4, v[0:1], off
13140 ; GFX10-NEXT: s_mov_b32 s4, 0
13141 ; GFX10-NEXT: .LBB45_1: ; %atomicrmw.start
13142 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
13143 ; GFX10-NEXT: s_waitcnt vmcnt(0)
13144 ; GFX10-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13145 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
13146 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
13147 ; GFX10-NEXT: s_waitcnt vmcnt(0)
13148 ; GFX10-NEXT: buffer_gl1_inv
13149 ; GFX10-NEXT: buffer_gl0_inv
13150 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13151 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
13152 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
13153 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
13154 ; GFX10-NEXT: s_cbranch_execnz .LBB45_1
13155 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
13156 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
13157 ; GFX10-NEXT: s_setpc_b64 s[30:31]
13158 ;
13159 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_v2f16:
13160 ; GFX90A: ; %bb.0:
13161 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13162 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
13163 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
13164 ; GFX90A-NEXT: .LBB45_1: ; %atomicrmw.start
13165 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
13166 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
13167 ; GFX90A-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
13168 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
13169 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
13170 ; GFX90A-NEXT: buffer_wbinvl1
13171 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
13172 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13173 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
13174 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
13175 ; GFX90A-NEXT: s_cbranch_execnz .LBB45_1
13176 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
13177 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
13178 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
13179 ;
13180 ; GFX908-LABEL: global_agent_atomic_fsub_noret_v2f16:
13181 ; GFX908: ; %bb.0:
13182 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13183 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
13184 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
13185 ; GFX908-NEXT: .LBB45_1: ; %atomicrmw.start
13186 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
13187 ; GFX908-NEXT: s_waitcnt vmcnt(0)
13188 ; GFX908-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13189 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
13190 ; GFX908-NEXT: s_waitcnt vmcnt(0)
13191 ; GFX908-NEXT: buffer_wbinvl1
13192 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
13193 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13194 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
13195 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
13196 ; GFX908-NEXT: s_cbranch_execnz .LBB45_1
13197 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
13198 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
13199 ; GFX908-NEXT: s_setpc_b64 s[30:31]
13200 ;
13201 ; GFX8-LABEL: global_agent_atomic_fsub_noret_v2f16:
13202 ; GFX8: ; %bb.0:
13203 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13204 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
13205 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
13206 ; GFX8-NEXT: .LBB45_1: ; %atomicrmw.start
13207 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
13208 ; GFX8-NEXT: s_waitcnt vmcnt(0)
13209 ; GFX8-NEXT: v_sub_f16_sdwa v3, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
13210 ; GFX8-NEXT: v_sub_f16_e32 v5, v4, v2
13211 ; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
13212 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
13213 ; GFX8-NEXT: s_waitcnt vmcnt(0)
13214 ; GFX8-NEXT: buffer_wbinvl1
13215 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
13216 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13217 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
13218 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
13219 ; GFX8-NEXT: s_cbranch_execnz .LBB45_1
13220 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
13221 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
13222 ; GFX8-NEXT: s_setpc_b64 s[30:31]
13223 ;
13224 ; GFX7-LABEL: global_agent_atomic_fsub_noret_v2f16:
13225 ; GFX7: ; %bb.0:
13226 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13227 ; GFX7-NEXT: s_mov_b32 s6, 0
13228 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
13229 ; GFX7-NEXT: s_mov_b32 s4, s6
13230 ; GFX7-NEXT: s_mov_b32 s5, s6
13231 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
13232 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
13233 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v2
13234 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
13235 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
13236 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13237 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
13238 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
13239 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
13240 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
13241 ; GFX7-NEXT: .LBB45_1: ; %atomicrmw.start
13242 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
13243 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
13244 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
13245 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v5
13246 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v4
13247 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
13248 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
13249 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
13250 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v6
13251 ; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v7
13252 ; GFX7-NEXT: v_or_b32_e32 v6, v4, v5
13253 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v8
13254 ; GFX7-NEXT: v_or_b32_e32 v5, v7, v4
13255 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
13256 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
13257 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
13258 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13259 ; GFX7-NEXT: buffer_wbinvl1
13260 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v7
13261 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v7
13262 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
13263 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
13264 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
13265 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
13266 ; GFX7-NEXT: s_cbranch_execnz .LBB45_1
13267 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
13268 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
13269 ; GFX7-NEXT: s_setpc_b64 s[30:31]
13270 ;
13271 ; GFX6-LABEL: global_agent_atomic_fsub_noret_v2f16:
13272 ; GFX6: ; %bb.0:
13273 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13274 ; GFX6-NEXT: s_mov_b32 s6, 0
13275 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
13276 ; GFX6-NEXT: s_mov_b32 s4, s6
13277 ; GFX6-NEXT: s_mov_b32 s5, s6
13278 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
13279 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
13280 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v2
13281 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
13282 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v3
13283 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13284 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
13285 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v5
13286 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v3
13287 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v6
13288 ; GFX6-NEXT: .LBB45_1: ; %atomicrmw.start
13289 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
13290 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
13291 ; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
13292 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v5
13293 ; GFX6-NEXT: s_waitcnt expcnt(0)
13294 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v4
13295 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
13296 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
13297 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
13298 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v6
13299 ; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v7
13300 ; GFX6-NEXT: v_or_b32_e32 v6, v4, v5
13301 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v8
13302 ; GFX6-NEXT: v_or_b32_e32 v5, v7, v4
13303 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
13304 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
13305 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
13306 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13307 ; GFX6-NEXT: buffer_wbinvl1
13308 ; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v7
13309 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v7
13310 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
13311 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
13312 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
13313 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
13314 ; GFX6-NEXT: s_cbranch_execnz .LBB45_1
13315 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
13316 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
13317 ; GFX6-NEXT: s_waitcnt expcnt(0)
13318 ; GFX6-NEXT: s_setpc_b64 s[30:31]
13319 %unused = atomicrmw fsub ptr addrspace(1) %ptr, <2 x half> %val syncscope("agent") seq_cst
13320 ret void
13321 }
13323 define void @global_agent_atomic_fsub_noret_v2f16__offset12b_pos(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
13324 ; GFX12-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_pos:
13325 ; GFX12: ; %bb.0:
13326 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
13327 ; GFX12-NEXT: s_wait_expcnt 0x0
13328 ; GFX12-NEXT: s_wait_samplecnt 0x0
13329 ; GFX12-NEXT: s_wait_bvhcnt 0x0
13330 ; GFX12-NEXT: s_wait_kmcnt 0x0
13331 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:2044
13332 ; GFX12-NEXT: s_mov_b32 s0, 0
13333 ; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start
13334 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
13335 ; GFX12-NEXT: s_wait_loadcnt 0x0
13336 ; GFX12-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13337 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
13338 ; GFX12-NEXT: s_wait_storecnt 0x0
13339 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
13340 ; GFX12-NEXT: s_wait_loadcnt 0x0
13341 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
13342 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13343 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
13344 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
13345 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
13346 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
13347 ; GFX12-NEXT: s_cbranch_execnz .LBB46_1
13348 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
13349 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
13350 ; GFX12-NEXT: s_setpc_b64 s[30:31]
13351 ;
13352 ; GFX940-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_pos:
13353 ; GFX940: ; %bb.0:
13354 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13355 ; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:2044
13356 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
13357 ; GFX940-NEXT: .LBB46_1: ; %atomicrmw.start
13358 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
13359 ; GFX940-NEXT: s_waitcnt vmcnt(0)
13360 ; GFX940-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
13361 ; GFX940-NEXT: buffer_wbl2 sc1
13362 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
13363 ; GFX940-NEXT: s_waitcnt vmcnt(0)
13364 ; GFX940-NEXT: buffer_inv sc1
13365 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
13366 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
13367 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
13368 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
13369 ; GFX940-NEXT: s_cbranch_execnz .LBB46_1
13370 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
13371 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
13372 ; GFX940-NEXT: s_setpc_b64 s[30:31]
13373 ;
13374 ; GFX11-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_pos:
13375 ; GFX11: ; %bb.0:
13376 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13377 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:2044
13378 ; GFX11-NEXT: s_mov_b32 s0, 0
13379 ; GFX11-NEXT: .LBB46_1: ; %atomicrmw.start
13380 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
13381 ; GFX11-NEXT: s_waitcnt vmcnt(0)
13382 ; GFX11-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13383 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
13384 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
13385 ; GFX11-NEXT: s_waitcnt vmcnt(0)
13386 ; GFX11-NEXT: buffer_gl1_inv
13387 ; GFX11-NEXT: buffer_gl0_inv
13388 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13389 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
13390 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
13391 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
13392 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
13393 ; GFX11-NEXT: s_cbranch_execnz .LBB46_1
13394 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
13395 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
13396 ; GFX11-NEXT: s_setpc_b64 s[30:31]
13397 ;
13398 ; GFX10-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_pos:
13399 ; GFX10: ; %bb.0:
13400 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13401 ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
13402 ; GFX10-NEXT: s_mov_b32 s4, 0
13403 ; GFX10-NEXT: .LBB46_1: ; %atomicrmw.start
13404 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
13405 ; GFX10-NEXT: s_waitcnt vmcnt(0)
13406 ; GFX10-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13407 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
13408 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
13409 ; GFX10-NEXT: s_waitcnt vmcnt(0)
13410 ; GFX10-NEXT: buffer_gl1_inv
13411 ; GFX10-NEXT: buffer_gl0_inv
13412 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13413 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
13414 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
13415 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
13416 ; GFX10-NEXT: s_cbranch_execnz .LBB46_1
13417 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
13418 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
13419 ; GFX10-NEXT: s_setpc_b64 s[30:31]
13420 ;
13421 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_pos:
13422 ; GFX90A: ; %bb.0:
13423 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13424 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
13425 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
13426 ; GFX90A-NEXT: .LBB46_1: ; %atomicrmw.start
13427 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
13428 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
13429 ; GFX90A-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
13430 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
13431 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
13432 ; GFX90A-NEXT: buffer_wbinvl1
13433 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
13434 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13435 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
13436 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
13437 ; GFX90A-NEXT: s_cbranch_execnz .LBB46_1
13438 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
13439 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
13440 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
13441 ;
13442 ; GFX908-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_pos:
13443 ; GFX908: ; %bb.0:
13444 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13445 ; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
13446 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
13447 ; GFX908-NEXT: .LBB46_1: ; %atomicrmw.start
13448 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
13449 ; GFX908-NEXT: s_waitcnt vmcnt(0)
13450 ; GFX908-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13451 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
13452 ; GFX908-NEXT: s_waitcnt vmcnt(0)
13453 ; GFX908-NEXT: buffer_wbinvl1
13454 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
13455 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13456 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
13457 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
13458 ; GFX908-NEXT: s_cbranch_execnz .LBB46_1
13459 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
13460 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
13461 ; GFX908-NEXT: s_setpc_b64 s[30:31]
13462 ;
13463 ; GFX8-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_pos:
13464 ; GFX8: ; %bb.0:
13465 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13466 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
13467 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
13468 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
13469 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
13470 ; GFX8-NEXT: .LBB46_1: ; %atomicrmw.start
13471 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
13472 ; GFX8-NEXT: s_waitcnt vmcnt(0)
13473 ; GFX8-NEXT: v_sub_f16_sdwa v3, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
13474 ; GFX8-NEXT: v_sub_f16_e32 v5, v4, v2
13475 ; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
13476 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
13477 ; GFX8-NEXT: s_waitcnt vmcnt(0)
13478 ; GFX8-NEXT: buffer_wbinvl1
13479 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
13480 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13481 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
13482 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
13483 ; GFX8-NEXT: s_cbranch_execnz .LBB46_1
13484 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
13485 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
13486 ; GFX8-NEXT: s_setpc_b64 s[30:31]
13487 ;
13488 ; GFX7-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_pos:
13489 ; GFX7: ; %bb.0:
13490 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13491 ; GFX7-NEXT: s_mov_b32 s6, 0
13492 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
13493 ; GFX7-NEXT: s_mov_b32 s4, s6
13494 ; GFX7-NEXT: s_mov_b32 s5, s6
13495 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
13496 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
13497 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v2
13498 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
13499 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
13500 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13501 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
13502 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
13503 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
13504 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
13505 ; GFX7-NEXT: .LBB46_1: ; %atomicrmw.start
13506 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
13507 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
13508 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
13509 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v5
13510 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v4
13511 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
13512 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
13513 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
13514 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v6
13515 ; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v7
13516 ; GFX7-NEXT: v_or_b32_e32 v6, v4, v5
13517 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v8
13518 ; GFX7-NEXT: v_or_b32_e32 v5, v7, v4
13519 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
13520 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
13521 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 offset:2044 glc
13522 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13523 ; GFX7-NEXT: buffer_wbinvl1
13524 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v7
13525 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v7
13526 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
13527 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
13528 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
13529 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
13530 ; GFX7-NEXT: s_cbranch_execnz .LBB46_1
13531 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
13532 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
13533 ; GFX7-NEXT: s_setpc_b64 s[30:31]
13534 ;
13535 ; GFX6-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_pos:
13536 ; GFX6: ; %bb.0:
13537 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13538 ; GFX6-NEXT: s_mov_b32 s6, 0
13539 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
13540 ; GFX6-NEXT: s_mov_b32 s4, s6
13541 ; GFX6-NEXT: s_mov_b32 s5, s6
13542 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
13543 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
13544 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v2
13545 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
13546 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v3
13547 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13548 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
13549 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v5
13550 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v3
13551 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v6
13552 ; GFX6-NEXT: .LBB46_1: ; %atomicrmw.start
13553 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
13554 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
13555 ; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
13556 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v5
13557 ; GFX6-NEXT: s_waitcnt expcnt(0)
13558 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v4
13559 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
13560 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
13561 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
13562 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v6
13563 ; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v7
13564 ; GFX6-NEXT: v_or_b32_e32 v6, v4, v5
13565 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v8
13566 ; GFX6-NEXT: v_or_b32_e32 v5, v7, v4
13567 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
13568 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
13569 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 offset:2044 glc
13570 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13571 ; GFX6-NEXT: buffer_wbinvl1
13572 ; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v7
13573 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v7
13574 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
13575 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
13576 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
13577 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
13578 ; GFX6-NEXT: s_cbranch_execnz .LBB46_1
13579 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
13580 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
13581 ; GFX6-NEXT: s_waitcnt expcnt(0)
13582 ; GFX6-NEXT: s_setpc_b64 s[30:31]
13583 %gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 511
13584 %unused = atomicrmw fsub ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst
13585 ret void
13586 }
13588 define void @global_agent_atomic_fsub_noret_v2f16__offset12b_neg(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
13589 ; GFX12-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_neg:
13590 ; GFX12: ; %bb.0:
13591 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
13592 ; GFX12-NEXT: s_wait_expcnt 0x0
13593 ; GFX12-NEXT: s_wait_samplecnt 0x0
13594 ; GFX12-NEXT: s_wait_bvhcnt 0x0
13595 ; GFX12-NEXT: s_wait_kmcnt 0x0
13596 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
13597 ; GFX12-NEXT: s_mov_b32 s0, 0
13598 ; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start
13599 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
13600 ; GFX12-NEXT: s_wait_loadcnt 0x0
13601 ; GFX12-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13602 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
13603 ; GFX12-NEXT: s_wait_storecnt 0x0
13604 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
13605 ; GFX12-NEXT: s_wait_loadcnt 0x0
13606 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
13607 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13608 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
13609 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
13610 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
13611 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
13612 ; GFX12-NEXT: s_cbranch_execnz .LBB47_1
13613 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
13614 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
13615 ; GFX12-NEXT: s_setpc_b64 s[30:31]
13616 ;
13617 ; GFX940-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_neg:
13618 ; GFX940: ; %bb.0:
13619 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13620 ; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:-2048
13621 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
13622 ; GFX940-NEXT: .LBB47_1: ; %atomicrmw.start
13623 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
13624 ; GFX940-NEXT: s_waitcnt vmcnt(0)
13625 ; GFX940-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
13626 ; GFX940-NEXT: buffer_wbl2 sc1
13627 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
13628 ; GFX940-NEXT: s_waitcnt vmcnt(0)
13629 ; GFX940-NEXT: buffer_inv sc1
13630 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
13631 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
13632 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
13633 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
13634 ; GFX940-NEXT: s_cbranch_execnz .LBB47_1
13635 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
13636 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
13637 ; GFX940-NEXT: s_setpc_b64 s[30:31]
13638 ;
13639 ; GFX11-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_neg:
13640 ; GFX11: ; %bb.0:
13641 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13642 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
13643 ; GFX11-NEXT: s_mov_b32 s0, 0
13644 ; GFX11-NEXT: .LBB47_1: ; %atomicrmw.start
13645 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
13646 ; GFX11-NEXT: s_waitcnt vmcnt(0)
13647 ; GFX11-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13648 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
13649 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
13650 ; GFX11-NEXT: s_waitcnt vmcnt(0)
13651 ; GFX11-NEXT: buffer_gl1_inv
13652 ; GFX11-NEXT: buffer_gl0_inv
13653 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13654 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
13655 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
13656 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
13657 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
13658 ; GFX11-NEXT: s_cbranch_execnz .LBB47_1
13659 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
13660 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
13661 ; GFX11-NEXT: s_setpc_b64 s[30:31]
13662 ;
13663 ; GFX10-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_neg:
13664 ; GFX10: ; %bb.0:
13665 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13666 ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:-2048
13667 ; GFX10-NEXT: s_mov_b32 s4, 0
13668 ; GFX10-NEXT: .LBB47_1: ; %atomicrmw.start
13669 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
13670 ; GFX10-NEXT: s_waitcnt vmcnt(0)
13671 ; GFX10-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13672 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
13673 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
13674 ; GFX10-NEXT: s_waitcnt vmcnt(0)
13675 ; GFX10-NEXT: buffer_gl1_inv
13676 ; GFX10-NEXT: buffer_gl0_inv
13677 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13678 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
13679 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
13680 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
13681 ; GFX10-NEXT: s_cbranch_execnz .LBB47_1
13682 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
13683 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
13684 ; GFX10-NEXT: s_setpc_b64 s[30:31]
13685 ;
13686 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_neg:
13687 ; GFX90A: ; %bb.0:
13688 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13689 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
13690 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
13691 ; GFX90A-NEXT: .LBB47_1: ; %atomicrmw.start
13692 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
13693 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
13694 ; GFX90A-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
13695 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
13696 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
13697 ; GFX90A-NEXT: buffer_wbinvl1
13698 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
13699 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13700 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
13701 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
13702 ; GFX90A-NEXT: s_cbranch_execnz .LBB47_1
13703 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
13704 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
13705 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
13706 ;
13707 ; GFX908-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_neg:
13708 ; GFX908: ; %bb.0:
13709 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13710 ; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:-2048
13711 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
13712 ; GFX908-NEXT: .LBB47_1: ; %atomicrmw.start
13713 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
13714 ; GFX908-NEXT: s_waitcnt vmcnt(0)
13715 ; GFX908-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13716 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
13717 ; GFX908-NEXT: s_waitcnt vmcnt(0)
13718 ; GFX908-NEXT: buffer_wbinvl1
13719 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
13720 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13721 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
13722 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
13723 ; GFX908-NEXT: s_cbranch_execnz .LBB47_1
13724 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
13725 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
13726 ; GFX908-NEXT: s_setpc_b64 s[30:31]
13727 ;
13728 ; GFX8-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_neg:
13729 ; GFX8: ; %bb.0:
13730 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13731 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
13732 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
13733 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
13734 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
13735 ; GFX8-NEXT: .LBB47_1: ; %atomicrmw.start
13736 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
13737 ; GFX8-NEXT: s_waitcnt vmcnt(0)
13738 ; GFX8-NEXT: v_sub_f16_sdwa v3, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
13739 ; GFX8-NEXT: v_sub_f16_e32 v5, v4, v2
13740 ; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
13741 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
13742 ; GFX8-NEXT: s_waitcnt vmcnt(0)
13743 ; GFX8-NEXT: buffer_wbinvl1
13744 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
13745 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13746 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
13747 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
13748 ; GFX8-NEXT: s_cbranch_execnz .LBB47_1
13749 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
13750 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
13751 ; GFX8-NEXT: s_setpc_b64 s[30:31]
13752 ;
13753 ; GFX7-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_neg:
13754 ; GFX7: ; %bb.0:
13755 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13756 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
13757 ; GFX7-NEXT: s_mov_b32 s5, -1
13758 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
13759 ; GFX7-NEXT: s_mov_b32 s6, 0
13760 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
13761 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
13762 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v2
13763 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
13764 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
13765 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
13766 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
13767 ; GFX7-NEXT: s_mov_b32 s4, s6
13768 ; GFX7-NEXT: s_mov_b32 s5, s6
13769 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13770 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
13771 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
13772 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
13773 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
13774 ; GFX7-NEXT: .LBB47_1: ; %atomicrmw.start
13775 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
13776 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
13777 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
13778 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v5
13779 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v4
13780 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
13781 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
13782 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
13783 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v6
13784 ; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v7
13785 ; GFX7-NEXT: v_or_b32_e32 v6, v4, v5
13786 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v8
13787 ; GFX7-NEXT: v_or_b32_e32 v5, v7, v4
13788 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
13789 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
13790 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
13791 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13792 ; GFX7-NEXT: buffer_wbinvl1
13793 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v7
13794 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v7
13795 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
13796 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
13797 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
13798 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
13799 ; GFX7-NEXT: s_cbranch_execnz .LBB47_1
13800 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
13801 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
13802 ; GFX7-NEXT: s_setpc_b64 s[30:31]
13803 ;
13804 ; GFX6-LABEL: global_agent_atomic_fsub_noret_v2f16__offset12b_neg:
13805 ; GFX6: ; %bb.0:
13806 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13807 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
13808 ; GFX6-NEXT: s_mov_b32 s5, -1
13809 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
13810 ; GFX6-NEXT: s_mov_b32 s6, 0
13811 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
13812 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
13813 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v2
13814 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
13815 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v3
13816 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
13817 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
13818 ; GFX6-NEXT: s_mov_b32 s4, s6
13819 ; GFX6-NEXT: s_mov_b32 s5, s6
13820 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13821 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
13822 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v5
13823 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v3
13824 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v6
13825 ; GFX6-NEXT: .LBB47_1: ; %atomicrmw.start
13826 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
13827 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
13828 ; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
13829 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v5
13830 ; GFX6-NEXT: s_waitcnt expcnt(0)
13831 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v4
13832 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
13833 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
13834 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
13835 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v6
13836 ; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v7
13837 ; GFX6-NEXT: v_or_b32_e32 v6, v4, v5
13838 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v8
13839 ; GFX6-NEXT: v_or_b32_e32 v5, v7, v4
13840 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
13841 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
13842 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
13843 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13844 ; GFX6-NEXT: buffer_wbinvl1
13845 ; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v7
13846 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v7
13847 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
13848 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
13849 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
13850 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
13851 ; GFX6-NEXT: s_cbranch_execnz .LBB47_1
13852 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
13853 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
13854 ; GFX6-NEXT: s_waitcnt expcnt(0)
13855 ; GFX6-NEXT: s_setpc_b64 s[30:31]
13856 %gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 -512
13857 %unused = atomicrmw fsub ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst
13858 ret void
13859 }
13861 define <2 x half> @global_system_atomic_fsub_ret_v2f16__offset12b_pos(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
13862 ; GFX12-LABEL: global_system_atomic_fsub_ret_v2f16__offset12b_pos:
13863 ; GFX12: ; %bb.0:
13864 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
13865 ; GFX12-NEXT: s_wait_expcnt 0x0
13866 ; GFX12-NEXT: s_wait_samplecnt 0x0
13867 ; GFX12-NEXT: s_wait_bvhcnt 0x0
13868 ; GFX12-NEXT: s_wait_kmcnt 0x0
13869 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
13870 ; GFX12-NEXT: s_mov_b32 s0, 0
13871 ; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start
13872 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
13873 ; GFX12-NEXT: s_wait_loadcnt 0x0
13874 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
13875 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
13876 ; GFX12-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13877 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
13878 ; GFX12-NEXT: s_wait_storecnt 0x0
13879 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
13880 ; GFX12-NEXT: s_wait_loadcnt 0x0
13881 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
13882 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13883 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
13884 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
13885 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
13886 ; GFX12-NEXT: s_cbranch_execnz .LBB48_1
13887 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
13888 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
13889 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
13890 ; GFX12-NEXT: s_setpc_b64 s[30:31]
13891 ;
13892 ; GFX940-LABEL: global_system_atomic_fsub_ret_v2f16__offset12b_pos:
13893 ; GFX940: ; %bb.0:
13894 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13895 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
13896 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
13897 ; GFX940-NEXT: .LBB48_1: ; %atomicrmw.start
13898 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
13899 ; GFX940-NEXT: s_waitcnt vmcnt(0)
13900 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
13901 ; GFX940-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
13902 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
13903 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
13904 ; GFX940-NEXT: s_waitcnt vmcnt(0)
13905 ; GFX940-NEXT: buffer_inv sc0 sc1
13906 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
13907 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
13908 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
13909 ; GFX940-NEXT: s_cbranch_execnz .LBB48_1
13910 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
13911 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
13912 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
13913 ; GFX940-NEXT: s_setpc_b64 s[30:31]
13914 ;
13915 ; GFX11-LABEL: global_system_atomic_fsub_ret_v2f16__offset12b_pos:
13916 ; GFX11: ; %bb.0:
13917 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13918 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
13919 ; GFX11-NEXT: s_mov_b32 s0, 0
13920 ; GFX11-NEXT: .LBB48_1: ; %atomicrmw.start
13921 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
13922 ; GFX11-NEXT: s_waitcnt vmcnt(0)
13923 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
13924 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
13925 ; GFX11-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13926 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
13927 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
13928 ; GFX11-NEXT: s_waitcnt vmcnt(0)
13929 ; GFX11-NEXT: buffer_gl1_inv
13930 ; GFX11-NEXT: buffer_gl0_inv
13931 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13932 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
13933 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
13934 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
13935 ; GFX11-NEXT: s_cbranch_execnz .LBB48_1
13936 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
13937 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
13938 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
13939 ; GFX11-NEXT: s_setpc_b64 s[30:31]
13940 ;
13941 ; GFX10-LABEL: global_system_atomic_fsub_ret_v2f16__offset12b_pos:
13942 ; GFX10: ; %bb.0:
13943 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13944 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
13945 ; GFX10-NEXT: s_mov_b32 s4, 0
13946 ; GFX10-NEXT: .LBB48_1: ; %atomicrmw.start
13947 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
13948 ; GFX10-NEXT: s_waitcnt vmcnt(0)
13949 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
13950 ; GFX10-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13951 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
13952 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
13953 ; GFX10-NEXT: s_waitcnt vmcnt(0)
13954 ; GFX10-NEXT: buffer_gl1_inv
13955 ; GFX10-NEXT: buffer_gl0_inv
13956 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13957 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
13958 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
13959 ; GFX10-NEXT: s_cbranch_execnz .LBB48_1
13960 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
13961 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
13962 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
13963 ; GFX10-NEXT: s_setpc_b64 s[30:31]
13964 ;
13965 ; GFX90A-LABEL: global_system_atomic_fsub_ret_v2f16__offset12b_pos:
13966 ; GFX90A: ; %bb.0:
13967 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13968 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
13969 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
13970 ; GFX90A-NEXT: .LBB48_1: ; %atomicrmw.start
13971 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
13972 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
13973 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
13974 ; GFX90A-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
13975 ; GFX90A-NEXT: buffer_wbl2
13976 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
13977 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
13978 ; GFX90A-NEXT: buffer_invl2
13979 ; GFX90A-NEXT: buffer_wbinvl1
13980 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
13981 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13982 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
13983 ; GFX90A-NEXT: s_cbranch_execnz .LBB48_1
13984 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
13985 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
13986 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
13987 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
13988 ;
13989 ; GFX908-LABEL: global_system_atomic_fsub_ret_v2f16__offset12b_pos:
13990 ; GFX908: ; %bb.0:
13991 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13992 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
13993 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
13994 ; GFX908-NEXT: .LBB48_1: ; %atomicrmw.start
13995 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
13996 ; GFX908-NEXT: s_waitcnt vmcnt(0)
13997 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
13998 ; GFX908-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
13999 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
14000 ; GFX908-NEXT: s_waitcnt vmcnt(0)
14001 ; GFX908-NEXT: buffer_wbinvl1
14002 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
14003 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
14004 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
14005 ; GFX908-NEXT: s_cbranch_execnz .LBB48_1
14006 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
14007 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
14008 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
14009 ; GFX908-NEXT: s_setpc_b64 s[30:31]
14010 ;
14011 ; GFX8-LABEL: global_system_atomic_fsub_ret_v2f16__offset12b_pos:
14012 ; GFX8: ; %bb.0:
14013 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14014 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
14015 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
14016 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
14017 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
14018 ; GFX8-NEXT: .LBB48_1: ; %atomicrmw.start
14019 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
14020 ; GFX8-NEXT: s_waitcnt vmcnt(0)
14021 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
14022 ; GFX8-NEXT: v_sub_f16_sdwa v0, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
14023 ; GFX8-NEXT: v_sub_f16_e32 v5, v1, v2
14024 ; GFX8-NEXT: v_or_b32_e32 v0, v5, v0
14025 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
14026 ; GFX8-NEXT: s_waitcnt vmcnt(0)
14027 ; GFX8-NEXT: buffer_wbinvl1
14028 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
14029 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
14030 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
14031 ; GFX8-NEXT: s_cbranch_execnz .LBB48_1
14032 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
14033 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
14034 ; GFX8-NEXT: s_setpc_b64 s[30:31]
14035 ;
14036 ; GFX7-LABEL: global_system_atomic_fsub_ret_v2f16__offset12b_pos:
14037 ; GFX7: ; %bb.0:
14038 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14039 ; GFX7-NEXT: s_mov_b32 s6, 0
14040 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
14041 ; GFX7-NEXT: s_mov_b32 s4, s6
14042 ; GFX7-NEXT: s_mov_b32 s5, s6
14043 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
14044 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
14045 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v2
14046 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
14047 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v3
14048 ; GFX7-NEXT: s_waitcnt vmcnt(0)
14049 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
14050 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v5
14051 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
14052 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v6
14053 ; GFX7-NEXT: .LBB48_1: ; %atomicrmw.start
14054 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
14055 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
14056 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
14057 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v3
14058 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
14059 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
14060 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v4
14061 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v5
14062 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6
14063 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v7
14064 ; GFX7-NEXT: v_or_b32_e32 v7, v2, v3
14065 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
14066 ; GFX7-NEXT: v_or_b32_e32 v6, v8, v2
14067 ; GFX7-NEXT: v_mov_b32_e32 v9, v7
14068 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
14069 ; GFX7-NEXT: buffer_atomic_cmpswap v[8:9], v[0:1], s[4:7], 0 addr64 offset:2044 glc
14070 ; GFX7-NEXT: s_waitcnt vmcnt(0)
14071 ; GFX7-NEXT: buffer_wbinvl1
14072 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
14073 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v8
14074 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
14075 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
14076 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
14077 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
14078 ; GFX7-NEXT: s_cbranch_execnz .LBB48_1
14079 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
14080 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
14081 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
14082 ; GFX7-NEXT: v_mov_b32_e32 v1, v3
14083 ; GFX7-NEXT: s_setpc_b64 s[30:31]
14084 ;
14085 ; GFX6-LABEL: global_system_atomic_fsub_ret_v2f16__offset12b_pos:
14086 ; GFX6: ; %bb.0:
14087 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14088 ; GFX6-NEXT: s_mov_b32 s6, 0
14089 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
14090 ; GFX6-NEXT: s_mov_b32 s4, s6
14091 ; GFX6-NEXT: s_mov_b32 s5, s6
14092 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
14093 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
14094 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v2
14095 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
14096 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v3
14097 ; GFX6-NEXT: s_waitcnt vmcnt(0)
14098 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
14099 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v5
14100 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
14101 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v6
14102 ; GFX6-NEXT: .LBB48_1: ; %atomicrmw.start
14103 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
14104 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
14105 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
14106 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v3
14107 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
14108 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
14109 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v4
14110 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v5
14111 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
14112 ; GFX6-NEXT: s_waitcnt expcnt(0)
14113 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v7
14114 ; GFX6-NEXT: v_or_b32_e32 v7, v2, v3
14115 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v6
14116 ; GFX6-NEXT: v_or_b32_e32 v6, v8, v2
14117 ; GFX6-NEXT: v_mov_b32_e32 v9, v7
14118 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
14119 ; GFX6-NEXT: buffer_atomic_cmpswap v[8:9], v[0:1], s[4:7], 0 addr64 offset:2044 glc
14120 ; GFX6-NEXT: s_waitcnt vmcnt(0)
14121 ; GFX6-NEXT: buffer_wbinvl1
14122 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
14123 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v8
14124 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
14125 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
14126 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
14127 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
14128 ; GFX6-NEXT: s_cbranch_execnz .LBB48_1
14129 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
14130 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
14131 ; GFX6-NEXT: v_mov_b32_e32 v0, v2
14132 ; GFX6-NEXT: v_mov_b32_e32 v1, v3
14133 ; GFX6-NEXT: s_waitcnt expcnt(0)
14134 ; GFX6-NEXT: s_setpc_b64 s[30:31]
14135 %gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 511
14136 %result = atomicrmw fsub ptr addrspace(1) %gep, <2 x half> %val seq_cst
14137 ret <2 x half> %result
14138 }
14140 define void @global_system_atomic_fsub_noret_v2f16__offset12b_pos(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
14141 ; GFX12-LABEL: global_system_atomic_fsub_noret_v2f16__offset12b_pos:
14142 ; GFX12: ; %bb.0:
14143 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
14144 ; GFX12-NEXT: s_wait_expcnt 0x0
14145 ; GFX12-NEXT: s_wait_samplecnt 0x0
14146 ; GFX12-NEXT: s_wait_bvhcnt 0x0
14147 ; GFX12-NEXT: s_wait_kmcnt 0x0
14148 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:2044
14149 ; GFX12-NEXT: s_mov_b32 s0, 0
14150 ; GFX12-NEXT: .LBB49_1: ; %atomicrmw.start
14151 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
14152 ; GFX12-NEXT: s_wait_loadcnt 0x0
14153 ; GFX12-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
14154 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
14155 ; GFX12-NEXT: s_wait_storecnt 0x0
14156 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
14157 ; GFX12-NEXT: s_wait_loadcnt 0x0
14158 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
14159 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
14160 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
14161 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
14162 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
14163 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
14164 ; GFX12-NEXT: s_cbranch_execnz .LBB49_1
14165 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
14166 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
14167 ; GFX12-NEXT: s_setpc_b64 s[30:31]
14168 ;
14169 ; GFX940-LABEL: global_system_atomic_fsub_noret_v2f16__offset12b_pos:
14170 ; GFX940: ; %bb.0:
14171 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14172 ; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:2044
14173 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
14174 ; GFX940-NEXT: .LBB49_1: ; %atomicrmw.start
14175 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
14176 ; GFX940-NEXT: s_waitcnt vmcnt(0)
14177 ; GFX940-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
14178 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
14179 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
14180 ; GFX940-NEXT: s_waitcnt vmcnt(0)
14181 ; GFX940-NEXT: buffer_inv sc0 sc1
14182 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
14183 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
14184 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
14185 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
14186 ; GFX940-NEXT: s_cbranch_execnz .LBB49_1
14187 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
14188 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
14189 ; GFX940-NEXT: s_setpc_b64 s[30:31]
14190 ;
14191 ; GFX11-LABEL: global_system_atomic_fsub_noret_v2f16__offset12b_pos:
14192 ; GFX11: ; %bb.0:
14193 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14194 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:2044
14195 ; GFX11-NEXT: s_mov_b32 s0, 0
14196 ; GFX11-NEXT: .LBB49_1: ; %atomicrmw.start
14197 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
14198 ; GFX11-NEXT: s_waitcnt vmcnt(0)
14199 ; GFX11-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
14200 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
14201 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
14202 ; GFX11-NEXT: s_waitcnt vmcnt(0)
14203 ; GFX11-NEXT: buffer_gl1_inv
14204 ; GFX11-NEXT: buffer_gl0_inv
14205 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
14206 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
14207 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
14208 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
14209 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
14210 ; GFX11-NEXT: s_cbranch_execnz .LBB49_1
14211 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
14212 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
14213 ; GFX11-NEXT: s_setpc_b64 s[30:31]
14214 ;
14215 ; GFX10-LABEL: global_system_atomic_fsub_noret_v2f16__offset12b_pos:
14216 ; GFX10: ; %bb.0:
14217 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14218 ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
14219 ; GFX10-NEXT: s_mov_b32 s4, 0
14220 ; GFX10-NEXT: .LBB49_1: ; %atomicrmw.start
14221 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
14222 ; GFX10-NEXT: s_waitcnt vmcnt(0)
14223 ; GFX10-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
14224 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
14225 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
14226 ; GFX10-NEXT: s_waitcnt vmcnt(0)
14227 ; GFX10-NEXT: buffer_gl1_inv
14228 ; GFX10-NEXT: buffer_gl0_inv
14229 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
14230 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
14231 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
14232 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
14233 ; GFX10-NEXT: s_cbranch_execnz .LBB49_1
14234 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
14235 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
14236 ; GFX10-NEXT: s_setpc_b64 s[30:31]
14237 ;
14238 ; GFX90A-LABEL: global_system_atomic_fsub_noret_v2f16__offset12b_pos:
14239 ; GFX90A: ; %bb.0:
14240 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14241 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
14242 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
14243 ; GFX90A-NEXT: .LBB49_1: ; %atomicrmw.start
14244 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
14245 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
14246 ; GFX90A-NEXT: v_pk_add_f16 v4, v5, v2 neg_lo:[0,1] neg_hi:[0,1]
14247 ; GFX90A-NEXT: buffer_wbl2
14248 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
14249 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
14250 ; GFX90A-NEXT: buffer_invl2
14251 ; GFX90A-NEXT: buffer_wbinvl1
14252 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
14253 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
14254 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
14255 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
14256 ; GFX90A-NEXT: s_cbranch_execnz .LBB49_1
14257 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
14258 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
14259 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
14260 ;
14261 ; GFX908-LABEL: global_system_atomic_fsub_noret_v2f16__offset12b_pos:
14262 ; GFX908: ; %bb.0:
14263 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14264 ; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
14265 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
14266 ; GFX908-NEXT: .LBB49_1: ; %atomicrmw.start
14267 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
14268 ; GFX908-NEXT: s_waitcnt vmcnt(0)
14269 ; GFX908-NEXT: v_pk_add_f16 v3, v4, v2 neg_lo:[0,1] neg_hi:[0,1]
14270 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
14271 ; GFX908-NEXT: s_waitcnt vmcnt(0)
14272 ; GFX908-NEXT: buffer_wbinvl1
14273 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
14274 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
14275 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
14276 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
14277 ; GFX908-NEXT: s_cbranch_execnz .LBB49_1
14278 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
14279 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
14280 ; GFX908-NEXT: s_setpc_b64 s[30:31]
14281 ;
14282 ; GFX8-LABEL: global_system_atomic_fsub_noret_v2f16__offset12b_pos:
14283 ; GFX8: ; %bb.0:
14284 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14285 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
14286 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
14287 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
14288 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
14289 ; GFX8-NEXT: .LBB49_1: ; %atomicrmw.start
14290 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
14291 ; GFX8-NEXT: s_waitcnt vmcnt(0)
14292 ; GFX8-NEXT: v_sub_f16_sdwa v3, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
14293 ; GFX8-NEXT: v_sub_f16_e32 v5, v4, v2
14294 ; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
14295 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
14296 ; GFX8-NEXT: s_waitcnt vmcnt(0)
14297 ; GFX8-NEXT: buffer_wbinvl1
14298 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
14299 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
14300 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
14301 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
14302 ; GFX8-NEXT: s_cbranch_execnz .LBB49_1
14303 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
14304 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
14305 ; GFX8-NEXT: s_setpc_b64 s[30:31]
14306 ;
14307 ; GFX7-LABEL: global_system_atomic_fsub_noret_v2f16__offset12b_pos:
14308 ; GFX7: ; %bb.0:
14309 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14310 ; GFX7-NEXT: s_mov_b32 s6, 0
14311 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
14312 ; GFX7-NEXT: s_mov_b32 s4, s6
14313 ; GFX7-NEXT: s_mov_b32 s5, s6
14314 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
14315 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
14316 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v2
14317 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
14318 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
14319 ; GFX7-NEXT: s_waitcnt vmcnt(0)
14320 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
14321 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
14322 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
14323 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
14324 ; GFX7-NEXT: .LBB49_1: ; %atomicrmw.start
14325 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
14326 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
14327 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
14328 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v5
14329 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v4
14330 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
14331 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
14332 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
14333 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v6
14334 ; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v7
14335 ; GFX7-NEXT: v_or_b32_e32 v6, v4, v5
14336 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v8
14337 ; GFX7-NEXT: v_or_b32_e32 v5, v7, v4
14338 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
14339 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
14340 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 offset:2044 glc
14341 ; GFX7-NEXT: s_waitcnt vmcnt(0)
14342 ; GFX7-NEXT: buffer_wbinvl1
14343 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v7
14344 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v7
14345 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
14346 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
14347 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
14348 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
14349 ; GFX7-NEXT: s_cbranch_execnz .LBB49_1
14350 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
14351 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
14352 ; GFX7-NEXT: s_setpc_b64 s[30:31]
14353 ;
14354 ; GFX6-LABEL: global_system_atomic_fsub_noret_v2f16__offset12b_pos:
14355 ; GFX6: ; %bb.0:
14356 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14357 ; GFX6-NEXT: s_mov_b32 s6, 0
14358 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
14359 ; GFX6-NEXT: s_mov_b32 s4, s6
14360 ; GFX6-NEXT: s_mov_b32 s5, s6
14361 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
14362 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
14363 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v2
14364 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
14365 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v3
14366 ; GFX6-NEXT: s_waitcnt vmcnt(0)
14367 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
14368 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v5
14369 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v3
14370 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v6
14371 ; GFX6-NEXT: .LBB49_1: ; %atomicrmw.start
14372 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
14373 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
14374 ; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
14375 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v5
14376 ; GFX6-NEXT: s_waitcnt expcnt(0)
14377 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v4
14378 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
14379 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
14380 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
14381 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v6
14382 ; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v7
14383 ; GFX6-NEXT: v_or_b32_e32 v6, v4, v5
14384 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v8
14385 ; GFX6-NEXT: v_or_b32_e32 v5, v7, v4
14386 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
14387 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
14388 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 offset:2044 glc
14389 ; GFX6-NEXT: s_waitcnt vmcnt(0)
14390 ; GFX6-NEXT: buffer_wbinvl1
14391 ; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v7
14392 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v7
14393 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
14394 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
14395 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
14396 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
14397 ; GFX6-NEXT: s_cbranch_execnz .LBB49_1
14398 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
14399 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
14400 ; GFX6-NEXT: s_waitcnt expcnt(0)
14401 ; GFX6-NEXT: s_setpc_b64 s[30:31]
14402 %gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 511
14403 %unused = atomicrmw fsub ptr addrspace(1) %gep, <2 x half> %val seq_cst
14404 ret void
14405 }
14407 ; --------------------------------------------------------------------
14408 ; <2 x bfloat>
14409 ; --------------------------------------------------------------------
14411 define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
14412 ; GFX12-LABEL: global_agent_atomic_fsub_ret_v2bf16:
14413 ; GFX12: ; %bb.0:
14414 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
14415 ; GFX12-NEXT: s_wait_expcnt 0x0
14416 ; GFX12-NEXT: s_wait_samplecnt 0x0
14417 ; GFX12-NEXT: s_wait_bvhcnt 0x0
14418 ; GFX12-NEXT: s_wait_kmcnt 0x0
14419 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
14420 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14421 ; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14422 ; GFX12-NEXT: s_mov_b32 s1, 0
14423 ; GFX12-NEXT: .LBB50_1: ; %atomicrmw.start
14424 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
14425 ; GFX12-NEXT: s_wait_loadcnt 0x0
14426 ; GFX12-NEXT: v_mov_b32_e32 v6, v3
14427 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14428 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14429 ; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
14430 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14431 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
14432 ; GFX12-NEXT: v_bfe_u32 v8, v5, 16, 1
14433 ; GFX12-NEXT: v_sub_f32_e32 v3, v3, v4
14434 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v5
14435 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
14436 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14437 ; GFX12-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
14438 ; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
14439 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v3
14440 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v3, v3
14441 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14442 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
14443 ; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
14444 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14445 ; GFX12-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
14446 ; GFX12-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
14447 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
14448 ; GFX12-NEXT: s_wait_storecnt 0x0
14449 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
14450 ; GFX12-NEXT: s_wait_loadcnt 0x0
14451 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
14452 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
14453 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
14454 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
14455 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
14456 ; GFX12-NEXT: s_cbranch_execnz .LBB50_1
14457 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
14458 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
14459 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
14460 ; GFX12-NEXT: s_setpc_b64 s[30:31]
14461 ;
14462 ; GFX940-LABEL: global_agent_atomic_fsub_ret_v2bf16:
14463 ; GFX940: ; %bb.0:
14464 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14465 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
14466 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
14467 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14468 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
14469 ; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14470 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
14471 ; GFX940-NEXT: .LBB50_1: ; %atomicrmw.start
14472 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
14473 ; GFX940-NEXT: s_waitcnt vmcnt(0)
14474 ; GFX940-NEXT: v_mov_b32_e32 v7, v3
14475 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 16, v7
14476 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
14477 ; GFX940-NEXT: v_sub_f32_e32 v3, v3, v4
14478 ; GFX940-NEXT: v_sub_f32_e32 v5, v5, v2
14479 ; GFX940-NEXT: v_bfe_u32 v6, v3, 16, 1
14480 ; GFX940-NEXT: v_bfe_u32 v9, v5, 16, 1
14481 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v3
14482 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v5
14483 ; GFX940-NEXT: v_add3_u32 v6, v6, v3, s4
14484 ; GFX940-NEXT: v_add3_u32 v9, v9, v5, s4
14485 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
14486 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
14487 ; GFX940-NEXT: s_nop 0
14488 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
14489 ; GFX940-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[0:1]
14490 ; GFX940-NEXT: v_perm_b32 v6, v5, v3, s5
14491 ; GFX940-NEXT: buffer_wbl2 sc1
14492 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off sc0
14493 ; GFX940-NEXT: s_waitcnt vmcnt(0)
14494 ; GFX940-NEXT: buffer_inv sc1
14495 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
14496 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
14497 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
14498 ; GFX940-NEXT: s_cbranch_execnz .LBB50_1
14499 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
14500 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
14501 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
14502 ; GFX940-NEXT: s_setpc_b64 s[30:31]
14503 ;
14504 ; GFX11-LABEL: global_agent_atomic_fsub_ret_v2bf16:
14505 ; GFX11: ; %bb.0:
14506 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14507 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
14508 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14509 ; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14510 ; GFX11-NEXT: s_mov_b32 s1, 0
14511 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
14512 ; GFX11-NEXT: .p2align 6
14513 ; GFX11-NEXT: .LBB50_1: ; %atomicrmw.start
14514 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
14515 ; GFX11-NEXT: s_waitcnt vmcnt(0)
14516 ; GFX11-NEXT: v_mov_b32_e32 v6, v3
14517 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14518 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14519 ; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
14520 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14521 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
14522 ; GFX11-NEXT: v_bfe_u32 v8, v5, 16, 1
14523 ; GFX11-NEXT: v_sub_f32_e32 v3, v3, v4
14524 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v5
14525 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
14526 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14527 ; GFX11-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
14528 ; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
14529 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v3
14530 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v3, v3
14531 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14532 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
14533 ; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
14534 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14535 ; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
14536 ; GFX11-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
14537 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
14538 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
14539 ; GFX11-NEXT: s_waitcnt vmcnt(0)
14540 ; GFX11-NEXT: buffer_gl1_inv
14541 ; GFX11-NEXT: buffer_gl0_inv
14542 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
14543 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
14544 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
14545 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
14546 ; GFX11-NEXT: s_cbranch_execnz .LBB50_1
14547 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
14548 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
14549 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
14550 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
14551 ; GFX11-NEXT: s_setpc_b64 s[30:31]
14552 ;
14553 ; GFX10-LABEL: global_agent_atomic_fsub_ret_v2bf16:
14554 ; GFX10: ; %bb.0:
14555 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14556 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
14557 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14558 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14559 ; GFX10-NEXT: s_mov_b32 s5, 0
14560 ; GFX10-NEXT: .LBB50_1: ; %atomicrmw.start
14561 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
14562 ; GFX10-NEXT: s_waitcnt vmcnt(0)
14563 ; GFX10-NEXT: v_mov_b32_e32 v6, v3
14564 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14565 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14566 ; GFX10-NEXT: v_sub_f32_e32 v3, v3, v4
14567 ; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
14568 ; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
14569 ; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
14570 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
14571 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
14572 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
14573 ; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
14574 ; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
14575 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
14576 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
14577 ; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
14578 ; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
14579 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
14580 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
14581 ; GFX10-NEXT: s_waitcnt vmcnt(0)
14582 ; GFX10-NEXT: buffer_gl1_inv
14583 ; GFX10-NEXT: buffer_gl0_inv
14584 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
14585 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
14586 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
14587 ; GFX10-NEXT: s_cbranch_execnz .LBB50_1
14588 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
14589 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
14590 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
14591 ; GFX10-NEXT: s_setpc_b64 s[30:31]
14592 ;
14593 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_v2bf16:
14594 ; GFX90A: ; %bb.0:
14595 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14596 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
14597 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
14598 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14599 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
14600 ; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14601 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
14602 ; GFX90A-NEXT: .LBB50_1: ; %atomicrmw.start
14603 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
14604 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
14605 ; GFX90A-NEXT: v_mov_b32_e32 v7, v3
14606 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v7
14607 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
14608 ; GFX90A-NEXT: v_sub_f32_e32 v3, v3, v4
14609 ; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v2
14610 ; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
14611 ; GFX90A-NEXT: v_bfe_u32 v9, v5, 16, 1
14612 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v3
14613 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v5
14614 ; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
14615 ; GFX90A-NEXT: v_add3_u32 v9, v9, v5, s8
14616 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
14617 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
14618 ; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[4:5]
14619 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
14620 ; GFX90A-NEXT: v_perm_b32 v6, v5, v3, s9
14621 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off glc
14622 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
14623 ; GFX90A-NEXT: buffer_wbinvl1
14624 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
14625 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
14626 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
14627 ; GFX90A-NEXT: s_cbranch_execnz .LBB50_1
14628 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
14629 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
14630 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
14631 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
14632 ;
14633 ; GFX908-LABEL: global_agent_atomic_fsub_ret_v2bf16:
14634 ; GFX908: ; %bb.0:
14635 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14636 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
14637 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
14638 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14639 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
14640 ; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14641 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
14642 ; GFX908-NEXT: .LBB50_1: ; %atomicrmw.start
14643 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
14644 ; GFX908-NEXT: s_waitcnt vmcnt(0)
14645 ; GFX908-NEXT: v_mov_b32_e32 v6, v3
14646 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14647 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14648 ; GFX908-NEXT: v_sub_f32_e32 v3, v3, v4
14649 ; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
14650 ; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
14651 ; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
14652 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
14653 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
14654 ; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
14655 ; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
14656 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
14657 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
14658 ; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
14659 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
14660 ; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
14661 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
14662 ; GFX908-NEXT: s_waitcnt vmcnt(0)
14663 ; GFX908-NEXT: buffer_wbinvl1
14664 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
14665 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
14666 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
14667 ; GFX908-NEXT: s_cbranch_execnz .LBB50_1
14668 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
14669 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
14670 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
14671 ; GFX908-NEXT: s_setpc_b64 s[30:31]
14672 ;
14673 ; GFX8-LABEL: global_agent_atomic_fsub_ret_v2bf16:
14674 ; GFX8: ; %bb.0:
14675 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14676 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
14677 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
14678 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14679 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14680 ; GFX8-NEXT: .LBB50_1: ; %atomicrmw.start
14681 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
14682 ; GFX8-NEXT: s_waitcnt vmcnt(0)
14683 ; GFX8-NEXT: v_mov_b32_e32 v6, v3
14684 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14685 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14686 ; GFX8-NEXT: v_sub_f32_e32 v3, v3, v4
14687 ; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
14688 ; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
14689 ; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
14690 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
14691 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
14692 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
14693 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
14694 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
14695 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
14696 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v3
14697 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
14698 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
14699 ; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
14700 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
14701 ; GFX8-NEXT: v_alignbit_b32 v5, v5, v3, 16
14702 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
14703 ; GFX8-NEXT: s_waitcnt vmcnt(0)
14704 ; GFX8-NEXT: buffer_wbinvl1
14705 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
14706 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
14707 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
14708 ; GFX8-NEXT: s_cbranch_execnz .LBB50_1
14709 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
14710 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
14711 ; GFX8-NEXT: v_mov_b32_e32 v0, v3
14712 ; GFX8-NEXT: s_setpc_b64 s[30:31]
14713 ;
14714 ; GFX7-LABEL: global_agent_atomic_fsub_ret_v2bf16:
14715 ; GFX7: ; %bb.0:
14716 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14717 ; GFX7-NEXT: s_mov_b32 s6, 0
14718 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
14719 ; GFX7-NEXT: s_mov_b32 s4, s6
14720 ; GFX7-NEXT: s_mov_b32 s5, s6
14721 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
14722 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
14723 ; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v3
14724 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
14725 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
14726 ; GFX7-NEXT: s_waitcnt vmcnt(0)
14727 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
14728 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
14729 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14730 ; GFX7-NEXT: .LBB50_1: ; %atomicrmw.start
14731 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
14732 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
14733 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
14734 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
14735 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
14736 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
14737 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v5
14738 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v4
14739 ; GFX7-NEXT: v_alignbit_b32 v3, v2, v3, 16
14740 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v7
14741 ; GFX7-NEXT: v_alignbit_b32 v2, v2, v6, 16
14742 ; GFX7-NEXT: v_mov_b32_e32 v7, v3
14743 ; GFX7-NEXT: v_mov_b32_e32 v6, v2
14744 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
14745 ; GFX7-NEXT: s_waitcnt vmcnt(0)
14746 ; GFX7-NEXT: buffer_wbinvl1
14747 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v3
14748 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
14749 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
14750 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14751 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
14752 ; GFX7-NEXT: s_cbranch_execnz .LBB50_1
14753 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
14754 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
14755 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
14756 ; GFX7-NEXT: v_mov_b32_e32 v1, v2
14757 ; GFX7-NEXT: s_setpc_b64 s[30:31]
14758 ;
14759 ; GFX6-LABEL: global_agent_atomic_fsub_ret_v2bf16:
14760 ; GFX6: ; %bb.0:
14761 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14762 ; GFX6-NEXT: s_mov_b32 s6, 0
14763 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
14764 ; GFX6-NEXT: s_mov_b32 s4, s6
14765 ; GFX6-NEXT: s_mov_b32 s5, s6
14766 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
14767 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
14768 ; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v3
14769 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
14770 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
14771 ; GFX6-NEXT: s_waitcnt vmcnt(0)
14772 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
14773 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
14774 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14775 ; GFX6-NEXT: .LBB50_1: ; %atomicrmw.start
14776 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
14777 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
14778 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
14779 ; GFX6-NEXT: s_waitcnt expcnt(0)
14780 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
14781 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
14782 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
14783 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v5
14784 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v4
14785 ; GFX6-NEXT: v_alignbit_b32 v3, v2, v3, 16
14786 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v7
14787 ; GFX6-NEXT: v_alignbit_b32 v2, v2, v6, 16
14788 ; GFX6-NEXT: v_mov_b32_e32 v7, v3
14789 ; GFX6-NEXT: v_mov_b32_e32 v6, v2
14790 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
14791 ; GFX6-NEXT: s_waitcnt vmcnt(0)
14792 ; GFX6-NEXT: buffer_wbinvl1
14793 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v3
14794 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
14795 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
14796 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14797 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
14798 ; GFX6-NEXT: s_cbranch_execnz .LBB50_1
14799 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
14800 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
14801 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
14802 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
14803 ; GFX6-NEXT: s_waitcnt expcnt(0)
14804 ; GFX6-NEXT: s_setpc_b64 s[30:31]
14805 %result = atomicrmw fsub ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst
14806 ret <2 x bfloat> %result
14807 }
14809 define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
14810 ; GFX12-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
14811 ; GFX12: ; %bb.0:
14812 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
14813 ; GFX12-NEXT: s_wait_expcnt 0x0
14814 ; GFX12-NEXT: s_wait_samplecnt 0x0
14815 ; GFX12-NEXT: s_wait_bvhcnt 0x0
14816 ; GFX12-NEXT: s_wait_kmcnt 0x0
14817 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
14818 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14819 ; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14820 ; GFX12-NEXT: s_mov_b32 s1, 0
14821 ; GFX12-NEXT: .LBB51_1: ; %atomicrmw.start
14822 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
14823 ; GFX12-NEXT: s_wait_loadcnt 0x0
14824 ; GFX12-NEXT: v_mov_b32_e32 v6, v3
14825 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14826 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14827 ; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
14828 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14829 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
14830 ; GFX12-NEXT: v_bfe_u32 v8, v5, 16, 1
14831 ; GFX12-NEXT: v_sub_f32_e32 v3, v3, v4
14832 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v5
14833 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
14834 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14835 ; GFX12-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
14836 ; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
14837 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v3
14838 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v3, v3
14839 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14840 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
14841 ; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
14842 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14843 ; GFX12-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
14844 ; GFX12-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
14845 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
14846 ; GFX12-NEXT: s_wait_storecnt 0x0
14847 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
14848 ; GFX12-NEXT: s_wait_loadcnt 0x0
14849 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
14850 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
14851 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
14852 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
14853 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
14854 ; GFX12-NEXT: s_cbranch_execnz .LBB51_1
14855 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
14856 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
14857 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
14858 ; GFX12-NEXT: s_setpc_b64 s[30:31]
14859 ;
14860 ; GFX940-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
14861 ; GFX940: ; %bb.0:
14862 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14863 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
14864 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
14865 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14866 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
14867 ; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14868 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
14869 ; GFX940-NEXT: .LBB51_1: ; %atomicrmw.start
14870 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
14871 ; GFX940-NEXT: s_waitcnt vmcnt(0)
14872 ; GFX940-NEXT: v_mov_b32_e32 v7, v3
14873 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 16, v7
14874 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
14875 ; GFX940-NEXT: v_sub_f32_e32 v3, v3, v4
14876 ; GFX940-NEXT: v_sub_f32_e32 v5, v5, v2
14877 ; GFX940-NEXT: v_bfe_u32 v6, v3, 16, 1
14878 ; GFX940-NEXT: v_bfe_u32 v9, v5, 16, 1
14879 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v3
14880 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v5
14881 ; GFX940-NEXT: v_add3_u32 v6, v6, v3, s4
14882 ; GFX940-NEXT: v_add3_u32 v9, v9, v5, s4
14883 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
14884 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
14885 ; GFX940-NEXT: s_nop 0
14886 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
14887 ; GFX940-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[0:1]
14888 ; GFX940-NEXT: v_perm_b32 v6, v5, v3, s5
14889 ; GFX940-NEXT: buffer_wbl2 sc1
14890 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off offset:2044 sc0
14891 ; GFX940-NEXT: s_waitcnt vmcnt(0)
14892 ; GFX940-NEXT: buffer_inv sc1
14893 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
14894 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
14895 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
14896 ; GFX940-NEXT: s_cbranch_execnz .LBB51_1
14897 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
14898 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
14899 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
14900 ; GFX940-NEXT: s_setpc_b64 s[30:31]
14901 ;
14902 ; GFX11-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
14903 ; GFX11: ; %bb.0:
14904 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14905 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
14906 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14907 ; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14908 ; GFX11-NEXT: s_mov_b32 s1, 0
14909 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
14910 ; GFX11-NEXT: .p2align 6
14911 ; GFX11-NEXT: .LBB51_1: ; %atomicrmw.start
14912 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
14913 ; GFX11-NEXT: s_waitcnt vmcnt(0)
14914 ; GFX11-NEXT: v_mov_b32_e32 v6, v3
14915 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14916 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14917 ; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
14918 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14919 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
14920 ; GFX11-NEXT: v_bfe_u32 v8, v5, 16, 1
14921 ; GFX11-NEXT: v_sub_f32_e32 v3, v3, v4
14922 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v5
14923 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
14924 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14925 ; GFX11-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
14926 ; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
14927 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v3
14928 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v3, v3
14929 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14930 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
14931 ; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
14932 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14933 ; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
14934 ; GFX11-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
14935 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
14936 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
14937 ; GFX11-NEXT: s_waitcnt vmcnt(0)
14938 ; GFX11-NEXT: buffer_gl1_inv
14939 ; GFX11-NEXT: buffer_gl0_inv
14940 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
14941 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
14942 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
14943 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
14944 ; GFX11-NEXT: s_cbranch_execnz .LBB51_1
14945 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
14946 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
14947 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
14948 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
14949 ; GFX11-NEXT: s_setpc_b64 s[30:31]
14950 ;
14951 ; GFX10-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
14952 ; GFX10: ; %bb.0:
14953 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14954 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
14955 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14956 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14957 ; GFX10-NEXT: s_mov_b32 s5, 0
14958 ; GFX10-NEXT: .LBB51_1: ; %atomicrmw.start
14959 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
14960 ; GFX10-NEXT: s_waitcnt vmcnt(0)
14961 ; GFX10-NEXT: v_mov_b32_e32 v6, v3
14962 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14963 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14964 ; GFX10-NEXT: v_sub_f32_e32 v3, v3, v4
14965 ; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
14966 ; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
14967 ; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
14968 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
14969 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
14970 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
14971 ; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
14972 ; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
14973 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
14974 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
14975 ; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
14976 ; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
14977 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
14978 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
14979 ; GFX10-NEXT: s_waitcnt vmcnt(0)
14980 ; GFX10-NEXT: buffer_gl1_inv
14981 ; GFX10-NEXT: buffer_gl0_inv
14982 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
14983 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
14984 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
14985 ; GFX10-NEXT: s_cbranch_execnz .LBB51_1
14986 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
14987 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
14988 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
14989 ; GFX10-NEXT: s_setpc_b64 s[30:31]
14990 ;
14991 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
14992 ; GFX90A: ; %bb.0:
14993 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14994 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
14995 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
14996 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14997 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
14998 ; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14999 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
15000 ; GFX90A-NEXT: .LBB51_1: ; %atomicrmw.start
15001 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
15002 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
15003 ; GFX90A-NEXT: v_mov_b32_e32 v7, v3
15004 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v7
15005 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
15006 ; GFX90A-NEXT: v_sub_f32_e32 v3, v3, v4
15007 ; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v2
15008 ; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
15009 ; GFX90A-NEXT: v_bfe_u32 v9, v5, 16, 1
15010 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v3
15011 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v5
15012 ; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
15013 ; GFX90A-NEXT: v_add3_u32 v9, v9, v5, s8
15014 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
15015 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
15016 ; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[4:5]
15017 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
15018 ; GFX90A-NEXT: v_perm_b32 v6, v5, v3, s9
15019 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off offset:2044 glc
15020 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
15021 ; GFX90A-NEXT: buffer_wbinvl1
15022 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
15023 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15024 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
15025 ; GFX90A-NEXT: s_cbranch_execnz .LBB51_1
15026 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
15027 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
15028 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
15029 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
15030 ;
15031 ; GFX908-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
15032 ; GFX908: ; %bb.0:
15033 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15034 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
15035 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
15036 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15037 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
15038 ; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15039 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
15040 ; GFX908-NEXT: .LBB51_1: ; %atomicrmw.start
15041 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
15042 ; GFX908-NEXT: s_waitcnt vmcnt(0)
15043 ; GFX908-NEXT: v_mov_b32_e32 v6, v3
15044 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
15045 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15046 ; GFX908-NEXT: v_sub_f32_e32 v3, v3, v4
15047 ; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
15048 ; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
15049 ; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
15050 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
15051 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
15052 ; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
15053 ; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
15054 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
15055 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
15056 ; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
15057 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
15058 ; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
15059 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
15060 ; GFX908-NEXT: s_waitcnt vmcnt(0)
15061 ; GFX908-NEXT: buffer_wbinvl1
15062 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
15063 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15064 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
15065 ; GFX908-NEXT: s_cbranch_execnz .LBB51_1
15066 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
15067 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
15068 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
15069 ; GFX908-NEXT: s_setpc_b64 s[30:31]
15070 ;
15071 ; GFX8-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
15072 ; GFX8: ; %bb.0:
15073 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15074 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
15075 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
15076 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
15077 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
15078 ; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
15079 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15080 ; GFX8-NEXT: .LBB51_1: ; %atomicrmw.start
15081 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
15082 ; GFX8-NEXT: s_waitcnt vmcnt(0)
15083 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
15084 ; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
15085 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15086 ; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1
15087 ; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
15088 ; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
15089 ; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
15090 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
15091 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
15092 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
15093 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
15094 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
15095 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
15096 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
15097 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
15098 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
15099 ; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
15100 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
15101 ; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
15102 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
15103 ; GFX8-NEXT: s_waitcnt vmcnt(0)
15104 ; GFX8-NEXT: buffer_wbinvl1
15105 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
15106 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15107 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
15108 ; GFX8-NEXT: s_cbranch_execnz .LBB51_1
15109 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
15110 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
15111 ; GFX8-NEXT: s_setpc_b64 s[30:31]
15112 ;
15113 ; GFX7-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
15114 ; GFX7: ; %bb.0:
15115 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15116 ; GFX7-NEXT: s_mov_b32 s6, 0
15117 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
15118 ; GFX7-NEXT: s_mov_b32 s4, s6
15119 ; GFX7-NEXT: s_mov_b32 s5, s6
15120 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
15121 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
15122 ; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v3
15123 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
15124 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
15125 ; GFX7-NEXT: s_waitcnt vmcnt(0)
15126 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
15127 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
15128 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15129 ; GFX7-NEXT: .LBB51_1: ; %atomicrmw.start
15130 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
15131 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
15132 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
15133 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
15134 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15135 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
15136 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v5
15137 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v4
15138 ; GFX7-NEXT: v_alignbit_b32 v3, v2, v3, 16
15139 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v7
15140 ; GFX7-NEXT: v_alignbit_b32 v2, v2, v6, 16
15141 ; GFX7-NEXT: v_mov_b32_e32 v7, v3
15142 ; GFX7-NEXT: v_mov_b32_e32 v6, v2
15143 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
15144 ; GFX7-NEXT: s_waitcnt vmcnt(0)
15145 ; GFX7-NEXT: buffer_wbinvl1
15146 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v3
15147 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
15148 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
15149 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v6
15150 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
15151 ; GFX7-NEXT: s_cbranch_execnz .LBB51_1
15152 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
15153 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
15154 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
15155 ; GFX7-NEXT: v_mov_b32_e32 v1, v2
15156 ; GFX7-NEXT: s_setpc_b64 s[30:31]
15157 ;
15158 ; GFX6-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
15159 ; GFX6: ; %bb.0:
15160 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15161 ; GFX6-NEXT: s_mov_b32 s6, 0
15162 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
15163 ; GFX6-NEXT: s_mov_b32 s4, s6
15164 ; GFX6-NEXT: s_mov_b32 s5, s6
15165 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
15166 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
15167 ; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v3
15168 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
15169 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
15170 ; GFX6-NEXT: s_waitcnt vmcnt(0)
15171 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
15172 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
15173 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15174 ; GFX6-NEXT: .LBB51_1: ; %atomicrmw.start
15175 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
15176 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
15177 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
15178 ; GFX6-NEXT: s_waitcnt expcnt(0)
15179 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
15180 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15181 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
15182 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v5
15183 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v4
15184 ; GFX6-NEXT: v_alignbit_b32 v3, v2, v3, 16
15185 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v7
15186 ; GFX6-NEXT: v_alignbit_b32 v2, v2, v6, 16
15187 ; GFX6-NEXT: v_mov_b32_e32 v7, v3
15188 ; GFX6-NEXT: v_mov_b32_e32 v6, v2
15189 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
15190 ; GFX6-NEXT: s_waitcnt vmcnt(0)
15191 ; GFX6-NEXT: buffer_wbinvl1
15192 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v3
15193 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
15194 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
15195 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v6
15196 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
15197 ; GFX6-NEXT: s_cbranch_execnz .LBB51_1
15198 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
15199 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
15200 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
15201 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
15202 ; GFX6-NEXT: s_waitcnt expcnt(0)
15203 ; GFX6-NEXT: s_setpc_b64 s[30:31]
15204 %gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
15205 %result = atomicrmw fsub ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
15206 ret <2 x bfloat> %result
15207 }
15209 define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
15210 ; GFX12-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
15211 ; GFX12: ; %bb.0:
15212 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
15213 ; GFX12-NEXT: s_wait_expcnt 0x0
15214 ; GFX12-NEXT: s_wait_samplecnt 0x0
15215 ; GFX12-NEXT: s_wait_bvhcnt 0x0
15216 ; GFX12-NEXT: s_wait_kmcnt 0x0
15217 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
15218 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15219 ; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15220 ; GFX12-NEXT: s_mov_b32 s1, 0
15221 ; GFX12-NEXT: .LBB52_1: ; %atomicrmw.start
15222 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
15223 ; GFX12-NEXT: s_wait_loadcnt 0x0
15224 ; GFX12-NEXT: v_mov_b32_e32 v6, v3
15225 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
15226 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15227 ; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
15228 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v6
15229 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15230 ; GFX12-NEXT: v_bfe_u32 v8, v5, 16, 1
15231 ; GFX12-NEXT: v_sub_f32_e32 v3, v3, v4
15232 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v5
15233 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
15234 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
15235 ; GFX12-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
15236 ; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
15237 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v3
15238 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v3, v3
15239 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
15240 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
15241 ; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
15242 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
15243 ; GFX12-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
15244 ; GFX12-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
15245 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
15246 ; GFX12-NEXT: s_wait_storecnt 0x0
15247 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
15248 ; GFX12-NEXT: s_wait_loadcnt 0x0
15249 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
15250 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
15251 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
15252 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
15253 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
15254 ; GFX12-NEXT: s_cbranch_execnz .LBB52_1
15255 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
15256 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
15257 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
15258 ; GFX12-NEXT: s_setpc_b64 s[30:31]
15259 ;
15260 ; GFX940-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
15261 ; GFX940: ; %bb.0:
15262 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15263 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
15264 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
15265 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15266 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
15267 ; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15268 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
15269 ; GFX940-NEXT: .LBB52_1: ; %atomicrmw.start
15270 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
15271 ; GFX940-NEXT: s_waitcnt vmcnt(0)
15272 ; GFX940-NEXT: v_mov_b32_e32 v7, v3
15273 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 16, v7
15274 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
15275 ; GFX940-NEXT: v_sub_f32_e32 v3, v3, v4
15276 ; GFX940-NEXT: v_sub_f32_e32 v5, v5, v2
15277 ; GFX940-NEXT: v_bfe_u32 v6, v3, 16, 1
15278 ; GFX940-NEXT: v_bfe_u32 v9, v5, 16, 1
15279 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v3
15280 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v5
15281 ; GFX940-NEXT: v_add3_u32 v6, v6, v3, s4
15282 ; GFX940-NEXT: v_add3_u32 v9, v9, v5, s4
15283 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
15284 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
15285 ; GFX940-NEXT: s_nop 0
15286 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
15287 ; GFX940-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[0:1]
15288 ; GFX940-NEXT: v_perm_b32 v6, v5, v3, s5
15289 ; GFX940-NEXT: buffer_wbl2 sc1
15290 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off offset:-2048 sc0
15291 ; GFX940-NEXT: s_waitcnt vmcnt(0)
15292 ; GFX940-NEXT: buffer_inv sc1
15293 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
15294 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
15295 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
15296 ; GFX940-NEXT: s_cbranch_execnz .LBB52_1
15297 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
15298 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
15299 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
15300 ; GFX940-NEXT: s_setpc_b64 s[30:31]
15301 ;
15302 ; GFX11-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
15303 ; GFX11: ; %bb.0:
15304 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15305 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
15306 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15307 ; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15308 ; GFX11-NEXT: s_mov_b32 s1, 0
15309 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
15310 ; GFX11-NEXT: .p2align 6
15311 ; GFX11-NEXT: .LBB52_1: ; %atomicrmw.start
15312 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
15313 ; GFX11-NEXT: s_waitcnt vmcnt(0)
15314 ; GFX11-NEXT: v_mov_b32_e32 v6, v3
15315 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
15316 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15317 ; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
15318 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v6
15319 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15320 ; GFX11-NEXT: v_bfe_u32 v8, v5, 16, 1
15321 ; GFX11-NEXT: v_sub_f32_e32 v3, v3, v4
15322 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v5
15323 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
15324 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
15325 ; GFX11-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
15326 ; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
15327 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v3
15328 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v3, v3
15329 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
15330 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
15331 ; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
15332 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
15333 ; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
15334 ; GFX11-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
15335 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
15336 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 glc
15337 ; GFX11-NEXT: s_waitcnt vmcnt(0)
15338 ; GFX11-NEXT: buffer_gl1_inv
15339 ; GFX11-NEXT: buffer_gl0_inv
15340 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
15341 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
15342 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
15343 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
15344 ; GFX11-NEXT: s_cbranch_execnz .LBB52_1
15345 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
15346 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
15347 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
15348 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
15349 ; GFX11-NEXT: s_setpc_b64 s[30:31]
15350 ;
15351 ; GFX10-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
15352 ; GFX10: ; %bb.0:
15353 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15354 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
15355 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15356 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15357 ; GFX10-NEXT: s_mov_b32 s5, 0
15358 ; GFX10-NEXT: .LBB52_1: ; %atomicrmw.start
15359 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
15360 ; GFX10-NEXT: s_waitcnt vmcnt(0)
15361 ; GFX10-NEXT: v_mov_b32_e32 v6, v3
15362 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
15363 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15364 ; GFX10-NEXT: v_sub_f32_e32 v3, v3, v4
15365 ; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
15366 ; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
15367 ; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
15368 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
15369 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
15370 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
15371 ; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
15372 ; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
15373 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
15374 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
15375 ; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
15376 ; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
15377 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
15378 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:-2048 glc
15379 ; GFX10-NEXT: s_waitcnt vmcnt(0)
15380 ; GFX10-NEXT: buffer_gl1_inv
15381 ; GFX10-NEXT: buffer_gl0_inv
15382 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
15383 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
15384 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
15385 ; GFX10-NEXT: s_cbranch_execnz .LBB52_1
15386 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
15387 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
15388 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
15389 ; GFX10-NEXT: s_setpc_b64 s[30:31]
15390 ;
15391 ; GFX90A-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
15392 ; GFX90A: ; %bb.0:
15393 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15394 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
15395 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
15396 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15397 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
15398 ; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15399 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
15400 ; GFX90A-NEXT: .LBB52_1: ; %atomicrmw.start
15401 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
15402 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
15403 ; GFX90A-NEXT: v_mov_b32_e32 v7, v3
15404 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v7
15405 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
15406 ; GFX90A-NEXT: v_sub_f32_e32 v3, v3, v4
15407 ; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v2
15408 ; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
15409 ; GFX90A-NEXT: v_bfe_u32 v9, v5, 16, 1
15410 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v3
15411 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v5
15412 ; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
15413 ; GFX90A-NEXT: v_add3_u32 v9, v9, v5, s8
15414 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
15415 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
15416 ; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[4:5]
15417 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
15418 ; GFX90A-NEXT: v_perm_b32 v6, v5, v3, s9
15419 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off offset:-2048 glc
15420 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
15421 ; GFX90A-NEXT: buffer_wbinvl1
15422 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
15423 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15424 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
15425 ; GFX90A-NEXT: s_cbranch_execnz .LBB52_1
15426 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
15427 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
15428 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
15429 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
15430 ;
15431 ; GFX908-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
15432 ; GFX908: ; %bb.0:
15433 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15434 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
15435 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
15436 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15437 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
15438 ; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15439 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
15440 ; GFX908-NEXT: .LBB52_1: ; %atomicrmw.start
15441 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
15442 ; GFX908-NEXT: s_waitcnt vmcnt(0)
15443 ; GFX908-NEXT: v_mov_b32_e32 v6, v3
15444 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
15445 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15446 ; GFX908-NEXT: v_sub_f32_e32 v3, v3, v4
15447 ; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
15448 ; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
15449 ; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
15450 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
15451 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
15452 ; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
15453 ; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
15454 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
15455 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
15456 ; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
15457 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
15458 ; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
15459 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:-2048 glc
15460 ; GFX908-NEXT: s_waitcnt vmcnt(0)
15461 ; GFX908-NEXT: buffer_wbinvl1
15462 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
15463 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15464 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
15465 ; GFX908-NEXT: s_cbranch_execnz .LBB52_1
15466 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
15467 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
15468 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
15469 ; GFX908-NEXT: s_setpc_b64 s[30:31]
15470 ;
15471 ; GFX8-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
15472 ; GFX8: ; %bb.0:
15473 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15474 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
15475 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
15476 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
15477 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
15478 ; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
15479 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15480 ; GFX8-NEXT: .LBB52_1: ; %atomicrmw.start
15481 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
15482 ; GFX8-NEXT: s_waitcnt vmcnt(0)
15483 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
15484 ; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
15485 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15486 ; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1
15487 ; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
15488 ; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
15489 ; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
15490 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
15491 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
15492 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
15493 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
15494 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
15495 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
15496 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
15497 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
15498 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
15499 ; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
15500 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
15501 ; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
15502 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
15503 ; GFX8-NEXT: s_waitcnt vmcnt(0)
15504 ; GFX8-NEXT: buffer_wbinvl1
15505 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
15506 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15507 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
15508 ; GFX8-NEXT: s_cbranch_execnz .LBB52_1
15509 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
15510 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
15511 ; GFX8-NEXT: s_setpc_b64 s[30:31]
15512 ;
15513 ; GFX7-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
15514 ; GFX7: ; %bb.0:
15515 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15516 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
15517 ; GFX7-NEXT: s_mov_b32 s5, -1
15518 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
15519 ; GFX7-NEXT: s_mov_b32 s6, 0
15520 ; GFX7-NEXT: buffer_load_dword v6, v[0:1], s[4:7], 0 addr64
15521 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
15522 ; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v2
15523 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
15524 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
15525 ; GFX7-NEXT: s_mov_b32 s4, s6
15526 ; GFX7-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
15527 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
15528 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
15529 ; GFX7-NEXT: s_mov_b32 s5, s6
15530 ; GFX7-NEXT: s_waitcnt vmcnt(0)
15531 ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
15532 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v6
15533 ; GFX7-NEXT: .LBB52_1: ; %atomicrmw.start
15534 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
15535 ; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
15536 ; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
15537 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
15538 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
15539 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
15540 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
15541 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
15542 ; GFX7-NEXT: v_alignbit_b32 v1, v1, v0, 16
15543 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v7
15544 ; GFX7-NEXT: v_alignbit_b32 v0, v0, v6, 16
15545 ; GFX7-NEXT: v_mov_b32_e32 v7, v1
15546 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
15547 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[4:5], s[4:7], 0 addr64 glc
15548 ; GFX7-NEXT: s_waitcnt vmcnt(0)
15549 ; GFX7-NEXT: buffer_wbinvl1
15550 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v1
15551 ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
15552 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
15553 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v6
15554 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
15555 ; GFX7-NEXT: s_cbranch_execnz .LBB52_1
15556 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
15557 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
15558 ; GFX7-NEXT: s_setpc_b64 s[30:31]
15559 ;
15560 ; GFX6-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
15561 ; GFX6: ; %bb.0:
15562 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15563 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
15564 ; GFX6-NEXT: s_mov_b32 s5, -1
15565 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
15566 ; GFX6-NEXT: s_mov_b32 s6, 0
15567 ; GFX6-NEXT: buffer_load_dword v6, v[0:1], s[4:7], 0 addr64
15568 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
15569 ; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v2
15570 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
15571 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
15572 ; GFX6-NEXT: s_mov_b32 s4, s6
15573 ; GFX6-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
15574 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
15575 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
15576 ; GFX6-NEXT: s_mov_b32 s5, s6
15577 ; GFX6-NEXT: s_waitcnt vmcnt(0)
15578 ; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
15579 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v6
15580 ; GFX6-NEXT: .LBB52_1: ; %atomicrmw.start
15581 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
15582 ; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
15583 ; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
15584 ; GFX6-NEXT: s_waitcnt expcnt(0)
15585 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
15586 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
15587 ; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
15588 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
15589 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
15590 ; GFX6-NEXT: v_alignbit_b32 v1, v1, v0, 16
15591 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v7
15592 ; GFX6-NEXT: v_alignbit_b32 v0, v0, v6, 16
15593 ; GFX6-NEXT: v_mov_b32_e32 v7, v1
15594 ; GFX6-NEXT: v_mov_b32_e32 v6, v0
15595 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[4:5], s[4:7], 0 addr64 glc
15596 ; GFX6-NEXT: s_waitcnt vmcnt(0)
15597 ; GFX6-NEXT: buffer_wbinvl1
15598 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v1
15599 ; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
15600 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
15601 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v6
15602 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
15603 ; GFX6-NEXT: s_cbranch_execnz .LBB52_1
15604 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
15605 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
15606 ; GFX6-NEXT: s_waitcnt expcnt(0)
15607 ; GFX6-NEXT: s_setpc_b64 s[30:31]
15608 %gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 -512
15609 %result = atomicrmw fsub ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
15610 ret <2 x bfloat> %result
15611 }
15613 define void @global_agent_atomic_fsub_noret_v2bf16(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
15614 ; GFX12-LABEL: global_agent_atomic_fsub_noret_v2bf16:
15615 ; GFX12: ; %bb.0:
15616 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
15617 ; GFX12-NEXT: s_wait_expcnt 0x0
15618 ; GFX12-NEXT: s_wait_samplecnt 0x0
15619 ; GFX12-NEXT: s_wait_bvhcnt 0x0
15620 ; GFX12-NEXT: s_wait_kmcnt 0x0
15621 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
15622 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15623 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15624 ; GFX12-NEXT: s_mov_b32 s1, 0
15625 ; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start
15626 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
15627 ; GFX12-NEXT: s_wait_loadcnt 0x0
15628 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15629 ; GFX12-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15630 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15631 ; GFX12-NEXT: v_sub_f32_e32 v2, v2, v4
15632 ; GFX12-NEXT: v_sub_f32_e32 v6, v6, v5
15633 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15634 ; GFX12-NEXT: v_bfe_u32 v7, v2, 16, 1
15635 ; GFX12-NEXT: v_bfe_u32 v8, v6, 16, 1
15636 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v2
15637 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v6
15638 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
15639 ; GFX12-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
15640 ; GFX12-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
15641 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v2, v2
15642 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15643 ; GFX12-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
15644 ; GFX12-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
15645 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
15646 ; GFX12-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
15647 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
15648 ; GFX12-NEXT: s_wait_storecnt 0x0
15649 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
15650 ; GFX12-NEXT: s_wait_loadcnt 0x0
15651 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
15652 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
15653 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
15654 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
15655 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
15656 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
15657 ; GFX12-NEXT: s_cbranch_execnz .LBB53_1
15658 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
15659 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
15660 ; GFX12-NEXT: s_setpc_b64 s[30:31]
15661 ;
15662 ; GFX940-LABEL: global_agent_atomic_fsub_noret_v2bf16:
15663 ; GFX940: ; %bb.0:
15664 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15665 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
15666 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
15667 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15668 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
15669 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15670 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
15671 ; GFX940-NEXT: .LBB53_1: ; %atomicrmw.start
15672 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
15673 ; GFX940-NEXT: s_waitcnt vmcnt(0)
15674 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15675 ; GFX940-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15676 ; GFX940-NEXT: v_sub_f32_e32 v2, v2, v4
15677 ; GFX940-NEXT: v_sub_f32_e32 v6, v6, v5
15678 ; GFX940-NEXT: v_bfe_u32 v7, v2, 16, 1
15679 ; GFX940-NEXT: v_bfe_u32 v9, v6, 16, 1
15680 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v2
15681 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v6
15682 ; GFX940-NEXT: v_add3_u32 v7, v7, v2, s4
15683 ; GFX940-NEXT: v_add3_u32 v9, v9, v6, s4
15684 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
15685 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
15686 ; GFX940-NEXT: s_nop 0
15687 ; GFX940-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
15688 ; GFX940-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
15689 ; GFX940-NEXT: v_perm_b32 v2, v6, v2, s5
15690 ; GFX940-NEXT: buffer_wbl2 sc1
15691 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
15692 ; GFX940-NEXT: s_waitcnt vmcnt(0)
15693 ; GFX940-NEXT: buffer_inv sc1
15694 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
15695 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
15696 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
15697 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
15698 ; GFX940-NEXT: s_cbranch_execnz .LBB53_1
15699 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
15700 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
15701 ; GFX940-NEXT: s_setpc_b64 s[30:31]
15702 ;
15703 ; GFX11-LABEL: global_agent_atomic_fsub_noret_v2bf16:
15704 ; GFX11: ; %bb.0:
15705 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15706 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
15707 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15708 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15709 ; GFX11-NEXT: s_mov_b32 s1, 0
15710 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
15711 ; GFX11-NEXT: .p2align 6
15712 ; GFX11-NEXT: .LBB53_1: ; %atomicrmw.start
15713 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
15714 ; GFX11-NEXT: s_waitcnt vmcnt(0)
15715 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15716 ; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15717 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15718 ; GFX11-NEXT: v_sub_f32_e32 v2, v2, v4
15719 ; GFX11-NEXT: v_sub_f32_e32 v6, v6, v5
15720 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15721 ; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
15722 ; GFX11-NEXT: v_bfe_u32 v8, v6, 16, 1
15723 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v2
15724 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v6
15725 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
15726 ; GFX11-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
15727 ; GFX11-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
15728 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v2, v2
15729 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15730 ; GFX11-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
15731 ; GFX11-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
15732 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
15733 ; GFX11-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
15734 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
15735 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
15736 ; GFX11-NEXT: s_waitcnt vmcnt(0)
15737 ; GFX11-NEXT: buffer_gl1_inv
15738 ; GFX11-NEXT: buffer_gl0_inv
15739 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
15740 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
15741 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
15742 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
15743 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
15744 ; GFX11-NEXT: s_cbranch_execnz .LBB53_1
15745 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
15746 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
15747 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
15748 ; GFX11-NEXT: s_setpc_b64 s[30:31]
15749 ;
15750 ; GFX10-LABEL: global_agent_atomic_fsub_noret_v2bf16:
15751 ; GFX10: ; %bb.0:
15752 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15753 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
15754 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15755 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15756 ; GFX10-NEXT: s_mov_b32 s5, 0
15757 ; GFX10-NEXT: .LBB53_1: ; %atomicrmw.start
15758 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
15759 ; GFX10-NEXT: s_waitcnt vmcnt(0)
15760 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15761 ; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15762 ; GFX10-NEXT: v_sub_f32_e32 v2, v2, v4
15763 ; GFX10-NEXT: v_sub_f32_e32 v6, v6, v5
15764 ; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
15765 ; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
15766 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
15767 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
15768 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
15769 ; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
15770 ; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
15771 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
15772 ; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
15773 ; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
15774 ; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
15775 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
15776 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
15777 ; GFX10-NEXT: s_waitcnt vmcnt(0)
15778 ; GFX10-NEXT: buffer_gl1_inv
15779 ; GFX10-NEXT: buffer_gl0_inv
15780 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
15781 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
15782 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
15783 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
15784 ; GFX10-NEXT: s_cbranch_execnz .LBB53_1
15785 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
15786 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
15787 ; GFX10-NEXT: s_setpc_b64 s[30:31]
15788 ;
15789 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_v2bf16:
15790 ; GFX90A: ; %bb.0:
15791 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15792 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
15793 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
15794 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15795 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
15796 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15797 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
15798 ; GFX90A-NEXT: .LBB53_1: ; %atomicrmw.start
15799 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
15800 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
15801 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15802 ; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15803 ; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
15804 ; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
15805 ; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
15806 ; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
15807 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
15808 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
15809 ; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
15810 ; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
15811 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
15812 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
15813 ; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
15814 ; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
15815 ; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
15816 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
15817 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
15818 ; GFX90A-NEXT: buffer_wbinvl1
15819 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
15820 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15821 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
15822 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
15823 ; GFX90A-NEXT: s_cbranch_execnz .LBB53_1
15824 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
15825 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
15826 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
15827 ;
15828 ; GFX908-LABEL: global_agent_atomic_fsub_noret_v2bf16:
15829 ; GFX908: ; %bb.0:
15830 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15831 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
15832 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
15833 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15834 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
15835 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15836 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
15837 ; GFX908-NEXT: .LBB53_1: ; %atomicrmw.start
15838 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
15839 ; GFX908-NEXT: s_waitcnt vmcnt(0)
15840 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15841 ; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15842 ; GFX908-NEXT: v_sub_f32_e32 v2, v2, v4
15843 ; GFX908-NEXT: v_sub_f32_e32 v6, v6, v5
15844 ; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
15845 ; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
15846 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
15847 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
15848 ; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
15849 ; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
15850 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
15851 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
15852 ; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
15853 ; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
15854 ; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
15855 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
15856 ; GFX908-NEXT: s_waitcnt vmcnt(0)
15857 ; GFX908-NEXT: buffer_wbinvl1
15858 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
15859 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15860 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
15861 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
15862 ; GFX908-NEXT: s_cbranch_execnz .LBB53_1
15863 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
15864 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
15865 ; GFX908-NEXT: s_setpc_b64 s[30:31]
15866 ;
15867 ; GFX8-LABEL: global_agent_atomic_fsub_noret_v2bf16:
15868 ; GFX8: ; %bb.0:
15869 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15870 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
15871 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
15872 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15873 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15874 ; GFX8-NEXT: .LBB53_1: ; %atomicrmw.start
15875 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
15876 ; GFX8-NEXT: s_waitcnt vmcnt(0)
15877 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15878 ; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15879 ; GFX8-NEXT: v_sub_f32_e32 v2, v2, v4
15880 ; GFX8-NEXT: v_sub_f32_e32 v6, v6, v5
15881 ; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
15882 ; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
15883 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
15884 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
15885 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
15886 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
15887 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
15888 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
15889 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
15890 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
15891 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
15892 ; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
15893 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
15894 ; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
15895 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
15896 ; GFX8-NEXT: s_waitcnt vmcnt(0)
15897 ; GFX8-NEXT: buffer_wbinvl1
15898 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
15899 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15900 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
15901 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
15902 ; GFX8-NEXT: s_cbranch_execnz .LBB53_1
15903 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
15904 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
15905 ; GFX8-NEXT: s_setpc_b64 s[30:31]
15906 ;
15907 ; GFX7-LABEL: global_agent_atomic_fsub_noret_v2bf16:
15908 ; GFX7: ; %bb.0:
15909 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15910 ; GFX7-NEXT: s_mov_b32 s6, 0
15911 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
15912 ; GFX7-NEXT: s_mov_b32 s4, s6
15913 ; GFX7-NEXT: s_mov_b32 s5, s6
15914 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
15915 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
15916 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
15917 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
15918 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15919 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
15920 ; GFX7-NEXT: s_waitcnt vmcnt(0)
15921 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
15922 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
15923 ; GFX7-NEXT: .LBB53_1: ; %atomicrmw.start
15924 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
15925 ; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
15926 ; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
15927 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
15928 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
15929 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
15930 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
15931 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
15932 ; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
15933 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
15934 ; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
15935 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
15936 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
15937 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
15938 ; GFX7-NEXT: s_waitcnt vmcnt(0)
15939 ; GFX7-NEXT: buffer_wbinvl1
15940 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
15941 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
15942 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
15943 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
15944 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
15945 ; GFX7-NEXT: s_cbranch_execnz .LBB53_1
15946 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
15947 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
15948 ; GFX7-NEXT: s_setpc_b64 s[30:31]
15949 ;
15950 ; GFX6-LABEL: global_agent_atomic_fsub_noret_v2bf16:
15951 ; GFX6: ; %bb.0:
15952 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15953 ; GFX6-NEXT: s_mov_b32 s6, 0
15954 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
15955 ; GFX6-NEXT: s_mov_b32 s4, s6
15956 ; GFX6-NEXT: s_mov_b32 s5, s6
15957 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
15958 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
15959 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
15960 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
15961 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15962 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
15963 ; GFX6-NEXT: s_waitcnt vmcnt(0)
15964 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
15965 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
15966 ; GFX6-NEXT: .LBB53_1: ; %atomicrmw.start
15967 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
15968 ; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
15969 ; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
15970 ; GFX6-NEXT: s_waitcnt expcnt(0)
15971 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
15972 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
15973 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
15974 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
15975 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
15976 ; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
15977 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
15978 ; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
15979 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
15980 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
15981 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
15982 ; GFX6-NEXT: s_waitcnt vmcnt(0)
15983 ; GFX6-NEXT: buffer_wbinvl1
15984 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
15985 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
15986 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
15987 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
15988 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
15989 ; GFX6-NEXT: s_cbranch_execnz .LBB53_1
15990 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
15991 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
15992 ; GFX6-NEXT: s_waitcnt expcnt(0)
15993 ; GFX6-NEXT: s_setpc_b64 s[30:31]
15994 %unused = atomicrmw fsub ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst
15995 ret void
15996 }
15998 define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
15999 ; GFX12-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
16000 ; GFX12: ; %bb.0:
16001 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
16002 ; GFX12-NEXT: s_wait_expcnt 0x0
16003 ; GFX12-NEXT: s_wait_samplecnt 0x0
16004 ; GFX12-NEXT: s_wait_bvhcnt 0x0
16005 ; GFX12-NEXT: s_wait_kmcnt 0x0
16006 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
16007 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16008 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16009 ; GFX12-NEXT: s_mov_b32 s1, 0
16010 ; GFX12-NEXT: .LBB54_1: ; %atomicrmw.start
16011 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
16012 ; GFX12-NEXT: s_wait_loadcnt 0x0
16013 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16014 ; GFX12-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16015 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16016 ; GFX12-NEXT: v_sub_f32_e32 v2, v2, v4
16017 ; GFX12-NEXT: v_sub_f32_e32 v6, v6, v5
16018 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16019 ; GFX12-NEXT: v_bfe_u32 v7, v2, 16, 1
16020 ; GFX12-NEXT: v_bfe_u32 v8, v6, 16, 1
16021 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v2
16022 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v6
16023 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
16024 ; GFX12-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
16025 ; GFX12-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
16026 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v2, v2
16027 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16028 ; GFX12-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
16029 ; GFX12-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
16030 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
16031 ; GFX12-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
16032 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
16033 ; GFX12-NEXT: s_wait_storecnt 0x0
16034 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
16035 ; GFX12-NEXT: s_wait_loadcnt 0x0
16036 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
16037 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
16038 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
16039 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
16040 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
16041 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
16042 ; GFX12-NEXT: s_cbranch_execnz .LBB54_1
16043 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
16044 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
16045 ; GFX12-NEXT: s_setpc_b64 s[30:31]
16046 ;
16047 ; GFX940-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
16048 ; GFX940: ; %bb.0:
16049 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16050 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
16051 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
16052 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16053 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
16054 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16055 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
16056 ; GFX940-NEXT: .LBB54_1: ; %atomicrmw.start
16057 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
16058 ; GFX940-NEXT: s_waitcnt vmcnt(0)
16059 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16060 ; GFX940-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16061 ; GFX940-NEXT: v_sub_f32_e32 v2, v2, v4
16062 ; GFX940-NEXT: v_sub_f32_e32 v6, v6, v5
16063 ; GFX940-NEXT: v_bfe_u32 v7, v2, 16, 1
16064 ; GFX940-NEXT: v_bfe_u32 v9, v6, 16, 1
16065 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v2
16066 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v6
16067 ; GFX940-NEXT: v_add3_u32 v7, v7, v2, s4
16068 ; GFX940-NEXT: v_add3_u32 v9, v9, v6, s4
16069 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
16070 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
16071 ; GFX940-NEXT: s_nop 0
16072 ; GFX940-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
16073 ; GFX940-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
16074 ; GFX940-NEXT: v_perm_b32 v2, v6, v2, s5
16075 ; GFX940-NEXT: buffer_wbl2 sc1
16076 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
16077 ; GFX940-NEXT: s_waitcnt vmcnt(0)
16078 ; GFX940-NEXT: buffer_inv sc1
16079 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
16080 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
16081 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
16082 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
16083 ; GFX940-NEXT: s_cbranch_execnz .LBB54_1
16084 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
16085 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
16086 ; GFX940-NEXT: s_setpc_b64 s[30:31]
16087 ;
16088 ; GFX11-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
16089 ; GFX11: ; %bb.0:
16090 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16091 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
16092 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16093 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16094 ; GFX11-NEXT: s_mov_b32 s1, 0
16095 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
16096 ; GFX11-NEXT: .p2align 6
16097 ; GFX11-NEXT: .LBB54_1: ; %atomicrmw.start
16098 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
16099 ; GFX11-NEXT: s_waitcnt vmcnt(0)
16100 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16101 ; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16102 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16103 ; GFX11-NEXT: v_sub_f32_e32 v2, v2, v4
16104 ; GFX11-NEXT: v_sub_f32_e32 v6, v6, v5
16105 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16106 ; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
16107 ; GFX11-NEXT: v_bfe_u32 v8, v6, 16, 1
16108 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v2
16109 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v6
16110 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
16111 ; GFX11-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
16112 ; GFX11-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
16113 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v2, v2
16114 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16115 ; GFX11-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
16116 ; GFX11-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
16117 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
16118 ; GFX11-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
16119 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
16120 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
16121 ; GFX11-NEXT: s_waitcnt vmcnt(0)
16122 ; GFX11-NEXT: buffer_gl1_inv
16123 ; GFX11-NEXT: buffer_gl0_inv
16124 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
16125 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
16126 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
16127 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
16128 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
16129 ; GFX11-NEXT: s_cbranch_execnz .LBB54_1
16130 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
16131 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
16132 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
16133 ; GFX11-NEXT: s_setpc_b64 s[30:31]
16134 ;
16135 ; GFX10-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
16136 ; GFX10: ; %bb.0:
16137 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16138 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
16139 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16140 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16141 ; GFX10-NEXT: s_mov_b32 s5, 0
16142 ; GFX10-NEXT: .LBB54_1: ; %atomicrmw.start
16143 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
16144 ; GFX10-NEXT: s_waitcnt vmcnt(0)
16145 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16146 ; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16147 ; GFX10-NEXT: v_sub_f32_e32 v2, v2, v4
16148 ; GFX10-NEXT: v_sub_f32_e32 v6, v6, v5
16149 ; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
16150 ; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
16151 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
16152 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
16153 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
16154 ; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
16155 ; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
16156 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
16157 ; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
16158 ; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
16159 ; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
16160 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
16161 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
16162 ; GFX10-NEXT: s_waitcnt vmcnt(0)
16163 ; GFX10-NEXT: buffer_gl1_inv
16164 ; GFX10-NEXT: buffer_gl0_inv
16165 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
16166 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
16167 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
16168 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
16169 ; GFX10-NEXT: s_cbranch_execnz .LBB54_1
16170 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
16171 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
16172 ; GFX10-NEXT: s_setpc_b64 s[30:31]
16173 ;
16174 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
16175 ; GFX90A: ; %bb.0:
16176 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16177 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
16178 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
16179 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16180 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
16181 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16182 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
16183 ; GFX90A-NEXT: .LBB54_1: ; %atomicrmw.start
16184 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
16185 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
16186 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16187 ; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16188 ; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
16189 ; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
16190 ; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
16191 ; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
16192 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
16193 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
16194 ; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
16195 ; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
16196 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
16197 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
16198 ; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
16199 ; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
16200 ; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
16201 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
16202 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
16203 ; GFX90A-NEXT: buffer_wbinvl1
16204 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
16205 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
16206 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
16207 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
16208 ; GFX90A-NEXT: s_cbranch_execnz .LBB54_1
16209 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
16210 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
16211 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
16212 ;
16213 ; GFX908-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
16214 ; GFX908: ; %bb.0:
16215 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16216 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
16217 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
16218 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16219 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
16220 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16221 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
16222 ; GFX908-NEXT: .LBB54_1: ; %atomicrmw.start
16223 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
16224 ; GFX908-NEXT: s_waitcnt vmcnt(0)
16225 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16226 ; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16227 ; GFX908-NEXT: v_sub_f32_e32 v2, v2, v4
16228 ; GFX908-NEXT: v_sub_f32_e32 v6, v6, v5
16229 ; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
16230 ; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
16231 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
16232 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
16233 ; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
16234 ; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
16235 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
16236 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
16237 ; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
16238 ; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
16239 ; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
16240 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
16241 ; GFX908-NEXT: s_waitcnt vmcnt(0)
16242 ; GFX908-NEXT: buffer_wbinvl1
16243 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
16244 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
16245 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
16246 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
16247 ; GFX908-NEXT: s_cbranch_execnz .LBB54_1
16248 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
16249 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
16250 ; GFX908-NEXT: s_setpc_b64 s[30:31]
16251 ;
16252 ; GFX8-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
16253 ; GFX8: ; %bb.0:
16254 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16255 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
16256 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
16257 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
16258 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
16259 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16260 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16261 ; GFX8-NEXT: .LBB54_1: ; %atomicrmw.start
16262 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
16263 ; GFX8-NEXT: s_waitcnt vmcnt(0)
16264 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16265 ; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16266 ; GFX8-NEXT: v_sub_f32_e32 v2, v2, v4
16267 ; GFX8-NEXT: v_sub_f32_e32 v6, v6, v5
16268 ; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
16269 ; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
16270 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
16271 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
16272 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
16273 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
16274 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
16275 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
16276 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
16277 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
16278 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
16279 ; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
16280 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
16281 ; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
16282 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
16283 ; GFX8-NEXT: s_waitcnt vmcnt(0)
16284 ; GFX8-NEXT: buffer_wbinvl1
16285 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
16286 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
16287 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
16288 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
16289 ; GFX8-NEXT: s_cbranch_execnz .LBB54_1
16290 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
16291 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
16292 ; GFX8-NEXT: s_setpc_b64 s[30:31]
16293 ;
16294 ; GFX7-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
16295 ; GFX7: ; %bb.0:
16296 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16297 ; GFX7-NEXT: s_mov_b32 s6, 0
16298 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
16299 ; GFX7-NEXT: s_mov_b32 s4, s6
16300 ; GFX7-NEXT: s_mov_b32 s5, s6
16301 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
16302 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
16303 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
16304 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
16305 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16306 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
16307 ; GFX7-NEXT: s_waitcnt vmcnt(0)
16308 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
16309 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
16310 ; GFX7-NEXT: .LBB54_1: ; %atomicrmw.start
16311 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
16312 ; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
16313 ; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
16314 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
16315 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
16316 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
16317 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
16318 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
16319 ; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
16320 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
16321 ; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
16322 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
16323 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
16324 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
16325 ; GFX7-NEXT: s_waitcnt vmcnt(0)
16326 ; GFX7-NEXT: buffer_wbinvl1
16327 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
16328 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
16329 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
16330 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
16331 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
16332 ; GFX7-NEXT: s_cbranch_execnz .LBB54_1
16333 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
16334 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
16335 ; GFX7-NEXT: s_setpc_b64 s[30:31]
16336 ;
16337 ; GFX6-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
16338 ; GFX6: ; %bb.0:
16339 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16340 ; GFX6-NEXT: s_mov_b32 s6, 0
16341 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
16342 ; GFX6-NEXT: s_mov_b32 s4, s6
16343 ; GFX6-NEXT: s_mov_b32 s5, s6
16344 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
16345 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
16346 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
16347 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
16348 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16349 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
16350 ; GFX6-NEXT: s_waitcnt vmcnt(0)
16351 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
16352 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
16353 ; GFX6-NEXT: .LBB54_1: ; %atomicrmw.start
16354 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
16355 ; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
16356 ; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
16357 ; GFX6-NEXT: s_waitcnt expcnt(0)
16358 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
16359 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
16360 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
16361 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
16362 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
16363 ; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
16364 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
16365 ; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
16366 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
16367 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
16368 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
16369 ; GFX6-NEXT: s_waitcnt vmcnt(0)
16370 ; GFX6-NEXT: buffer_wbinvl1
16371 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
16372 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
16373 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
16374 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
16375 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
16376 ; GFX6-NEXT: s_cbranch_execnz .LBB54_1
16377 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
16378 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
16379 ; GFX6-NEXT: s_waitcnt expcnt(0)
16380 ; GFX6-NEXT: s_setpc_b64 s[30:31]
16381 %gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
16382 %unused = atomicrmw fsub ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
16383 ret void
16384 }
16386 define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
16387 ; GFX12-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
16388 ; GFX12: ; %bb.0:
16389 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
16390 ; GFX12-NEXT: s_wait_expcnt 0x0
16391 ; GFX12-NEXT: s_wait_samplecnt 0x0
16392 ; GFX12-NEXT: s_wait_bvhcnt 0x0
16393 ; GFX12-NEXT: s_wait_kmcnt 0x0
16394 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
16395 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16396 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16397 ; GFX12-NEXT: s_mov_b32 s1, 0
16398 ; GFX12-NEXT: .LBB55_1: ; %atomicrmw.start
16399 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
16400 ; GFX12-NEXT: s_wait_loadcnt 0x0
16401 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16402 ; GFX12-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16403 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16404 ; GFX12-NEXT: v_sub_f32_e32 v2, v2, v4
16405 ; GFX12-NEXT: v_sub_f32_e32 v6, v6, v5
16406 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16407 ; GFX12-NEXT: v_bfe_u32 v7, v2, 16, 1
16408 ; GFX12-NEXT: v_bfe_u32 v8, v6, 16, 1
16409 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v2
16410 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v6
16411 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
16412 ; GFX12-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
16413 ; GFX12-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
16414 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v2, v2
16415 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16416 ; GFX12-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
16417 ; GFX12-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
16418 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
16419 ; GFX12-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
16420 ; GFX12-NEXT: global_wb scope:SCOPE_DEV
16421 ; GFX12-NEXT: s_wait_storecnt 0x0
16422 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
16423 ; GFX12-NEXT: s_wait_loadcnt 0x0
16424 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
16425 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
16426 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
16427 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
16428 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
16429 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
16430 ; GFX12-NEXT: s_cbranch_execnz .LBB55_1
16431 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
16432 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
16433 ; GFX12-NEXT: s_setpc_b64 s[30:31]
16434 ;
16435 ; GFX940-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
16436 ; GFX940: ; %bb.0:
16437 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16438 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
16439 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
16440 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16441 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
16442 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16443 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
16444 ; GFX940-NEXT: .LBB55_1: ; %atomicrmw.start
16445 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
16446 ; GFX940-NEXT: s_waitcnt vmcnt(0)
16447 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16448 ; GFX940-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16449 ; GFX940-NEXT: v_sub_f32_e32 v2, v2, v4
16450 ; GFX940-NEXT: v_sub_f32_e32 v6, v6, v5
16451 ; GFX940-NEXT: v_bfe_u32 v7, v2, 16, 1
16452 ; GFX940-NEXT: v_bfe_u32 v9, v6, 16, 1
16453 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v2
16454 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v6
16455 ; GFX940-NEXT: v_add3_u32 v7, v7, v2, s4
16456 ; GFX940-NEXT: v_add3_u32 v9, v9, v6, s4
16457 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
16458 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
16459 ; GFX940-NEXT: s_nop 0
16460 ; GFX940-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
16461 ; GFX940-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
16462 ; GFX940-NEXT: v_perm_b32 v2, v6, v2, s5
16463 ; GFX940-NEXT: buffer_wbl2 sc1
16464 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
16465 ; GFX940-NEXT: s_waitcnt vmcnt(0)
16466 ; GFX940-NEXT: buffer_inv sc1
16467 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
16468 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
16469 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
16470 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
16471 ; GFX940-NEXT: s_cbranch_execnz .LBB55_1
16472 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
16473 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
16474 ; GFX940-NEXT: s_setpc_b64 s[30:31]
16475 ;
16476 ; GFX11-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
16477 ; GFX11: ; %bb.0:
16478 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16479 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
16480 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16481 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16482 ; GFX11-NEXT: s_mov_b32 s1, 0
16483 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
16484 ; GFX11-NEXT: .p2align 6
16485 ; GFX11-NEXT: .LBB55_1: ; %atomicrmw.start
16486 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
16487 ; GFX11-NEXT: s_waitcnt vmcnt(0)
16488 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16489 ; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16490 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16491 ; GFX11-NEXT: v_sub_f32_e32 v2, v2, v4
16492 ; GFX11-NEXT: v_sub_f32_e32 v6, v6, v5
16493 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16494 ; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
16495 ; GFX11-NEXT: v_bfe_u32 v8, v6, 16, 1
16496 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v2
16497 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v6
16498 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
16499 ; GFX11-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
16500 ; GFX11-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
16501 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v2, v2
16502 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16503 ; GFX11-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
16504 ; GFX11-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
16505 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
16506 ; GFX11-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
16507 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
16508 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 glc
16509 ; GFX11-NEXT: s_waitcnt vmcnt(0)
16510 ; GFX11-NEXT: buffer_gl1_inv
16511 ; GFX11-NEXT: buffer_gl0_inv
16512 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
16513 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
16514 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
16515 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
16516 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
16517 ; GFX11-NEXT: s_cbranch_execnz .LBB55_1
16518 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
16519 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
16520 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
16521 ; GFX11-NEXT: s_setpc_b64 s[30:31]
16522 ;
16523 ; GFX10-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
16524 ; GFX10: ; %bb.0:
16525 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16526 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
16527 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16528 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16529 ; GFX10-NEXT: s_mov_b32 s5, 0
16530 ; GFX10-NEXT: .LBB55_1: ; %atomicrmw.start
16531 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
16532 ; GFX10-NEXT: s_waitcnt vmcnt(0)
16533 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16534 ; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16535 ; GFX10-NEXT: v_sub_f32_e32 v2, v2, v4
16536 ; GFX10-NEXT: v_sub_f32_e32 v6, v6, v5
16537 ; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
16538 ; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
16539 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
16540 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
16541 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
16542 ; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
16543 ; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
16544 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
16545 ; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
16546 ; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
16547 ; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
16548 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
16549 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
16550 ; GFX10-NEXT: s_waitcnt vmcnt(0)
16551 ; GFX10-NEXT: buffer_gl1_inv
16552 ; GFX10-NEXT: buffer_gl0_inv
16553 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
16554 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
16555 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
16556 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
16557 ; GFX10-NEXT: s_cbranch_execnz .LBB55_1
16558 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
16559 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
16560 ; GFX10-NEXT: s_setpc_b64 s[30:31]
16561 ;
16562 ; GFX90A-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
16563 ; GFX90A: ; %bb.0:
16564 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16565 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
16566 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
16567 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16568 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
16569 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16570 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
16571 ; GFX90A-NEXT: .LBB55_1: ; %atomicrmw.start
16572 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
16573 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
16574 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16575 ; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16576 ; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
16577 ; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
16578 ; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
16579 ; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
16580 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
16581 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
16582 ; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
16583 ; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
16584 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
16585 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
16586 ; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
16587 ; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
16588 ; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
16589 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
16590 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
16591 ; GFX90A-NEXT: buffer_wbinvl1
16592 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
16593 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
16594 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
16595 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
16596 ; GFX90A-NEXT: s_cbranch_execnz .LBB55_1
16597 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
16598 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
16599 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
16600 ;
16601 ; GFX908-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
16602 ; GFX908: ; %bb.0:
16603 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16604 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
16605 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
16606 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16607 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
16608 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16609 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
16610 ; GFX908-NEXT: .LBB55_1: ; %atomicrmw.start
16611 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
16612 ; GFX908-NEXT: s_waitcnt vmcnt(0)
16613 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16614 ; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16615 ; GFX908-NEXT: v_sub_f32_e32 v2, v2, v4
16616 ; GFX908-NEXT: v_sub_f32_e32 v6, v6, v5
16617 ; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
16618 ; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
16619 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
16620 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
16621 ; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
16622 ; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
16623 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
16624 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
16625 ; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
16626 ; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
16627 ; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
16628 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
16629 ; GFX908-NEXT: s_waitcnt vmcnt(0)
16630 ; GFX908-NEXT: buffer_wbinvl1
16631 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
16632 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
16633 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
16634 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
16635 ; GFX908-NEXT: s_cbranch_execnz .LBB55_1
16636 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
16637 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
16638 ; GFX908-NEXT: s_setpc_b64 s[30:31]
16639 ;
16640 ; GFX8-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
16641 ; GFX8: ; %bb.0:
16642 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16643 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
16644 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
16645 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
16646 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
16647 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16648 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16649 ; GFX8-NEXT: .LBB55_1: ; %atomicrmw.start
16650 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
16651 ; GFX8-NEXT: s_waitcnt vmcnt(0)
16652 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16653 ; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16654 ; GFX8-NEXT: v_sub_f32_e32 v2, v2, v4
16655 ; GFX8-NEXT: v_sub_f32_e32 v6, v6, v5
16656 ; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
16657 ; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
16658 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
16659 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
16660 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
16661 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
16662 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
16663 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
16664 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
16665 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
16666 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
16667 ; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
16668 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
16669 ; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
16670 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
16671 ; GFX8-NEXT: s_waitcnt vmcnt(0)
16672 ; GFX8-NEXT: buffer_wbinvl1
16673 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
16674 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
16675 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
16676 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
16677 ; GFX8-NEXT: s_cbranch_execnz .LBB55_1
16678 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
16679 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
16680 ; GFX8-NEXT: s_setpc_b64 s[30:31]
16681 ;
16682 ; GFX7-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
16683 ; GFX7: ; %bb.0:
16684 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16685 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
16686 ; GFX7-NEXT: s_mov_b32 s5, -1
16687 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
16688 ; GFX7-NEXT: s_mov_b32 s6, 0
16689 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
16690 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
16691 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
16692 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
16693 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
16694 ; GFX7-NEXT: s_mov_b32 s4, s6
16695 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
16696 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16697 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
16698 ; GFX7-NEXT: s_mov_b32 s5, s6
16699 ; GFX7-NEXT: s_waitcnt vmcnt(0)
16700 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
16701 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
16702 ; GFX7-NEXT: .LBB55_1: ; %atomicrmw.start
16703 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
16704 ; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
16705 ; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
16706 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
16707 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
16708 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
16709 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
16710 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
16711 ; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
16712 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
16713 ; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
16714 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
16715 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
16716 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
16717 ; GFX7-NEXT: s_waitcnt vmcnt(0)
16718 ; GFX7-NEXT: buffer_wbinvl1
16719 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
16720 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
16721 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
16722 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
16723 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
16724 ; GFX7-NEXT: s_cbranch_execnz .LBB55_1
16725 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
16726 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
16727 ; GFX7-NEXT: s_setpc_b64 s[30:31]
16728 ;
16729 ; GFX6-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
16730 ; GFX6: ; %bb.0:
16731 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16732 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
16733 ; GFX6-NEXT: s_mov_b32 s5, -1
16734 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
16735 ; GFX6-NEXT: s_mov_b32 s6, 0
16736 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
16737 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
16738 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
16739 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
16740 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
16741 ; GFX6-NEXT: s_mov_b32 s4, s6
16742 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
16743 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16744 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
16745 ; GFX6-NEXT: s_mov_b32 s5, s6
16746 ; GFX6-NEXT: s_waitcnt vmcnt(0)
16747 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
16748 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
16749 ; GFX6-NEXT: .LBB55_1: ; %atomicrmw.start
16750 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
16751 ; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
16752 ; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
16753 ; GFX6-NEXT: s_waitcnt expcnt(0)
16754 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
16755 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
16756 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
16757 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
16758 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
16759 ; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
16760 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
16761 ; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
16762 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
16763 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
16764 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
16765 ; GFX6-NEXT: s_waitcnt vmcnt(0)
16766 ; GFX6-NEXT: buffer_wbinvl1
16767 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
16768 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
16769 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
16770 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
16771 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
16772 ; GFX6-NEXT: s_cbranch_execnz .LBB55_1
16773 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
16774 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
16775 ; GFX6-NEXT: s_waitcnt expcnt(0)
16776 ; GFX6-NEXT: s_setpc_b64 s[30:31]
16777 %gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 -512
16778 %unused = atomicrmw fsub ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst
16779 ret void
16780 }
16782 define <2 x bfloat> @global_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
16783 ; GFX12-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
16784 ; GFX12: ; %bb.0:
16785 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
16786 ; GFX12-NEXT: s_wait_expcnt 0x0
16787 ; GFX12-NEXT: s_wait_samplecnt 0x0
16788 ; GFX12-NEXT: s_wait_bvhcnt 0x0
16789 ; GFX12-NEXT: s_wait_kmcnt 0x0
16790 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
16791 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16792 ; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16793 ; GFX12-NEXT: s_mov_b32 s1, 0
16794 ; GFX12-NEXT: .LBB56_1: ; %atomicrmw.start
16795 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
16796 ; GFX12-NEXT: s_wait_loadcnt 0x0
16797 ; GFX12-NEXT: v_mov_b32_e32 v6, v3
16798 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
16799 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
16800 ; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
16801 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v6
16802 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16803 ; GFX12-NEXT: v_bfe_u32 v8, v5, 16, 1
16804 ; GFX12-NEXT: v_sub_f32_e32 v3, v3, v4
16805 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v5
16806 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
16807 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
16808 ; GFX12-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
16809 ; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
16810 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v3
16811 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v3, v3
16812 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
16813 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
16814 ; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
16815 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
16816 ; GFX12-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
16817 ; GFX12-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
16818 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
16819 ; GFX12-NEXT: s_wait_storecnt 0x0
16820 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
16821 ; GFX12-NEXT: s_wait_loadcnt 0x0
16822 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
16823 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
16824 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
16825 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
16826 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
16827 ; GFX12-NEXT: s_cbranch_execnz .LBB56_1
16828 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
16829 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
16830 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
16831 ; GFX12-NEXT: s_setpc_b64 s[30:31]
16832 ;
16833 ; GFX940-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
16834 ; GFX940: ; %bb.0:
16835 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16836 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
16837 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
16838 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16839 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
16840 ; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16841 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
16842 ; GFX940-NEXT: .LBB56_1: ; %atomicrmw.start
16843 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
16844 ; GFX940-NEXT: s_waitcnt vmcnt(0)
16845 ; GFX940-NEXT: v_mov_b32_e32 v7, v3
16846 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 16, v7
16847 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
16848 ; GFX940-NEXT: v_sub_f32_e32 v3, v3, v4
16849 ; GFX940-NEXT: v_sub_f32_e32 v5, v5, v2
16850 ; GFX940-NEXT: v_bfe_u32 v6, v3, 16, 1
16851 ; GFX940-NEXT: v_bfe_u32 v9, v5, 16, 1
16852 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v3
16853 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v5
16854 ; GFX940-NEXT: v_add3_u32 v6, v6, v3, s4
16855 ; GFX940-NEXT: v_add3_u32 v9, v9, v5, s4
16856 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
16857 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
16858 ; GFX940-NEXT: s_nop 0
16859 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
16860 ; GFX940-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[0:1]
16861 ; GFX940-NEXT: v_perm_b32 v6, v5, v3, s5
16862 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
16863 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off offset:2044 sc0 sc1
16864 ; GFX940-NEXT: s_waitcnt vmcnt(0)
16865 ; GFX940-NEXT: buffer_inv sc0 sc1
16866 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
16867 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
16868 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
16869 ; GFX940-NEXT: s_cbranch_execnz .LBB56_1
16870 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
16871 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
16872 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
16873 ; GFX940-NEXT: s_setpc_b64 s[30:31]
16874 ;
16875 ; GFX11-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
16876 ; GFX11: ; %bb.0:
16877 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16878 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
16879 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16880 ; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16881 ; GFX11-NEXT: s_mov_b32 s1, 0
16882 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
16883 ; GFX11-NEXT: .p2align 6
16884 ; GFX11-NEXT: .LBB56_1: ; %atomicrmw.start
16885 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
16886 ; GFX11-NEXT: s_waitcnt vmcnt(0)
16887 ; GFX11-NEXT: v_mov_b32_e32 v6, v3
16888 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
16889 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
16890 ; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
16891 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v6
16892 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16893 ; GFX11-NEXT: v_bfe_u32 v8, v5, 16, 1
16894 ; GFX11-NEXT: v_sub_f32_e32 v3, v3, v4
16895 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v5
16896 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
16897 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
16898 ; GFX11-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
16899 ; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
16900 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v3
16901 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v3, v3
16902 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
16903 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
16904 ; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
16905 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
16906 ; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
16907 ; GFX11-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
16908 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
16909 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
16910 ; GFX11-NEXT: s_waitcnt vmcnt(0)
16911 ; GFX11-NEXT: buffer_gl1_inv
16912 ; GFX11-NEXT: buffer_gl0_inv
16913 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
16914 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
16915 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
16916 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
16917 ; GFX11-NEXT: s_cbranch_execnz .LBB56_1
16918 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
16919 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
16920 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
16921 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
16922 ; GFX11-NEXT: s_setpc_b64 s[30:31]
16923 ;
16924 ; GFX10-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
16925 ; GFX10: ; %bb.0:
16926 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16927 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
16928 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16929 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16930 ; GFX10-NEXT: s_mov_b32 s5, 0
16931 ; GFX10-NEXT: .LBB56_1: ; %atomicrmw.start
16932 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
16933 ; GFX10-NEXT: s_waitcnt vmcnt(0)
16934 ; GFX10-NEXT: v_mov_b32_e32 v6, v3
16935 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
16936 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
16937 ; GFX10-NEXT: v_sub_f32_e32 v3, v3, v4
16938 ; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
16939 ; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
16940 ; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
16941 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
16942 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
16943 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
16944 ; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
16945 ; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
16946 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
16947 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
16948 ; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
16949 ; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
16950 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
16951 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
16952 ; GFX10-NEXT: s_waitcnt vmcnt(0)
16953 ; GFX10-NEXT: buffer_gl1_inv
16954 ; GFX10-NEXT: buffer_gl0_inv
16955 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
16956 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
16957 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
16958 ; GFX10-NEXT: s_cbranch_execnz .LBB56_1
16959 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
16960 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
16961 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
16962 ; GFX10-NEXT: s_setpc_b64 s[30:31]
16963 ;
16964 ; GFX90A-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
16965 ; GFX90A: ; %bb.0:
16966 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16967 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
16968 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
16969 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16970 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
16971 ; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16972 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
16973 ; GFX90A-NEXT: .LBB56_1: ; %atomicrmw.start
16974 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
16975 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
16976 ; GFX90A-NEXT: v_mov_b32_e32 v7, v3
16977 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v7
16978 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
16979 ; GFX90A-NEXT: v_sub_f32_e32 v3, v3, v4
16980 ; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v2
16981 ; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
16982 ; GFX90A-NEXT: v_bfe_u32 v9, v5, 16, 1
16983 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v3
16984 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v5
16985 ; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
16986 ; GFX90A-NEXT: v_add3_u32 v9, v9, v5, s8
16987 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
16988 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
16989 ; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[4:5]
16990 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
16991 ; GFX90A-NEXT: v_perm_b32 v6, v5, v3, s9
16992 ; GFX90A-NEXT: buffer_wbl2
16993 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off offset:2044 glc
16994 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
16995 ; GFX90A-NEXT: buffer_invl2
16996 ; GFX90A-NEXT: buffer_wbinvl1
16997 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
16998 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
16999 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
17000 ; GFX90A-NEXT: s_cbranch_execnz .LBB56_1
17001 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
17002 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
17003 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
17004 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
17005 ;
17006 ; GFX908-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
17007 ; GFX908: ; %bb.0:
17008 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17009 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
17010 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
17011 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
17012 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
17013 ; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
17014 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
17015 ; GFX908-NEXT: .LBB56_1: ; %atomicrmw.start
17016 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
17017 ; GFX908-NEXT: s_waitcnt vmcnt(0)
17018 ; GFX908-NEXT: v_mov_b32_e32 v6, v3
17019 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
17020 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
17021 ; GFX908-NEXT: v_sub_f32_e32 v3, v3, v4
17022 ; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
17023 ; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
17024 ; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
17025 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
17026 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
17027 ; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
17028 ; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
17029 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
17030 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
17031 ; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
17032 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
17033 ; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
17034 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
17035 ; GFX908-NEXT: s_waitcnt vmcnt(0)
17036 ; GFX908-NEXT: buffer_wbinvl1
17037 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
17038 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
17039 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
17040 ; GFX908-NEXT: s_cbranch_execnz .LBB56_1
17041 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
17042 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
17043 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
17044 ; GFX908-NEXT: s_setpc_b64 s[30:31]
17045 ;
17046 ; GFX8-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
17047 ; GFX8: ; %bb.0:
17048 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17049 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
17050 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
17051 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
17052 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
17053 ; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
17054 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
17055 ; GFX8-NEXT: .LBB56_1: ; %atomicrmw.start
17056 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
17057 ; GFX8-NEXT: s_waitcnt vmcnt(0)
17058 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
17059 ; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
17060 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
17061 ; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1
17062 ; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
17063 ; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
17064 ; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
17065 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
17066 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
17067 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
17068 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
17069 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
17070 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
17071 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
17072 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
17073 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
17074 ; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
17075 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
17076 ; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
17077 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
17078 ; GFX8-NEXT: s_waitcnt vmcnt(0)
17079 ; GFX8-NEXT: buffer_wbinvl1
17080 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
17081 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
17082 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
17083 ; GFX8-NEXT: s_cbranch_execnz .LBB56_1
17084 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
17085 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
17086 ; GFX8-NEXT: s_setpc_b64 s[30:31]
17087 ;
17088 ; GFX7-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
17089 ; GFX7: ; %bb.0:
17090 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17091 ; GFX7-NEXT: s_mov_b32 s6, 0
17092 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
17093 ; GFX7-NEXT: s_mov_b32 s4, s6
17094 ; GFX7-NEXT: s_mov_b32 s5, s6
17095 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
17096 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
17097 ; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v3
17098 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
17099 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
17100 ; GFX7-NEXT: s_waitcnt vmcnt(0)
17101 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
17102 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
17103 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
17104 ; GFX7-NEXT: .LBB56_1: ; %atomicrmw.start
17105 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
17106 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
17107 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
17108 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
17109 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
17110 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
17111 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v5
17112 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v4
17113 ; GFX7-NEXT: v_alignbit_b32 v3, v2, v3, 16
17114 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v7
17115 ; GFX7-NEXT: v_alignbit_b32 v2, v2, v6, 16
17116 ; GFX7-NEXT: v_mov_b32_e32 v7, v3
17117 ; GFX7-NEXT: v_mov_b32_e32 v6, v2
17118 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
17119 ; GFX7-NEXT: s_waitcnt vmcnt(0)
17120 ; GFX7-NEXT: buffer_wbinvl1
17121 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v3
17122 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
17123 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
17124 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v6
17125 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
17126 ; GFX7-NEXT: s_cbranch_execnz .LBB56_1
17127 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
17128 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
17129 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
17130 ; GFX7-NEXT: v_mov_b32_e32 v1, v2
17131 ; GFX7-NEXT: s_setpc_b64 s[30:31]
17132 ;
17133 ; GFX6-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
17134 ; GFX6: ; %bb.0:
17135 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17136 ; GFX6-NEXT: s_mov_b32 s6, 0
17137 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
17138 ; GFX6-NEXT: s_mov_b32 s4, s6
17139 ; GFX6-NEXT: s_mov_b32 s5, s6
17140 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
17141 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
17142 ; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v3
17143 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
17144 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
17145 ; GFX6-NEXT: s_waitcnt vmcnt(0)
17146 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
17147 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
17148 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
17149 ; GFX6-NEXT: .LBB56_1: ; %atomicrmw.start
17150 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
17151 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
17152 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
17153 ; GFX6-NEXT: s_waitcnt expcnt(0)
17154 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
17155 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
17156 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
17157 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v5
17158 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v4
17159 ; GFX6-NEXT: v_alignbit_b32 v3, v2, v3, 16
17160 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v7
17161 ; GFX6-NEXT: v_alignbit_b32 v2, v2, v6, 16
17162 ; GFX6-NEXT: v_mov_b32_e32 v7, v3
17163 ; GFX6-NEXT: v_mov_b32_e32 v6, v2
17164 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
17165 ; GFX6-NEXT: s_waitcnt vmcnt(0)
17166 ; GFX6-NEXT: buffer_wbinvl1
17167 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v3
17168 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
17169 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
17170 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v6
17171 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
17172 ; GFX6-NEXT: s_cbranch_execnz .LBB56_1
17173 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
17174 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
17175 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
17176 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
17177 ; GFX6-NEXT: s_waitcnt expcnt(0)
17178 ; GFX6-NEXT: s_setpc_b64 s[30:31]
17179 %gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
17180 %result = atomicrmw fsub ptr addrspace(1) %gep, <2 x bfloat> %val seq_cst
17181 ret <2 x bfloat> %result
17182 }
17184 define void @global_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
17185 ; GFX12-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
17186 ; GFX12: ; %bb.0:
17187 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
17188 ; GFX12-NEXT: s_wait_expcnt 0x0
17189 ; GFX12-NEXT: s_wait_samplecnt 0x0
17190 ; GFX12-NEXT: s_wait_bvhcnt 0x0
17191 ; GFX12-NEXT: s_wait_kmcnt 0x0
17192 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
17193 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
17194 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
17195 ; GFX12-NEXT: s_mov_b32 s1, 0
17196 ; GFX12-NEXT: .LBB57_1: ; %atomicrmw.start
17197 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
17198 ; GFX12-NEXT: s_wait_loadcnt 0x0
17199 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
17200 ; GFX12-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
17201 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
17202 ; GFX12-NEXT: v_sub_f32_e32 v2, v2, v4
17203 ; GFX12-NEXT: v_sub_f32_e32 v6, v6, v5
17204 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
17205 ; GFX12-NEXT: v_bfe_u32 v7, v2, 16, 1
17206 ; GFX12-NEXT: v_bfe_u32 v8, v6, 16, 1
17207 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v2
17208 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v6
17209 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
17210 ; GFX12-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
17211 ; GFX12-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
17212 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v2, v2
17213 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
17214 ; GFX12-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
17215 ; GFX12-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
17216 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
17217 ; GFX12-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
17218 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
17219 ; GFX12-NEXT: s_wait_storecnt 0x0
17220 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
17221 ; GFX12-NEXT: s_wait_loadcnt 0x0
17222 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
17223 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
17224 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
17225 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
17226 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
17227 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
17228 ; GFX12-NEXT: s_cbranch_execnz .LBB57_1
17229 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
17230 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
17231 ; GFX12-NEXT: s_setpc_b64 s[30:31]
17232 ;
17233 ; GFX940-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
17234 ; GFX940: ; %bb.0:
17235 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17236 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
17237 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
17238 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
17239 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
17240 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
17241 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
17242 ; GFX940-NEXT: .LBB57_1: ; %atomicrmw.start
17243 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
17244 ; GFX940-NEXT: s_waitcnt vmcnt(0)
17245 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
17246 ; GFX940-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
17247 ; GFX940-NEXT: v_sub_f32_e32 v2, v2, v4
17248 ; GFX940-NEXT: v_sub_f32_e32 v6, v6, v5
17249 ; GFX940-NEXT: v_bfe_u32 v7, v2, 16, 1
17250 ; GFX940-NEXT: v_bfe_u32 v9, v6, 16, 1
17251 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v2
17252 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v6
17253 ; GFX940-NEXT: v_add3_u32 v7, v7, v2, s4
17254 ; GFX940-NEXT: v_add3_u32 v9, v9, v6, s4
17255 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
17256 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
17257 ; GFX940-NEXT: s_nop 0
17258 ; GFX940-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
17259 ; GFX940-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
17260 ; GFX940-NEXT: v_perm_b32 v2, v6, v2, s5
17261 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
17262 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
17263 ; GFX940-NEXT: s_waitcnt vmcnt(0)
17264 ; GFX940-NEXT: buffer_inv sc0 sc1
17265 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
17266 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
17267 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
17268 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
17269 ; GFX940-NEXT: s_cbranch_execnz .LBB57_1
17270 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
17271 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
17272 ; GFX940-NEXT: s_setpc_b64 s[30:31]
17273 ;
17274 ; GFX11-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
17275 ; GFX11: ; %bb.0:
17276 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17277 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
17278 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
17279 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
17280 ; GFX11-NEXT: s_mov_b32 s1, 0
17281 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
17282 ; GFX11-NEXT: .p2align 6
17283 ; GFX11-NEXT: .LBB57_1: ; %atomicrmw.start
17284 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
17285 ; GFX11-NEXT: s_waitcnt vmcnt(0)
17286 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
17287 ; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
17288 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
17289 ; GFX11-NEXT: v_sub_f32_e32 v2, v2, v4
17290 ; GFX11-NEXT: v_sub_f32_e32 v6, v6, v5
17291 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
17292 ; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
17293 ; GFX11-NEXT: v_bfe_u32 v8, v6, 16, 1
17294 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v2
17295 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v6
17296 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
17297 ; GFX11-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
17298 ; GFX11-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
17299 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v2, v2
17300 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
17301 ; GFX11-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
17302 ; GFX11-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
17303 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
17304 ; GFX11-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
17305 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
17306 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
17307 ; GFX11-NEXT: s_waitcnt vmcnt(0)
17308 ; GFX11-NEXT: buffer_gl1_inv
17309 ; GFX11-NEXT: buffer_gl0_inv
17310 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
17311 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
17312 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
17313 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
17314 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
17315 ; GFX11-NEXT: s_cbranch_execnz .LBB57_1
17316 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
17317 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
17318 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
17319 ; GFX11-NEXT: s_setpc_b64 s[30:31]
17320 ;
17321 ; GFX10-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
17322 ; GFX10: ; %bb.0:
17323 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17324 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
17325 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
17326 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
17327 ; GFX10-NEXT: s_mov_b32 s5, 0
17328 ; GFX10-NEXT: .LBB57_1: ; %atomicrmw.start
17329 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
17330 ; GFX10-NEXT: s_waitcnt vmcnt(0)
17331 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
17332 ; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
17333 ; GFX10-NEXT: v_sub_f32_e32 v2, v2, v4
17334 ; GFX10-NEXT: v_sub_f32_e32 v6, v6, v5
17335 ; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
17336 ; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
17337 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
17338 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
17339 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
17340 ; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
17341 ; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
17342 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
17343 ; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
17344 ; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
17345 ; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
17346 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
17347 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
17348 ; GFX10-NEXT: s_waitcnt vmcnt(0)
17349 ; GFX10-NEXT: buffer_gl1_inv
17350 ; GFX10-NEXT: buffer_gl0_inv
17351 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
17352 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
17353 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
17354 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
17355 ; GFX10-NEXT: s_cbranch_execnz .LBB57_1
17356 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
17357 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
17358 ; GFX10-NEXT: s_setpc_b64 s[30:31]
17359 ;
17360 ; GFX90A-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
17361 ; GFX90A: ; %bb.0:
17362 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17363 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
17364 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
17365 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
17366 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
17367 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
17368 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
17369 ; GFX90A-NEXT: .LBB57_1: ; %atomicrmw.start
17370 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
17371 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
17372 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
17373 ; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
17374 ; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
17375 ; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
17376 ; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
17377 ; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
17378 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
17379 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
17380 ; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
17381 ; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
17382 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
17383 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
17384 ; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
17385 ; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
17386 ; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
17387 ; GFX90A-NEXT: buffer_wbl2
17388 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
17389 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
17390 ; GFX90A-NEXT: buffer_invl2
17391 ; GFX90A-NEXT: buffer_wbinvl1
17392 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
17393 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
17394 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
17395 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
17396 ; GFX90A-NEXT: s_cbranch_execnz .LBB57_1
17397 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
17398 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
17399 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
17400 ;
17401 ; GFX908-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
17402 ; GFX908: ; %bb.0:
17403 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17404 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
17405 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
17406 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
17407 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
17408 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
17409 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
17410 ; GFX908-NEXT: .LBB57_1: ; %atomicrmw.start
17411 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
17412 ; GFX908-NEXT: s_waitcnt vmcnt(0)
17413 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
17414 ; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
17415 ; GFX908-NEXT: v_sub_f32_e32 v2, v2, v4
17416 ; GFX908-NEXT: v_sub_f32_e32 v6, v6, v5
17417 ; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
17418 ; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
17419 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
17420 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
17421 ; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
17422 ; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
17423 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
17424 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
17425 ; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
17426 ; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
17427 ; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
17428 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
17429 ; GFX908-NEXT: s_waitcnt vmcnt(0)
17430 ; GFX908-NEXT: buffer_wbinvl1
17431 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
17432 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
17433 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
17434 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
17435 ; GFX908-NEXT: s_cbranch_execnz .LBB57_1
17436 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
17437 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
17438 ; GFX908-NEXT: s_setpc_b64 s[30:31]
17439 ;
17440 ; GFX8-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
17441 ; GFX8: ; %bb.0:
17442 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17443 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
17444 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
17445 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
17446 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
17447 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
17448 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
17449 ; GFX8-NEXT: .LBB57_1: ; %atomicrmw.start
17450 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
17451 ; GFX8-NEXT: s_waitcnt vmcnt(0)
17452 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
17453 ; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
17454 ; GFX8-NEXT: v_sub_f32_e32 v2, v2, v4
17455 ; GFX8-NEXT: v_sub_f32_e32 v6, v6, v5
17456 ; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
17457 ; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
17458 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
17459 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
17460 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
17461 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
17462 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
17463 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
17464 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
17465 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
17466 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
17467 ; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
17468 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
17469 ; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
17470 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
17471 ; GFX8-NEXT: s_waitcnt vmcnt(0)
17472 ; GFX8-NEXT: buffer_wbinvl1
17473 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
17474 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
17475 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
17476 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
17477 ; GFX8-NEXT: s_cbranch_execnz .LBB57_1
17478 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
17479 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
17480 ; GFX8-NEXT: s_setpc_b64 s[30:31]
17481 ;
17482 ; GFX7-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
17483 ; GFX7: ; %bb.0:
17484 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17485 ; GFX7-NEXT: s_mov_b32 s6, 0
17486 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
17487 ; GFX7-NEXT: s_mov_b32 s4, s6
17488 ; GFX7-NEXT: s_mov_b32 s5, s6
17489 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
17490 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
17491 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
17492 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
17493 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
17494 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
17495 ; GFX7-NEXT: s_waitcnt vmcnt(0)
17496 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
17497 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
17498 ; GFX7-NEXT: .LBB57_1: ; %atomicrmw.start
17499 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
17500 ; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
17501 ; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
17502 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
17503 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
17504 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
17505 ; GFX7-NEXT: v_sub_f32_e32 v7, v7, v3
17506 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
17507 ; GFX7-NEXT: v_alignbit_b32 v5, v4, v5, 16
17508 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v7
17509 ; GFX7-NEXT: v_alignbit_b32 v4, v4, v6, 16
17510 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
17511 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
17512 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
17513 ; GFX7-NEXT: s_waitcnt vmcnt(0)
17514 ; GFX7-NEXT: buffer_wbinvl1
17515 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
17516 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
17517 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
17518 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
17519 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
17520 ; GFX7-NEXT: s_cbranch_execnz .LBB57_1
17521 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
17522 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
17523 ; GFX7-NEXT: s_setpc_b64 s[30:31]
17524 ;
17525 ; GFX6-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
17526 ; GFX6: ; %bb.0:
17527 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17528 ; GFX6-NEXT: s_mov_b32 s6, 0
17529 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
17530 ; GFX6-NEXT: s_mov_b32 s4, s6
17531 ; GFX6-NEXT: s_mov_b32 s5, s6
17532 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
17533 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
17534 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
17535 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
17536 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
17537 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
17538 ; GFX6-NEXT: s_waitcnt vmcnt(0)
17539 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
17540 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
17541 ; GFX6-NEXT: .LBB57_1: ; %atomicrmw.start
17542 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
17543 ; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
17544 ; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
17545 ; GFX6-NEXT: s_waitcnt expcnt(0)
17546 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
17547 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
17548 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
17549 ; GFX6-NEXT: v_sub_f32_e32 v7, v7, v3
17550 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
17551 ; GFX6-NEXT: v_alignbit_b32 v5, v4, v5, 16
17552 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
17553 ; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
17554 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
17555 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
17556 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
17557 ; GFX6-NEXT: s_waitcnt vmcnt(0)
17558 ; GFX6-NEXT: buffer_wbinvl1
17559 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
17560 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
17561 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
17562 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v6
17563 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
17564 ; GFX6-NEXT: s_cbranch_execnz .LBB57_1
17565 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
17566 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
17567 ; GFX6-NEXT: s_waitcnt expcnt(0)
17568 ; GFX6-NEXT: s_setpc_b64 s[30:31]
17569 %gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
17570 %unused = atomicrmw fsub ptr addrspace(1) %gep, <2 x bfloat> %val seq_cst
17571 ret void
17572 }
17574 attributes #0 = { nounwind "amdgpu-unsafe-fp-atomics"="true" }
17575 attributes #1 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" }