| blob | 65e00c50292dc3150c0d489591d8ee1f1bd966da |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
2 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
3 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx940 < %s | FileCheck -check-prefix=GFX940 %s
4 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 < %s | FileCheck -check-prefix=GFX11 %s
5 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
6 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx90a < %s | FileCheck -check-prefix=GFX90A %s
7 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx908 < %s | FileCheck -check-prefix=GFX908 %s
8 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=tonga < %s | FileCheck -check-prefix=GFX8 %s
9 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=hawaii < %s | FileCheck -check-prefix=GFX7 %s
10 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=tahiti < %s | FileCheck -check-prefix=GFX6 %s
12 ; --------------------------------------------------------------------
13 ; float
14 ; --------------------------------------------------------------------
16 define float @local_atomic_fsub_ret_f32(ptr addrspace(3) %ptr) nounwind {
17 ; GFX12-LABEL: local_atomic_fsub_ret_f32:
18 ; GFX12: ; %bb.0:
19 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
20 ; GFX12-NEXT: s_wait_expcnt 0x0
21 ; GFX12-NEXT: s_wait_samplecnt 0x0
22 ; GFX12-NEXT: s_wait_bvhcnt 0x0
23 ; GFX12-NEXT: s_wait_kmcnt 0x0
24 ; GFX12-NEXT: ds_load_b32 v1, v0
25 ; GFX12-NEXT: s_mov_b32 s0, 0
26 ; GFX12-NEXT: .LBB0_1: ; %atomicrmw.start
27 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
28 ; GFX12-NEXT: s_wait_dscnt 0x0
29 ; GFX12-NEXT: v_mov_b32_e32 v2, v1
30 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
31 ; GFX12-NEXT: v_add_f32_e32 v1, -4.0, v2
32 ; GFX12-NEXT: s_wait_storecnt 0x0
33 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2
34 ; GFX12-NEXT: s_wait_dscnt 0x0
35 ; GFX12-NEXT: global_inv scope:SCOPE_SE
36 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
37 ; GFX12-NEXT: s_wait_alu 0xfffe
38 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
39 ; GFX12-NEXT: s_wait_alu 0xfffe
40 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
41 ; GFX12-NEXT: s_cbranch_execnz .LBB0_1
42 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
43 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
44 ; GFX12-NEXT: v_mov_b32_e32 v0, v1
45 ; GFX12-NEXT: s_setpc_b64 s[30:31]
46 ;
47 ; GFX940-LABEL: local_atomic_fsub_ret_f32:
48 ; GFX940: ; %bb.0:
49 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
50 ; GFX940-NEXT: ds_read_b32 v1, v0
51 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
52 ; GFX940-NEXT: .LBB0_1: ; %atomicrmw.start
53 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
54 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
55 ; GFX940-NEXT: v_mov_b32_e32 v2, v1
56 ; GFX940-NEXT: v_add_f32_e32 v1, -4.0, v2
57 ; GFX940-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1
58 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
59 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
60 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
61 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
62 ; GFX940-NEXT: s_cbranch_execnz .LBB0_1
63 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
64 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
65 ; GFX940-NEXT: v_mov_b32_e32 v0, v1
66 ; GFX940-NEXT: s_setpc_b64 s[30:31]
67 ;
68 ; GFX11-LABEL: local_atomic_fsub_ret_f32:
69 ; GFX11: ; %bb.0:
70 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
71 ; GFX11-NEXT: ds_load_b32 v1, v0
72 ; GFX11-NEXT: s_mov_b32 s0, 0
73 ; GFX11-NEXT: .LBB0_1: ; %atomicrmw.start
74 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
75 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
76 ; GFX11-NEXT: v_mov_b32_e32 v2, v1
77 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
78 ; GFX11-NEXT: v_add_f32_e32 v1, -4.0, v2
79 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
80 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2
81 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
82 ; GFX11-NEXT: buffer_gl0_inv
83 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
84 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
85 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
86 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
87 ; GFX11-NEXT: s_cbranch_execnz .LBB0_1
88 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
89 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
90 ; GFX11-NEXT: v_mov_b32_e32 v0, v1
91 ; GFX11-NEXT: s_setpc_b64 s[30:31]
92 ;
93 ; GFX10-LABEL: local_atomic_fsub_ret_f32:
94 ; GFX10: ; %bb.0:
95 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
96 ; GFX10-NEXT: ds_read_b32 v1, v0
97 ; GFX10-NEXT: s_mov_b32 s4, 0
98 ; GFX10-NEXT: .LBB0_1: ; %atomicrmw.start
99 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
100 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
101 ; GFX10-NEXT: v_mov_b32_e32 v2, v1
102 ; GFX10-NEXT: v_add_f32_e32 v1, -4.0, v2
103 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
104 ; GFX10-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1
105 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
106 ; GFX10-NEXT: buffer_gl0_inv
107 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
108 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
109 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
110 ; GFX10-NEXT: s_cbranch_execnz .LBB0_1
111 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
112 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
113 ; GFX10-NEXT: v_mov_b32_e32 v0, v1
114 ; GFX10-NEXT: s_setpc_b64 s[30:31]
115 ;
116 ; GFX90A-LABEL: local_atomic_fsub_ret_f32:
117 ; GFX90A: ; %bb.0:
118 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
119 ; GFX90A-NEXT: ds_read_b32 v1, v0
120 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
121 ; GFX90A-NEXT: .LBB0_1: ; %atomicrmw.start
122 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
123 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
124 ; GFX90A-NEXT: v_mov_b32_e32 v2, v1
125 ; GFX90A-NEXT: v_add_f32_e32 v1, -4.0, v2
126 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1
127 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
128 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
129 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
130 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
131 ; GFX90A-NEXT: s_cbranch_execnz .LBB0_1
132 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
133 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
134 ; GFX90A-NEXT: v_mov_b32_e32 v0, v1
135 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
136 ;
137 ; GFX908-LABEL: local_atomic_fsub_ret_f32:
138 ; GFX908: ; %bb.0:
139 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
140 ; GFX908-NEXT: ds_read_b32 v1, v0
141 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
142 ; GFX908-NEXT: .LBB0_1: ; %atomicrmw.start
143 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
144 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
145 ; GFX908-NEXT: v_mov_b32_e32 v2, v1
146 ; GFX908-NEXT: v_add_f32_e32 v1, -4.0, v2
147 ; GFX908-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1
148 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
149 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
150 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
151 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
152 ; GFX908-NEXT: s_cbranch_execnz .LBB0_1
153 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
154 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
155 ; GFX908-NEXT: v_mov_b32_e32 v0, v1
156 ; GFX908-NEXT: s_setpc_b64 s[30:31]
157 ;
158 ; GFX8-LABEL: local_atomic_fsub_ret_f32:
159 ; GFX8: ; %bb.0:
160 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
161 ; GFX8-NEXT: s_mov_b32 m0, -1
162 ; GFX8-NEXT: ds_read_b32 v1, v0
163 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
164 ; GFX8-NEXT: .LBB0_1: ; %atomicrmw.start
165 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
166 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
167 ; GFX8-NEXT: v_mov_b32_e32 v2, v1
168 ; GFX8-NEXT: v_add_f32_e32 v1, -4.0, v2
169 ; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1
170 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
171 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
172 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
173 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
174 ; GFX8-NEXT: s_cbranch_execnz .LBB0_1
175 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
176 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
177 ; GFX8-NEXT: v_mov_b32_e32 v0, v1
178 ; GFX8-NEXT: s_setpc_b64 s[30:31]
179 ;
180 ; GFX7-LABEL: local_atomic_fsub_ret_f32:
181 ; GFX7: ; %bb.0:
182 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
183 ; GFX7-NEXT: s_mov_b32 m0, -1
184 ; GFX7-NEXT: ds_read_b32 v1, v0
185 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
186 ; GFX7-NEXT: .LBB0_1: ; %atomicrmw.start
187 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
188 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
189 ; GFX7-NEXT: v_mov_b32_e32 v2, v1
190 ; GFX7-NEXT: v_add_f32_e32 v1, -4.0, v2
191 ; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1
192 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
193 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
194 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
195 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
196 ; GFX7-NEXT: s_cbranch_execnz .LBB0_1
197 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
198 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
199 ; GFX7-NEXT: v_mov_b32_e32 v0, v1
200 ; GFX7-NEXT: s_setpc_b64 s[30:31]
201 ;
202 ; GFX6-LABEL: local_atomic_fsub_ret_f32:
203 ; GFX6: ; %bb.0:
204 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
205 ; GFX6-NEXT: s_mov_b32 m0, -1
206 ; GFX6-NEXT: ds_read_b32 v1, v0
207 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
208 ; GFX6-NEXT: .LBB0_1: ; %atomicrmw.start
209 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
210 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
211 ; GFX6-NEXT: v_mov_b32_e32 v2, v1
212 ; GFX6-NEXT: v_add_f32_e32 v1, -4.0, v2
213 ; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1
214 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
215 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
216 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
217 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
218 ; GFX6-NEXT: s_cbranch_execnz .LBB0_1
219 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
220 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
221 ; GFX6-NEXT: v_mov_b32_e32 v0, v1
222 ; GFX6-NEXT: s_setpc_b64 s[30:31]
223 %result = atomicrmw fsub ptr addrspace(3) %ptr, float 4.0 seq_cst
224 ret float %result
225 }
227 define float @local_atomic_fsub_ret_f32__offset(ptr addrspace(3) %ptr) nounwind {
228 ; GFX12-LABEL: local_atomic_fsub_ret_f32__offset:
229 ; GFX12: ; %bb.0:
230 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
231 ; GFX12-NEXT: s_wait_expcnt 0x0
232 ; GFX12-NEXT: s_wait_samplecnt 0x0
233 ; GFX12-NEXT: s_wait_bvhcnt 0x0
234 ; GFX12-NEXT: s_wait_kmcnt 0x0
235 ; GFX12-NEXT: ds_load_b32 v1, v0 offset:65532
236 ; GFX12-NEXT: s_mov_b32 s0, 0
237 ; GFX12-NEXT: .LBB1_1: ; %atomicrmw.start
238 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
239 ; GFX12-NEXT: s_wait_dscnt 0x0
240 ; GFX12-NEXT: v_mov_b32_e32 v2, v1
241 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
242 ; GFX12-NEXT: v_add_f32_e32 v1, -4.0, v2
243 ; GFX12-NEXT: s_wait_storecnt 0x0
244 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:65532
245 ; GFX12-NEXT: s_wait_dscnt 0x0
246 ; GFX12-NEXT: global_inv scope:SCOPE_SE
247 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
248 ; GFX12-NEXT: s_wait_alu 0xfffe
249 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
250 ; GFX12-NEXT: s_wait_alu 0xfffe
251 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
252 ; GFX12-NEXT: s_cbranch_execnz .LBB1_1
253 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
254 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
255 ; GFX12-NEXT: v_mov_b32_e32 v0, v1
256 ; GFX12-NEXT: s_setpc_b64 s[30:31]
257 ;
258 ; GFX940-LABEL: local_atomic_fsub_ret_f32__offset:
259 ; GFX940: ; %bb.0:
260 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
261 ; GFX940-NEXT: ds_read_b32 v1, v0 offset:65532
262 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
263 ; GFX940-NEXT: .LBB1_1: ; %atomicrmw.start
264 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
265 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
266 ; GFX940-NEXT: v_mov_b32_e32 v2, v1
267 ; GFX940-NEXT: v_add_f32_e32 v1, -4.0, v2
268 ; GFX940-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:65532
269 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
270 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
271 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
272 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
273 ; GFX940-NEXT: s_cbranch_execnz .LBB1_1
274 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
275 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
276 ; GFX940-NEXT: v_mov_b32_e32 v0, v1
277 ; GFX940-NEXT: s_setpc_b64 s[30:31]
278 ;
279 ; GFX11-LABEL: local_atomic_fsub_ret_f32__offset:
280 ; GFX11: ; %bb.0:
281 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
282 ; GFX11-NEXT: ds_load_b32 v1, v0 offset:65532
283 ; GFX11-NEXT: s_mov_b32 s0, 0
284 ; GFX11-NEXT: .LBB1_1: ; %atomicrmw.start
285 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
286 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
287 ; GFX11-NEXT: v_mov_b32_e32 v2, v1
288 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
289 ; GFX11-NEXT: v_add_f32_e32 v1, -4.0, v2
290 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
291 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:65532
292 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
293 ; GFX11-NEXT: buffer_gl0_inv
294 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
295 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
296 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
297 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
298 ; GFX11-NEXT: s_cbranch_execnz .LBB1_1
299 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
300 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
301 ; GFX11-NEXT: v_mov_b32_e32 v0, v1
302 ; GFX11-NEXT: s_setpc_b64 s[30:31]
303 ;
304 ; GFX10-LABEL: local_atomic_fsub_ret_f32__offset:
305 ; GFX10: ; %bb.0:
306 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
307 ; GFX10-NEXT: ds_read_b32 v1, v0 offset:65532
308 ; GFX10-NEXT: s_mov_b32 s4, 0
309 ; GFX10-NEXT: .LBB1_1: ; %atomicrmw.start
310 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
311 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
312 ; GFX10-NEXT: v_mov_b32_e32 v2, v1
313 ; GFX10-NEXT: v_add_f32_e32 v1, -4.0, v2
314 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
315 ; GFX10-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:65532
316 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
317 ; GFX10-NEXT: buffer_gl0_inv
318 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
319 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
320 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
321 ; GFX10-NEXT: s_cbranch_execnz .LBB1_1
322 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
323 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
324 ; GFX10-NEXT: v_mov_b32_e32 v0, v1
325 ; GFX10-NEXT: s_setpc_b64 s[30:31]
326 ;
327 ; GFX90A-LABEL: local_atomic_fsub_ret_f32__offset:
328 ; GFX90A: ; %bb.0:
329 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
330 ; GFX90A-NEXT: ds_read_b32 v1, v0 offset:65532
331 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
332 ; GFX90A-NEXT: .LBB1_1: ; %atomicrmw.start
333 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
334 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
335 ; GFX90A-NEXT: v_mov_b32_e32 v2, v1
336 ; GFX90A-NEXT: v_add_f32_e32 v1, -4.0, v2
337 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:65532
338 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
339 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
340 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
341 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
342 ; GFX90A-NEXT: s_cbranch_execnz .LBB1_1
343 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
344 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
345 ; GFX90A-NEXT: v_mov_b32_e32 v0, v1
346 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
347 ;
348 ; GFX908-LABEL: local_atomic_fsub_ret_f32__offset:
349 ; GFX908: ; %bb.0:
350 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
351 ; GFX908-NEXT: ds_read_b32 v1, v0 offset:65532
352 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
353 ; GFX908-NEXT: .LBB1_1: ; %atomicrmw.start
354 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
355 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
356 ; GFX908-NEXT: v_mov_b32_e32 v2, v1
357 ; GFX908-NEXT: v_add_f32_e32 v1, -4.0, v2
358 ; GFX908-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:65532
359 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
360 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
361 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
362 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
363 ; GFX908-NEXT: s_cbranch_execnz .LBB1_1
364 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
365 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
366 ; GFX908-NEXT: v_mov_b32_e32 v0, v1
367 ; GFX908-NEXT: s_setpc_b64 s[30:31]
368 ;
369 ; GFX8-LABEL: local_atomic_fsub_ret_f32__offset:
370 ; GFX8: ; %bb.0:
371 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
372 ; GFX8-NEXT: s_mov_b32 m0, -1
373 ; GFX8-NEXT: ds_read_b32 v1, v0 offset:65532
374 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
375 ; GFX8-NEXT: .LBB1_1: ; %atomicrmw.start
376 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
377 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
378 ; GFX8-NEXT: v_mov_b32_e32 v2, v1
379 ; GFX8-NEXT: v_add_f32_e32 v1, -4.0, v2
380 ; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:65532
381 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
382 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
383 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
384 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
385 ; GFX8-NEXT: s_cbranch_execnz .LBB1_1
386 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
387 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
388 ; GFX8-NEXT: v_mov_b32_e32 v0, v1
389 ; GFX8-NEXT: s_setpc_b64 s[30:31]
390 ;
391 ; GFX7-LABEL: local_atomic_fsub_ret_f32__offset:
392 ; GFX7: ; %bb.0:
393 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
394 ; GFX7-NEXT: s_mov_b32 m0, -1
395 ; GFX7-NEXT: ds_read_b32 v1, v0 offset:65532
396 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
397 ; GFX7-NEXT: .LBB1_1: ; %atomicrmw.start
398 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
399 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
400 ; GFX7-NEXT: v_mov_b32_e32 v2, v1
401 ; GFX7-NEXT: v_add_f32_e32 v1, -4.0, v2
402 ; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:65532
403 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
404 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
405 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
406 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
407 ; GFX7-NEXT: s_cbranch_execnz .LBB1_1
408 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
409 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
410 ; GFX7-NEXT: v_mov_b32_e32 v0, v1
411 ; GFX7-NEXT: s_setpc_b64 s[30:31]
412 ;
413 ; GFX6-LABEL: local_atomic_fsub_ret_f32__offset:
414 ; GFX6: ; %bb.0:
415 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
416 ; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffc, v0
417 ; GFX6-NEXT: s_mov_b32 m0, -1
418 ; GFX6-NEXT: ds_read_b32 v0, v1
419 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
420 ; GFX6-NEXT: .LBB1_1: ; %atomicrmw.start
421 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
422 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
423 ; GFX6-NEXT: v_mov_b32_e32 v2, v0
424 ; GFX6-NEXT: v_add_f32_e32 v0, -4.0, v2
425 ; GFX6-NEXT: ds_cmpst_rtn_b32 v0, v1, v2, v0
426 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
427 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
428 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
429 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
430 ; GFX6-NEXT: s_cbranch_execnz .LBB1_1
431 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
432 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
433 ; GFX6-NEXT: s_setpc_b64 s[30:31]
434 %gep = getelementptr float, ptr addrspace(3) %ptr, i32 16383
435 %result = atomicrmw fsub ptr addrspace(3) %gep, float 4.0 seq_cst
436 ret float %result
437 }
439 define void @local_atomic_fsub_noret_f32(ptr addrspace(3) %ptr) nounwind {
440 ; GFX12-LABEL: local_atomic_fsub_noret_f32:
441 ; GFX12: ; %bb.0:
442 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
443 ; GFX12-NEXT: s_wait_expcnt 0x0
444 ; GFX12-NEXT: s_wait_samplecnt 0x0
445 ; GFX12-NEXT: s_wait_bvhcnt 0x0
446 ; GFX12-NEXT: s_wait_kmcnt 0x0
447 ; GFX12-NEXT: ds_load_b32 v1, v0
448 ; GFX12-NEXT: s_mov_b32 s0, 0
449 ; GFX12-NEXT: .LBB2_1: ; %atomicrmw.start
450 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
451 ; GFX12-NEXT: s_wait_dscnt 0x0
452 ; GFX12-NEXT: v_add_f32_e32 v2, -4.0, v1
453 ; GFX12-NEXT: s_wait_storecnt 0x0
454 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1
455 ; GFX12-NEXT: s_wait_dscnt 0x0
456 ; GFX12-NEXT: global_inv scope:SCOPE_SE
457 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
458 ; GFX12-NEXT: v_mov_b32_e32 v1, v2
459 ; GFX12-NEXT: s_wait_alu 0xfffe
460 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
461 ; GFX12-NEXT: s_wait_alu 0xfffe
462 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
463 ; GFX12-NEXT: s_cbranch_execnz .LBB2_1
464 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
465 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
466 ; GFX12-NEXT: s_setpc_b64 s[30:31]
467 ;
468 ; GFX940-LABEL: local_atomic_fsub_noret_f32:
469 ; GFX940: ; %bb.0:
470 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
471 ; GFX940-NEXT: ds_read_b32 v1, v0
472 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
473 ; GFX940-NEXT: .LBB2_1: ; %atomicrmw.start
474 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
475 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
476 ; GFX940-NEXT: v_add_f32_e32 v2, -4.0, v1
477 ; GFX940-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
478 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
479 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
480 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
481 ; GFX940-NEXT: v_mov_b32_e32 v1, v2
482 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
483 ; GFX940-NEXT: s_cbranch_execnz .LBB2_1
484 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
485 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
486 ; GFX940-NEXT: s_setpc_b64 s[30:31]
487 ;
488 ; GFX11-LABEL: local_atomic_fsub_noret_f32:
489 ; GFX11: ; %bb.0:
490 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
491 ; GFX11-NEXT: ds_load_b32 v1, v0
492 ; GFX11-NEXT: s_mov_b32 s0, 0
493 ; GFX11-NEXT: .LBB2_1: ; %atomicrmw.start
494 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
495 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
496 ; GFX11-NEXT: v_add_f32_e32 v2, -4.0, v1
497 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
498 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1
499 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
500 ; GFX11-NEXT: buffer_gl0_inv
501 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
502 ; GFX11-NEXT: v_mov_b32_e32 v1, v2
503 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
504 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
505 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
506 ; GFX11-NEXT: s_cbranch_execnz .LBB2_1
507 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
508 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
509 ; GFX11-NEXT: s_setpc_b64 s[30:31]
510 ;
511 ; GFX10-LABEL: local_atomic_fsub_noret_f32:
512 ; GFX10: ; %bb.0:
513 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
514 ; GFX10-NEXT: ds_read_b32 v1, v0
515 ; GFX10-NEXT: s_mov_b32 s4, 0
516 ; GFX10-NEXT: .LBB2_1: ; %atomicrmw.start
517 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
518 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
519 ; GFX10-NEXT: v_add_f32_e32 v2, -4.0, v1
520 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
521 ; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
522 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
523 ; GFX10-NEXT: buffer_gl0_inv
524 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
525 ; GFX10-NEXT: v_mov_b32_e32 v1, v2
526 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
527 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
528 ; GFX10-NEXT: s_cbranch_execnz .LBB2_1
529 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
530 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
531 ; GFX10-NEXT: s_setpc_b64 s[30:31]
532 ;
533 ; GFX90A-LABEL: local_atomic_fsub_noret_f32:
534 ; GFX90A: ; %bb.0:
535 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
536 ; GFX90A-NEXT: ds_read_b32 v1, v0
537 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
538 ; GFX90A-NEXT: .LBB2_1: ; %atomicrmw.start
539 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
540 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
541 ; GFX90A-NEXT: v_add_f32_e32 v2, -4.0, v1
542 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
543 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
544 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
545 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
546 ; GFX90A-NEXT: v_mov_b32_e32 v1, v2
547 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
548 ; GFX90A-NEXT: s_cbranch_execnz .LBB2_1
549 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
550 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
551 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
552 ;
553 ; GFX908-LABEL: local_atomic_fsub_noret_f32:
554 ; GFX908: ; %bb.0:
555 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
556 ; GFX908-NEXT: ds_read_b32 v1, v0
557 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
558 ; GFX908-NEXT: .LBB2_1: ; %atomicrmw.start
559 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
560 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
561 ; GFX908-NEXT: v_add_f32_e32 v2, -4.0, v1
562 ; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
563 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
564 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
565 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
566 ; GFX908-NEXT: v_mov_b32_e32 v1, v2
567 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
568 ; GFX908-NEXT: s_cbranch_execnz .LBB2_1
569 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
570 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
571 ; GFX908-NEXT: s_setpc_b64 s[30:31]
572 ;
573 ; GFX8-LABEL: local_atomic_fsub_noret_f32:
574 ; GFX8: ; %bb.0:
575 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
576 ; GFX8-NEXT: s_mov_b32 m0, -1
577 ; GFX8-NEXT: ds_read_b32 v1, v0
578 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
579 ; GFX8-NEXT: .LBB2_1: ; %atomicrmw.start
580 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
581 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
582 ; GFX8-NEXT: v_add_f32_e32 v2, -4.0, v1
583 ; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
584 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
585 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
586 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
587 ; GFX8-NEXT: v_mov_b32_e32 v1, v2
588 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
589 ; GFX8-NEXT: s_cbranch_execnz .LBB2_1
590 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
591 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
592 ; GFX8-NEXT: s_setpc_b64 s[30:31]
593 ;
594 ; GFX7-LABEL: local_atomic_fsub_noret_f32:
595 ; GFX7: ; %bb.0:
596 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
597 ; GFX7-NEXT: s_mov_b32 m0, -1
598 ; GFX7-NEXT: ds_read_b32 v1, v0
599 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
600 ; GFX7-NEXT: .LBB2_1: ; %atomicrmw.start
601 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
602 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
603 ; GFX7-NEXT: v_add_f32_e32 v2, -4.0, v1
604 ; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
605 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
606 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
607 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
608 ; GFX7-NEXT: v_mov_b32_e32 v1, v2
609 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
610 ; GFX7-NEXT: s_cbranch_execnz .LBB2_1
611 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
612 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
613 ; GFX7-NEXT: s_setpc_b64 s[30:31]
614 ;
615 ; GFX6-LABEL: local_atomic_fsub_noret_f32:
616 ; GFX6: ; %bb.0:
617 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
618 ; GFX6-NEXT: s_mov_b32 m0, -1
619 ; GFX6-NEXT: ds_read_b32 v1, v0
620 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
621 ; GFX6-NEXT: .LBB2_1: ; %atomicrmw.start
622 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
623 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
624 ; GFX6-NEXT: v_add_f32_e32 v2, -4.0, v1
625 ; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
626 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
627 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
628 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
629 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
630 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
631 ; GFX6-NEXT: s_cbranch_execnz .LBB2_1
632 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
633 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
634 ; GFX6-NEXT: s_setpc_b64 s[30:31]
635 %result = atomicrmw fsub ptr addrspace(3) %ptr, float 4.0 seq_cst
636 ret void
637 }
639 define void @local_atomic_fsub_noret_f32__offset(ptr addrspace(3) %ptr) nounwind {
640 ; GFX12-LABEL: local_atomic_fsub_noret_f32__offset:
641 ; GFX12: ; %bb.0:
642 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
643 ; GFX12-NEXT: s_wait_expcnt 0x0
644 ; GFX12-NEXT: s_wait_samplecnt 0x0
645 ; GFX12-NEXT: s_wait_bvhcnt 0x0
646 ; GFX12-NEXT: s_wait_kmcnt 0x0
647 ; GFX12-NEXT: ds_load_b32 v1, v0 offset:65532
648 ; GFX12-NEXT: s_mov_b32 s0, 0
649 ; GFX12-NEXT: .LBB3_1: ; %atomicrmw.start
650 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
651 ; GFX12-NEXT: s_wait_dscnt 0x0
652 ; GFX12-NEXT: v_add_f32_e32 v2, -4.0, v1
653 ; GFX12-NEXT: s_wait_storecnt 0x0
654 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65532
655 ; GFX12-NEXT: s_wait_dscnt 0x0
656 ; GFX12-NEXT: global_inv scope:SCOPE_SE
657 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
658 ; GFX12-NEXT: v_mov_b32_e32 v1, v2
659 ; GFX12-NEXT: s_wait_alu 0xfffe
660 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
661 ; GFX12-NEXT: s_wait_alu 0xfffe
662 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
663 ; GFX12-NEXT: s_cbranch_execnz .LBB3_1
664 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
665 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
666 ; GFX12-NEXT: s_setpc_b64 s[30:31]
667 ;
668 ; GFX940-LABEL: local_atomic_fsub_noret_f32__offset:
669 ; GFX940: ; %bb.0:
670 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
671 ; GFX940-NEXT: ds_read_b32 v1, v0 offset:65532
672 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
673 ; GFX940-NEXT: .LBB3_1: ; %atomicrmw.start
674 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
675 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
676 ; GFX940-NEXT: v_add_f32_e32 v2, -4.0, v1
677 ; GFX940-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:65532
678 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
679 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
680 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
681 ; GFX940-NEXT: v_mov_b32_e32 v1, v2
682 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
683 ; GFX940-NEXT: s_cbranch_execnz .LBB3_1
684 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
685 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
686 ; GFX940-NEXT: s_setpc_b64 s[30:31]
687 ;
688 ; GFX11-LABEL: local_atomic_fsub_noret_f32__offset:
689 ; GFX11: ; %bb.0:
690 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
691 ; GFX11-NEXT: ds_load_b32 v1, v0 offset:65532
692 ; GFX11-NEXT: s_mov_b32 s0, 0
693 ; GFX11-NEXT: .LBB3_1: ; %atomicrmw.start
694 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
695 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
696 ; GFX11-NEXT: v_add_f32_e32 v2, -4.0, v1
697 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
698 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65532
699 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
700 ; GFX11-NEXT: buffer_gl0_inv
701 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
702 ; GFX11-NEXT: v_mov_b32_e32 v1, v2
703 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
704 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
705 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
706 ; GFX11-NEXT: s_cbranch_execnz .LBB3_1
707 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
708 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
709 ; GFX11-NEXT: s_setpc_b64 s[30:31]
710 ;
711 ; GFX10-LABEL: local_atomic_fsub_noret_f32__offset:
712 ; GFX10: ; %bb.0:
713 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
714 ; GFX10-NEXT: ds_read_b32 v1, v0 offset:65532
715 ; GFX10-NEXT: s_mov_b32 s4, 0
716 ; GFX10-NEXT: .LBB3_1: ; %atomicrmw.start
717 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
718 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
719 ; GFX10-NEXT: v_add_f32_e32 v2, -4.0, v1
720 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
721 ; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:65532
722 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
723 ; GFX10-NEXT: buffer_gl0_inv
724 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
725 ; GFX10-NEXT: v_mov_b32_e32 v1, v2
726 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
727 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
728 ; GFX10-NEXT: s_cbranch_execnz .LBB3_1
729 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
730 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
731 ; GFX10-NEXT: s_setpc_b64 s[30:31]
732 ;
733 ; GFX90A-LABEL: local_atomic_fsub_noret_f32__offset:
734 ; GFX90A: ; %bb.0:
735 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
736 ; GFX90A-NEXT: ds_read_b32 v1, v0 offset:65532
737 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
738 ; GFX90A-NEXT: .LBB3_1: ; %atomicrmw.start
739 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
740 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
741 ; GFX90A-NEXT: v_add_f32_e32 v2, -4.0, v1
742 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:65532
743 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
744 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
745 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
746 ; GFX90A-NEXT: v_mov_b32_e32 v1, v2
747 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
748 ; GFX90A-NEXT: s_cbranch_execnz .LBB3_1
749 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
750 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
751 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
752 ;
753 ; GFX908-LABEL: local_atomic_fsub_noret_f32__offset:
754 ; GFX908: ; %bb.0:
755 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
756 ; GFX908-NEXT: ds_read_b32 v1, v0 offset:65532
757 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
758 ; GFX908-NEXT: .LBB3_1: ; %atomicrmw.start
759 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
760 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
761 ; GFX908-NEXT: v_add_f32_e32 v2, -4.0, v1
762 ; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:65532
763 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
764 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
765 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
766 ; GFX908-NEXT: v_mov_b32_e32 v1, v2
767 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
768 ; GFX908-NEXT: s_cbranch_execnz .LBB3_1
769 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
770 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
771 ; GFX908-NEXT: s_setpc_b64 s[30:31]
772 ;
773 ; GFX8-LABEL: local_atomic_fsub_noret_f32__offset:
774 ; GFX8: ; %bb.0:
775 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
776 ; GFX8-NEXT: s_mov_b32 m0, -1
777 ; GFX8-NEXT: ds_read_b32 v1, v0 offset:65532
778 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
779 ; GFX8-NEXT: .LBB3_1: ; %atomicrmw.start
780 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
781 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
782 ; GFX8-NEXT: v_add_f32_e32 v2, -4.0, v1
783 ; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:65532
784 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
785 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
786 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
787 ; GFX8-NEXT: v_mov_b32_e32 v1, v2
788 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
789 ; GFX8-NEXT: s_cbranch_execnz .LBB3_1
790 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
791 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
792 ; GFX8-NEXT: s_setpc_b64 s[30:31]
793 ;
794 ; GFX7-LABEL: local_atomic_fsub_noret_f32__offset:
795 ; GFX7: ; %bb.0:
796 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
797 ; GFX7-NEXT: s_mov_b32 m0, -1
798 ; GFX7-NEXT: ds_read_b32 v1, v0 offset:65532
799 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
800 ; GFX7-NEXT: .LBB3_1: ; %atomicrmw.start
801 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
802 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
803 ; GFX7-NEXT: v_add_f32_e32 v2, -4.0, v1
804 ; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:65532
805 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
806 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
807 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
808 ; GFX7-NEXT: v_mov_b32_e32 v1, v2
809 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
810 ; GFX7-NEXT: s_cbranch_execnz .LBB3_1
811 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
812 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
813 ; GFX7-NEXT: s_setpc_b64 s[30:31]
814 ;
815 ; GFX6-LABEL: local_atomic_fsub_noret_f32__offset:
816 ; GFX6: ; %bb.0:
817 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
818 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v0
819 ; GFX6-NEXT: s_mov_b32 m0, -1
820 ; GFX6-NEXT: ds_read_b32 v1, v0
821 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
822 ; GFX6-NEXT: .LBB3_1: ; %atomicrmw.start
823 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
824 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
825 ; GFX6-NEXT: v_add_f32_e32 v2, -4.0, v1
826 ; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
827 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
828 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
829 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
830 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
831 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
832 ; GFX6-NEXT: s_cbranch_execnz .LBB3_1
833 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
834 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
835 ; GFX6-NEXT: s_setpc_b64 s[30:31]
836 %gep = getelementptr float, ptr addrspace(3) %ptr, i32 16383
837 %unused = atomicrmw fsub ptr addrspace(3) %gep, float 4.0 seq_cst
838 ret void
839 }
841 ; --------------------------------------------------------------------
842 ; double
843 ; --------------------------------------------------------------------
845 define double @local_atomic_fsub_ret_f64(ptr addrspace(3) %ptr) nounwind {
846 ; GFX12-LABEL: local_atomic_fsub_ret_f64:
847 ; GFX12: ; %bb.0:
848 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
849 ; GFX12-NEXT: s_wait_expcnt 0x0
850 ; GFX12-NEXT: s_wait_samplecnt 0x0
851 ; GFX12-NEXT: s_wait_bvhcnt 0x0
852 ; GFX12-NEXT: s_wait_kmcnt 0x0
853 ; GFX12-NEXT: v_mov_b32_e32 v2, v0
854 ; GFX12-NEXT: ds_load_b64 v[0:1], v0
855 ; GFX12-NEXT: s_mov_b32 s0, 0
856 ; GFX12-NEXT: .LBB4_1: ; %atomicrmw.start
857 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
858 ; GFX12-NEXT: s_wait_dscnt 0x0
859 ; GFX12-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
860 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
861 ; GFX12-NEXT: v_add_f64_e32 v[0:1], -4.0, v[3:4]
862 ; GFX12-NEXT: s_wait_storecnt 0x0
863 ; GFX12-NEXT: ds_cmpstore_rtn_b64 v[0:1], v2, v[0:1], v[3:4]
864 ; GFX12-NEXT: s_wait_dscnt 0x0
865 ; GFX12-NEXT: global_inv scope:SCOPE_SE
866 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[3:4]
867 ; GFX12-NEXT: s_wait_alu 0xfffe
868 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
869 ; GFX12-NEXT: s_wait_alu 0xfffe
870 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
871 ; GFX12-NEXT: s_cbranch_execnz .LBB4_1
872 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
873 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
874 ; GFX12-NEXT: s_setpc_b64 s[30:31]
875 ;
876 ; GFX940-LABEL: local_atomic_fsub_ret_f64:
877 ; GFX940: ; %bb.0:
878 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
879 ; GFX940-NEXT: v_mov_b32_e32 v2, v0
880 ; GFX940-NEXT: ds_read_b64 v[0:1], v0
881 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
882 ; GFX940-NEXT: .LBB4_1: ; %atomicrmw.start
883 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
884 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
885 ; GFX940-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
886 ; GFX940-NEXT: v_add_f64 v[0:1], v[4:5], -4.0
887 ; GFX940-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[4:5], v[0:1]
888 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
889 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
890 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
891 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
892 ; GFX940-NEXT: s_cbranch_execnz .LBB4_1
893 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
894 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
895 ; GFX940-NEXT: s_setpc_b64 s[30:31]
896 ;
897 ; GFX11-LABEL: local_atomic_fsub_ret_f64:
898 ; GFX11: ; %bb.0:
899 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
900 ; GFX11-NEXT: v_mov_b32_e32 v2, v0
901 ; GFX11-NEXT: ds_load_b64 v[0:1], v0
902 ; GFX11-NEXT: s_mov_b32 s0, 0
903 ; GFX11-NEXT: .LBB4_1: ; %atomicrmw.start
904 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
905 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
906 ; GFX11-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
907 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
908 ; GFX11-NEXT: v_add_f64 v[0:1], v[3:4], -4.0
909 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
910 ; GFX11-NEXT: ds_cmpstore_rtn_b64 v[0:1], v2, v[0:1], v[3:4]
911 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
912 ; GFX11-NEXT: buffer_gl0_inv
913 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[3:4]
914 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
915 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
916 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
917 ; GFX11-NEXT: s_cbranch_execnz .LBB4_1
918 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
919 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
920 ; GFX11-NEXT: s_setpc_b64 s[30:31]
921 ;
922 ; GFX10-LABEL: local_atomic_fsub_ret_f64:
923 ; GFX10: ; %bb.0:
924 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
925 ; GFX10-NEXT: v_mov_b32_e32 v2, v0
926 ; GFX10-NEXT: ds_read_b64 v[0:1], v0
927 ; GFX10-NEXT: s_mov_b32 s4, 0
928 ; GFX10-NEXT: .LBB4_1: ; %atomicrmw.start
929 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
930 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
931 ; GFX10-NEXT: v_mov_b32_e32 v4, v1
932 ; GFX10-NEXT: v_mov_b32_e32 v3, v0
933 ; GFX10-NEXT: v_add_f64 v[0:1], v[3:4], -4.0
934 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
935 ; GFX10-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1]
936 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
937 ; GFX10-NEXT: buffer_gl0_inv
938 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[3:4]
939 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
940 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
941 ; GFX10-NEXT: s_cbranch_execnz .LBB4_1
942 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
943 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
944 ; GFX10-NEXT: s_setpc_b64 s[30:31]
945 ;
946 ; GFX90A-LABEL: local_atomic_fsub_ret_f64:
947 ; GFX90A: ; %bb.0:
948 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
949 ; GFX90A-NEXT: v_mov_b32_e32 v2, v0
950 ; GFX90A-NEXT: ds_read_b64 v[0:1], v0
951 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
952 ; GFX90A-NEXT: .LBB4_1: ; %atomicrmw.start
953 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
954 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
955 ; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
956 ; GFX90A-NEXT: v_add_f64 v[0:1], v[4:5], -4.0
957 ; GFX90A-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[4:5], v[0:1]
958 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
959 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
960 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
961 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
962 ; GFX90A-NEXT: s_cbranch_execnz .LBB4_1
963 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
964 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
965 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
966 ;
967 ; GFX908-LABEL: local_atomic_fsub_ret_f64:
968 ; GFX908: ; %bb.0:
969 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
970 ; GFX908-NEXT: v_mov_b32_e32 v2, v0
971 ; GFX908-NEXT: ds_read_b64 v[0:1], v0
972 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
973 ; GFX908-NEXT: .LBB4_1: ; %atomicrmw.start
974 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
975 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
976 ; GFX908-NEXT: v_mov_b32_e32 v4, v1
977 ; GFX908-NEXT: v_mov_b32_e32 v3, v0
978 ; GFX908-NEXT: v_add_f64 v[0:1], v[3:4], -4.0
979 ; GFX908-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1]
980 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
981 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[3:4]
982 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
983 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
984 ; GFX908-NEXT: s_cbranch_execnz .LBB4_1
985 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
986 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
987 ; GFX908-NEXT: s_setpc_b64 s[30:31]
988 ;
989 ; GFX8-LABEL: local_atomic_fsub_ret_f64:
990 ; GFX8: ; %bb.0:
991 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
992 ; GFX8-NEXT: s_mov_b32 m0, -1
993 ; GFX8-NEXT: v_mov_b32_e32 v2, v0
994 ; GFX8-NEXT: ds_read_b64 v[0:1], v0
995 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
996 ; GFX8-NEXT: .LBB4_1: ; %atomicrmw.start
997 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
998 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
999 ; GFX8-NEXT: v_mov_b32_e32 v4, v1
1000 ; GFX8-NEXT: v_mov_b32_e32 v3, v0
1001 ; GFX8-NEXT: v_add_f64 v[0:1], v[3:4], -4.0
1002 ; GFX8-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1]
1003 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1004 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[3:4]
1005 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1006 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1007 ; GFX8-NEXT: s_cbranch_execnz .LBB4_1
1008 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1009 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1010 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1011 ;
1012 ; GFX7-LABEL: local_atomic_fsub_ret_f64:
1013 ; GFX7: ; %bb.0:
1014 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1015 ; GFX7-NEXT: s_mov_b32 m0, -1
1016 ; GFX7-NEXT: v_mov_b32_e32 v2, v0
1017 ; GFX7-NEXT: ds_read_b64 v[0:1], v0
1018 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
1019 ; GFX7-NEXT: .LBB4_1: ; %atomicrmw.start
1020 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
1021 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1022 ; GFX7-NEXT: v_mov_b32_e32 v4, v1
1023 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
1024 ; GFX7-NEXT: v_add_f64 v[0:1], v[3:4], -4.0
1025 ; GFX7-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1]
1026 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1027 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[3:4]
1028 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1029 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
1030 ; GFX7-NEXT: s_cbranch_execnz .LBB4_1
1031 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
1032 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
1033 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1034 ;
1035 ; GFX6-LABEL: local_atomic_fsub_ret_f64:
1036 ; GFX6: ; %bb.0:
1037 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1038 ; GFX6-NEXT: s_mov_b32 m0, -1
1039 ; GFX6-NEXT: v_mov_b32_e32 v2, v0
1040 ; GFX6-NEXT: ds_read_b64 v[0:1], v0
1041 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
1042 ; GFX6-NEXT: .LBB4_1: ; %atomicrmw.start
1043 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
1044 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
1045 ; GFX6-NEXT: v_mov_b32_e32 v4, v1
1046 ; GFX6-NEXT: v_mov_b32_e32 v3, v0
1047 ; GFX6-NEXT: v_add_f64 v[0:1], v[3:4], -4.0
1048 ; GFX6-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1]
1049 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
1050 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[3:4]
1051 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1052 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
1053 ; GFX6-NEXT: s_cbranch_execnz .LBB4_1
1054 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
1055 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
1056 ; GFX6-NEXT: s_setpc_b64 s[30:31]
1057 %result = atomicrmw fsub ptr addrspace(3) %ptr, double 4.0 seq_cst
1058 ret double %result
1059 }
1061 define double @local_atomic_fsub_ret_f64__offset(ptr addrspace(3) %ptr) nounwind {
1062 ; GFX12-LABEL: local_atomic_fsub_ret_f64__offset:
1063 ; GFX12: ; %bb.0:
1064 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
1065 ; GFX12-NEXT: s_wait_expcnt 0x0
1066 ; GFX12-NEXT: s_wait_samplecnt 0x0
1067 ; GFX12-NEXT: s_wait_bvhcnt 0x0
1068 ; GFX12-NEXT: s_wait_kmcnt 0x0
1069 ; GFX12-NEXT: v_mov_b32_e32 v2, v0
1070 ; GFX12-NEXT: ds_load_b64 v[0:1], v0 offset:65528
1071 ; GFX12-NEXT: s_mov_b32 s0, 0
1072 ; GFX12-NEXT: .LBB5_1: ; %atomicrmw.start
1073 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
1074 ; GFX12-NEXT: s_wait_dscnt 0x0
1075 ; GFX12-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
1076 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
1077 ; GFX12-NEXT: v_add_f64_e32 v[0:1], -4.0, v[3:4]
1078 ; GFX12-NEXT: s_wait_storecnt 0x0
1079 ; GFX12-NEXT: ds_cmpstore_rtn_b64 v[0:1], v2, v[0:1], v[3:4] offset:65528
1080 ; GFX12-NEXT: s_wait_dscnt 0x0
1081 ; GFX12-NEXT: global_inv scope:SCOPE_SE
1082 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[3:4]
1083 ; GFX12-NEXT: s_wait_alu 0xfffe
1084 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
1085 ; GFX12-NEXT: s_wait_alu 0xfffe
1086 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1087 ; GFX12-NEXT: s_cbranch_execnz .LBB5_1
1088 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
1089 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
1090 ; GFX12-NEXT: s_setpc_b64 s[30:31]
1091 ;
1092 ; GFX940-LABEL: local_atomic_fsub_ret_f64__offset:
1093 ; GFX940: ; %bb.0:
1094 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1095 ; GFX940-NEXT: v_mov_b32_e32 v2, v0
1096 ; GFX940-NEXT: ds_read_b64 v[0:1], v0 offset:65528
1097 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
1098 ; GFX940-NEXT: .LBB5_1: ; %atomicrmw.start
1099 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
1100 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
1101 ; GFX940-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
1102 ; GFX940-NEXT: v_add_f64 v[0:1], v[4:5], -4.0
1103 ; GFX940-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[4:5], v[0:1] offset:65528
1104 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
1105 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
1106 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
1107 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
1108 ; GFX940-NEXT: s_cbranch_execnz .LBB5_1
1109 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
1110 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
1111 ; GFX940-NEXT: s_setpc_b64 s[30:31]
1112 ;
1113 ; GFX11-LABEL: local_atomic_fsub_ret_f64__offset:
1114 ; GFX11: ; %bb.0:
1115 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1116 ; GFX11-NEXT: v_mov_b32_e32 v2, v0
1117 ; GFX11-NEXT: ds_load_b64 v[0:1], v0 offset:65528
1118 ; GFX11-NEXT: s_mov_b32 s0, 0
1119 ; GFX11-NEXT: .LBB5_1: ; %atomicrmw.start
1120 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
1121 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
1122 ; GFX11-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
1123 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
1124 ; GFX11-NEXT: v_add_f64 v[0:1], v[3:4], -4.0
1125 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1126 ; GFX11-NEXT: ds_cmpstore_rtn_b64 v[0:1], v2, v[0:1], v[3:4] offset:65528
1127 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
1128 ; GFX11-NEXT: buffer_gl0_inv
1129 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[3:4]
1130 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
1131 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1132 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1133 ; GFX11-NEXT: s_cbranch_execnz .LBB5_1
1134 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
1135 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
1136 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1137 ;
1138 ; GFX10-LABEL: local_atomic_fsub_ret_f64__offset:
1139 ; GFX10: ; %bb.0:
1140 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1141 ; GFX10-NEXT: v_mov_b32_e32 v2, v0
1142 ; GFX10-NEXT: ds_read_b64 v[0:1], v0 offset:65528
1143 ; GFX10-NEXT: s_mov_b32 s4, 0
1144 ; GFX10-NEXT: .LBB5_1: ; %atomicrmw.start
1145 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
1146 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
1147 ; GFX10-NEXT: v_mov_b32_e32 v4, v1
1148 ; GFX10-NEXT: v_mov_b32_e32 v3, v0
1149 ; GFX10-NEXT: v_add_f64 v[0:1], v[3:4], -4.0
1150 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1151 ; GFX10-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1] offset:65528
1152 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
1153 ; GFX10-NEXT: buffer_gl0_inv
1154 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[3:4]
1155 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
1156 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
1157 ; GFX10-NEXT: s_cbranch_execnz .LBB5_1
1158 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
1159 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
1160 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1161 ;
1162 ; GFX90A-LABEL: local_atomic_fsub_ret_f64__offset:
1163 ; GFX90A: ; %bb.0:
1164 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1165 ; GFX90A-NEXT: v_mov_b32_e32 v2, v0
1166 ; GFX90A-NEXT: ds_read_b64 v[0:1], v0 offset:65528
1167 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
1168 ; GFX90A-NEXT: .LBB5_1: ; %atomicrmw.start
1169 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
1170 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
1171 ; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
1172 ; GFX90A-NEXT: v_add_f64 v[0:1], v[4:5], -4.0
1173 ; GFX90A-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[4:5], v[0:1] offset:65528
1174 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
1175 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
1176 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1177 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
1178 ; GFX90A-NEXT: s_cbranch_execnz .LBB5_1
1179 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
1180 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
1181 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1182 ;
1183 ; GFX908-LABEL: local_atomic_fsub_ret_f64__offset:
1184 ; GFX908: ; %bb.0:
1185 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1186 ; GFX908-NEXT: v_mov_b32_e32 v2, v0
1187 ; GFX908-NEXT: ds_read_b64 v[0:1], v0 offset:65528
1188 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
1189 ; GFX908-NEXT: .LBB5_1: ; %atomicrmw.start
1190 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
1191 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
1192 ; GFX908-NEXT: v_mov_b32_e32 v4, v1
1193 ; GFX908-NEXT: v_mov_b32_e32 v3, v0
1194 ; GFX908-NEXT: v_add_f64 v[0:1], v[3:4], -4.0
1195 ; GFX908-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1] offset:65528
1196 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
1197 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[3:4]
1198 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1199 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
1200 ; GFX908-NEXT: s_cbranch_execnz .LBB5_1
1201 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
1202 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
1203 ; GFX908-NEXT: s_setpc_b64 s[30:31]
1204 ;
1205 ; GFX8-LABEL: local_atomic_fsub_ret_f64__offset:
1206 ; GFX8: ; %bb.0:
1207 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1208 ; GFX8-NEXT: s_mov_b32 m0, -1
1209 ; GFX8-NEXT: v_mov_b32_e32 v2, v0
1210 ; GFX8-NEXT: ds_read_b64 v[0:1], v0 offset:65528
1211 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
1212 ; GFX8-NEXT: .LBB5_1: ; %atomicrmw.start
1213 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
1214 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1215 ; GFX8-NEXT: v_mov_b32_e32 v4, v1
1216 ; GFX8-NEXT: v_mov_b32_e32 v3, v0
1217 ; GFX8-NEXT: v_add_f64 v[0:1], v[3:4], -4.0
1218 ; GFX8-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1] offset:65528
1219 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1220 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[3:4]
1221 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1222 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1223 ; GFX8-NEXT: s_cbranch_execnz .LBB5_1
1224 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1225 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1226 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1227 ;
1228 ; GFX7-LABEL: local_atomic_fsub_ret_f64__offset:
1229 ; GFX7: ; %bb.0:
1230 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1231 ; GFX7-NEXT: s_mov_b32 m0, -1
1232 ; GFX7-NEXT: v_mov_b32_e32 v2, v0
1233 ; GFX7-NEXT: ds_read_b64 v[0:1], v0 offset:65528
1234 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
1235 ; GFX7-NEXT: .LBB5_1: ; %atomicrmw.start
1236 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
1237 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1238 ; GFX7-NEXT: v_mov_b32_e32 v4, v1
1239 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
1240 ; GFX7-NEXT: v_add_f64 v[0:1], v[3:4], -4.0
1241 ; GFX7-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1] offset:65528
1242 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1243 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[3:4]
1244 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1245 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
1246 ; GFX7-NEXT: s_cbranch_execnz .LBB5_1
1247 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
1248 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
1249 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1250 ;
1251 ; GFX6-LABEL: local_atomic_fsub_ret_f64__offset:
1252 ; GFX6: ; %bb.0:
1253 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1254 ; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfff8, v0
1255 ; GFX6-NEXT: s_mov_b32 m0, -1
1256 ; GFX6-NEXT: ds_read_b64 v[0:1], v2
1257 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
1258 ; GFX6-NEXT: .LBB5_1: ; %atomicrmw.start
1259 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
1260 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
1261 ; GFX6-NEXT: v_mov_b32_e32 v4, v1
1262 ; GFX6-NEXT: v_mov_b32_e32 v3, v0
1263 ; GFX6-NEXT: v_add_f64 v[0:1], v[3:4], -4.0
1264 ; GFX6-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1]
1265 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
1266 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[3:4]
1267 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1268 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
1269 ; GFX6-NEXT: s_cbranch_execnz .LBB5_1
1270 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
1271 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
1272 ; GFX6-NEXT: s_setpc_b64 s[30:31]
1273 %gep = getelementptr double, ptr addrspace(3) %ptr, i32 8191
1274 %result = atomicrmw fsub ptr addrspace(3) %gep, double 4.0 seq_cst
1275 ret double %result
1276 }
1278 define void @local_atomic_fsub_noret_f64(ptr addrspace(3) %ptr) nounwind {
1279 ; GFX12-LABEL: local_atomic_fsub_noret_f64:
1280 ; GFX12: ; %bb.0:
1281 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
1282 ; GFX12-NEXT: s_wait_expcnt 0x0
1283 ; GFX12-NEXT: s_wait_samplecnt 0x0
1284 ; GFX12-NEXT: s_wait_bvhcnt 0x0
1285 ; GFX12-NEXT: s_wait_kmcnt 0x0
1286 ; GFX12-NEXT: ds_load_b64 v[1:2], v0
1287 ; GFX12-NEXT: s_mov_b32 s0, 0
1288 ; GFX12-NEXT: .LBB6_1: ; %atomicrmw.start
1289 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
1290 ; GFX12-NEXT: s_wait_dscnt 0x0
1291 ; GFX12-NEXT: v_add_f64_e32 v[3:4], -4.0, v[1:2]
1292 ; GFX12-NEXT: s_wait_storecnt 0x0
1293 ; GFX12-NEXT: ds_cmpstore_rtn_b64 v[3:4], v0, v[3:4], v[1:2]
1294 ; GFX12-NEXT: s_wait_dscnt 0x0
1295 ; GFX12-NEXT: global_inv scope:SCOPE_SE
1296 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[3:4], v[1:2]
1297 ; GFX12-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
1298 ; GFX12-NEXT: s_wait_alu 0xfffe
1299 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
1300 ; GFX12-NEXT: s_wait_alu 0xfffe
1301 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1302 ; GFX12-NEXT: s_cbranch_execnz .LBB6_1
1303 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
1304 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
1305 ; GFX12-NEXT: s_setpc_b64 s[30:31]
1306 ;
1307 ; GFX940-LABEL: local_atomic_fsub_noret_f64:
1308 ; GFX940: ; %bb.0:
1309 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1310 ; GFX940-NEXT: ds_read_b64 v[2:3], v0
1311 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
1312 ; GFX940-NEXT: .LBB6_1: ; %atomicrmw.start
1313 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
1314 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
1315 ; GFX940-NEXT: v_add_f64 v[4:5], v[2:3], -4.0
1316 ; GFX940-NEXT: ds_cmpst_rtn_b64 v[4:5], v0, v[2:3], v[4:5]
1317 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
1318 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
1319 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
1320 ; GFX940-NEXT: v_mov_b64_e32 v[2:3], v[4:5]
1321 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
1322 ; GFX940-NEXT: s_cbranch_execnz .LBB6_1
1323 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
1324 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
1325 ; GFX940-NEXT: s_setpc_b64 s[30:31]
1326 ;
1327 ; GFX11-LABEL: local_atomic_fsub_noret_f64:
1328 ; GFX11: ; %bb.0:
1329 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1330 ; GFX11-NEXT: ds_load_b64 v[1:2], v0
1331 ; GFX11-NEXT: s_mov_b32 s0, 0
1332 ; GFX11-NEXT: .LBB6_1: ; %atomicrmw.start
1333 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
1334 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
1335 ; GFX11-NEXT: v_add_f64 v[3:4], v[1:2], -4.0
1336 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1337 ; GFX11-NEXT: ds_cmpstore_rtn_b64 v[3:4], v0, v[3:4], v[1:2]
1338 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
1339 ; GFX11-NEXT: buffer_gl0_inv
1340 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[3:4], v[1:2]
1341 ; GFX11-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
1342 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
1343 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1344 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1345 ; GFX11-NEXT: s_cbranch_execnz .LBB6_1
1346 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
1347 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
1348 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1349 ;
1350 ; GFX10-LABEL: local_atomic_fsub_noret_f64:
1351 ; GFX10: ; %bb.0:
1352 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1353 ; GFX10-NEXT: ds_read_b64 v[1:2], v0
1354 ; GFX10-NEXT: s_mov_b32 s4, 0
1355 ; GFX10-NEXT: .LBB6_1: ; %atomicrmw.start
1356 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
1357 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
1358 ; GFX10-NEXT: v_add_f64 v[3:4], v[1:2], -4.0
1359 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1360 ; GFX10-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[1:2], v[3:4]
1361 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
1362 ; GFX10-NEXT: buffer_gl0_inv
1363 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[3:4], v[1:2]
1364 ; GFX10-NEXT: v_mov_b32_e32 v1, v3
1365 ; GFX10-NEXT: v_mov_b32_e32 v2, v4
1366 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
1367 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
1368 ; GFX10-NEXT: s_cbranch_execnz .LBB6_1
1369 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
1370 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
1371 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1372 ;
1373 ; GFX90A-LABEL: local_atomic_fsub_noret_f64:
1374 ; GFX90A: ; %bb.0:
1375 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1376 ; GFX90A-NEXT: ds_read_b64 v[2:3], v0
1377 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
1378 ; GFX90A-NEXT: .LBB6_1: ; %atomicrmw.start
1379 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
1380 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
1381 ; GFX90A-NEXT: v_add_f64 v[4:5], v[2:3], -4.0
1382 ; GFX90A-NEXT: ds_cmpst_rtn_b64 v[4:5], v0, v[2:3], v[4:5]
1383 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
1384 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
1385 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1386 ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[4:5], v[4:5] op_sel:[0,1]
1387 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
1388 ; GFX90A-NEXT: s_cbranch_execnz .LBB6_1
1389 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
1390 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
1391 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1392 ;
1393 ; GFX908-LABEL: local_atomic_fsub_noret_f64:
1394 ; GFX908: ; %bb.0:
1395 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1396 ; GFX908-NEXT: ds_read_b64 v[1:2], v0
1397 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
1398 ; GFX908-NEXT: .LBB6_1: ; %atomicrmw.start
1399 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
1400 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
1401 ; GFX908-NEXT: v_add_f64 v[3:4], v[1:2], -4.0
1402 ; GFX908-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[1:2], v[3:4]
1403 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
1404 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[1:2]
1405 ; GFX908-NEXT: v_mov_b32_e32 v1, v3
1406 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1407 ; GFX908-NEXT: v_mov_b32_e32 v2, v4
1408 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
1409 ; GFX908-NEXT: s_cbranch_execnz .LBB6_1
1410 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
1411 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
1412 ; GFX908-NEXT: s_setpc_b64 s[30:31]
1413 ;
1414 ; GFX8-LABEL: local_atomic_fsub_noret_f64:
1415 ; GFX8: ; %bb.0:
1416 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1417 ; GFX8-NEXT: s_mov_b32 m0, -1
1418 ; GFX8-NEXT: ds_read_b64 v[1:2], v0
1419 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
1420 ; GFX8-NEXT: .LBB6_1: ; %atomicrmw.start
1421 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
1422 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1423 ; GFX8-NEXT: v_add_f64 v[3:4], v[1:2], -4.0
1424 ; GFX8-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[1:2], v[3:4]
1425 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1426 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[1:2]
1427 ; GFX8-NEXT: v_mov_b32_e32 v1, v3
1428 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1429 ; GFX8-NEXT: v_mov_b32_e32 v2, v4
1430 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1431 ; GFX8-NEXT: s_cbranch_execnz .LBB6_1
1432 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1433 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1434 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1435 ;
1436 ; GFX7-LABEL: local_atomic_fsub_noret_f64:
1437 ; GFX7: ; %bb.0:
1438 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1439 ; GFX7-NEXT: s_mov_b32 m0, -1
1440 ; GFX7-NEXT: ds_read_b64 v[1:2], v0
1441 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
1442 ; GFX7-NEXT: .LBB6_1: ; %atomicrmw.start
1443 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
1444 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1445 ; GFX7-NEXT: v_add_f64 v[3:4], v[1:2], -4.0
1446 ; GFX7-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[1:2], v[3:4]
1447 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1448 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[1:2]
1449 ; GFX7-NEXT: v_mov_b32_e32 v1, v3
1450 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1451 ; GFX7-NEXT: v_mov_b32_e32 v2, v4
1452 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
1453 ; GFX7-NEXT: s_cbranch_execnz .LBB6_1
1454 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
1455 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
1456 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1457 ;
1458 ; GFX6-LABEL: local_atomic_fsub_noret_f64:
1459 ; GFX6: ; %bb.0:
1460 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1461 ; GFX6-NEXT: s_mov_b32 m0, -1
1462 ; GFX6-NEXT: ds_read_b64 v[1:2], v0
1463 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
1464 ; GFX6-NEXT: .LBB6_1: ; %atomicrmw.start
1465 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
1466 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
1467 ; GFX6-NEXT: v_add_f64 v[3:4], v[1:2], -4.0
1468 ; GFX6-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[1:2], v[3:4]
1469 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
1470 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[1:2]
1471 ; GFX6-NEXT: v_mov_b32_e32 v1, v3
1472 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1473 ; GFX6-NEXT: v_mov_b32_e32 v2, v4
1474 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
1475 ; GFX6-NEXT: s_cbranch_execnz .LBB6_1
1476 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
1477 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
1478 ; GFX6-NEXT: s_setpc_b64 s[30:31]
1479 %result = atomicrmw fsub ptr addrspace(3) %ptr, double 4.0 seq_cst
1480 ret void
1481 }
1483 define void @local_atomic_fsub_noret_f64__offset(ptr addrspace(3) %ptr) nounwind {
1484 ; GFX12-LABEL: local_atomic_fsub_noret_f64__offset:
1485 ; GFX12: ; %bb.0:
1486 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
1487 ; GFX12-NEXT: s_wait_expcnt 0x0
1488 ; GFX12-NEXT: s_wait_samplecnt 0x0
1489 ; GFX12-NEXT: s_wait_bvhcnt 0x0
1490 ; GFX12-NEXT: s_wait_kmcnt 0x0
1491 ; GFX12-NEXT: ds_load_b64 v[1:2], v0 offset:65528
1492 ; GFX12-NEXT: s_mov_b32 s0, 0
1493 ; GFX12-NEXT: .LBB7_1: ; %atomicrmw.start
1494 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
1495 ; GFX12-NEXT: s_wait_dscnt 0x0
1496 ; GFX12-NEXT: v_add_f64_e32 v[3:4], -4.0, v[1:2]
1497 ; GFX12-NEXT: s_wait_storecnt 0x0
1498 ; GFX12-NEXT: ds_cmpstore_rtn_b64 v[3:4], v0, v[3:4], v[1:2] offset:65528
1499 ; GFX12-NEXT: s_wait_dscnt 0x0
1500 ; GFX12-NEXT: global_inv scope:SCOPE_SE
1501 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[3:4], v[1:2]
1502 ; GFX12-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
1503 ; GFX12-NEXT: s_wait_alu 0xfffe
1504 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
1505 ; GFX12-NEXT: s_wait_alu 0xfffe
1506 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1507 ; GFX12-NEXT: s_cbranch_execnz .LBB7_1
1508 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
1509 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
1510 ; GFX12-NEXT: s_setpc_b64 s[30:31]
1511 ;
1512 ; GFX940-LABEL: local_atomic_fsub_noret_f64__offset:
1513 ; GFX940: ; %bb.0:
1514 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1515 ; GFX940-NEXT: ds_read_b64 v[2:3], v0 offset:65528
1516 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
1517 ; GFX940-NEXT: .LBB7_1: ; %atomicrmw.start
1518 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
1519 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
1520 ; GFX940-NEXT: v_add_f64 v[4:5], v[2:3], -4.0
1521 ; GFX940-NEXT: ds_cmpst_rtn_b64 v[4:5], v0, v[2:3], v[4:5] offset:65528
1522 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
1523 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
1524 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
1525 ; GFX940-NEXT: v_mov_b64_e32 v[2:3], v[4:5]
1526 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
1527 ; GFX940-NEXT: s_cbranch_execnz .LBB7_1
1528 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
1529 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
1530 ; GFX940-NEXT: s_setpc_b64 s[30:31]
1531 ;
1532 ; GFX11-LABEL: local_atomic_fsub_noret_f64__offset:
1533 ; GFX11: ; %bb.0:
1534 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1535 ; GFX11-NEXT: ds_load_b64 v[1:2], v0 offset:65528
1536 ; GFX11-NEXT: s_mov_b32 s0, 0
1537 ; GFX11-NEXT: .LBB7_1: ; %atomicrmw.start
1538 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
1539 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
1540 ; GFX11-NEXT: v_add_f64 v[3:4], v[1:2], -4.0
1541 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1542 ; GFX11-NEXT: ds_cmpstore_rtn_b64 v[3:4], v0, v[3:4], v[1:2] offset:65528
1543 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
1544 ; GFX11-NEXT: buffer_gl0_inv
1545 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[3:4], v[1:2]
1546 ; GFX11-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
1547 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
1548 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1549 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1550 ; GFX11-NEXT: s_cbranch_execnz .LBB7_1
1551 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
1552 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
1553 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1554 ;
1555 ; GFX10-LABEL: local_atomic_fsub_noret_f64__offset:
1556 ; GFX10: ; %bb.0:
1557 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1558 ; GFX10-NEXT: ds_read_b64 v[1:2], v0 offset:65528
1559 ; GFX10-NEXT: s_mov_b32 s4, 0
1560 ; GFX10-NEXT: .LBB7_1: ; %atomicrmw.start
1561 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
1562 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
1563 ; GFX10-NEXT: v_add_f64 v[3:4], v[1:2], -4.0
1564 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1565 ; GFX10-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[1:2], v[3:4] offset:65528
1566 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
1567 ; GFX10-NEXT: buffer_gl0_inv
1568 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[3:4], v[1:2]
1569 ; GFX10-NEXT: v_mov_b32_e32 v1, v3
1570 ; GFX10-NEXT: v_mov_b32_e32 v2, v4
1571 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
1572 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
1573 ; GFX10-NEXT: s_cbranch_execnz .LBB7_1
1574 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
1575 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
1576 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1577 ;
1578 ; GFX90A-LABEL: local_atomic_fsub_noret_f64__offset:
1579 ; GFX90A: ; %bb.0:
1580 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1581 ; GFX90A-NEXT: ds_read_b64 v[2:3], v0 offset:65528
1582 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
1583 ; GFX90A-NEXT: .LBB7_1: ; %atomicrmw.start
1584 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
1585 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
1586 ; GFX90A-NEXT: v_add_f64 v[4:5], v[2:3], -4.0
1587 ; GFX90A-NEXT: ds_cmpst_rtn_b64 v[4:5], v0, v[2:3], v[4:5] offset:65528
1588 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
1589 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
1590 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1591 ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[4:5], v[4:5] op_sel:[0,1]
1592 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
1593 ; GFX90A-NEXT: s_cbranch_execnz .LBB7_1
1594 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
1595 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
1596 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1597 ;
1598 ; GFX908-LABEL: local_atomic_fsub_noret_f64__offset:
1599 ; GFX908: ; %bb.0:
1600 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1601 ; GFX908-NEXT: ds_read_b64 v[1:2], v0 offset:65528
1602 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
1603 ; GFX908-NEXT: .LBB7_1: ; %atomicrmw.start
1604 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
1605 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
1606 ; GFX908-NEXT: v_add_f64 v[3:4], v[1:2], -4.0
1607 ; GFX908-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[1:2], v[3:4] offset:65528
1608 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
1609 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[1:2]
1610 ; GFX908-NEXT: v_mov_b32_e32 v1, v3
1611 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1612 ; GFX908-NEXT: v_mov_b32_e32 v2, v4
1613 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
1614 ; GFX908-NEXT: s_cbranch_execnz .LBB7_1
1615 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
1616 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
1617 ; GFX908-NEXT: s_setpc_b64 s[30:31]
1618 ;
1619 ; GFX8-LABEL: local_atomic_fsub_noret_f64__offset:
1620 ; GFX8: ; %bb.0:
1621 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1622 ; GFX8-NEXT: s_mov_b32 m0, -1
1623 ; GFX8-NEXT: ds_read_b64 v[1:2], v0 offset:65528
1624 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
1625 ; GFX8-NEXT: .LBB7_1: ; %atomicrmw.start
1626 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
1627 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1628 ; GFX8-NEXT: v_add_f64 v[3:4], v[1:2], -4.0
1629 ; GFX8-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[1:2], v[3:4] offset:65528
1630 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1631 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[1:2]
1632 ; GFX8-NEXT: v_mov_b32_e32 v1, v3
1633 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1634 ; GFX8-NEXT: v_mov_b32_e32 v2, v4
1635 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1636 ; GFX8-NEXT: s_cbranch_execnz .LBB7_1
1637 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1638 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1639 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1640 ;
1641 ; GFX7-LABEL: local_atomic_fsub_noret_f64__offset:
1642 ; GFX7: ; %bb.0:
1643 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1644 ; GFX7-NEXT: s_mov_b32 m0, -1
1645 ; GFX7-NEXT: ds_read_b64 v[1:2], v0 offset:65528
1646 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
1647 ; GFX7-NEXT: .LBB7_1: ; %atomicrmw.start
1648 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
1649 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1650 ; GFX7-NEXT: v_add_f64 v[3:4], v[1:2], -4.0
1651 ; GFX7-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[1:2], v[3:4] offset:65528
1652 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1653 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[1:2]
1654 ; GFX7-NEXT: v_mov_b32_e32 v1, v3
1655 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1656 ; GFX7-NEXT: v_mov_b32_e32 v2, v4
1657 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
1658 ; GFX7-NEXT: s_cbranch_execnz .LBB7_1
1659 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
1660 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
1661 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1662 ;
1663 ; GFX6-LABEL: local_atomic_fsub_noret_f64__offset:
1664 ; GFX6: ; %bb.0:
1665 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1666 ; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfff8, v0
1667 ; GFX6-NEXT: s_mov_b32 m0, -1
1668 ; GFX6-NEXT: ds_read_b64 v[0:1], v2
1669 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
1670 ; GFX6-NEXT: .LBB7_1: ; %atomicrmw.start
1671 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
1672 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
1673 ; GFX6-NEXT: v_add_f64 v[3:4], v[0:1], -4.0
1674 ; GFX6-NEXT: ds_cmpst_rtn_b64 v[3:4], v2, v[0:1], v[3:4]
1675 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
1676 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[0:1]
1677 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
1678 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1679 ; GFX6-NEXT: v_mov_b32_e32 v1, v4
1680 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
1681 ; GFX6-NEXT: s_cbranch_execnz .LBB7_1
1682 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
1683 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
1684 ; GFX6-NEXT: s_setpc_b64 s[30:31]
1685 %gep = getelementptr double, ptr addrspace(3) %ptr, i32 8191
1686 %unused = atomicrmw fsub ptr addrspace(3) %gep, double 4.0 seq_cst
1687 ret void
1688 }
1690 ; --------------------------------------------------------------------
1691 ; half
1692 ; --------------------------------------------------------------------
1694 define half @local_atomic_fsub_ret_f16(ptr addrspace(3) %ptr) nounwind {
1695 ; GFX12-LABEL: local_atomic_fsub_ret_f16:
1696 ; GFX12: ; %bb.0:
1697 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
1698 ; GFX12-NEXT: s_wait_expcnt 0x0
1699 ; GFX12-NEXT: s_wait_samplecnt 0x0
1700 ; GFX12-NEXT: s_wait_bvhcnt 0x0
1701 ; GFX12-NEXT: s_wait_kmcnt 0x0
1702 ; GFX12-NEXT: v_and_b32_e32 v1, -4, v0
1703 ; GFX12-NEXT: v_lshlrev_b32_e32 v0, 3, v0
1704 ; GFX12-NEXT: s_mov_b32 s0, 0
1705 ; GFX12-NEXT: ds_load_b32 v2, v1
1706 ; GFX12-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff
1707 ; GFX12-NEXT: v_and_b32_e32 v0, 24, v0
1708 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
1709 ; GFX12-NEXT: v_not_b32_e32 v3, v3
1710 ; GFX12-NEXT: .LBB8_1: ; %atomicrmw.start
1711 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
1712 ; GFX12-NEXT: s_wait_dscnt 0x0
1713 ; GFX12-NEXT: v_mov_b32_e32 v4, v2
1714 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
1715 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, v0, v4
1716 ; GFX12-NEXT: v_add_f16_e32 v2, -4.0, v2
1717 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
1718 ; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
1719 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, v0, v2
1720 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
1721 ; GFX12-NEXT: v_and_or_b32 v2, v4, v3, v2
1722 ; GFX12-NEXT: s_wait_storecnt 0x0
1723 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v2, v1, v2, v4
1724 ; GFX12-NEXT: s_wait_dscnt 0x0
1725 ; GFX12-NEXT: global_inv scope:SCOPE_SE
1726 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
1727 ; GFX12-NEXT: s_wait_alu 0xfffe
1728 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
1729 ; GFX12-NEXT: s_wait_alu 0xfffe
1730 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1731 ; GFX12-NEXT: s_cbranch_execnz .LBB8_1
1732 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
1733 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
1734 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v0, v2
1735 ; GFX12-NEXT: s_setpc_b64 s[30:31]
1736 ;
1737 ; GFX940-LABEL: local_atomic_fsub_ret_f16:
1738 ; GFX940: ; %bb.0:
1739 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1740 ; GFX940-NEXT: v_and_b32_e32 v1, -4, v0
1741 ; GFX940-NEXT: ds_read_b32 v2, v1
1742 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v0
1743 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
1744 ; GFX940-NEXT: v_and_b32_e32 v0, 24, v3
1745 ; GFX940-NEXT: v_lshlrev_b32_e64 v3, v3, s0
1746 ; GFX940-NEXT: v_not_b32_e32 v3, v3
1747 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
1748 ; GFX940-NEXT: .LBB8_1: ; %atomicrmw.start
1749 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
1750 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
1751 ; GFX940-NEXT: v_mov_b32_e32 v4, v2
1752 ; GFX940-NEXT: v_lshrrev_b32_e32 v2, v0, v4
1753 ; GFX940-NEXT: v_add_f16_e32 v2, -4.0, v2
1754 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, v0, v2
1755 ; GFX940-NEXT: v_and_or_b32 v2, v4, v3, v2
1756 ; GFX940-NEXT: ds_cmpst_rtn_b32 v2, v1, v4, v2
1757 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
1758 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
1759 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
1760 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
1761 ; GFX940-NEXT: s_cbranch_execnz .LBB8_1
1762 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
1763 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
1764 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v0, v2
1765 ; GFX940-NEXT: s_setpc_b64 s[30:31]
1766 ;
1767 ; GFX11-LABEL: local_atomic_fsub_ret_f16:
1768 ; GFX11: ; %bb.0:
1769 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1770 ; GFX11-NEXT: v_and_b32_e32 v1, -4, v0
1771 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, 3, v0
1772 ; GFX11-NEXT: s_mov_b32 s0, 0
1773 ; GFX11-NEXT: ds_load_b32 v2, v1
1774 ; GFX11-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff
1775 ; GFX11-NEXT: v_and_b32_e32 v0, 24, v0
1776 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
1777 ; GFX11-NEXT: v_not_b32_e32 v3, v3
1778 ; GFX11-NEXT: .LBB8_1: ; %atomicrmw.start
1779 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
1780 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
1781 ; GFX11-NEXT: v_mov_b32_e32 v4, v2
1782 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
1783 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, v0, v4
1784 ; GFX11-NEXT: v_add_f16_e32 v2, -4.0, v2
1785 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
1786 ; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
1787 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, v0, v2
1788 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
1789 ; GFX11-NEXT: v_and_or_b32 v2, v4, v3, v2
1790 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1791 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v2, v1, v2, v4
1792 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
1793 ; GFX11-NEXT: buffer_gl0_inv
1794 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
1795 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
1796 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1797 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1798 ; GFX11-NEXT: s_cbranch_execnz .LBB8_1
1799 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
1800 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
1801 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v0, v2
1802 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1803 ;
1804 ; GFX10-LABEL: local_atomic_fsub_ret_f16:
1805 ; GFX10: ; %bb.0:
1806 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1807 ; GFX10-NEXT: v_and_b32_e32 v1, -4, v0
1808 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v0
1809 ; GFX10-NEXT: s_mov_b32 s4, 0
1810 ; GFX10-NEXT: ds_read_b32 v2, v1
1811 ; GFX10-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff
1812 ; GFX10-NEXT: v_and_b32_e32 v0, 24, v0
1813 ; GFX10-NEXT: v_not_b32_e32 v3, v3
1814 ; GFX10-NEXT: .LBB8_1: ; %atomicrmw.start
1815 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
1816 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
1817 ; GFX10-NEXT: v_mov_b32_e32 v4, v2
1818 ; GFX10-NEXT: v_lshrrev_b32_e32 v2, v0, v4
1819 ; GFX10-NEXT: v_add_f16_e32 v2, -4.0, v2
1820 ; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
1821 ; GFX10-NEXT: v_and_or_b32 v2, v4, v3, v2
1822 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1823 ; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v1, v4, v2
1824 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
1825 ; GFX10-NEXT: buffer_gl0_inv
1826 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
1827 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
1828 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
1829 ; GFX10-NEXT: s_cbranch_execnz .LBB8_1
1830 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
1831 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
1832 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v0, v2
1833 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1834 ;
1835 ; GFX90A-LABEL: local_atomic_fsub_ret_f16:
1836 ; GFX90A: ; %bb.0:
1837 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1838 ; GFX90A-NEXT: v_and_b32_e32 v1, -4, v0
1839 ; GFX90A-NEXT: ds_read_b32 v2, v1
1840 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v0
1841 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
1842 ; GFX90A-NEXT: v_and_b32_e32 v0, 24, v3
1843 ; GFX90A-NEXT: v_lshlrev_b32_e64 v3, v3, s4
1844 ; GFX90A-NEXT: v_not_b32_e32 v3, v3
1845 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
1846 ; GFX90A-NEXT: .LBB8_1: ; %atomicrmw.start
1847 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
1848 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
1849 ; GFX90A-NEXT: v_mov_b32_e32 v4, v2
1850 ; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v0, v4
1851 ; GFX90A-NEXT: v_add_f16_e32 v2, -4.0, v2
1852 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v0, v2
1853 ; GFX90A-NEXT: v_and_or_b32 v2, v4, v3, v2
1854 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v1, v4, v2
1855 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
1856 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
1857 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1858 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
1859 ; GFX90A-NEXT: s_cbranch_execnz .LBB8_1
1860 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
1861 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
1862 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v0, v2
1863 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1864 ;
1865 ; GFX908-LABEL: local_atomic_fsub_ret_f16:
1866 ; GFX908: ; %bb.0:
1867 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1868 ; GFX908-NEXT: v_and_b32_e32 v1, -4, v0
1869 ; GFX908-NEXT: ds_read_b32 v2, v1
1870 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v0
1871 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
1872 ; GFX908-NEXT: v_and_b32_e32 v0, 24, v3
1873 ; GFX908-NEXT: v_lshlrev_b32_e64 v3, v3, s4
1874 ; GFX908-NEXT: v_not_b32_e32 v3, v3
1875 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
1876 ; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start
1877 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
1878 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
1879 ; GFX908-NEXT: v_mov_b32_e32 v4, v2
1880 ; GFX908-NEXT: v_lshrrev_b32_e32 v2, v0, v4
1881 ; GFX908-NEXT: v_add_f16_e32 v2, -4.0, v2
1882 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, v0, v2
1883 ; GFX908-NEXT: v_and_or_b32 v2, v4, v3, v2
1884 ; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v1, v4, v2
1885 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
1886 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
1887 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1888 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
1889 ; GFX908-NEXT: s_cbranch_execnz .LBB8_1
1890 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
1891 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
1892 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v0, v2
1893 ; GFX908-NEXT: s_setpc_b64 s[30:31]
1894 ;
1895 ; GFX8-LABEL: local_atomic_fsub_ret_f16:
1896 ; GFX8: ; %bb.0:
1897 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1898 ; GFX8-NEXT: v_and_b32_e32 v1, -4, v0
1899 ; GFX8-NEXT: s_mov_b32 m0, -1
1900 ; GFX8-NEXT: ds_read_b32 v3, v1
1901 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v0
1902 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
1903 ; GFX8-NEXT: v_and_b32_e32 v0, 24, v2
1904 ; GFX8-NEXT: v_lshlrev_b32_e64 v2, v2, s4
1905 ; GFX8-NEXT: v_not_b32_e32 v2, v2
1906 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
1907 ; GFX8-NEXT: .LBB8_1: ; %atomicrmw.start
1908 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
1909 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1910 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
1911 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, v0, v4
1912 ; GFX8-NEXT: v_add_f16_e32 v3, -4.0, v3
1913 ; GFX8-NEXT: v_and_b32_e32 v5, v4, v2
1914 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, v0, v3
1915 ; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
1916 ; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v3
1917 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1918 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1919 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1920 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1921 ; GFX8-NEXT: s_cbranch_execnz .LBB8_1
1922 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1923 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1924 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v0, v3
1925 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1926 ;
1927 ; GFX7-LABEL: local_atomic_fsub_ret_f16:
1928 ; GFX7: ; %bb.0:
1929 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1930 ; GFX7-NEXT: v_and_b32_e32 v1, -4, v0
1931 ; GFX7-NEXT: s_mov_b32 m0, -1
1932 ; GFX7-NEXT: ds_read_b32 v3, v1
1933 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v0
1934 ; GFX7-NEXT: v_and_b32_e32 v0, 24, v2
1935 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v2
1936 ; GFX7-NEXT: v_not_b32_e32 v2, v2
1937 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
1938 ; GFX7-NEXT: .LBB8_1: ; %atomicrmw.start
1939 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
1940 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1941 ; GFX7-NEXT: v_mov_b32_e32 v4, v3
1942 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, v0, v4
1943 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
1944 ; GFX7-NEXT: v_and_b32_e32 v5, v4, v2
1945 ; GFX7-NEXT: v_add_f32_e32 v3, -4.0, v3
1946 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
1947 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, v0, v3
1948 ; GFX7-NEXT: v_or_b32_e32 v3, v5, v3
1949 ; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v3
1950 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1951 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1952 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1953 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
1954 ; GFX7-NEXT: s_cbranch_execnz .LBB8_1
1955 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
1956 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
1957 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v0, v3
1958 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
1959 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1960 ;
1961 ; GFX6-LABEL: local_atomic_fsub_ret_f16:
1962 ; GFX6: ; %bb.0:
1963 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1964 ; GFX6-NEXT: v_and_b32_e32 v1, -4, v0
1965 ; GFX6-NEXT: s_mov_b32 m0, -1
1966 ; GFX6-NEXT: ds_read_b32 v3, v1
1967 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 3, v0
1968 ; GFX6-NEXT: v_and_b32_e32 v0, 24, v2
1969 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v2
1970 ; GFX6-NEXT: v_not_b32_e32 v2, v2
1971 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
1972 ; GFX6-NEXT: .LBB8_1: ; %atomicrmw.start
1973 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
1974 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
1975 ; GFX6-NEXT: v_mov_b32_e32 v4, v3
1976 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, v0, v4
1977 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
1978 ; GFX6-NEXT: v_and_b32_e32 v5, v4, v2
1979 ; GFX6-NEXT: v_add_f32_e32 v3, -4.0, v3
1980 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
1981 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, v0, v3
1982 ; GFX6-NEXT: v_or_b32_e32 v3, v5, v3
1983 ; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v3
1984 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
1985 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1986 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1987 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
1988 ; GFX6-NEXT: s_cbranch_execnz .LBB8_1
1989 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
1990 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
1991 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v0, v3
1992 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
1993 ; GFX6-NEXT: s_setpc_b64 s[30:31]
1994 %result = atomicrmw fsub ptr addrspace(3) %ptr, half 4.0 seq_cst
1995 ret half %result
1996 }
1998 define half @local_atomic_fsub_ret_f16__offset(ptr addrspace(3) %ptr) nounwind {
1999 ; GFX12-LABEL: local_atomic_fsub_ret_f16__offset:
2000 ; GFX12: ; %bb.0:
2001 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
2002 ; GFX12-NEXT: s_wait_expcnt 0x0
2003 ; GFX12-NEXT: s_wait_samplecnt 0x0
2004 ; GFX12-NEXT: s_wait_bvhcnt 0x0
2005 ; GFX12-NEXT: s_wait_kmcnt 0x0
2006 ; GFX12-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v0
2007 ; GFX12-NEXT: s_mov_b32 s0, 0
2008 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
2009 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v1
2010 ; GFX12-NEXT: v_and_b32_e32 v1, 3, v1
2011 ; GFX12-NEXT: ds_load_b32 v3, v0
2012 ; GFX12-NEXT: v_lshlrev_b32_e32 v1, 3, v1
2013 ; GFX12-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff
2014 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
2015 ; GFX12-NEXT: v_not_b32_e32 v2, v2
2016 ; GFX12-NEXT: .LBB9_1: ; %atomicrmw.start
2017 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
2018 ; GFX12-NEXT: s_wait_dscnt 0x0
2019 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
2020 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2021 ; GFX12-NEXT: v_lshrrev_b32_e32 v3, v1, v4
2022 ; GFX12-NEXT: v_add_f16_e32 v3, -4.0, v3
2023 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2024 ; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
2025 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, v1, v3
2026 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
2027 ; GFX12-NEXT: v_and_or_b32 v3, v4, v2, v3
2028 ; GFX12-NEXT: s_wait_storecnt 0x0
2029 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v4
2030 ; GFX12-NEXT: s_wait_dscnt 0x0
2031 ; GFX12-NEXT: global_inv scope:SCOPE_SE
2032 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2033 ; GFX12-NEXT: s_wait_alu 0xfffe
2034 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
2035 ; GFX12-NEXT: s_wait_alu 0xfffe
2036 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2037 ; GFX12-NEXT: s_cbranch_execnz .LBB9_1
2038 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
2039 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
2040 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v1, v3
2041 ; GFX12-NEXT: s_setpc_b64 s[30:31]
2042 ;
2043 ; GFX940-LABEL: local_atomic_fsub_ret_f16__offset:
2044 ; GFX940: ; %bb.0:
2045 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2046 ; GFX940-NEXT: v_add_u32_e32 v0, 0xfffe, v0
2047 ; GFX940-NEXT: v_and_b32_e32 v1, -4, v0
2048 ; GFX940-NEXT: ds_read_b32 v2, v1
2049 ; GFX940-NEXT: v_and_b32_e32 v0, 3, v0
2050 ; GFX940-NEXT: v_lshlrev_b32_e32 v0, 3, v0
2051 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
2052 ; GFX940-NEXT: v_lshlrev_b32_e64 v3, v0, s0
2053 ; GFX940-NEXT: v_not_b32_e32 v3, v3
2054 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
2055 ; GFX940-NEXT: .LBB9_1: ; %atomicrmw.start
2056 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
2057 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
2058 ; GFX940-NEXT: v_mov_b32_e32 v4, v2
2059 ; GFX940-NEXT: v_lshrrev_b32_e32 v2, v0, v4
2060 ; GFX940-NEXT: v_add_f16_e32 v2, -4.0, v2
2061 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, v0, v2
2062 ; GFX940-NEXT: v_and_or_b32 v2, v4, v3, v2
2063 ; GFX940-NEXT: ds_cmpst_rtn_b32 v2, v1, v4, v2
2064 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
2065 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
2066 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
2067 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
2068 ; GFX940-NEXT: s_cbranch_execnz .LBB9_1
2069 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
2070 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
2071 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v0, v2
2072 ; GFX940-NEXT: s_setpc_b64 s[30:31]
2073 ;
2074 ; GFX11-LABEL: local_atomic_fsub_ret_f16__offset:
2075 ; GFX11: ; %bb.0:
2076 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2077 ; GFX11-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v0
2078 ; GFX11-NEXT: s_mov_b32 s0, 0
2079 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
2080 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v1
2081 ; GFX11-NEXT: v_and_b32_e32 v1, 3, v1
2082 ; GFX11-NEXT: ds_load_b32 v3, v0
2083 ; GFX11-NEXT: v_lshlrev_b32_e32 v1, 3, v1
2084 ; GFX11-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff
2085 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
2086 ; GFX11-NEXT: v_not_b32_e32 v2, v2
2087 ; GFX11-NEXT: .LBB9_1: ; %atomicrmw.start
2088 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
2089 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
2090 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
2091 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2092 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, v1, v4
2093 ; GFX11-NEXT: v_add_f16_e32 v3, -4.0, v3
2094 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2095 ; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
2096 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, v1, v3
2097 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
2098 ; GFX11-NEXT: v_and_or_b32 v3, v4, v2, v3
2099 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2100 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v4
2101 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
2102 ; GFX11-NEXT: buffer_gl0_inv
2103 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
2104 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
2105 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
2106 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2107 ; GFX11-NEXT: s_cbranch_execnz .LBB9_1
2108 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
2109 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
2110 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v1, v3
2111 ; GFX11-NEXT: s_setpc_b64 s[30:31]
2112 ;
2113 ; GFX10-LABEL: local_atomic_fsub_ret_f16__offset:
2114 ; GFX10: ; %bb.0:
2115 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2116 ; GFX10-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v0
2117 ; GFX10-NEXT: s_mov_b32 s4, 0
2118 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v1
2119 ; GFX10-NEXT: v_and_b32_e32 v1, 3, v1
2120 ; GFX10-NEXT: ds_read_b32 v2, v0
2121 ; GFX10-NEXT: v_lshlrev_b32_e32 v1, 3, v1
2122 ; GFX10-NEXT: v_lshlrev_b32_e64 v3, v1, 0xffff
2123 ; GFX10-NEXT: v_not_b32_e32 v3, v3
2124 ; GFX10-NEXT: .LBB9_1: ; %atomicrmw.start
2125 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
2126 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
2127 ; GFX10-NEXT: v_mov_b32_e32 v4, v2
2128 ; GFX10-NEXT: v_lshrrev_b32_e32 v2, v1, v4
2129 ; GFX10-NEXT: v_add_f16_e32 v2, -4.0, v2
2130 ; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
2131 ; GFX10-NEXT: v_and_or_b32 v2, v4, v3, v2
2132 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2133 ; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
2134 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
2135 ; GFX10-NEXT: buffer_gl0_inv
2136 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
2137 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
2138 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
2139 ; GFX10-NEXT: s_cbranch_execnz .LBB9_1
2140 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
2141 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
2142 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v1, v2
2143 ; GFX10-NEXT: s_setpc_b64 s[30:31]
2144 ;
2145 ; GFX90A-LABEL: local_atomic_fsub_ret_f16__offset:
2146 ; GFX90A: ; %bb.0:
2147 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2148 ; GFX90A-NEXT: v_add_u32_e32 v0, 0xfffe, v0
2149 ; GFX90A-NEXT: v_and_b32_e32 v1, -4, v0
2150 ; GFX90A-NEXT: ds_read_b32 v2, v1
2151 ; GFX90A-NEXT: v_and_b32_e32 v0, 3, v0
2152 ; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 3, v0
2153 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
2154 ; GFX90A-NEXT: v_lshlrev_b32_e64 v3, v0, s4
2155 ; GFX90A-NEXT: v_not_b32_e32 v3, v3
2156 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
2157 ; GFX90A-NEXT: .LBB9_1: ; %atomicrmw.start
2158 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
2159 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
2160 ; GFX90A-NEXT: v_mov_b32_e32 v4, v2
2161 ; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v0, v4
2162 ; GFX90A-NEXT: v_add_f16_e32 v2, -4.0, v2
2163 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v0, v2
2164 ; GFX90A-NEXT: v_and_or_b32 v2, v4, v3, v2
2165 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v1, v4, v2
2166 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
2167 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
2168 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2169 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
2170 ; GFX90A-NEXT: s_cbranch_execnz .LBB9_1
2171 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
2172 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
2173 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v0, v2
2174 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
2175 ;
2176 ; GFX908-LABEL: local_atomic_fsub_ret_f16__offset:
2177 ; GFX908: ; %bb.0:
2178 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2179 ; GFX908-NEXT: v_add_u32_e32 v0, 0xfffe, v0
2180 ; GFX908-NEXT: v_and_b32_e32 v1, -4, v0
2181 ; GFX908-NEXT: ds_read_b32 v2, v1
2182 ; GFX908-NEXT: v_and_b32_e32 v0, 3, v0
2183 ; GFX908-NEXT: v_lshlrev_b32_e32 v0, 3, v0
2184 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
2185 ; GFX908-NEXT: v_lshlrev_b32_e64 v3, v0, s4
2186 ; GFX908-NEXT: v_not_b32_e32 v3, v3
2187 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
2188 ; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
2189 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
2190 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
2191 ; GFX908-NEXT: v_mov_b32_e32 v4, v2
2192 ; GFX908-NEXT: v_lshrrev_b32_e32 v2, v0, v4
2193 ; GFX908-NEXT: v_add_f16_e32 v2, -4.0, v2
2194 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, v0, v2
2195 ; GFX908-NEXT: v_and_or_b32 v2, v4, v3, v2
2196 ; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v1, v4, v2
2197 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
2198 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
2199 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2200 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
2201 ; GFX908-NEXT: s_cbranch_execnz .LBB9_1
2202 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
2203 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
2204 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v0, v2
2205 ; GFX908-NEXT: s_setpc_b64 s[30:31]
2206 ;
2207 ; GFX8-LABEL: local_atomic_fsub_ret_f16__offset:
2208 ; GFX8: ; %bb.0:
2209 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2210 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffe, v0
2211 ; GFX8-NEXT: v_and_b32_e32 v1, -4, v0
2212 ; GFX8-NEXT: s_mov_b32 m0, -1
2213 ; GFX8-NEXT: ds_read_b32 v3, v1
2214 ; GFX8-NEXT: v_and_b32_e32 v0, 3, v0
2215 ; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v0
2216 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
2217 ; GFX8-NEXT: v_lshlrev_b32_e64 v2, v0, s4
2218 ; GFX8-NEXT: v_not_b32_e32 v2, v2
2219 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2220 ; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start
2221 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2222 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
2223 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
2224 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, v0, v4
2225 ; GFX8-NEXT: v_add_f16_e32 v3, -4.0, v3
2226 ; GFX8-NEXT: v_and_b32_e32 v5, v4, v2
2227 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, v0, v3
2228 ; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
2229 ; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v3
2230 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
2231 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
2232 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2233 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2234 ; GFX8-NEXT: s_cbranch_execnz .LBB9_1
2235 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2236 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2237 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v0, v3
2238 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2239 ;
2240 ; GFX7-LABEL: local_atomic_fsub_ret_f16__offset:
2241 ; GFX7: ; %bb.0:
2242 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2243 ; GFX7-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
2244 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v1
2245 ; GFX7-NEXT: s_mov_b32 m0, -1
2246 ; GFX7-NEXT: ds_read_b32 v3, v0
2247 ; GFX7-NEXT: v_and_b32_e32 v1, 3, v1
2248 ; GFX7-NEXT: v_lshlrev_b32_e32 v1, 3, v1
2249 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v1
2250 ; GFX7-NEXT: v_not_b32_e32 v2, v2
2251 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
2252 ; GFX7-NEXT: .LBB9_1: ; %atomicrmw.start
2253 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
2254 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2255 ; GFX7-NEXT: v_mov_b32_e32 v4, v3
2256 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, v1, v4
2257 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
2258 ; GFX7-NEXT: v_and_b32_e32 v5, v4, v2
2259 ; GFX7-NEXT: v_add_f32_e32 v3, -4.0, v3
2260 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
2261 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, v1, v3
2262 ; GFX7-NEXT: v_or_b32_e32 v3, v5, v3
2263 ; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3
2264 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2265 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
2266 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2267 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
2268 ; GFX7-NEXT: s_cbranch_execnz .LBB9_1
2269 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
2270 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
2271 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v1, v3
2272 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
2273 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2274 ;
2275 ; GFX6-LABEL: local_atomic_fsub_ret_f16__offset:
2276 ; GFX6: ; %bb.0:
2277 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2278 ; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
2279 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v1
2280 ; GFX6-NEXT: s_mov_b32 m0, -1
2281 ; GFX6-NEXT: ds_read_b32 v3, v0
2282 ; GFX6-NEXT: v_and_b32_e32 v1, 3, v1
2283 ; GFX6-NEXT: v_lshlrev_b32_e32 v1, 3, v1
2284 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v1
2285 ; GFX6-NEXT: v_not_b32_e32 v2, v2
2286 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
2287 ; GFX6-NEXT: .LBB9_1: ; %atomicrmw.start
2288 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
2289 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
2290 ; GFX6-NEXT: v_mov_b32_e32 v4, v3
2291 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, v1, v4
2292 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
2293 ; GFX6-NEXT: v_and_b32_e32 v5, v4, v2
2294 ; GFX6-NEXT: v_add_f32_e32 v3, -4.0, v3
2295 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
2296 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, v1, v3
2297 ; GFX6-NEXT: v_or_b32_e32 v3, v5, v3
2298 ; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3
2299 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
2300 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
2301 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2302 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
2303 ; GFX6-NEXT: s_cbranch_execnz .LBB9_1
2304 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
2305 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
2306 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v1, v3
2307 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
2308 ; GFX6-NEXT: s_setpc_b64 s[30:31]
2309 %gep = getelementptr half, ptr addrspace(3) %ptr, i32 32767
2310 %result = atomicrmw fsub ptr addrspace(3) %gep, half 4.0 seq_cst
2311 ret half %result
2312 }
2314 define void @local_atomic_fsub_noret_f16(ptr addrspace(3) %ptr) nounwind {
2315 ; GFX12-LABEL: local_atomic_fsub_noret_f16:
2316 ; GFX12: ; %bb.0:
2317 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
2318 ; GFX12-NEXT: s_wait_expcnt 0x0
2319 ; GFX12-NEXT: s_wait_samplecnt 0x0
2320 ; GFX12-NEXT: s_wait_bvhcnt 0x0
2321 ; GFX12-NEXT: s_wait_kmcnt 0x0
2322 ; GFX12-NEXT: v_and_b32_e32 v1, -4, v0
2323 ; GFX12-NEXT: v_lshlrev_b32_e32 v0, 3, v0
2324 ; GFX12-NEXT: s_mov_b32 s0, 0
2325 ; GFX12-NEXT: ds_load_b32 v2, v1
2326 ; GFX12-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff
2327 ; GFX12-NEXT: v_and_b32_e32 v0, 24, v0
2328 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
2329 ; GFX12-NEXT: v_not_b32_e32 v3, v3
2330 ; GFX12-NEXT: .LBB10_1: ; %atomicrmw.start
2331 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
2332 ; GFX12-NEXT: s_wait_dscnt 0x0
2333 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
2334 ; GFX12-NEXT: v_lshrrev_b32_e32 v4, v0, v2
2335 ; GFX12-NEXT: v_add_f16_e32 v4, -4.0, v4
2336 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2337 ; GFX12-NEXT: v_and_b32_e32 v4, 0xffff, v4
2338 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, v0, v4
2339 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
2340 ; GFX12-NEXT: v_and_or_b32 v4, v2, v3, v4
2341 ; GFX12-NEXT: s_wait_storecnt 0x0
2342 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v4, v1, v4, v2
2343 ; GFX12-NEXT: s_wait_dscnt 0x0
2344 ; GFX12-NEXT: global_inv scope:SCOPE_SE
2345 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
2346 ; GFX12-NEXT: v_mov_b32_e32 v2, v4
2347 ; GFX12-NEXT: s_wait_alu 0xfffe
2348 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
2349 ; GFX12-NEXT: s_wait_alu 0xfffe
2350 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2351 ; GFX12-NEXT: s_cbranch_execnz .LBB10_1
2352 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
2353 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
2354 ; GFX12-NEXT: s_setpc_b64 s[30:31]
2355 ;
2356 ; GFX940-LABEL: local_atomic_fsub_noret_f16:
2357 ; GFX940: ; %bb.0:
2358 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2359 ; GFX940-NEXT: v_and_b32_e32 v1, -4, v0
2360 ; GFX940-NEXT: ds_read_b32 v3, v1
2361 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 3, v0
2362 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
2363 ; GFX940-NEXT: v_and_b32_e32 v0, 24, v2
2364 ; GFX940-NEXT: v_lshlrev_b32_e64 v2, v2, s0
2365 ; GFX940-NEXT: v_not_b32_e32 v2, v2
2366 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
2367 ; GFX940-NEXT: .LBB10_1: ; %atomicrmw.start
2368 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
2369 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
2370 ; GFX940-NEXT: v_lshrrev_b32_e32 v4, v0, v3
2371 ; GFX940-NEXT: v_add_f16_e32 v4, -4.0, v4
2372 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, v0, v4
2373 ; GFX940-NEXT: v_and_or_b32 v4, v3, v2, v4
2374 ; GFX940-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
2375 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
2376 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
2377 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
2378 ; GFX940-NEXT: v_mov_b32_e32 v3, v4
2379 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
2380 ; GFX940-NEXT: s_cbranch_execnz .LBB10_1
2381 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
2382 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
2383 ; GFX940-NEXT: s_setpc_b64 s[30:31]
2384 ;
2385 ; GFX11-LABEL: local_atomic_fsub_noret_f16:
2386 ; GFX11: ; %bb.0:
2387 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2388 ; GFX11-NEXT: v_and_b32_e32 v1, -4, v0
2389 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, 3, v0
2390 ; GFX11-NEXT: s_mov_b32 s0, 0
2391 ; GFX11-NEXT: ds_load_b32 v2, v1
2392 ; GFX11-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff
2393 ; GFX11-NEXT: v_and_b32_e32 v0, 24, v0
2394 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
2395 ; GFX11-NEXT: v_not_b32_e32 v3, v3
2396 ; GFX11-NEXT: .LBB10_1: ; %atomicrmw.start
2397 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
2398 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
2399 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
2400 ; GFX11-NEXT: v_lshrrev_b32_e32 v4, v0, v2
2401 ; GFX11-NEXT: v_add_f16_e32 v4, -4.0, v4
2402 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2403 ; GFX11-NEXT: v_and_b32_e32 v4, 0xffff, v4
2404 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, v0, v4
2405 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
2406 ; GFX11-NEXT: v_and_or_b32 v4, v2, v3, v4
2407 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2408 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v4, v1, v4, v2
2409 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
2410 ; GFX11-NEXT: buffer_gl0_inv
2411 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
2412 ; GFX11-NEXT: v_mov_b32_e32 v2, v4
2413 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
2414 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
2415 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2416 ; GFX11-NEXT: s_cbranch_execnz .LBB10_1
2417 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
2418 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
2419 ; GFX11-NEXT: s_setpc_b64 s[30:31]
2420 ;
2421 ; GFX10-LABEL: local_atomic_fsub_noret_f16:
2422 ; GFX10: ; %bb.0:
2423 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2424 ; GFX10-NEXT: v_and_b32_e32 v1, -4, v0
2425 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v0
2426 ; GFX10-NEXT: s_mov_b32 s4, 0
2427 ; GFX10-NEXT: ds_read_b32 v2, v1
2428 ; GFX10-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff
2429 ; GFX10-NEXT: v_and_b32_e32 v0, 24, v0
2430 ; GFX10-NEXT: v_not_b32_e32 v3, v3
2431 ; GFX10-NEXT: .LBB10_1: ; %atomicrmw.start
2432 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
2433 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
2434 ; GFX10-NEXT: v_lshrrev_b32_e32 v4, v0, v2
2435 ; GFX10-NEXT: v_add_f16_e32 v4, -4.0, v4
2436 ; GFX10-NEXT: v_lshlrev_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
2437 ; GFX10-NEXT: v_and_or_b32 v4, v2, v3, v4
2438 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2439 ; GFX10-NEXT: ds_cmpst_rtn_b32 v4, v1, v2, v4
2440 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
2441 ; GFX10-NEXT: buffer_gl0_inv
2442 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
2443 ; GFX10-NEXT: v_mov_b32_e32 v2, v4
2444 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
2445 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
2446 ; GFX10-NEXT: s_cbranch_execnz .LBB10_1
2447 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
2448 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
2449 ; GFX10-NEXT: s_setpc_b64 s[30:31]
2450 ;
2451 ; GFX90A-LABEL: local_atomic_fsub_noret_f16:
2452 ; GFX90A: ; %bb.0:
2453 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2454 ; GFX90A-NEXT: v_and_b32_e32 v1, -4, v0
2455 ; GFX90A-NEXT: ds_read_b32 v3, v1
2456 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 3, v0
2457 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
2458 ; GFX90A-NEXT: v_and_b32_e32 v0, 24, v2
2459 ; GFX90A-NEXT: v_lshlrev_b32_e64 v2, v2, s4
2460 ; GFX90A-NEXT: v_not_b32_e32 v2, v2
2461 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
2462 ; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start
2463 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
2464 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
2465 ; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v0, v3
2466 ; GFX90A-NEXT: v_add_f16_e32 v4, -4.0, v4
2467 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v0, v4
2468 ; GFX90A-NEXT: v_and_or_b32 v4, v3, v2, v4
2469 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
2470 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
2471 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
2472 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2473 ; GFX90A-NEXT: v_mov_b32_e32 v3, v4
2474 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
2475 ; GFX90A-NEXT: s_cbranch_execnz .LBB10_1
2476 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
2477 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
2478 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
2479 ;
2480 ; GFX908-LABEL: local_atomic_fsub_noret_f16:
2481 ; GFX908: ; %bb.0:
2482 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2483 ; GFX908-NEXT: v_and_b32_e32 v1, -4, v0
2484 ; GFX908-NEXT: ds_read_b32 v3, v1
2485 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 3, v0
2486 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
2487 ; GFX908-NEXT: v_and_b32_e32 v0, 24, v2
2488 ; GFX908-NEXT: v_lshlrev_b32_e64 v2, v2, s4
2489 ; GFX908-NEXT: v_not_b32_e32 v2, v2
2490 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
2491 ; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
2492 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
2493 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
2494 ; GFX908-NEXT: v_lshrrev_b32_e32 v4, v0, v3
2495 ; GFX908-NEXT: v_add_f16_e32 v4, -4.0, v4
2496 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, v0, v4
2497 ; GFX908-NEXT: v_and_or_b32 v4, v3, v2, v4
2498 ; GFX908-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
2499 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
2500 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
2501 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2502 ; GFX908-NEXT: v_mov_b32_e32 v3, v4
2503 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
2504 ; GFX908-NEXT: s_cbranch_execnz .LBB10_1
2505 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
2506 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
2507 ; GFX908-NEXT: s_setpc_b64 s[30:31]
2508 ;
2509 ; GFX8-LABEL: local_atomic_fsub_noret_f16:
2510 ; GFX8: ; %bb.0:
2511 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2512 ; GFX8-NEXT: v_and_b32_e32 v1, -4, v0
2513 ; GFX8-NEXT: s_mov_b32 m0, -1
2514 ; GFX8-NEXT: ds_read_b32 v3, v1
2515 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v0
2516 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
2517 ; GFX8-NEXT: v_and_b32_e32 v0, 24, v2
2518 ; GFX8-NEXT: v_lshlrev_b32_e64 v2, v2, s4
2519 ; GFX8-NEXT: v_not_b32_e32 v2, v2
2520 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2521 ; GFX8-NEXT: .LBB10_1: ; %atomicrmw.start
2522 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2523 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
2524 ; GFX8-NEXT: v_lshrrev_b32_e32 v4, v0, v3
2525 ; GFX8-NEXT: v_add_f16_e32 v4, -4.0, v4
2526 ; GFX8-NEXT: v_and_b32_e32 v5, v3, v2
2527 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, v0, v4
2528 ; GFX8-NEXT: v_or_b32_e32 v4, v5, v4
2529 ; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
2530 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
2531 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
2532 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2533 ; GFX8-NEXT: v_mov_b32_e32 v3, v4
2534 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2535 ; GFX8-NEXT: s_cbranch_execnz .LBB10_1
2536 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2537 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2538 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2539 ;
2540 ; GFX7-LABEL: local_atomic_fsub_noret_f16:
2541 ; GFX7: ; %bb.0:
2542 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2543 ; GFX7-NEXT: v_and_b32_e32 v1, -4, v0
2544 ; GFX7-NEXT: s_mov_b32 m0, -1
2545 ; GFX7-NEXT: ds_read_b32 v3, v1
2546 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v0
2547 ; GFX7-NEXT: v_and_b32_e32 v0, 24, v2
2548 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v2
2549 ; GFX7-NEXT: v_not_b32_e32 v2, v2
2550 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
2551 ; GFX7-NEXT: .LBB10_1: ; %atomicrmw.start
2552 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
2553 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2554 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, v0, v3
2555 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v4
2556 ; GFX7-NEXT: v_and_b32_e32 v5, v3, v2
2557 ; GFX7-NEXT: v_add_f32_e32 v4, -4.0, v4
2558 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
2559 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, v0, v4
2560 ; GFX7-NEXT: v_or_b32_e32 v4, v5, v4
2561 ; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
2562 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2563 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
2564 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2565 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
2566 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
2567 ; GFX7-NEXT: s_cbranch_execnz .LBB10_1
2568 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
2569 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
2570 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2571 ;
2572 ; GFX6-LABEL: local_atomic_fsub_noret_f16:
2573 ; GFX6: ; %bb.0:
2574 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2575 ; GFX6-NEXT: v_and_b32_e32 v1, -4, v0
2576 ; GFX6-NEXT: s_mov_b32 m0, -1
2577 ; GFX6-NEXT: ds_read_b32 v3, v1
2578 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 3, v0
2579 ; GFX6-NEXT: v_and_b32_e32 v0, 24, v2
2580 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v2
2581 ; GFX6-NEXT: v_not_b32_e32 v2, v2
2582 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
2583 ; GFX6-NEXT: .LBB10_1: ; %atomicrmw.start
2584 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
2585 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
2586 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, v0, v3
2587 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v4
2588 ; GFX6-NEXT: v_and_b32_e32 v5, v3, v2
2589 ; GFX6-NEXT: v_add_f32_e32 v4, -4.0, v4
2590 ; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
2591 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, v0, v4
2592 ; GFX6-NEXT: v_or_b32_e32 v4, v5, v4
2593 ; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
2594 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
2595 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
2596 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2597 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
2598 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
2599 ; GFX6-NEXT: s_cbranch_execnz .LBB10_1
2600 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
2601 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
2602 ; GFX6-NEXT: s_setpc_b64 s[30:31]
2603 %result = atomicrmw fsub ptr addrspace(3) %ptr, half 4.0 seq_cst
2604 ret void
2605 }
2607 define void @local_atomic_fsub_noret_f16__offset(ptr addrspace(3) %ptr) nounwind {
2608 ; GFX12-LABEL: local_atomic_fsub_noret_f16__offset:
2609 ; GFX12: ; %bb.0:
2610 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
2611 ; GFX12-NEXT: s_wait_expcnt 0x0
2612 ; GFX12-NEXT: s_wait_samplecnt 0x0
2613 ; GFX12-NEXT: s_wait_bvhcnt 0x0
2614 ; GFX12-NEXT: s_wait_kmcnt 0x0
2615 ; GFX12-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v0
2616 ; GFX12-NEXT: s_mov_b32 s0, 0
2617 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
2618 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v1
2619 ; GFX12-NEXT: v_and_b32_e32 v1, 3, v1
2620 ; GFX12-NEXT: ds_load_b32 v3, v0
2621 ; GFX12-NEXT: v_lshlrev_b32_e32 v1, 3, v1
2622 ; GFX12-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff
2623 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
2624 ; GFX12-NEXT: v_not_b32_e32 v2, v2
2625 ; GFX12-NEXT: .LBB11_1: ; %atomicrmw.start
2626 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
2627 ; GFX12-NEXT: s_wait_dscnt 0x0
2628 ; GFX12-NEXT: v_lshrrev_b32_e32 v4, v1, v3
2629 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2630 ; GFX12-NEXT: v_add_f16_e32 v4, -4.0, v4
2631 ; GFX12-NEXT: v_and_b32_e32 v4, 0xffff, v4
2632 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2633 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, v1, v4
2634 ; GFX12-NEXT: v_and_or_b32 v4, v3, v2, v4
2635 ; GFX12-NEXT: s_wait_storecnt 0x0
2636 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
2637 ; GFX12-NEXT: s_wait_dscnt 0x0
2638 ; GFX12-NEXT: global_inv scope:SCOPE_SE
2639 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
2640 ; GFX12-NEXT: v_mov_b32_e32 v3, v4
2641 ; GFX12-NEXT: s_wait_alu 0xfffe
2642 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
2643 ; GFX12-NEXT: s_wait_alu 0xfffe
2644 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2645 ; GFX12-NEXT: s_cbranch_execnz .LBB11_1
2646 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
2647 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
2648 ; GFX12-NEXT: s_setpc_b64 s[30:31]
2649 ;
2650 ; GFX940-LABEL: local_atomic_fsub_noret_f16__offset:
2651 ; GFX940: ; %bb.0:
2652 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2653 ; GFX940-NEXT: v_add_u32_e32 v1, 0xfffe, v0
2654 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v1
2655 ; GFX940-NEXT: ds_read_b32 v3, v0
2656 ; GFX940-NEXT: v_and_b32_e32 v1, 3, v1
2657 ; GFX940-NEXT: v_lshlrev_b32_e32 v1, 3, v1
2658 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
2659 ; GFX940-NEXT: v_lshlrev_b32_e64 v2, v1, s0
2660 ; GFX940-NEXT: v_not_b32_e32 v2, v2
2661 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
2662 ; GFX940-NEXT: .LBB11_1: ; %atomicrmw.start
2663 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
2664 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
2665 ; GFX940-NEXT: v_lshrrev_b32_e32 v4, v1, v3
2666 ; GFX940-NEXT: v_add_f16_e32 v4, -4.0, v4
2667 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, v1, v4
2668 ; GFX940-NEXT: v_and_or_b32 v4, v3, v2, v4
2669 ; GFX940-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
2670 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
2671 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
2672 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
2673 ; GFX940-NEXT: v_mov_b32_e32 v3, v4
2674 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
2675 ; GFX940-NEXT: s_cbranch_execnz .LBB11_1
2676 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
2677 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
2678 ; GFX940-NEXT: s_setpc_b64 s[30:31]
2679 ;
2680 ; GFX11-LABEL: local_atomic_fsub_noret_f16__offset:
2681 ; GFX11: ; %bb.0:
2682 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2683 ; GFX11-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v0
2684 ; GFX11-NEXT: s_mov_b32 s0, 0
2685 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
2686 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v1
2687 ; GFX11-NEXT: v_and_b32_e32 v1, 3, v1
2688 ; GFX11-NEXT: ds_load_b32 v3, v0
2689 ; GFX11-NEXT: v_lshlrev_b32_e32 v1, 3, v1
2690 ; GFX11-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff
2691 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
2692 ; GFX11-NEXT: v_not_b32_e32 v2, v2
2693 ; GFX11-NEXT: .LBB11_1: ; %atomicrmw.start
2694 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
2695 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
2696 ; GFX11-NEXT: v_lshrrev_b32_e32 v4, v1, v3
2697 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2698 ; GFX11-NEXT: v_add_f16_e32 v4, -4.0, v4
2699 ; GFX11-NEXT: v_and_b32_e32 v4, 0xffff, v4
2700 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2701 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, v1, v4
2702 ; GFX11-NEXT: v_and_or_b32 v4, v3, v2, v4
2703 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2704 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
2705 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
2706 ; GFX11-NEXT: buffer_gl0_inv
2707 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
2708 ; GFX11-NEXT: v_mov_b32_e32 v3, v4
2709 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
2710 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
2711 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2712 ; GFX11-NEXT: s_cbranch_execnz .LBB11_1
2713 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
2714 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
2715 ; GFX11-NEXT: s_setpc_b64 s[30:31]
2716 ;
2717 ; GFX10-LABEL: local_atomic_fsub_noret_f16__offset:
2718 ; GFX10: ; %bb.0:
2719 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2720 ; GFX10-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v0
2721 ; GFX10-NEXT: s_mov_b32 s4, 0
2722 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v1
2723 ; GFX10-NEXT: v_and_b32_e32 v1, 3, v1
2724 ; GFX10-NEXT: ds_read_b32 v3, v0
2725 ; GFX10-NEXT: v_lshlrev_b32_e32 v1, 3, v1
2726 ; GFX10-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff
2727 ; GFX10-NEXT: v_not_b32_e32 v2, v2
2728 ; GFX10-NEXT: .LBB11_1: ; %atomicrmw.start
2729 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
2730 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
2731 ; GFX10-NEXT: v_lshrrev_b32_e32 v4, v1, v3
2732 ; GFX10-NEXT: v_add_f16_e32 v4, -4.0, v4
2733 ; GFX10-NEXT: v_lshlrev_b32_sdwa v4, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
2734 ; GFX10-NEXT: v_and_or_b32 v4, v3, v2, v4
2735 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2736 ; GFX10-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
2737 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
2738 ; GFX10-NEXT: buffer_gl0_inv
2739 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
2740 ; GFX10-NEXT: v_mov_b32_e32 v3, v4
2741 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
2742 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
2743 ; GFX10-NEXT: s_cbranch_execnz .LBB11_1
2744 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
2745 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
2746 ; GFX10-NEXT: s_setpc_b64 s[30:31]
2747 ;
2748 ; GFX90A-LABEL: local_atomic_fsub_noret_f16__offset:
2749 ; GFX90A: ; %bb.0:
2750 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2751 ; GFX90A-NEXT: v_add_u32_e32 v1, 0xfffe, v0
2752 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v1
2753 ; GFX90A-NEXT: ds_read_b32 v3, v0
2754 ; GFX90A-NEXT: v_and_b32_e32 v1, 3, v1
2755 ; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 3, v1
2756 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
2757 ; GFX90A-NEXT: v_lshlrev_b32_e64 v2, v1, s4
2758 ; GFX90A-NEXT: v_not_b32_e32 v2, v2
2759 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
2760 ; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start
2761 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
2762 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
2763 ; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v1, v3
2764 ; GFX90A-NEXT: v_add_f16_e32 v4, -4.0, v4
2765 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v1, v4
2766 ; GFX90A-NEXT: v_and_or_b32 v4, v3, v2, v4
2767 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
2768 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
2769 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
2770 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2771 ; GFX90A-NEXT: v_mov_b32_e32 v3, v4
2772 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
2773 ; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
2774 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
2775 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
2776 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
2777 ;
2778 ; GFX908-LABEL: local_atomic_fsub_noret_f16__offset:
2779 ; GFX908: ; %bb.0:
2780 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2781 ; GFX908-NEXT: v_add_u32_e32 v1, 0xfffe, v0
2782 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v1
2783 ; GFX908-NEXT: ds_read_b32 v3, v0
2784 ; GFX908-NEXT: v_and_b32_e32 v1, 3, v1
2785 ; GFX908-NEXT: v_lshlrev_b32_e32 v1, 3, v1
2786 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
2787 ; GFX908-NEXT: v_lshlrev_b32_e64 v2, v1, s4
2788 ; GFX908-NEXT: v_not_b32_e32 v2, v2
2789 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
2790 ; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
2791 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
2792 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
2793 ; GFX908-NEXT: v_lshrrev_b32_e32 v4, v1, v3
2794 ; GFX908-NEXT: v_add_f16_e32 v4, -4.0, v4
2795 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, v1, v4
2796 ; GFX908-NEXT: v_and_or_b32 v4, v3, v2, v4
2797 ; GFX908-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
2798 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
2799 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
2800 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2801 ; GFX908-NEXT: v_mov_b32_e32 v3, v4
2802 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
2803 ; GFX908-NEXT: s_cbranch_execnz .LBB11_1
2804 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
2805 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
2806 ; GFX908-NEXT: s_setpc_b64 s[30:31]
2807 ;
2808 ; GFX8-LABEL: local_atomic_fsub_noret_f16__offset:
2809 ; GFX8: ; %bb.0:
2810 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2811 ; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0xfffe, v0
2812 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v1
2813 ; GFX8-NEXT: s_mov_b32 m0, -1
2814 ; GFX8-NEXT: ds_read_b32 v3, v0
2815 ; GFX8-NEXT: v_and_b32_e32 v1, 3, v1
2816 ; GFX8-NEXT: v_lshlrev_b32_e32 v1, 3, v1
2817 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
2818 ; GFX8-NEXT: v_lshlrev_b32_e64 v2, v1, s4
2819 ; GFX8-NEXT: v_not_b32_e32 v2, v2
2820 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2821 ; GFX8-NEXT: .LBB11_1: ; %atomicrmw.start
2822 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2823 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
2824 ; GFX8-NEXT: v_lshrrev_b32_e32 v4, v1, v3
2825 ; GFX8-NEXT: v_add_f16_e32 v4, -4.0, v4
2826 ; GFX8-NEXT: v_and_b32_e32 v5, v3, v2
2827 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, v1, v4
2828 ; GFX8-NEXT: v_or_b32_e32 v4, v5, v4
2829 ; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
2830 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
2831 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
2832 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2833 ; GFX8-NEXT: v_mov_b32_e32 v3, v4
2834 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2835 ; GFX8-NEXT: s_cbranch_execnz .LBB11_1
2836 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2837 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2838 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2839 ;
2840 ; GFX7-LABEL: local_atomic_fsub_noret_f16__offset:
2841 ; GFX7: ; %bb.0:
2842 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2843 ; GFX7-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
2844 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v1
2845 ; GFX7-NEXT: s_mov_b32 m0, -1
2846 ; GFX7-NEXT: ds_read_b32 v3, v0
2847 ; GFX7-NEXT: v_and_b32_e32 v1, 3, v1
2848 ; GFX7-NEXT: v_lshlrev_b32_e32 v1, 3, v1
2849 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v1
2850 ; GFX7-NEXT: v_not_b32_e32 v2, v2
2851 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
2852 ; GFX7-NEXT: .LBB11_1: ; %atomicrmw.start
2853 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
2854 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2855 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, v1, v3
2856 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v4
2857 ; GFX7-NEXT: v_and_b32_e32 v5, v3, v2
2858 ; GFX7-NEXT: v_add_f32_e32 v4, -4.0, v4
2859 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
2860 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, v1, v4
2861 ; GFX7-NEXT: v_or_b32_e32 v4, v5, v4
2862 ; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
2863 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2864 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
2865 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2866 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
2867 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
2868 ; GFX7-NEXT: s_cbranch_execnz .LBB11_1
2869 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
2870 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
2871 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2872 ;
2873 ; GFX6-LABEL: local_atomic_fsub_noret_f16__offset:
2874 ; GFX6: ; %bb.0:
2875 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2876 ; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
2877 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v1
2878 ; GFX6-NEXT: s_mov_b32 m0, -1
2879 ; GFX6-NEXT: ds_read_b32 v3, v0
2880 ; GFX6-NEXT: v_and_b32_e32 v1, 3, v1
2881 ; GFX6-NEXT: v_lshlrev_b32_e32 v1, 3, v1
2882 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v1
2883 ; GFX6-NEXT: v_not_b32_e32 v2, v2
2884 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
2885 ; GFX6-NEXT: .LBB11_1: ; %atomicrmw.start
2886 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
2887 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
2888 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, v1, v3
2889 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v4
2890 ; GFX6-NEXT: v_and_b32_e32 v5, v3, v2
2891 ; GFX6-NEXT: v_add_f32_e32 v4, -4.0, v4
2892 ; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
2893 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, v1, v4
2894 ; GFX6-NEXT: v_or_b32_e32 v4, v5, v4
2895 ; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
2896 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
2897 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
2898 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2899 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
2900 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
2901 ; GFX6-NEXT: s_cbranch_execnz .LBB11_1
2902 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
2903 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
2904 ; GFX6-NEXT: s_setpc_b64 s[30:31]
2905 %gep = getelementptr half, ptr addrspace(3) %ptr, i32 32767
2906 %unused = atomicrmw fsub ptr addrspace(3) %gep, half 4.0 seq_cst
2907 ret void
2908 }
2910 define half @local_atomic_fsub_ret_f16__offset__align4(ptr addrspace(3) %ptr) nounwind {
2911 ; GFX12-LABEL: local_atomic_fsub_ret_f16__offset__align4:
2912 ; GFX12: ; %bb.0:
2913 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
2914 ; GFX12-NEXT: s_wait_expcnt 0x0
2915 ; GFX12-NEXT: s_wait_samplecnt 0x0
2916 ; GFX12-NEXT: s_wait_bvhcnt 0x0
2917 ; GFX12-NEXT: s_wait_kmcnt 0x0
2918 ; GFX12-NEXT: ds_load_b32 v1, v0 offset:65534
2919 ; GFX12-NEXT: s_mov_b32 s0, 0
2920 ; GFX12-NEXT: .LBB12_1: ; %atomicrmw.start
2921 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
2922 ; GFX12-NEXT: s_wait_dscnt 0x0
2923 ; GFX12-NEXT: v_mov_b32_e32 v2, v1
2924 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2925 ; GFX12-NEXT: v_add_f16_e32 v1, -4.0, v2
2926 ; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
2927 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
2928 ; GFX12-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v1
2929 ; GFX12-NEXT: s_wait_storecnt 0x0
2930 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:65534
2931 ; GFX12-NEXT: s_wait_dscnt 0x0
2932 ; GFX12-NEXT: global_inv scope:SCOPE_SE
2933 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
2934 ; GFX12-NEXT: s_wait_alu 0xfffe
2935 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
2936 ; GFX12-NEXT: s_wait_alu 0xfffe
2937 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2938 ; GFX12-NEXT: s_cbranch_execnz .LBB12_1
2939 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
2940 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
2941 ; GFX12-NEXT: v_mov_b32_e32 v0, v1
2942 ; GFX12-NEXT: s_setpc_b64 s[30:31]
2943 ;
2944 ; GFX940-LABEL: local_atomic_fsub_ret_f16__offset__align4:
2945 ; GFX940: ; %bb.0:
2946 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2947 ; GFX940-NEXT: ds_read_b32 v1, v0 offset:65534
2948 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
2949 ; GFX940-NEXT: s_mov_b32 s2, 0xffff0000
2950 ; GFX940-NEXT: .LBB12_1: ; %atomicrmw.start
2951 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
2952 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
2953 ; GFX940-NEXT: v_mov_b32_e32 v2, v1
2954 ; GFX940-NEXT: v_add_f16_e32 v1, -4.0, v2
2955 ; GFX940-NEXT: v_and_or_b32 v1, v2, s2, v1
2956 ; GFX940-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:65534
2957 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
2958 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
2959 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
2960 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
2961 ; GFX940-NEXT: s_cbranch_execnz .LBB12_1
2962 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
2963 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
2964 ; GFX940-NEXT: v_mov_b32_e32 v0, v1
2965 ; GFX940-NEXT: s_setpc_b64 s[30:31]
2966 ;
2967 ; GFX11-LABEL: local_atomic_fsub_ret_f16__offset__align4:
2968 ; GFX11: ; %bb.0:
2969 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2970 ; GFX11-NEXT: ds_load_b32 v1, v0 offset:65534
2971 ; GFX11-NEXT: s_mov_b32 s0, 0
2972 ; GFX11-NEXT: .LBB12_1: ; %atomicrmw.start
2973 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
2974 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
2975 ; GFX11-NEXT: v_mov_b32_e32 v2, v1
2976 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2977 ; GFX11-NEXT: v_add_f16_e32 v1, -4.0, v2
2978 ; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
2979 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
2980 ; GFX11-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v1
2981 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2982 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:65534
2983 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
2984 ; GFX11-NEXT: buffer_gl0_inv
2985 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
2986 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
2987 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
2988 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
2989 ; GFX11-NEXT: s_cbranch_execnz .LBB12_1
2990 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
2991 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
2992 ; GFX11-NEXT: v_mov_b32_e32 v0, v1
2993 ; GFX11-NEXT: s_setpc_b64 s[30:31]
2994 ;
2995 ; GFX10-LABEL: local_atomic_fsub_ret_f16__offset__align4:
2996 ; GFX10: ; %bb.0:
2997 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2998 ; GFX10-NEXT: ds_read_b32 v1, v0 offset:65534
2999 ; GFX10-NEXT: s_mov_b32 s4, 0
3000 ; GFX10-NEXT: .LBB12_1: ; %atomicrmw.start
3001 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
3002 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
3003 ; GFX10-NEXT: v_mov_b32_e32 v2, v1
3004 ; GFX10-NEXT: v_add_f16_e32 v1, -4.0, v2
3005 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
3006 ; GFX10-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v1
3007 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
3008 ; GFX10-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:65534
3009 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
3010 ; GFX10-NEXT: buffer_gl0_inv
3011 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
3012 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
3013 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
3014 ; GFX10-NEXT: s_cbranch_execnz .LBB12_1
3015 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
3016 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
3017 ; GFX10-NEXT: v_mov_b32_e32 v0, v1
3018 ; GFX10-NEXT: s_setpc_b64 s[30:31]
3019 ;
3020 ; GFX90A-LABEL: local_atomic_fsub_ret_f16__offset__align4:
3021 ; GFX90A: ; %bb.0:
3022 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3023 ; GFX90A-NEXT: ds_read_b32 v1, v0 offset:65534
3024 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
3025 ; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
3026 ; GFX90A-NEXT: .LBB12_1: ; %atomicrmw.start
3027 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
3028 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
3029 ; GFX90A-NEXT: v_mov_b32_e32 v2, v1
3030 ; GFX90A-NEXT: v_add_f16_e32 v1, -4.0, v2
3031 ; GFX90A-NEXT: v_and_or_b32 v1, v2, s6, v1
3032 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:65534
3033 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
3034 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
3035 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3036 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
3037 ; GFX90A-NEXT: s_cbranch_execnz .LBB12_1
3038 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
3039 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
3040 ; GFX90A-NEXT: v_mov_b32_e32 v0, v1
3041 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
3042 ;
3043 ; GFX908-LABEL: local_atomic_fsub_ret_f16__offset__align4:
3044 ; GFX908: ; %bb.0:
3045 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3046 ; GFX908-NEXT: ds_read_b32 v1, v0 offset:65534
3047 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
3048 ; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
3049 ; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start
3050 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
3051 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
3052 ; GFX908-NEXT: v_mov_b32_e32 v2, v1
3053 ; GFX908-NEXT: v_add_f16_e32 v1, -4.0, v2
3054 ; GFX908-NEXT: v_and_or_b32 v1, v2, s6, v1
3055 ; GFX908-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:65534
3056 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
3057 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
3058 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3059 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
3060 ; GFX908-NEXT: s_cbranch_execnz .LBB12_1
3061 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
3062 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
3063 ; GFX908-NEXT: v_mov_b32_e32 v0, v1
3064 ; GFX908-NEXT: s_setpc_b64 s[30:31]
3065 ;
3066 ; GFX8-LABEL: local_atomic_fsub_ret_f16__offset__align4:
3067 ; GFX8: ; %bb.0:
3068 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3069 ; GFX8-NEXT: s_mov_b32 m0, -1
3070 ; GFX8-NEXT: ds_read_b32 v1, v0 offset:65534
3071 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
3072 ; GFX8-NEXT: .LBB12_1: ; %atomicrmw.start
3073 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3074 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
3075 ; GFX8-NEXT: v_mov_b32_e32 v2, v1
3076 ; GFX8-NEXT: v_add_f16_e32 v1, -4.0, v2
3077 ; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
3078 ; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
3079 ; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:65534
3080 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
3081 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
3082 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3083 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
3084 ; GFX8-NEXT: s_cbranch_execnz .LBB12_1
3085 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
3086 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
3087 ; GFX8-NEXT: v_mov_b32_e32 v0, v1
3088 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3089 ;
3090 ; GFX7-LABEL: local_atomic_fsub_ret_f16__offset__align4:
3091 ; GFX7: ; %bb.0:
3092 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3093 ; GFX7-NEXT: s_mov_b32 m0, -1
3094 ; GFX7-NEXT: ds_read_b32 v1, v0 offset:65534
3095 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
3096 ; GFX7-NEXT: .LBB12_1: ; %atomicrmw.start
3097 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
3098 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3099 ; GFX7-NEXT: v_mov_b32_e32 v2, v1
3100 ; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v2
3101 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
3102 ; GFX7-NEXT: v_add_f32_e32 v1, -4.0, v1
3103 ; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
3104 ; GFX7-NEXT: v_or_b32_e32 v1, v3, v1
3105 ; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:65534
3106 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3107 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
3108 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3109 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
3110 ; GFX7-NEXT: s_cbranch_execnz .LBB12_1
3111 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
3112 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
3113 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v1
3114 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3115 ;
3116 ; GFX6-LABEL: local_atomic_fsub_ret_f16__offset__align4:
3117 ; GFX6: ; %bb.0:
3118 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3119 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffe, v0
3120 ; GFX6-NEXT: s_mov_b32 m0, -1
3121 ; GFX6-NEXT: ds_read_b32 v1, v0
3122 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
3123 ; GFX6-NEXT: .LBB12_1: ; %atomicrmw.start
3124 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
3125 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
3126 ; GFX6-NEXT: v_mov_b32_e32 v2, v1
3127 ; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v2
3128 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
3129 ; GFX6-NEXT: v_add_f32_e32 v1, -4.0, v1
3130 ; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
3131 ; GFX6-NEXT: v_or_b32_e32 v1, v3, v1
3132 ; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1
3133 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
3134 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
3135 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3136 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
3137 ; GFX6-NEXT: s_cbranch_execnz .LBB12_1
3138 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
3139 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
3140 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v1
3141 ; GFX6-NEXT: s_setpc_b64 s[30:31]
3142 %gep = getelementptr half, ptr addrspace(3) %ptr, i32 32767
3143 %result = atomicrmw fsub ptr addrspace(3) %gep, half 4.0 seq_cst, align 4
3144 ret half %result
3145 }
3147 define void @local_atomic_fsub_noret_f16__offset__align4(ptr addrspace(3) %ptr) nounwind {
3148 ; GFX12-LABEL: local_atomic_fsub_noret_f16__offset__align4:
3149 ; GFX12: ; %bb.0:
3150 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
3151 ; GFX12-NEXT: s_wait_expcnt 0x0
3152 ; GFX12-NEXT: s_wait_samplecnt 0x0
3153 ; GFX12-NEXT: s_wait_bvhcnt 0x0
3154 ; GFX12-NEXT: s_wait_kmcnt 0x0
3155 ; GFX12-NEXT: ds_load_b32 v1, v0 offset:65534
3156 ; GFX12-NEXT: s_mov_b32 s0, 0
3157 ; GFX12-NEXT: .LBB13_1: ; %atomicrmw.start
3158 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
3159 ; GFX12-NEXT: s_wait_dscnt 0x0
3160 ; GFX12-NEXT: v_add_f16_e32 v2, -4.0, v1
3161 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3162 ; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
3163 ; GFX12-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v2
3164 ; GFX12-NEXT: s_wait_storecnt 0x0
3165 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
3166 ; GFX12-NEXT: s_wait_dscnt 0x0
3167 ; GFX12-NEXT: global_inv scope:SCOPE_SE
3168 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
3169 ; GFX12-NEXT: v_mov_b32_e32 v1, v2
3170 ; GFX12-NEXT: s_wait_alu 0xfffe
3171 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
3172 ; GFX12-NEXT: s_wait_alu 0xfffe
3173 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3174 ; GFX12-NEXT: s_cbranch_execnz .LBB13_1
3175 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
3176 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
3177 ; GFX12-NEXT: s_setpc_b64 s[30:31]
3178 ;
3179 ; GFX940-LABEL: local_atomic_fsub_noret_f16__offset__align4:
3180 ; GFX940: ; %bb.0:
3181 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3182 ; GFX940-NEXT: ds_read_b32 v1, v0 offset:65534
3183 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
3184 ; GFX940-NEXT: s_mov_b32 s2, 0xffff0000
3185 ; GFX940-NEXT: .LBB13_1: ; %atomicrmw.start
3186 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
3187 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
3188 ; GFX940-NEXT: v_add_f16_e32 v2, -4.0, v1
3189 ; GFX940-NEXT: v_and_or_b32 v2, v1, s2, v2
3190 ; GFX940-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:65534
3191 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
3192 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
3193 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
3194 ; GFX940-NEXT: v_mov_b32_e32 v1, v2
3195 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
3196 ; GFX940-NEXT: s_cbranch_execnz .LBB13_1
3197 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
3198 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
3199 ; GFX940-NEXT: s_setpc_b64 s[30:31]
3200 ;
3201 ; GFX11-LABEL: local_atomic_fsub_noret_f16__offset__align4:
3202 ; GFX11: ; %bb.0:
3203 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3204 ; GFX11-NEXT: ds_load_b32 v1, v0 offset:65534
3205 ; GFX11-NEXT: s_mov_b32 s0, 0
3206 ; GFX11-NEXT: .LBB13_1: ; %atomicrmw.start
3207 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
3208 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
3209 ; GFX11-NEXT: v_add_f16_e32 v2, -4.0, v1
3210 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3211 ; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
3212 ; GFX11-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v2
3213 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
3214 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
3215 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
3216 ; GFX11-NEXT: buffer_gl0_inv
3217 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
3218 ; GFX11-NEXT: v_mov_b32_e32 v1, v2
3219 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
3220 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3221 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3222 ; GFX11-NEXT: s_cbranch_execnz .LBB13_1
3223 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
3224 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
3225 ; GFX11-NEXT: s_setpc_b64 s[30:31]
3226 ;
3227 ; GFX10-LABEL: local_atomic_fsub_noret_f16__offset__align4:
3228 ; GFX10: ; %bb.0:
3229 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3230 ; GFX10-NEXT: ds_read_b32 v1, v0 offset:65534
3231 ; GFX10-NEXT: s_mov_b32 s4, 0
3232 ; GFX10-NEXT: .LBB13_1: ; %atomicrmw.start
3233 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
3234 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
3235 ; GFX10-NEXT: v_add_f16_e32 v2, -4.0, v1
3236 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
3237 ; GFX10-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v2
3238 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
3239 ; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:65534
3240 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
3241 ; GFX10-NEXT: buffer_gl0_inv
3242 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
3243 ; GFX10-NEXT: v_mov_b32_e32 v1, v2
3244 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
3245 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
3246 ; GFX10-NEXT: s_cbranch_execnz .LBB13_1
3247 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
3248 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
3249 ; GFX10-NEXT: s_setpc_b64 s[30:31]
3250 ;
3251 ; GFX90A-LABEL: local_atomic_fsub_noret_f16__offset__align4:
3252 ; GFX90A: ; %bb.0:
3253 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3254 ; GFX90A-NEXT: ds_read_b32 v1, v0 offset:65534
3255 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
3256 ; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
3257 ; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
3258 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
3259 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
3260 ; GFX90A-NEXT: v_add_f16_e32 v2, -4.0, v1
3261 ; GFX90A-NEXT: v_and_or_b32 v2, v1, s6, v2
3262 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:65534
3263 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
3264 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
3265 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3266 ; GFX90A-NEXT: v_mov_b32_e32 v1, v2
3267 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
3268 ; GFX90A-NEXT: s_cbranch_execnz .LBB13_1
3269 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
3270 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
3271 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
3272 ;
3273 ; GFX908-LABEL: local_atomic_fsub_noret_f16__offset__align4:
3274 ; GFX908: ; %bb.0:
3275 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3276 ; GFX908-NEXT: ds_read_b32 v1, v0 offset:65534
3277 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
3278 ; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
3279 ; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
3280 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
3281 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
3282 ; GFX908-NEXT: v_add_f16_e32 v2, -4.0, v1
3283 ; GFX908-NEXT: v_and_or_b32 v2, v1, s6, v2
3284 ; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:65534
3285 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
3286 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
3287 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3288 ; GFX908-NEXT: v_mov_b32_e32 v1, v2
3289 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
3290 ; GFX908-NEXT: s_cbranch_execnz .LBB13_1
3291 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
3292 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
3293 ; GFX908-NEXT: s_setpc_b64 s[30:31]
3294 ;
3295 ; GFX8-LABEL: local_atomic_fsub_noret_f16__offset__align4:
3296 ; GFX8: ; %bb.0:
3297 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3298 ; GFX8-NEXT: s_mov_b32 m0, -1
3299 ; GFX8-NEXT: ds_read_b32 v1, v0 offset:65534
3300 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
3301 ; GFX8-NEXT: .LBB13_1: ; %atomicrmw.start
3302 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3303 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
3304 ; GFX8-NEXT: v_add_f16_e32 v2, -4.0, v1
3305 ; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
3306 ; GFX8-NEXT: v_or_b32_e32 v2, v3, v2
3307 ; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:65534
3308 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
3309 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
3310 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3311 ; GFX8-NEXT: v_mov_b32_e32 v1, v2
3312 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
3313 ; GFX8-NEXT: s_cbranch_execnz .LBB13_1
3314 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
3315 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
3316 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3317 ;
3318 ; GFX7-LABEL: local_atomic_fsub_noret_f16__offset__align4:
3319 ; GFX7: ; %bb.0:
3320 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3321 ; GFX7-NEXT: s_mov_b32 m0, -1
3322 ; GFX7-NEXT: ds_read_b32 v1, v0 offset:65534
3323 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
3324 ; GFX7-NEXT: .LBB13_1: ; %atomicrmw.start
3325 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
3326 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3327 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v1
3328 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
3329 ; GFX7-NEXT: v_add_f32_e32 v2, -4.0, v2
3330 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
3331 ; GFX7-NEXT: v_or_b32_e32 v2, v3, v2
3332 ; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:65534
3333 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3334 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
3335 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3336 ; GFX7-NEXT: v_mov_b32_e32 v1, v2
3337 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
3338 ; GFX7-NEXT: s_cbranch_execnz .LBB13_1
3339 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
3340 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
3341 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3342 ;
3343 ; GFX6-LABEL: local_atomic_fsub_noret_f16__offset__align4:
3344 ; GFX6: ; %bb.0:
3345 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3346 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffe, v0
3347 ; GFX6-NEXT: s_mov_b32 m0, -1
3348 ; GFX6-NEXT: ds_read_b32 v1, v0
3349 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
3350 ; GFX6-NEXT: .LBB13_1: ; %atomicrmw.start
3351 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
3352 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
3353 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v1
3354 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
3355 ; GFX6-NEXT: v_add_f32_e32 v2, -4.0, v2
3356 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
3357 ; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
3358 ; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
3359 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
3360 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
3361 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3362 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
3363 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
3364 ; GFX6-NEXT: s_cbranch_execnz .LBB13_1
3365 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
3366 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
3367 ; GFX6-NEXT: s_setpc_b64 s[30:31]
3368 %gep = getelementptr half, ptr addrspace(3) %ptr, i32 32767
3369 %unused = atomicrmw fsub ptr addrspace(3) %gep, half 4.0 seq_cst, align 4
3370 ret void
3371 }
3373 ; --------------------------------------------------------------------
3374 ; bfloat
3375 ; --------------------------------------------------------------------
3377 define bfloat @local_atomic_fsub_ret_bf16(ptr addrspace(3) %ptr) nounwind {
3378 ; GFX12-LABEL: local_atomic_fsub_ret_bf16:
3379 ; GFX12: ; %bb.0:
3380 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
3381 ; GFX12-NEXT: s_wait_expcnt 0x0
3382 ; GFX12-NEXT: s_wait_samplecnt 0x0
3383 ; GFX12-NEXT: s_wait_bvhcnt 0x0
3384 ; GFX12-NEXT: s_wait_kmcnt 0x0
3385 ; GFX12-NEXT: v_and_b32_e32 v1, -4, v0
3386 ; GFX12-NEXT: v_lshlrev_b32_e32 v0, 3, v0
3387 ; GFX12-NEXT: s_mov_b32 s0, 0
3388 ; GFX12-NEXT: ds_load_b32 v3, v1
3389 ; GFX12-NEXT: v_lshlrev_b32_e64 v2, v0, 0xffff
3390 ; GFX12-NEXT: v_and_b32_e32 v0, 24, v0
3391 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
3392 ; GFX12-NEXT: v_not_b32_e32 v2, v2
3393 ; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
3394 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
3395 ; GFX12-NEXT: s_wait_dscnt 0x0
3396 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
3397 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3398 ; GFX12-NEXT: v_lshrrev_b32_e32 v3, v0, v4
3399 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
3400 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3401 ; GFX12-NEXT: v_add_f32_e32 v3, -4.0, v3
3402 ; GFX12-NEXT: v_bfe_u32 v5, v3, 16, 1
3403 ; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v3
3404 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
3405 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
3406 ; GFX12-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
3407 ; GFX12-NEXT: s_wait_alu 0xfffd
3408 ; GFX12-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
3409 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3410 ; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
3411 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, v0, v3
3412 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
3413 ; GFX12-NEXT: v_and_or_b32 v3, v4, v2, v3
3414 ; GFX12-NEXT: s_wait_storecnt 0x0
3415 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v3, v1, v3, v4
3416 ; GFX12-NEXT: s_wait_dscnt 0x0
3417 ; GFX12-NEXT: global_inv scope:SCOPE_SE
3418 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3419 ; GFX12-NEXT: s_wait_alu 0xfffe
3420 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
3421 ; GFX12-NEXT: s_wait_alu 0xfffe
3422 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3423 ; GFX12-NEXT: s_cbranch_execnz .LBB14_1
3424 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
3425 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
3426 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v0, v3
3427 ; GFX12-NEXT: s_setpc_b64 s[30:31]
3428 ;
3429 ; GFX940-LABEL: local_atomic_fsub_ret_bf16:
3430 ; GFX940: ; %bb.0:
3431 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3432 ; GFX940-NEXT: v_and_b32_e32 v1, -4, v0
3433 ; GFX940-NEXT: ds_read_b32 v3, v1
3434 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 3, v0
3435 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
3436 ; GFX940-NEXT: v_and_b32_e32 v0, 24, v2
3437 ; GFX940-NEXT: v_lshlrev_b32_e64 v2, v2, s0
3438 ; GFX940-NEXT: v_not_b32_e32 v2, v2
3439 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
3440 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
3441 ; GFX940-NEXT: .LBB14_1: ; %atomicrmw.start
3442 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
3443 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
3444 ; GFX940-NEXT: v_mov_b32_e32 v4, v3
3445 ; GFX940-NEXT: v_lshrrev_b32_sdwa v3, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
3446 ; GFX940-NEXT: s_nop 0
3447 ; GFX940-NEXT: v_add_f32_e32 v3, -4.0, v3
3448 ; GFX940-NEXT: v_bfe_u32 v5, v3, 16, 1
3449 ; GFX940-NEXT: v_or_b32_e32 v6, 0x400000, v3
3450 ; GFX940-NEXT: v_add3_u32 v5, v5, v3, s2
3451 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
3452 ; GFX940-NEXT: s_nop 1
3453 ; GFX940-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
3454 ; GFX940-NEXT: v_lshlrev_b32_sdwa v3, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
3455 ; GFX940-NEXT: v_and_or_b32 v3, v4, v2, v3
3456 ; GFX940-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v3
3457 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
3458 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
3459 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
3460 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
3461 ; GFX940-NEXT: s_cbranch_execnz .LBB14_1
3462 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
3463 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
3464 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v0, v3
3465 ; GFX940-NEXT: s_setpc_b64 s[30:31]
3466 ;
3467 ; GFX11-LABEL: local_atomic_fsub_ret_bf16:
3468 ; GFX11: ; %bb.0:
3469 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3470 ; GFX11-NEXT: v_and_b32_e32 v1, -4, v0
3471 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, 3, v0
3472 ; GFX11-NEXT: s_mov_b32 s0, 0
3473 ; GFX11-NEXT: ds_load_b32 v3, v1
3474 ; GFX11-NEXT: v_lshlrev_b32_e64 v2, v0, 0xffff
3475 ; GFX11-NEXT: v_and_b32_e32 v0, 24, v0
3476 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
3477 ; GFX11-NEXT: v_not_b32_e32 v2, v2
3478 ; GFX11-NEXT: .p2align 6
3479 ; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
3480 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
3481 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
3482 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
3483 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3484 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, v0, v4
3485 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
3486 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3487 ; GFX11-NEXT: v_add_f32_e32 v3, -4.0, v3
3488 ; GFX11-NEXT: v_bfe_u32 v5, v3, 16, 1
3489 ; GFX11-NEXT: v_or_b32_e32 v6, 0x400000, v3
3490 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
3491 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
3492 ; GFX11-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
3493 ; GFX11-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
3494 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3495 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
3496 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, v0, v3
3497 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
3498 ; GFX11-NEXT: v_and_or_b32 v3, v4, v2, v3
3499 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
3500 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v3, v1, v3, v4
3501 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
3502 ; GFX11-NEXT: buffer_gl0_inv
3503 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3504 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
3505 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3506 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3507 ; GFX11-NEXT: s_cbranch_execnz .LBB14_1
3508 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
3509 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
3510 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v0, v3
3511 ; GFX11-NEXT: s_setpc_b64 s[30:31]
3512 ;
3513 ; GFX10-LABEL: local_atomic_fsub_ret_bf16:
3514 ; GFX10: ; %bb.0:
3515 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3516 ; GFX10-NEXT: v_and_b32_e32 v1, -4, v0
3517 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v0
3518 ; GFX10-NEXT: s_mov_b32 s4, 0
3519 ; GFX10-NEXT: ds_read_b32 v3, v1
3520 ; GFX10-NEXT: v_lshlrev_b32_e64 v2, v0, 0xffff
3521 ; GFX10-NEXT: v_and_b32_e32 v0, 24, v0
3522 ; GFX10-NEXT: v_not_b32_e32 v2, v2
3523 ; GFX10-NEXT: .LBB14_1: ; %atomicrmw.start
3524 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
3525 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
3526 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
3527 ; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
3528 ; GFX10-NEXT: v_add_f32_e32 v3, -4.0, v3
3529 ; GFX10-NEXT: v_bfe_u32 v5, v3, 16, 1
3530 ; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v3
3531 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
3532 ; GFX10-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
3533 ; GFX10-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
3534 ; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
3535 ; GFX10-NEXT: v_and_or_b32 v3, v4, v2, v3
3536 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
3537 ; GFX10-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v3
3538 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
3539 ; GFX10-NEXT: buffer_gl0_inv
3540 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3541 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
3542 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
3543 ; GFX10-NEXT: s_cbranch_execnz .LBB14_1
3544 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
3545 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
3546 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v0, v3
3547 ; GFX10-NEXT: s_setpc_b64 s[30:31]
3548 ;
3549 ; GFX90A-LABEL: local_atomic_fsub_ret_bf16:
3550 ; GFX90A: ; %bb.0:
3551 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3552 ; GFX90A-NEXT: v_and_b32_e32 v1, -4, v0
3553 ; GFX90A-NEXT: ds_read_b32 v3, v1
3554 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 3, v0
3555 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
3556 ; GFX90A-NEXT: v_and_b32_e32 v0, 24, v2
3557 ; GFX90A-NEXT: v_lshlrev_b32_e64 v2, v2, s4
3558 ; GFX90A-NEXT: v_not_b32_e32 v2, v2
3559 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
3560 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
3561 ; GFX90A-NEXT: .LBB14_1: ; %atomicrmw.start
3562 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
3563 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
3564 ; GFX90A-NEXT: v_mov_b32_e32 v4, v3
3565 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v3, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
3566 ; GFX90A-NEXT: v_add_f32_e32 v3, -4.0, v3
3567 ; GFX90A-NEXT: v_bfe_u32 v5, v3, 16, 1
3568 ; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v3
3569 ; GFX90A-NEXT: v_add3_u32 v5, v5, v3, s6
3570 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
3571 ; GFX90A-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
3572 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v3, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
3573 ; GFX90A-NEXT: v_and_or_b32 v3, v4, v2, v3
3574 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v3
3575 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
3576 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
3577 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3578 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
3579 ; GFX90A-NEXT: s_cbranch_execnz .LBB14_1
3580 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
3581 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
3582 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v0, v3
3583 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
3584 ;
3585 ; GFX908-LABEL: local_atomic_fsub_ret_bf16:
3586 ; GFX908: ; %bb.0:
3587 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3588 ; GFX908-NEXT: v_and_b32_e32 v1, -4, v0
3589 ; GFX908-NEXT: ds_read_b32 v3, v1
3590 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 3, v0
3591 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
3592 ; GFX908-NEXT: v_and_b32_e32 v0, 24, v2
3593 ; GFX908-NEXT: v_lshlrev_b32_e64 v2, v2, s4
3594 ; GFX908-NEXT: v_not_b32_e32 v2, v2
3595 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
3596 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
3597 ; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
3598 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
3599 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
3600 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
3601 ; GFX908-NEXT: v_lshrrev_b32_sdwa v3, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
3602 ; GFX908-NEXT: v_add_f32_e32 v3, -4.0, v3
3603 ; GFX908-NEXT: v_bfe_u32 v5, v3, 16, 1
3604 ; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v3
3605 ; GFX908-NEXT: v_add3_u32 v5, v5, v3, s6
3606 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
3607 ; GFX908-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
3608 ; GFX908-NEXT: v_lshlrev_b32_sdwa v3, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
3609 ; GFX908-NEXT: v_and_or_b32 v3, v4, v2, v3
3610 ; GFX908-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v3
3611 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
3612 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
3613 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3614 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
3615 ; GFX908-NEXT: s_cbranch_execnz .LBB14_1
3616 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
3617 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
3618 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v0, v3
3619 ; GFX908-NEXT: s_setpc_b64 s[30:31]
3620 ;
3621 ; GFX8-LABEL: local_atomic_fsub_ret_bf16:
3622 ; GFX8: ; %bb.0:
3623 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3624 ; GFX8-NEXT: v_and_b32_e32 v1, -4, v0
3625 ; GFX8-NEXT: s_mov_b32 m0, -1
3626 ; GFX8-NEXT: ds_read_b32 v3, v1
3627 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v0
3628 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
3629 ; GFX8-NEXT: v_and_b32_e32 v0, 24, v2
3630 ; GFX8-NEXT: v_lshlrev_b32_e64 v2, v2, s4
3631 ; GFX8-NEXT: v_not_b32_e32 v2, v2
3632 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
3633 ; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
3634 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3635 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
3636 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
3637 ; GFX8-NEXT: v_lshrrev_b32_sdwa v3, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
3638 ; GFX8-NEXT: v_add_f32_e32 v3, -4.0, v3
3639 ; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
3640 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
3641 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
3642 ; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
3643 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
3644 ; GFX8-NEXT: v_cndmask_b32_e32 v3, v6, v7, vcc
3645 ; GFX8-NEXT: v_and_b32_e32 v5, v4, v2
3646 ; GFX8-NEXT: v_lshlrev_b32_sdwa v3, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
3647 ; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
3648 ; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v3
3649 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
3650 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
3651 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3652 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
3653 ; GFX8-NEXT: s_cbranch_execnz .LBB14_1
3654 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
3655 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
3656 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v0, v3
3657 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3658 ;
3659 ; GFX7-LABEL: local_atomic_fsub_ret_bf16:
3660 ; GFX7: ; %bb.0:
3661 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3662 ; GFX7-NEXT: v_and_b32_e32 v1, -4, v0
3663 ; GFX7-NEXT: s_mov_b32 m0, -1
3664 ; GFX7-NEXT: ds_read_b32 v3, v1
3665 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v0
3666 ; GFX7-NEXT: v_and_b32_e32 v0, 24, v2
3667 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v2
3668 ; GFX7-NEXT: v_not_b32_e32 v2, v2
3669 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
3670 ; GFX7-NEXT: .LBB14_1: ; %atomicrmw.start
3671 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
3672 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3673 ; GFX7-NEXT: v_mov_b32_e32 v4, v3
3674 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, v0, v4
3675 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
3676 ; GFX7-NEXT: v_add_f32_e32 v3, -4.0, v3
3677 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
3678 ; GFX7-NEXT: v_and_b32_e32 v5, v4, v2
3679 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, v0, v3
3680 ; GFX7-NEXT: v_or_b32_e32 v3, v5, v3
3681 ; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v3
3682 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3683 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
3684 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3685 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
3686 ; GFX7-NEXT: s_cbranch_execnz .LBB14_1
3687 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
3688 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
3689 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v0, v3
3690 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
3691 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3692 ;
3693 ; GFX6-LABEL: local_atomic_fsub_ret_bf16:
3694 ; GFX6: ; %bb.0:
3695 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3696 ; GFX6-NEXT: v_and_b32_e32 v1, -4, v0
3697 ; GFX6-NEXT: s_mov_b32 m0, -1
3698 ; GFX6-NEXT: ds_read_b32 v3, v1
3699 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 3, v0
3700 ; GFX6-NEXT: v_and_b32_e32 v0, 24, v2
3701 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v2
3702 ; GFX6-NEXT: v_not_b32_e32 v2, v2
3703 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
3704 ; GFX6-NEXT: .LBB14_1: ; %atomicrmw.start
3705 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
3706 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
3707 ; GFX6-NEXT: v_mov_b32_e32 v4, v3
3708 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, v0, v4
3709 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
3710 ; GFX6-NEXT: v_add_f32_e32 v3, -4.0, v3
3711 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
3712 ; GFX6-NEXT: v_and_b32_e32 v5, v4, v2
3713 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, v0, v3
3714 ; GFX6-NEXT: v_or_b32_e32 v3, v5, v3
3715 ; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v3
3716 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
3717 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
3718 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3719 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
3720 ; GFX6-NEXT: s_cbranch_execnz .LBB14_1
3721 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
3722 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
3723 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v0, v3
3724 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
3725 ; GFX6-NEXT: s_setpc_b64 s[30:31]
3726 %result = atomicrmw fsub ptr addrspace(3) %ptr, bfloat 4.0 seq_cst
3727 ret bfloat %result
3728 }
3730 define bfloat @local_atomic_fsub_ret_bf16__offset(ptr addrspace(3) %ptr) nounwind {
3731 ; GFX12-LABEL: local_atomic_fsub_ret_bf16__offset:
3732 ; GFX12: ; %bb.0:
3733 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
3734 ; GFX12-NEXT: s_wait_expcnt 0x0
3735 ; GFX12-NEXT: s_wait_samplecnt 0x0
3736 ; GFX12-NEXT: s_wait_bvhcnt 0x0
3737 ; GFX12-NEXT: s_wait_kmcnt 0x0
3738 ; GFX12-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v0
3739 ; GFX12-NEXT: s_mov_b32 s0, 0
3740 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
3741 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v1
3742 ; GFX12-NEXT: v_and_b32_e32 v1, 3, v1
3743 ; GFX12-NEXT: ds_load_b32 v3, v0
3744 ; GFX12-NEXT: v_lshlrev_b32_e32 v1, 3, v1
3745 ; GFX12-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff
3746 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
3747 ; GFX12-NEXT: v_not_b32_e32 v2, v2
3748 ; GFX12-NEXT: .LBB15_1: ; %atomicrmw.start
3749 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
3750 ; GFX12-NEXT: s_wait_dscnt 0x0
3751 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
3752 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3753 ; GFX12-NEXT: v_lshrrev_b32_e32 v3, v1, v4
3754 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
3755 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3756 ; GFX12-NEXT: v_add_f32_e32 v3, -4.0, v3
3757 ; GFX12-NEXT: v_bfe_u32 v5, v3, 16, 1
3758 ; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v3
3759 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
3760 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
3761 ; GFX12-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
3762 ; GFX12-NEXT: s_wait_alu 0xfffd
3763 ; GFX12-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
3764 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3765 ; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
3766 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, v1, v3
3767 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
3768 ; GFX12-NEXT: v_and_or_b32 v3, v4, v2, v3
3769 ; GFX12-NEXT: s_wait_storecnt 0x0
3770 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v4
3771 ; GFX12-NEXT: s_wait_dscnt 0x0
3772 ; GFX12-NEXT: global_inv scope:SCOPE_SE
3773 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3774 ; GFX12-NEXT: s_wait_alu 0xfffe
3775 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
3776 ; GFX12-NEXT: s_wait_alu 0xfffe
3777 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3778 ; GFX12-NEXT: s_cbranch_execnz .LBB15_1
3779 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
3780 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
3781 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v1, v3
3782 ; GFX12-NEXT: s_setpc_b64 s[30:31]
3783 ;
3784 ; GFX940-LABEL: local_atomic_fsub_ret_bf16__offset:
3785 ; GFX940: ; %bb.0:
3786 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3787 ; GFX940-NEXT: v_add_u32_e32 v0, 0xfffe, v0
3788 ; GFX940-NEXT: v_and_b32_e32 v1, -4, v0
3789 ; GFX940-NEXT: ds_read_b32 v3, v1
3790 ; GFX940-NEXT: v_and_b32_e32 v0, 3, v0
3791 ; GFX940-NEXT: v_lshlrev_b32_e32 v0, 3, v0
3792 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
3793 ; GFX940-NEXT: v_lshlrev_b32_e64 v2, v0, s0
3794 ; GFX940-NEXT: v_not_b32_e32 v2, v2
3795 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
3796 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
3797 ; GFX940-NEXT: .LBB15_1: ; %atomicrmw.start
3798 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
3799 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
3800 ; GFX940-NEXT: v_mov_b32_e32 v4, v3
3801 ; GFX940-NEXT: v_lshrrev_b32_sdwa v3, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
3802 ; GFX940-NEXT: s_nop 0
3803 ; GFX940-NEXT: v_add_f32_e32 v3, -4.0, v3
3804 ; GFX940-NEXT: v_bfe_u32 v5, v3, 16, 1
3805 ; GFX940-NEXT: v_or_b32_e32 v6, 0x400000, v3
3806 ; GFX940-NEXT: v_add3_u32 v5, v5, v3, s2
3807 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
3808 ; GFX940-NEXT: s_nop 1
3809 ; GFX940-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
3810 ; GFX940-NEXT: v_lshlrev_b32_sdwa v3, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
3811 ; GFX940-NEXT: v_and_or_b32 v3, v4, v2, v3
3812 ; GFX940-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v3
3813 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
3814 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
3815 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
3816 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
3817 ; GFX940-NEXT: s_cbranch_execnz .LBB15_1
3818 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
3819 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
3820 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v0, v3
3821 ; GFX940-NEXT: s_setpc_b64 s[30:31]
3822 ;
3823 ; GFX11-LABEL: local_atomic_fsub_ret_bf16__offset:
3824 ; GFX11: ; %bb.0:
3825 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3826 ; GFX11-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v0
3827 ; GFX11-NEXT: s_mov_b32 s0, 0
3828 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
3829 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v1
3830 ; GFX11-NEXT: v_and_b32_e32 v1, 3, v1
3831 ; GFX11-NEXT: ds_load_b32 v3, v0
3832 ; GFX11-NEXT: v_lshlrev_b32_e32 v1, 3, v1
3833 ; GFX11-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff
3834 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
3835 ; GFX11-NEXT: v_not_b32_e32 v2, v2
3836 ; GFX11-NEXT: .p2align 6
3837 ; GFX11-NEXT: .LBB15_1: ; %atomicrmw.start
3838 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
3839 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
3840 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
3841 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3842 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, v1, v4
3843 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
3844 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3845 ; GFX11-NEXT: v_add_f32_e32 v3, -4.0, v3
3846 ; GFX11-NEXT: v_bfe_u32 v5, v3, 16, 1
3847 ; GFX11-NEXT: v_or_b32_e32 v6, 0x400000, v3
3848 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
3849 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
3850 ; GFX11-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
3851 ; GFX11-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
3852 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3853 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
3854 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, v1, v3
3855 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
3856 ; GFX11-NEXT: v_and_or_b32 v3, v4, v2, v3
3857 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
3858 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v4
3859 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
3860 ; GFX11-NEXT: buffer_gl0_inv
3861 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3862 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
3863 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3864 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3865 ; GFX11-NEXT: s_cbranch_execnz .LBB15_1
3866 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
3867 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
3868 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v1, v3
3869 ; GFX11-NEXT: s_setpc_b64 s[30:31]
3870 ;
3871 ; GFX10-LABEL: local_atomic_fsub_ret_bf16__offset:
3872 ; GFX10: ; %bb.0:
3873 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3874 ; GFX10-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v0
3875 ; GFX10-NEXT: s_mov_b32 s4, 0
3876 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v1
3877 ; GFX10-NEXT: v_and_b32_e32 v1, 3, v1
3878 ; GFX10-NEXT: ds_read_b32 v3, v0
3879 ; GFX10-NEXT: v_lshlrev_b32_e32 v1, 3, v1
3880 ; GFX10-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff
3881 ; GFX10-NEXT: v_not_b32_e32 v2, v2
3882 ; GFX10-NEXT: .LBB15_1: ; %atomicrmw.start
3883 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
3884 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
3885 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
3886 ; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
3887 ; GFX10-NEXT: v_add_f32_e32 v3, -4.0, v3
3888 ; GFX10-NEXT: v_bfe_u32 v5, v3, 16, 1
3889 ; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v3
3890 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
3891 ; GFX10-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
3892 ; GFX10-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
3893 ; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
3894 ; GFX10-NEXT: v_and_or_b32 v3, v4, v2, v3
3895 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
3896 ; GFX10-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3
3897 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
3898 ; GFX10-NEXT: buffer_gl0_inv
3899 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
3900 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
3901 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
3902 ; GFX10-NEXT: s_cbranch_execnz .LBB15_1
3903 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
3904 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
3905 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v1, v3
3906 ; GFX10-NEXT: s_setpc_b64 s[30:31]
3907 ;
3908 ; GFX90A-LABEL: local_atomic_fsub_ret_bf16__offset:
3909 ; GFX90A: ; %bb.0:
3910 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3911 ; GFX90A-NEXT: v_add_u32_e32 v0, 0xfffe, v0
3912 ; GFX90A-NEXT: v_and_b32_e32 v1, -4, v0
3913 ; GFX90A-NEXT: ds_read_b32 v3, v1
3914 ; GFX90A-NEXT: v_and_b32_e32 v0, 3, v0
3915 ; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 3, v0
3916 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
3917 ; GFX90A-NEXT: v_lshlrev_b32_e64 v2, v0, s4
3918 ; GFX90A-NEXT: v_not_b32_e32 v2, v2
3919 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
3920 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
3921 ; GFX90A-NEXT: .LBB15_1: ; %atomicrmw.start
3922 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
3923 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
3924 ; GFX90A-NEXT: v_mov_b32_e32 v4, v3
3925 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v3, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
3926 ; GFX90A-NEXT: v_add_f32_e32 v3, -4.0, v3
3927 ; GFX90A-NEXT: v_bfe_u32 v5, v3, 16, 1
3928 ; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v3
3929 ; GFX90A-NEXT: v_add3_u32 v5, v5, v3, s6
3930 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
3931 ; GFX90A-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
3932 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v3, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
3933 ; GFX90A-NEXT: v_and_or_b32 v3, v4, v2, v3
3934 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v3
3935 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
3936 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
3937 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3938 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
3939 ; GFX90A-NEXT: s_cbranch_execnz .LBB15_1
3940 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
3941 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
3942 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v0, v3
3943 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
3944 ;
3945 ; GFX908-LABEL: local_atomic_fsub_ret_bf16__offset:
3946 ; GFX908: ; %bb.0:
3947 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3948 ; GFX908-NEXT: v_add_u32_e32 v0, 0xfffe, v0
3949 ; GFX908-NEXT: v_and_b32_e32 v1, -4, v0
3950 ; GFX908-NEXT: ds_read_b32 v3, v1
3951 ; GFX908-NEXT: v_and_b32_e32 v0, 3, v0
3952 ; GFX908-NEXT: v_lshlrev_b32_e32 v0, 3, v0
3953 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
3954 ; GFX908-NEXT: v_lshlrev_b32_e64 v2, v0, s4
3955 ; GFX908-NEXT: v_not_b32_e32 v2, v2
3956 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
3957 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
3958 ; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start
3959 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
3960 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
3961 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
3962 ; GFX908-NEXT: v_lshrrev_b32_sdwa v3, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
3963 ; GFX908-NEXT: v_add_f32_e32 v3, -4.0, v3
3964 ; GFX908-NEXT: v_bfe_u32 v5, v3, 16, 1
3965 ; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v3
3966 ; GFX908-NEXT: v_add3_u32 v5, v5, v3, s6
3967 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
3968 ; GFX908-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
3969 ; GFX908-NEXT: v_lshlrev_b32_sdwa v3, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
3970 ; GFX908-NEXT: v_and_or_b32 v3, v4, v2, v3
3971 ; GFX908-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v3
3972 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
3973 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
3974 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3975 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
3976 ; GFX908-NEXT: s_cbranch_execnz .LBB15_1
3977 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
3978 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
3979 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v0, v3
3980 ; GFX908-NEXT: s_setpc_b64 s[30:31]
3981 ;
3982 ; GFX8-LABEL: local_atomic_fsub_ret_bf16__offset:
3983 ; GFX8: ; %bb.0:
3984 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3985 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffe, v0
3986 ; GFX8-NEXT: v_and_b32_e32 v1, -4, v0
3987 ; GFX8-NEXT: s_mov_b32 m0, -1
3988 ; GFX8-NEXT: ds_read_b32 v3, v1
3989 ; GFX8-NEXT: v_and_b32_e32 v0, 3, v0
3990 ; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v0
3991 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
3992 ; GFX8-NEXT: v_lshlrev_b32_e64 v2, v0, s4
3993 ; GFX8-NEXT: v_not_b32_e32 v2, v2
3994 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
3995 ; GFX8-NEXT: .LBB15_1: ; %atomicrmw.start
3996 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3997 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
3998 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
3999 ; GFX8-NEXT: v_lshrrev_b32_sdwa v3, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4000 ; GFX8-NEXT: v_add_f32_e32 v3, -4.0, v3
4001 ; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
4002 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
4003 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
4004 ; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
4005 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
4006 ; GFX8-NEXT: v_cndmask_b32_e32 v3, v6, v7, vcc
4007 ; GFX8-NEXT: v_and_b32_e32 v5, v4, v2
4008 ; GFX8-NEXT: v_lshlrev_b32_sdwa v3, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4009 ; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
4010 ; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v3
4011 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
4012 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
4013 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4014 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
4015 ; GFX8-NEXT: s_cbranch_execnz .LBB15_1
4016 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4017 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
4018 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v0, v3
4019 ; GFX8-NEXT: s_setpc_b64 s[30:31]
4020 ;
4021 ; GFX7-LABEL: local_atomic_fsub_ret_bf16__offset:
4022 ; GFX7: ; %bb.0:
4023 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4024 ; GFX7-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
4025 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v1
4026 ; GFX7-NEXT: s_mov_b32 m0, -1
4027 ; GFX7-NEXT: ds_read_b32 v3, v0
4028 ; GFX7-NEXT: v_and_b32_e32 v1, 3, v1
4029 ; GFX7-NEXT: v_lshlrev_b32_e32 v1, 3, v1
4030 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v1
4031 ; GFX7-NEXT: v_not_b32_e32 v2, v2
4032 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
4033 ; GFX7-NEXT: .LBB15_1: ; %atomicrmw.start
4034 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4035 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4036 ; GFX7-NEXT: v_mov_b32_e32 v4, v3
4037 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, v1, v4
4038 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
4039 ; GFX7-NEXT: v_add_f32_e32 v3, -4.0, v3
4040 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
4041 ; GFX7-NEXT: v_and_b32_e32 v5, v4, v2
4042 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, v1, v3
4043 ; GFX7-NEXT: v_or_b32_e32 v3, v5, v3
4044 ; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3
4045 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4046 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
4047 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4048 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
4049 ; GFX7-NEXT: s_cbranch_execnz .LBB15_1
4050 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4051 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
4052 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v1, v3
4053 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
4054 ; GFX7-NEXT: s_setpc_b64 s[30:31]
4055 ;
4056 ; GFX6-LABEL: local_atomic_fsub_ret_bf16__offset:
4057 ; GFX6: ; %bb.0:
4058 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4059 ; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
4060 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v1
4061 ; GFX6-NEXT: s_mov_b32 m0, -1
4062 ; GFX6-NEXT: ds_read_b32 v3, v0
4063 ; GFX6-NEXT: v_and_b32_e32 v1, 3, v1
4064 ; GFX6-NEXT: v_lshlrev_b32_e32 v1, 3, v1
4065 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v1
4066 ; GFX6-NEXT: v_not_b32_e32 v2, v2
4067 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
4068 ; GFX6-NEXT: .LBB15_1: ; %atomicrmw.start
4069 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
4070 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
4071 ; GFX6-NEXT: v_mov_b32_e32 v4, v3
4072 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, v1, v4
4073 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
4074 ; GFX6-NEXT: v_add_f32_e32 v3, -4.0, v3
4075 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
4076 ; GFX6-NEXT: v_and_b32_e32 v5, v4, v2
4077 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, v1, v3
4078 ; GFX6-NEXT: v_or_b32_e32 v3, v5, v3
4079 ; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3
4080 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
4081 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
4082 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4083 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
4084 ; GFX6-NEXT: s_cbranch_execnz .LBB15_1
4085 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
4086 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
4087 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v1, v3
4088 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
4089 ; GFX6-NEXT: s_setpc_b64 s[30:31]
4090 %gep = getelementptr bfloat, ptr addrspace(3) %ptr, i32 32767
4091 %result = atomicrmw fsub ptr addrspace(3) %gep, bfloat 4.0 seq_cst
4092 ret bfloat %result
4093 }
4095 define void @local_atomic_fsub_noret_bf16(ptr addrspace(3) %ptr) nounwind {
4096 ; GFX12-LABEL: local_atomic_fsub_noret_bf16:
4097 ; GFX12: ; %bb.0:
4098 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
4099 ; GFX12-NEXT: s_wait_expcnt 0x0
4100 ; GFX12-NEXT: s_wait_samplecnt 0x0
4101 ; GFX12-NEXT: s_wait_bvhcnt 0x0
4102 ; GFX12-NEXT: s_wait_kmcnt 0x0
4103 ; GFX12-NEXT: v_and_b32_e32 v1, -4, v0
4104 ; GFX12-NEXT: v_lshlrev_b32_e32 v0, 3, v0
4105 ; GFX12-NEXT: s_mov_b32 s0, 0
4106 ; GFX12-NEXT: ds_load_b32 v2, v1
4107 ; GFX12-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff
4108 ; GFX12-NEXT: v_and_b32_e32 v0, 24, v0
4109 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
4110 ; GFX12-NEXT: v_not_b32_e32 v3, v3
4111 ; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start
4112 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
4113 ; GFX12-NEXT: s_wait_dscnt 0x0
4114 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
4115 ; GFX12-NEXT: v_lshrrev_b32_e32 v4, v0, v2
4116 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v4
4117 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4118 ; GFX12-NEXT: v_add_f32_e32 v4, -4.0, v4
4119 ; GFX12-NEXT: v_bfe_u32 v5, v4, 16, 1
4120 ; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v4
4121 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
4122 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
4123 ; GFX12-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
4124 ; GFX12-NEXT: s_wait_alu 0xfffd
4125 ; GFX12-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
4126 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4127 ; GFX12-NEXT: v_lshrrev_b32_e32 v4, 16, v4
4128 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, v0, v4
4129 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
4130 ; GFX12-NEXT: v_and_or_b32 v4, v2, v3, v4
4131 ; GFX12-NEXT: s_wait_storecnt 0x0
4132 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v4, v1, v4, v2
4133 ; GFX12-NEXT: s_wait_dscnt 0x0
4134 ; GFX12-NEXT: global_inv scope:SCOPE_SE
4135 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
4136 ; GFX12-NEXT: v_mov_b32_e32 v2, v4
4137 ; GFX12-NEXT: s_wait_alu 0xfffe
4138 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
4139 ; GFX12-NEXT: s_wait_alu 0xfffe
4140 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4141 ; GFX12-NEXT: s_cbranch_execnz .LBB16_1
4142 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
4143 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
4144 ; GFX12-NEXT: s_setpc_b64 s[30:31]
4145 ;
4146 ; GFX940-LABEL: local_atomic_fsub_noret_bf16:
4147 ; GFX940: ; %bb.0:
4148 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4149 ; GFX940-NEXT: v_and_b32_e32 v1, -4, v0
4150 ; GFX940-NEXT: ds_read_b32 v3, v1
4151 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 3, v0
4152 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
4153 ; GFX940-NEXT: v_and_b32_e32 v0, 24, v2
4154 ; GFX940-NEXT: v_lshlrev_b32_e64 v2, v2, s0
4155 ; GFX940-NEXT: v_not_b32_e32 v2, v2
4156 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
4157 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
4158 ; GFX940-NEXT: .LBB16_1: ; %atomicrmw.start
4159 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
4160 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
4161 ; GFX940-NEXT: v_lshrrev_b32_sdwa v4, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4162 ; GFX940-NEXT: s_nop 0
4163 ; GFX940-NEXT: v_add_f32_e32 v4, -4.0, v4
4164 ; GFX940-NEXT: v_bfe_u32 v5, v4, 16, 1
4165 ; GFX940-NEXT: v_or_b32_e32 v6, 0x400000, v4
4166 ; GFX940-NEXT: v_add3_u32 v5, v5, v4, s2
4167 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
4168 ; GFX940-NEXT: s_nop 1
4169 ; GFX940-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
4170 ; GFX940-NEXT: v_lshlrev_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4171 ; GFX940-NEXT: v_and_or_b32 v4, v3, v2, v4
4172 ; GFX940-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
4173 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
4174 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
4175 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4176 ; GFX940-NEXT: v_mov_b32_e32 v3, v4
4177 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
4178 ; GFX940-NEXT: s_cbranch_execnz .LBB16_1
4179 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
4180 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
4181 ; GFX940-NEXT: s_setpc_b64 s[30:31]
4182 ;
4183 ; GFX11-LABEL: local_atomic_fsub_noret_bf16:
4184 ; GFX11: ; %bb.0:
4185 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4186 ; GFX11-NEXT: v_and_b32_e32 v1, -4, v0
4187 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, 3, v0
4188 ; GFX11-NEXT: s_mov_b32 s0, 0
4189 ; GFX11-NEXT: ds_load_b32 v2, v1
4190 ; GFX11-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff
4191 ; GFX11-NEXT: v_and_b32_e32 v0, 24, v0
4192 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
4193 ; GFX11-NEXT: v_not_b32_e32 v3, v3
4194 ; GFX11-NEXT: .p2align 6
4195 ; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start
4196 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
4197 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
4198 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
4199 ; GFX11-NEXT: v_lshrrev_b32_e32 v4, v0, v2
4200 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
4201 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4202 ; GFX11-NEXT: v_add_f32_e32 v4, -4.0, v4
4203 ; GFX11-NEXT: v_bfe_u32 v5, v4, 16, 1
4204 ; GFX11-NEXT: v_or_b32_e32 v6, 0x400000, v4
4205 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
4206 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
4207 ; GFX11-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
4208 ; GFX11-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
4209 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4210 ; GFX11-NEXT: v_lshrrev_b32_e32 v4, 16, v4
4211 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, v0, v4
4212 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
4213 ; GFX11-NEXT: v_and_or_b32 v4, v2, v3, v4
4214 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
4215 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v4, v1, v4, v2
4216 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
4217 ; GFX11-NEXT: buffer_gl0_inv
4218 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
4219 ; GFX11-NEXT: v_mov_b32_e32 v2, v4
4220 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
4221 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4222 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4223 ; GFX11-NEXT: s_cbranch_execnz .LBB16_1
4224 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
4225 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
4226 ; GFX11-NEXT: s_setpc_b64 s[30:31]
4227 ;
4228 ; GFX10-LABEL: local_atomic_fsub_noret_bf16:
4229 ; GFX10: ; %bb.0:
4230 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4231 ; GFX10-NEXT: v_and_b32_e32 v1, -4, v0
4232 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v0
4233 ; GFX10-NEXT: s_mov_b32 s4, 0
4234 ; GFX10-NEXT: ds_read_b32 v2, v1
4235 ; GFX10-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff
4236 ; GFX10-NEXT: v_and_b32_e32 v0, 24, v0
4237 ; GFX10-NEXT: v_not_b32_e32 v3, v3
4238 ; GFX10-NEXT: .LBB16_1: ; %atomicrmw.start
4239 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
4240 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
4241 ; GFX10-NEXT: v_lshrrev_b32_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4242 ; GFX10-NEXT: v_add_f32_e32 v4, -4.0, v4
4243 ; GFX10-NEXT: v_bfe_u32 v5, v4, 16, 1
4244 ; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v4
4245 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
4246 ; GFX10-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
4247 ; GFX10-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
4248 ; GFX10-NEXT: v_lshlrev_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4249 ; GFX10-NEXT: v_and_or_b32 v4, v2, v3, v4
4250 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
4251 ; GFX10-NEXT: ds_cmpst_rtn_b32 v4, v1, v2, v4
4252 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
4253 ; GFX10-NEXT: buffer_gl0_inv
4254 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
4255 ; GFX10-NEXT: v_mov_b32_e32 v2, v4
4256 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
4257 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
4258 ; GFX10-NEXT: s_cbranch_execnz .LBB16_1
4259 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
4260 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
4261 ; GFX10-NEXT: s_setpc_b64 s[30:31]
4262 ;
4263 ; GFX90A-LABEL: local_atomic_fsub_noret_bf16:
4264 ; GFX90A: ; %bb.0:
4265 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4266 ; GFX90A-NEXT: v_and_b32_e32 v1, -4, v0
4267 ; GFX90A-NEXT: ds_read_b32 v3, v1
4268 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 3, v0
4269 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
4270 ; GFX90A-NEXT: v_and_b32_e32 v0, 24, v2
4271 ; GFX90A-NEXT: v_lshlrev_b32_e64 v2, v2, s4
4272 ; GFX90A-NEXT: v_not_b32_e32 v2, v2
4273 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
4274 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
4275 ; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
4276 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
4277 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
4278 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v4, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4279 ; GFX90A-NEXT: v_add_f32_e32 v4, -4.0, v4
4280 ; GFX90A-NEXT: v_bfe_u32 v5, v4, 16, 1
4281 ; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v4
4282 ; GFX90A-NEXT: v_add3_u32 v5, v5, v4, s6
4283 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
4284 ; GFX90A-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
4285 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4286 ; GFX90A-NEXT: v_and_or_b32 v4, v3, v2, v4
4287 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
4288 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
4289 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
4290 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4291 ; GFX90A-NEXT: v_mov_b32_e32 v3, v4
4292 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
4293 ; GFX90A-NEXT: s_cbranch_execnz .LBB16_1
4294 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
4295 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
4296 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
4297 ;
4298 ; GFX908-LABEL: local_atomic_fsub_noret_bf16:
4299 ; GFX908: ; %bb.0:
4300 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4301 ; GFX908-NEXT: v_and_b32_e32 v1, -4, v0
4302 ; GFX908-NEXT: ds_read_b32 v3, v1
4303 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 3, v0
4304 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
4305 ; GFX908-NEXT: v_and_b32_e32 v0, 24, v2
4306 ; GFX908-NEXT: v_lshlrev_b32_e64 v2, v2, s4
4307 ; GFX908-NEXT: v_not_b32_e32 v2, v2
4308 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
4309 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
4310 ; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
4311 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
4312 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
4313 ; GFX908-NEXT: v_lshrrev_b32_sdwa v4, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4314 ; GFX908-NEXT: v_add_f32_e32 v4, -4.0, v4
4315 ; GFX908-NEXT: v_bfe_u32 v5, v4, 16, 1
4316 ; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v4
4317 ; GFX908-NEXT: v_add3_u32 v5, v5, v4, s6
4318 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
4319 ; GFX908-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
4320 ; GFX908-NEXT: v_lshlrev_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4321 ; GFX908-NEXT: v_and_or_b32 v4, v3, v2, v4
4322 ; GFX908-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
4323 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
4324 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
4325 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4326 ; GFX908-NEXT: v_mov_b32_e32 v3, v4
4327 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
4328 ; GFX908-NEXT: s_cbranch_execnz .LBB16_1
4329 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
4330 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
4331 ; GFX908-NEXT: s_setpc_b64 s[30:31]
4332 ;
4333 ; GFX8-LABEL: local_atomic_fsub_noret_bf16:
4334 ; GFX8: ; %bb.0:
4335 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4336 ; GFX8-NEXT: v_and_b32_e32 v1, -4, v0
4337 ; GFX8-NEXT: s_mov_b32 m0, -1
4338 ; GFX8-NEXT: ds_read_b32 v3, v1
4339 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v0
4340 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
4341 ; GFX8-NEXT: v_and_b32_e32 v0, 24, v2
4342 ; GFX8-NEXT: v_lshlrev_b32_e64 v2, v2, s4
4343 ; GFX8-NEXT: v_not_b32_e32 v2, v2
4344 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
4345 ; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start
4346 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4347 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
4348 ; GFX8-NEXT: v_lshrrev_b32_sdwa v4, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4349 ; GFX8-NEXT: v_add_f32_e32 v4, -4.0, v4
4350 ; GFX8-NEXT: v_bfe_u32 v6, v4, 16, 1
4351 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v4
4352 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
4353 ; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v4
4354 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
4355 ; GFX8-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc
4356 ; GFX8-NEXT: v_and_b32_e32 v5, v3, v2
4357 ; GFX8-NEXT: v_lshlrev_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4358 ; GFX8-NEXT: v_or_b32_e32 v4, v5, v4
4359 ; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
4360 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
4361 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
4362 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4363 ; GFX8-NEXT: v_mov_b32_e32 v3, v4
4364 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
4365 ; GFX8-NEXT: s_cbranch_execnz .LBB16_1
4366 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4367 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
4368 ; GFX8-NEXT: s_setpc_b64 s[30:31]
4369 ;
4370 ; GFX7-LABEL: local_atomic_fsub_noret_bf16:
4371 ; GFX7: ; %bb.0:
4372 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4373 ; GFX7-NEXT: v_and_b32_e32 v1, -4, v0
4374 ; GFX7-NEXT: s_mov_b32 m0, -1
4375 ; GFX7-NEXT: ds_read_b32 v3, v1
4376 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v0
4377 ; GFX7-NEXT: v_and_b32_e32 v0, 24, v2
4378 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v2
4379 ; GFX7-NEXT: v_not_b32_e32 v2, v2
4380 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
4381 ; GFX7-NEXT: .LBB16_1: ; %atomicrmw.start
4382 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4383 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4384 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, v0, v3
4385 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
4386 ; GFX7-NEXT: v_add_f32_e32 v4, -4.0, v4
4387 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
4388 ; GFX7-NEXT: v_and_b32_e32 v5, v3, v2
4389 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, v0, v4
4390 ; GFX7-NEXT: v_or_b32_e32 v4, v5, v4
4391 ; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
4392 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4393 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
4394 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4395 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
4396 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
4397 ; GFX7-NEXT: s_cbranch_execnz .LBB16_1
4398 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4399 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
4400 ; GFX7-NEXT: s_setpc_b64 s[30:31]
4401 ;
4402 ; GFX6-LABEL: local_atomic_fsub_noret_bf16:
4403 ; GFX6: ; %bb.0:
4404 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4405 ; GFX6-NEXT: v_and_b32_e32 v1, -4, v0
4406 ; GFX6-NEXT: s_mov_b32 m0, -1
4407 ; GFX6-NEXT: ds_read_b32 v3, v1
4408 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 3, v0
4409 ; GFX6-NEXT: v_and_b32_e32 v0, 24, v2
4410 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v2
4411 ; GFX6-NEXT: v_not_b32_e32 v2, v2
4412 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
4413 ; GFX6-NEXT: .LBB16_1: ; %atomicrmw.start
4414 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
4415 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
4416 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, v0, v3
4417 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
4418 ; GFX6-NEXT: v_add_f32_e32 v4, -4.0, v4
4419 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
4420 ; GFX6-NEXT: v_and_b32_e32 v5, v3, v2
4421 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, v0, v4
4422 ; GFX6-NEXT: v_or_b32_e32 v4, v5, v4
4423 ; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
4424 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
4425 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
4426 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4427 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
4428 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
4429 ; GFX6-NEXT: s_cbranch_execnz .LBB16_1
4430 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
4431 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
4432 ; GFX6-NEXT: s_setpc_b64 s[30:31]
4433 %result = atomicrmw fsub ptr addrspace(3) %ptr, bfloat 4.0 seq_cst
4434 ret void
4435 }
4437 define void @local_atomic_fsub_noret_bf16__offset(ptr addrspace(3) %ptr) nounwind {
4438 ; GFX12-LABEL: local_atomic_fsub_noret_bf16__offset:
4439 ; GFX12: ; %bb.0:
4440 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
4441 ; GFX12-NEXT: s_wait_expcnt 0x0
4442 ; GFX12-NEXT: s_wait_samplecnt 0x0
4443 ; GFX12-NEXT: s_wait_bvhcnt 0x0
4444 ; GFX12-NEXT: s_wait_kmcnt 0x0
4445 ; GFX12-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v0
4446 ; GFX12-NEXT: s_mov_b32 s0, 0
4447 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
4448 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v1
4449 ; GFX12-NEXT: v_and_b32_e32 v1, 3, v1
4450 ; GFX12-NEXT: ds_load_b32 v3, v0
4451 ; GFX12-NEXT: v_lshlrev_b32_e32 v1, 3, v1
4452 ; GFX12-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff
4453 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
4454 ; GFX12-NEXT: v_not_b32_e32 v2, v2
4455 ; GFX12-NEXT: .LBB17_1: ; %atomicrmw.start
4456 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
4457 ; GFX12-NEXT: s_wait_dscnt 0x0
4458 ; GFX12-NEXT: v_lshrrev_b32_e32 v4, v1, v3
4459 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4460 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v4
4461 ; GFX12-NEXT: v_add_f32_e32 v4, -4.0, v4
4462 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
4463 ; GFX12-NEXT: v_bfe_u32 v5, v4, 16, 1
4464 ; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v4
4465 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
4466 ; GFX12-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
4467 ; GFX12-NEXT: s_wait_alu 0xfffd
4468 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4469 ; GFX12-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
4470 ; GFX12-NEXT: v_lshrrev_b32_e32 v4, 16, v4
4471 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4472 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, v1, v4
4473 ; GFX12-NEXT: v_and_or_b32 v4, v3, v2, v4
4474 ; GFX12-NEXT: s_wait_storecnt 0x0
4475 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
4476 ; GFX12-NEXT: s_wait_dscnt 0x0
4477 ; GFX12-NEXT: global_inv scope:SCOPE_SE
4478 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
4479 ; GFX12-NEXT: v_mov_b32_e32 v3, v4
4480 ; GFX12-NEXT: s_wait_alu 0xfffe
4481 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
4482 ; GFX12-NEXT: s_wait_alu 0xfffe
4483 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4484 ; GFX12-NEXT: s_cbranch_execnz .LBB17_1
4485 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
4486 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
4487 ; GFX12-NEXT: s_setpc_b64 s[30:31]
4488 ;
4489 ; GFX940-LABEL: local_atomic_fsub_noret_bf16__offset:
4490 ; GFX940: ; %bb.0:
4491 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4492 ; GFX940-NEXT: v_add_u32_e32 v1, 0xfffe, v0
4493 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v1
4494 ; GFX940-NEXT: ds_read_b32 v3, v0
4495 ; GFX940-NEXT: v_and_b32_e32 v1, 3, v1
4496 ; GFX940-NEXT: v_lshlrev_b32_e32 v1, 3, v1
4497 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
4498 ; GFX940-NEXT: v_lshlrev_b32_e64 v2, v1, s0
4499 ; GFX940-NEXT: v_not_b32_e32 v2, v2
4500 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
4501 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
4502 ; GFX940-NEXT: .LBB17_1: ; %atomicrmw.start
4503 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
4504 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
4505 ; GFX940-NEXT: v_lshrrev_b32_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4506 ; GFX940-NEXT: s_nop 0
4507 ; GFX940-NEXT: v_add_f32_e32 v4, -4.0, v4
4508 ; GFX940-NEXT: v_bfe_u32 v5, v4, 16, 1
4509 ; GFX940-NEXT: v_or_b32_e32 v6, 0x400000, v4
4510 ; GFX940-NEXT: v_add3_u32 v5, v5, v4, s2
4511 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
4512 ; GFX940-NEXT: s_nop 1
4513 ; GFX940-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
4514 ; GFX940-NEXT: v_lshlrev_b32_sdwa v4, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4515 ; GFX940-NEXT: v_and_or_b32 v4, v3, v2, v4
4516 ; GFX940-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
4517 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
4518 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
4519 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4520 ; GFX940-NEXT: v_mov_b32_e32 v3, v4
4521 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
4522 ; GFX940-NEXT: s_cbranch_execnz .LBB17_1
4523 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
4524 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
4525 ; GFX940-NEXT: s_setpc_b64 s[30:31]
4526 ;
4527 ; GFX11-LABEL: local_atomic_fsub_noret_bf16__offset:
4528 ; GFX11: ; %bb.0:
4529 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4530 ; GFX11-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v0
4531 ; GFX11-NEXT: s_mov_b32 s0, 0
4532 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
4533 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v1
4534 ; GFX11-NEXT: v_and_b32_e32 v1, 3, v1
4535 ; GFX11-NEXT: ds_load_b32 v3, v0
4536 ; GFX11-NEXT: v_lshlrev_b32_e32 v1, 3, v1
4537 ; GFX11-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff
4538 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
4539 ; GFX11-NEXT: v_not_b32_e32 v2, v2
4540 ; GFX11-NEXT: .p2align 6
4541 ; GFX11-NEXT: .LBB17_1: ; %atomicrmw.start
4542 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
4543 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
4544 ; GFX11-NEXT: v_lshrrev_b32_e32 v4, v1, v3
4545 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4546 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
4547 ; GFX11-NEXT: v_add_f32_e32 v4, -4.0, v4
4548 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
4549 ; GFX11-NEXT: v_bfe_u32 v5, v4, 16, 1
4550 ; GFX11-NEXT: v_or_b32_e32 v6, 0x400000, v4
4551 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
4552 ; GFX11-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
4553 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4554 ; GFX11-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
4555 ; GFX11-NEXT: v_lshrrev_b32_e32 v4, 16, v4
4556 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4557 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, v1, v4
4558 ; GFX11-NEXT: v_and_or_b32 v4, v3, v2, v4
4559 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
4560 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
4561 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
4562 ; GFX11-NEXT: buffer_gl0_inv
4563 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
4564 ; GFX11-NEXT: v_mov_b32_e32 v3, v4
4565 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
4566 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4567 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4568 ; GFX11-NEXT: s_cbranch_execnz .LBB17_1
4569 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
4570 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
4571 ; GFX11-NEXT: s_setpc_b64 s[30:31]
4572 ;
4573 ; GFX10-LABEL: local_atomic_fsub_noret_bf16__offset:
4574 ; GFX10: ; %bb.0:
4575 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4576 ; GFX10-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v0
4577 ; GFX10-NEXT: s_mov_b32 s4, 0
4578 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v1
4579 ; GFX10-NEXT: v_and_b32_e32 v1, 3, v1
4580 ; GFX10-NEXT: ds_read_b32 v3, v0
4581 ; GFX10-NEXT: v_lshlrev_b32_e32 v1, 3, v1
4582 ; GFX10-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff
4583 ; GFX10-NEXT: v_not_b32_e32 v2, v2
4584 ; GFX10-NEXT: .LBB17_1: ; %atomicrmw.start
4585 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
4586 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
4587 ; GFX10-NEXT: v_lshrrev_b32_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4588 ; GFX10-NEXT: v_add_f32_e32 v4, -4.0, v4
4589 ; GFX10-NEXT: v_bfe_u32 v5, v4, 16, 1
4590 ; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v4
4591 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
4592 ; GFX10-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
4593 ; GFX10-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
4594 ; GFX10-NEXT: v_lshlrev_b32_sdwa v4, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4595 ; GFX10-NEXT: v_and_or_b32 v4, v3, v2, v4
4596 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
4597 ; GFX10-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
4598 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
4599 ; GFX10-NEXT: buffer_gl0_inv
4600 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
4601 ; GFX10-NEXT: v_mov_b32_e32 v3, v4
4602 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
4603 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
4604 ; GFX10-NEXT: s_cbranch_execnz .LBB17_1
4605 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
4606 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
4607 ; GFX10-NEXT: s_setpc_b64 s[30:31]
4608 ;
4609 ; GFX90A-LABEL: local_atomic_fsub_noret_bf16__offset:
4610 ; GFX90A: ; %bb.0:
4611 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4612 ; GFX90A-NEXT: v_add_u32_e32 v1, 0xfffe, v0
4613 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v1
4614 ; GFX90A-NEXT: ds_read_b32 v3, v0
4615 ; GFX90A-NEXT: v_and_b32_e32 v1, 3, v1
4616 ; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 3, v1
4617 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
4618 ; GFX90A-NEXT: v_lshlrev_b32_e64 v2, v1, s4
4619 ; GFX90A-NEXT: v_not_b32_e32 v2, v2
4620 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
4621 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
4622 ; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
4623 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
4624 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
4625 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4626 ; GFX90A-NEXT: v_add_f32_e32 v4, -4.0, v4
4627 ; GFX90A-NEXT: v_bfe_u32 v5, v4, 16, 1
4628 ; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v4
4629 ; GFX90A-NEXT: v_add3_u32 v5, v5, v4, s6
4630 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
4631 ; GFX90A-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
4632 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v4, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4633 ; GFX90A-NEXT: v_and_or_b32 v4, v3, v2, v4
4634 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
4635 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
4636 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
4637 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4638 ; GFX90A-NEXT: v_mov_b32_e32 v3, v4
4639 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
4640 ; GFX90A-NEXT: s_cbranch_execnz .LBB17_1
4641 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
4642 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
4643 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
4644 ;
4645 ; GFX908-LABEL: local_atomic_fsub_noret_bf16__offset:
4646 ; GFX908: ; %bb.0:
4647 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4648 ; GFX908-NEXT: v_add_u32_e32 v1, 0xfffe, v0
4649 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v1
4650 ; GFX908-NEXT: ds_read_b32 v3, v0
4651 ; GFX908-NEXT: v_and_b32_e32 v1, 3, v1
4652 ; GFX908-NEXT: v_lshlrev_b32_e32 v1, 3, v1
4653 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
4654 ; GFX908-NEXT: v_lshlrev_b32_e64 v2, v1, s4
4655 ; GFX908-NEXT: v_not_b32_e32 v2, v2
4656 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
4657 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
4658 ; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
4659 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
4660 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
4661 ; GFX908-NEXT: v_lshrrev_b32_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4662 ; GFX908-NEXT: v_add_f32_e32 v4, -4.0, v4
4663 ; GFX908-NEXT: v_bfe_u32 v5, v4, 16, 1
4664 ; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v4
4665 ; GFX908-NEXT: v_add3_u32 v5, v5, v4, s6
4666 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
4667 ; GFX908-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
4668 ; GFX908-NEXT: v_lshlrev_b32_sdwa v4, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4669 ; GFX908-NEXT: v_and_or_b32 v4, v3, v2, v4
4670 ; GFX908-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
4671 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
4672 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
4673 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4674 ; GFX908-NEXT: v_mov_b32_e32 v3, v4
4675 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
4676 ; GFX908-NEXT: s_cbranch_execnz .LBB17_1
4677 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
4678 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
4679 ; GFX908-NEXT: s_setpc_b64 s[30:31]
4680 ;
4681 ; GFX8-LABEL: local_atomic_fsub_noret_bf16__offset:
4682 ; GFX8: ; %bb.0:
4683 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4684 ; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0xfffe, v0
4685 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v1
4686 ; GFX8-NEXT: s_mov_b32 m0, -1
4687 ; GFX8-NEXT: ds_read_b32 v3, v0
4688 ; GFX8-NEXT: v_and_b32_e32 v1, 3, v1
4689 ; GFX8-NEXT: v_lshlrev_b32_e32 v1, 3, v1
4690 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
4691 ; GFX8-NEXT: v_lshlrev_b32_e64 v2, v1, s4
4692 ; GFX8-NEXT: v_not_b32_e32 v2, v2
4693 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
4694 ; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
4695 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4696 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
4697 ; GFX8-NEXT: v_lshrrev_b32_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4698 ; GFX8-NEXT: v_add_f32_e32 v4, -4.0, v4
4699 ; GFX8-NEXT: v_bfe_u32 v6, v4, 16, 1
4700 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v4
4701 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
4702 ; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v4
4703 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
4704 ; GFX8-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc
4705 ; GFX8-NEXT: v_and_b32_e32 v5, v3, v2
4706 ; GFX8-NEXT: v_lshlrev_b32_sdwa v4, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4707 ; GFX8-NEXT: v_or_b32_e32 v4, v5, v4
4708 ; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
4709 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
4710 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
4711 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4712 ; GFX8-NEXT: v_mov_b32_e32 v3, v4
4713 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
4714 ; GFX8-NEXT: s_cbranch_execnz .LBB17_1
4715 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4716 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
4717 ; GFX8-NEXT: s_setpc_b64 s[30:31]
4718 ;
4719 ; GFX7-LABEL: local_atomic_fsub_noret_bf16__offset:
4720 ; GFX7: ; %bb.0:
4721 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4722 ; GFX7-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
4723 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v1
4724 ; GFX7-NEXT: s_mov_b32 m0, -1
4725 ; GFX7-NEXT: ds_read_b32 v3, v0
4726 ; GFX7-NEXT: v_and_b32_e32 v1, 3, v1
4727 ; GFX7-NEXT: v_lshlrev_b32_e32 v1, 3, v1
4728 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v1
4729 ; GFX7-NEXT: v_not_b32_e32 v2, v2
4730 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
4731 ; GFX7-NEXT: .LBB17_1: ; %atomicrmw.start
4732 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4733 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4734 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, v1, v3
4735 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
4736 ; GFX7-NEXT: v_add_f32_e32 v4, -4.0, v4
4737 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
4738 ; GFX7-NEXT: v_and_b32_e32 v5, v3, v2
4739 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, v1, v4
4740 ; GFX7-NEXT: v_or_b32_e32 v4, v5, v4
4741 ; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
4742 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4743 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
4744 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4745 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
4746 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
4747 ; GFX7-NEXT: s_cbranch_execnz .LBB17_1
4748 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4749 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
4750 ; GFX7-NEXT: s_setpc_b64 s[30:31]
4751 ;
4752 ; GFX6-LABEL: local_atomic_fsub_noret_bf16__offset:
4753 ; GFX6: ; %bb.0:
4754 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4755 ; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
4756 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v1
4757 ; GFX6-NEXT: s_mov_b32 m0, -1
4758 ; GFX6-NEXT: ds_read_b32 v3, v0
4759 ; GFX6-NEXT: v_and_b32_e32 v1, 3, v1
4760 ; GFX6-NEXT: v_lshlrev_b32_e32 v1, 3, v1
4761 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v1
4762 ; GFX6-NEXT: v_not_b32_e32 v2, v2
4763 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
4764 ; GFX6-NEXT: .LBB17_1: ; %atomicrmw.start
4765 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
4766 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
4767 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, v1, v3
4768 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
4769 ; GFX6-NEXT: v_add_f32_e32 v4, -4.0, v4
4770 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
4771 ; GFX6-NEXT: v_and_b32_e32 v5, v3, v2
4772 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, v1, v4
4773 ; GFX6-NEXT: v_or_b32_e32 v4, v5, v4
4774 ; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
4775 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
4776 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
4777 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4778 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
4779 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
4780 ; GFX6-NEXT: s_cbranch_execnz .LBB17_1
4781 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
4782 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
4783 ; GFX6-NEXT: s_setpc_b64 s[30:31]
4784 %gep = getelementptr bfloat, ptr addrspace(3) %ptr, i32 32767
4785 %unused = atomicrmw fsub ptr addrspace(3) %gep, bfloat 4.0 seq_cst
4786 ret void
4787 }
4789 define bfloat @local_atomic_fsub_ret_bf16__offset__align4(ptr addrspace(3) %ptr) nounwind {
4790 ; GFX12-LABEL: local_atomic_fsub_ret_bf16__offset__align4:
4791 ; GFX12: ; %bb.0:
4792 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
4793 ; GFX12-NEXT: s_wait_expcnt 0x0
4794 ; GFX12-NEXT: s_wait_samplecnt 0x0
4795 ; GFX12-NEXT: s_wait_bvhcnt 0x0
4796 ; GFX12-NEXT: s_wait_kmcnt 0x0
4797 ; GFX12-NEXT: ds_load_b32 v1, v0 offset:65534
4798 ; GFX12-NEXT: s_mov_b32 s0, 0
4799 ; GFX12-NEXT: .LBB18_1: ; %atomicrmw.start
4800 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
4801 ; GFX12-NEXT: s_wait_dscnt 0x0
4802 ; GFX12-NEXT: v_mov_b32_e32 v2, v1
4803 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4804 ; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v2
4805 ; GFX12-NEXT: v_add_f32_e32 v1, -4.0, v1
4806 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
4807 ; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
4808 ; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v1
4809 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
4810 ; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
4811 ; GFX12-NEXT: s_wait_alu 0xfffd
4812 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4813 ; GFX12-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
4814 ; GFX12-NEXT: v_lshrrev_b32_e32 v1, 16, v1
4815 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
4816 ; GFX12-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v1
4817 ; GFX12-NEXT: s_wait_storecnt 0x0
4818 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:65534
4819 ; GFX12-NEXT: s_wait_dscnt 0x0
4820 ; GFX12-NEXT: global_inv scope:SCOPE_SE
4821 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
4822 ; GFX12-NEXT: s_wait_alu 0xfffe
4823 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
4824 ; GFX12-NEXT: s_wait_alu 0xfffe
4825 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4826 ; GFX12-NEXT: s_cbranch_execnz .LBB18_1
4827 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
4828 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
4829 ; GFX12-NEXT: v_mov_b32_e32 v0, v1
4830 ; GFX12-NEXT: s_setpc_b64 s[30:31]
4831 ;
4832 ; GFX940-LABEL: local_atomic_fsub_ret_bf16__offset__align4:
4833 ; GFX940: ; %bb.0:
4834 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4835 ; GFX940-NEXT: ds_read_b32 v1, v0 offset:65534
4836 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
4837 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
4838 ; GFX940-NEXT: s_mov_b32 s3, 0xffff0000
4839 ; GFX940-NEXT: .LBB18_1: ; %atomicrmw.start
4840 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
4841 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
4842 ; GFX940-NEXT: v_mov_b32_e32 v2, v1
4843 ; GFX940-NEXT: v_lshlrev_b32_e32 v1, 16, v2
4844 ; GFX940-NEXT: v_add_f32_e32 v1, -4.0, v1
4845 ; GFX940-NEXT: v_bfe_u32 v3, v1, 16, 1
4846 ; GFX940-NEXT: v_or_b32_e32 v4, 0x400000, v1
4847 ; GFX940-NEXT: v_add3_u32 v3, v3, v1, s2
4848 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
4849 ; GFX940-NEXT: s_nop 1
4850 ; GFX940-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
4851 ; GFX940-NEXT: v_lshrrev_b32_e32 v1, 16, v1
4852 ; GFX940-NEXT: v_and_or_b32 v1, v2, s3, v1
4853 ; GFX940-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:65534
4854 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
4855 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
4856 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4857 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
4858 ; GFX940-NEXT: s_cbranch_execnz .LBB18_1
4859 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
4860 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
4861 ; GFX940-NEXT: v_mov_b32_e32 v0, v1
4862 ; GFX940-NEXT: s_setpc_b64 s[30:31]
4863 ;
4864 ; GFX11-LABEL: local_atomic_fsub_ret_bf16__offset__align4:
4865 ; GFX11: ; %bb.0:
4866 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4867 ; GFX11-NEXT: ds_load_b32 v1, v0 offset:65534
4868 ; GFX11-NEXT: s_mov_b32 s0, 0
4869 ; GFX11-NEXT: .p2align 6
4870 ; GFX11-NEXT: .LBB18_1: ; %atomicrmw.start
4871 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
4872 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
4873 ; GFX11-NEXT: v_mov_b32_e32 v2, v1
4874 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4875 ; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v2
4876 ; GFX11-NEXT: v_add_f32_e32 v1, -4.0, v1
4877 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
4878 ; GFX11-NEXT: v_bfe_u32 v3, v1, 16, 1
4879 ; GFX11-NEXT: v_or_b32_e32 v4, 0x400000, v1
4880 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
4881 ; GFX11-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
4882 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4883 ; GFX11-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
4884 ; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v1
4885 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
4886 ; GFX11-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v1
4887 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
4888 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:65534
4889 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
4890 ; GFX11-NEXT: buffer_gl0_inv
4891 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
4892 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
4893 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4894 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4895 ; GFX11-NEXT: s_cbranch_execnz .LBB18_1
4896 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
4897 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
4898 ; GFX11-NEXT: v_mov_b32_e32 v0, v1
4899 ; GFX11-NEXT: s_setpc_b64 s[30:31]
4900 ;
4901 ; GFX10-LABEL: local_atomic_fsub_ret_bf16__offset__align4:
4902 ; GFX10: ; %bb.0:
4903 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4904 ; GFX10-NEXT: ds_read_b32 v1, v0 offset:65534
4905 ; GFX10-NEXT: s_mov_b32 s4, 0
4906 ; GFX10-NEXT: .LBB18_1: ; %atomicrmw.start
4907 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
4908 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
4909 ; GFX10-NEXT: v_mov_b32_e32 v2, v1
4910 ; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v2
4911 ; GFX10-NEXT: v_add_f32_e32 v1, -4.0, v1
4912 ; GFX10-NEXT: v_bfe_u32 v3, v1, 16, 1
4913 ; GFX10-NEXT: v_or_b32_e32 v4, 0x400000, v1
4914 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
4915 ; GFX10-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
4916 ; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
4917 ; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v1
4918 ; GFX10-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v1
4919 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
4920 ; GFX10-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:65534
4921 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
4922 ; GFX10-NEXT: buffer_gl0_inv
4923 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
4924 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
4925 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
4926 ; GFX10-NEXT: s_cbranch_execnz .LBB18_1
4927 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
4928 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
4929 ; GFX10-NEXT: v_mov_b32_e32 v0, v1
4930 ; GFX10-NEXT: s_setpc_b64 s[30:31]
4931 ;
4932 ; GFX90A-LABEL: local_atomic_fsub_ret_bf16__offset__align4:
4933 ; GFX90A: ; %bb.0:
4934 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4935 ; GFX90A-NEXT: ds_read_b32 v1, v0 offset:65534
4936 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
4937 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
4938 ; GFX90A-NEXT: s_mov_b32 s7, 0xffff0000
4939 ; GFX90A-NEXT: .LBB18_1: ; %atomicrmw.start
4940 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
4941 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
4942 ; GFX90A-NEXT: v_mov_b32_e32 v2, v1
4943 ; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v2
4944 ; GFX90A-NEXT: v_add_f32_e32 v1, -4.0, v1
4945 ; GFX90A-NEXT: v_bfe_u32 v3, v1, 16, 1
4946 ; GFX90A-NEXT: v_or_b32_e32 v4, 0x400000, v1
4947 ; GFX90A-NEXT: v_add3_u32 v3, v3, v1, s6
4948 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
4949 ; GFX90A-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
4950 ; GFX90A-NEXT: v_lshrrev_b32_e32 v1, 16, v1
4951 ; GFX90A-NEXT: v_and_or_b32 v1, v2, s7, v1
4952 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:65534
4953 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
4954 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
4955 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4956 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
4957 ; GFX90A-NEXT: s_cbranch_execnz .LBB18_1
4958 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
4959 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
4960 ; GFX90A-NEXT: v_mov_b32_e32 v0, v1
4961 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
4962 ;
4963 ; GFX908-LABEL: local_atomic_fsub_ret_bf16__offset__align4:
4964 ; GFX908: ; %bb.0:
4965 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4966 ; GFX908-NEXT: ds_read_b32 v1, v0 offset:65534
4967 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
4968 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
4969 ; GFX908-NEXT: s_mov_b32 s7, 0xffff0000
4970 ; GFX908-NEXT: .LBB18_1: ; %atomicrmw.start
4971 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
4972 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
4973 ; GFX908-NEXT: v_mov_b32_e32 v2, v1
4974 ; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v2
4975 ; GFX908-NEXT: v_add_f32_e32 v1, -4.0, v1
4976 ; GFX908-NEXT: v_bfe_u32 v3, v1, 16, 1
4977 ; GFX908-NEXT: v_or_b32_e32 v4, 0x400000, v1
4978 ; GFX908-NEXT: v_add3_u32 v3, v3, v1, s6
4979 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
4980 ; GFX908-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
4981 ; GFX908-NEXT: v_lshrrev_b32_e32 v1, 16, v1
4982 ; GFX908-NEXT: v_and_or_b32 v1, v2, s7, v1
4983 ; GFX908-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:65534
4984 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
4985 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
4986 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4987 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
4988 ; GFX908-NEXT: s_cbranch_execnz .LBB18_1
4989 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
4990 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
4991 ; GFX908-NEXT: v_mov_b32_e32 v0, v1
4992 ; GFX908-NEXT: s_setpc_b64 s[30:31]
4993 ;
4994 ; GFX8-LABEL: local_atomic_fsub_ret_bf16__offset__align4:
4995 ; GFX8: ; %bb.0:
4996 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4997 ; GFX8-NEXT: s_mov_b32 m0, -1
4998 ; GFX8-NEXT: ds_read_b32 v1, v0 offset:65534
4999 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
5000 ; GFX8-NEXT: .LBB18_1: ; %atomicrmw.start
5001 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5002 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
5003 ; GFX8-NEXT: v_mov_b32_e32 v2, v1
5004 ; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
5005 ; GFX8-NEXT: v_add_f32_e32 v1, -4.0, v1
5006 ; GFX8-NEXT: v_bfe_u32 v4, v1, 16, 1
5007 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v1
5008 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
5009 ; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v1
5010 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
5011 ; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
5012 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
5013 ; GFX8-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
5014 ; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:65534
5015 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
5016 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
5017 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5018 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
5019 ; GFX8-NEXT: s_cbranch_execnz .LBB18_1
5020 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5021 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
5022 ; GFX8-NEXT: v_mov_b32_e32 v0, v1
5023 ; GFX8-NEXT: s_setpc_b64 s[30:31]
5024 ;
5025 ; GFX7-LABEL: local_atomic_fsub_ret_bf16__offset__align4:
5026 ; GFX7: ; %bb.0:
5027 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5028 ; GFX7-NEXT: s_mov_b32 m0, -1
5029 ; GFX7-NEXT: ds_read_b32 v1, v0 offset:65534
5030 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
5031 ; GFX7-NEXT: .LBB18_1: ; %atomicrmw.start
5032 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5033 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5034 ; GFX7-NEXT: v_mov_b32_e32 v2, v1
5035 ; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v2
5036 ; GFX7-NEXT: v_add_f32_e32 v1, -4.0, v1
5037 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
5038 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
5039 ; GFX7-NEXT: v_or_b32_e32 v1, v3, v1
5040 ; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:65534
5041 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5042 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
5043 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5044 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
5045 ; GFX7-NEXT: s_cbranch_execnz .LBB18_1
5046 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5047 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
5048 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v1
5049 ; GFX7-NEXT: s_setpc_b64 s[30:31]
5050 ;
5051 ; GFX6-LABEL: local_atomic_fsub_ret_bf16__offset__align4:
5052 ; GFX6: ; %bb.0:
5053 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5054 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffe, v0
5055 ; GFX6-NEXT: s_mov_b32 m0, -1
5056 ; GFX6-NEXT: ds_read_b32 v1, v0
5057 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
5058 ; GFX6-NEXT: .LBB18_1: ; %atomicrmw.start
5059 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
5060 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
5061 ; GFX6-NEXT: v_mov_b32_e32 v2, v1
5062 ; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v2
5063 ; GFX6-NEXT: v_add_f32_e32 v1, -4.0, v1
5064 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
5065 ; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
5066 ; GFX6-NEXT: v_or_b32_e32 v1, v3, v1
5067 ; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1
5068 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
5069 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
5070 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5071 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
5072 ; GFX6-NEXT: s_cbranch_execnz .LBB18_1
5073 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
5074 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
5075 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v1
5076 ; GFX6-NEXT: s_setpc_b64 s[30:31]
5077 %gep = getelementptr bfloat, ptr addrspace(3) %ptr, i32 32767
5078 %result = atomicrmw fsub ptr addrspace(3) %gep, bfloat 4.0 seq_cst, align 4
5079 ret bfloat %result
5080 }
5082 define void @local_atomic_fsub_noret_bf16__offset__align4(ptr addrspace(3) %ptr) nounwind {
5083 ; GFX12-LABEL: local_atomic_fsub_noret_bf16__offset__align4:
5084 ; GFX12: ; %bb.0:
5085 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
5086 ; GFX12-NEXT: s_wait_expcnt 0x0
5087 ; GFX12-NEXT: s_wait_samplecnt 0x0
5088 ; GFX12-NEXT: s_wait_bvhcnt 0x0
5089 ; GFX12-NEXT: s_wait_kmcnt 0x0
5090 ; GFX12-NEXT: ds_load_b32 v1, v0 offset:65534
5091 ; GFX12-NEXT: s_mov_b32 s0, 0
5092 ; GFX12-NEXT: .LBB19_1: ; %atomicrmw.start
5093 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
5094 ; GFX12-NEXT: s_wait_dscnt 0x0
5095 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v1
5096 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5097 ; GFX12-NEXT: v_add_f32_e32 v2, -4.0, v2
5098 ; GFX12-NEXT: v_bfe_u32 v3, v2, 16, 1
5099 ; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v2
5100 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
5101 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
5102 ; GFX12-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
5103 ; GFX12-NEXT: s_wait_alu 0xfffd
5104 ; GFX12-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
5105 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5106 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, 16, v2
5107 ; GFX12-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v2
5108 ; GFX12-NEXT: s_wait_storecnt 0x0
5109 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
5110 ; GFX12-NEXT: s_wait_dscnt 0x0
5111 ; GFX12-NEXT: global_inv scope:SCOPE_SE
5112 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
5113 ; GFX12-NEXT: v_mov_b32_e32 v1, v2
5114 ; GFX12-NEXT: s_wait_alu 0xfffe
5115 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
5116 ; GFX12-NEXT: s_wait_alu 0xfffe
5117 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5118 ; GFX12-NEXT: s_cbranch_execnz .LBB19_1
5119 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
5120 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
5121 ; GFX12-NEXT: s_setpc_b64 s[30:31]
5122 ;
5123 ; GFX940-LABEL: local_atomic_fsub_noret_bf16__offset__align4:
5124 ; GFX940: ; %bb.0:
5125 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5126 ; GFX940-NEXT: ds_read_b32 v1, v0 offset:65534
5127 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
5128 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
5129 ; GFX940-NEXT: s_mov_b32 s3, 0xffff0000
5130 ; GFX940-NEXT: .LBB19_1: ; %atomicrmw.start
5131 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
5132 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
5133 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v1
5134 ; GFX940-NEXT: v_add_f32_e32 v2, -4.0, v2
5135 ; GFX940-NEXT: v_bfe_u32 v3, v2, 16, 1
5136 ; GFX940-NEXT: v_or_b32_e32 v4, 0x400000, v2
5137 ; GFX940-NEXT: v_add3_u32 v3, v3, v2, s2
5138 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
5139 ; GFX940-NEXT: s_nop 1
5140 ; GFX940-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
5141 ; GFX940-NEXT: v_lshrrev_b32_e32 v2, 16, v2
5142 ; GFX940-NEXT: v_and_or_b32 v2, v1, s3, v2
5143 ; GFX940-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:65534
5144 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
5145 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
5146 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
5147 ; GFX940-NEXT: v_mov_b32_e32 v1, v2
5148 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
5149 ; GFX940-NEXT: s_cbranch_execnz .LBB19_1
5150 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
5151 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
5152 ; GFX940-NEXT: s_setpc_b64 s[30:31]
5153 ;
5154 ; GFX11-LABEL: local_atomic_fsub_noret_bf16__offset__align4:
5155 ; GFX11: ; %bb.0:
5156 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5157 ; GFX11-NEXT: ds_load_b32 v1, v0 offset:65534
5158 ; GFX11-NEXT: s_mov_b32 s0, 0
5159 ; GFX11-NEXT: .p2align 6
5160 ; GFX11-NEXT: .LBB19_1: ; %atomicrmw.start
5161 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
5162 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
5163 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v1
5164 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5165 ; GFX11-NEXT: v_add_f32_e32 v2, -4.0, v2
5166 ; GFX11-NEXT: v_bfe_u32 v3, v2, 16, 1
5167 ; GFX11-NEXT: v_or_b32_e32 v4, 0x400000, v2
5168 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
5169 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
5170 ; GFX11-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
5171 ; GFX11-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
5172 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5173 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v2
5174 ; GFX11-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v2
5175 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
5176 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
5177 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
5178 ; GFX11-NEXT: buffer_gl0_inv
5179 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
5180 ; GFX11-NEXT: v_mov_b32_e32 v1, v2
5181 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
5182 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
5183 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5184 ; GFX11-NEXT: s_cbranch_execnz .LBB19_1
5185 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
5186 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
5187 ; GFX11-NEXT: s_setpc_b64 s[30:31]
5188 ;
5189 ; GFX10-LABEL: local_atomic_fsub_noret_bf16__offset__align4:
5190 ; GFX10: ; %bb.0:
5191 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5192 ; GFX10-NEXT: ds_read_b32 v1, v0 offset:65534
5193 ; GFX10-NEXT: s_mov_b32 s4, 0
5194 ; GFX10-NEXT: .LBB19_1: ; %atomicrmw.start
5195 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
5196 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
5197 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
5198 ; GFX10-NEXT: v_add_f32_e32 v2, -4.0, v2
5199 ; GFX10-NEXT: v_bfe_u32 v3, v2, 16, 1
5200 ; GFX10-NEXT: v_or_b32_e32 v4, 0x400000, v2
5201 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
5202 ; GFX10-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
5203 ; GFX10-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
5204 ; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v2
5205 ; GFX10-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v2
5206 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
5207 ; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:65534
5208 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
5209 ; GFX10-NEXT: buffer_gl0_inv
5210 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
5211 ; GFX10-NEXT: v_mov_b32_e32 v1, v2
5212 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
5213 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
5214 ; GFX10-NEXT: s_cbranch_execnz .LBB19_1
5215 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
5216 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
5217 ; GFX10-NEXT: s_setpc_b64 s[30:31]
5218 ;
5219 ; GFX90A-LABEL: local_atomic_fsub_noret_bf16__offset__align4:
5220 ; GFX90A: ; %bb.0:
5221 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5222 ; GFX90A-NEXT: ds_read_b32 v1, v0 offset:65534
5223 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
5224 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
5225 ; GFX90A-NEXT: s_mov_b32 s7, 0xffff0000
5226 ; GFX90A-NEXT: .LBB19_1: ; %atomicrmw.start
5227 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
5228 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
5229 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
5230 ; GFX90A-NEXT: v_add_f32_e32 v2, -4.0, v2
5231 ; GFX90A-NEXT: v_bfe_u32 v3, v2, 16, 1
5232 ; GFX90A-NEXT: v_or_b32_e32 v4, 0x400000, v2
5233 ; GFX90A-NEXT: v_add3_u32 v3, v3, v2, s6
5234 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
5235 ; GFX90A-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
5236 ; GFX90A-NEXT: v_lshrrev_b32_e32 v2, 16, v2
5237 ; GFX90A-NEXT: v_and_or_b32 v2, v1, s7, v2
5238 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:65534
5239 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
5240 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
5241 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5242 ; GFX90A-NEXT: v_mov_b32_e32 v1, v2
5243 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
5244 ; GFX90A-NEXT: s_cbranch_execnz .LBB19_1
5245 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
5246 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
5247 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
5248 ;
5249 ; GFX908-LABEL: local_atomic_fsub_noret_bf16__offset__align4:
5250 ; GFX908: ; %bb.0:
5251 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5252 ; GFX908-NEXT: ds_read_b32 v1, v0 offset:65534
5253 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
5254 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
5255 ; GFX908-NEXT: s_mov_b32 s7, 0xffff0000
5256 ; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
5257 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
5258 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
5259 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v1
5260 ; GFX908-NEXT: v_add_f32_e32 v2, -4.0, v2
5261 ; GFX908-NEXT: v_bfe_u32 v3, v2, 16, 1
5262 ; GFX908-NEXT: v_or_b32_e32 v4, 0x400000, v2
5263 ; GFX908-NEXT: v_add3_u32 v3, v3, v2, s6
5264 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
5265 ; GFX908-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
5266 ; GFX908-NEXT: v_lshrrev_b32_e32 v2, 16, v2
5267 ; GFX908-NEXT: v_and_or_b32 v2, v1, s7, v2
5268 ; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:65534
5269 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
5270 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
5271 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5272 ; GFX908-NEXT: v_mov_b32_e32 v1, v2
5273 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
5274 ; GFX908-NEXT: s_cbranch_execnz .LBB19_1
5275 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
5276 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
5277 ; GFX908-NEXT: s_setpc_b64 s[30:31]
5278 ;
5279 ; GFX8-LABEL: local_atomic_fsub_noret_bf16__offset__align4:
5280 ; GFX8: ; %bb.0:
5281 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5282 ; GFX8-NEXT: s_mov_b32 m0, -1
5283 ; GFX8-NEXT: ds_read_b32 v1, v0 offset:65534
5284 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
5285 ; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
5286 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5287 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
5288 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
5289 ; GFX8-NEXT: v_add_f32_e32 v2, -4.0, v2
5290 ; GFX8-NEXT: v_bfe_u32 v4, v2, 16, 1
5291 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v2
5292 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
5293 ; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v2
5294 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
5295 ; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
5296 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc
5297 ; GFX8-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
5298 ; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:65534
5299 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
5300 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
5301 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5302 ; GFX8-NEXT: v_mov_b32_e32 v1, v2
5303 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
5304 ; GFX8-NEXT: s_cbranch_execnz .LBB19_1
5305 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5306 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
5307 ; GFX8-NEXT: s_setpc_b64 s[30:31]
5308 ;
5309 ; GFX7-LABEL: local_atomic_fsub_noret_bf16__offset__align4:
5310 ; GFX7: ; %bb.0:
5311 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5312 ; GFX7-NEXT: s_mov_b32 m0, -1
5313 ; GFX7-NEXT: ds_read_b32 v1, v0 offset:65534
5314 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
5315 ; GFX7-NEXT: .LBB19_1: ; %atomicrmw.start
5316 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5317 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5318 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v1
5319 ; GFX7-NEXT: v_add_f32_e32 v2, -4.0, v2
5320 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
5321 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
5322 ; GFX7-NEXT: v_or_b32_e32 v2, v3, v2
5323 ; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:65534
5324 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5325 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
5326 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5327 ; GFX7-NEXT: v_mov_b32_e32 v1, v2
5328 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
5329 ; GFX7-NEXT: s_cbranch_execnz .LBB19_1
5330 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5331 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
5332 ; GFX7-NEXT: s_setpc_b64 s[30:31]
5333 ;
5334 ; GFX6-LABEL: local_atomic_fsub_noret_bf16__offset__align4:
5335 ; GFX6: ; %bb.0:
5336 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5337 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffe, v0
5338 ; GFX6-NEXT: s_mov_b32 m0, -1
5339 ; GFX6-NEXT: ds_read_b32 v1, v0
5340 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
5341 ; GFX6-NEXT: .LBB19_1: ; %atomicrmw.start
5342 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
5343 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
5344 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v1
5345 ; GFX6-NEXT: v_add_f32_e32 v2, -4.0, v2
5346 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
5347 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
5348 ; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
5349 ; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
5350 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
5351 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
5352 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5353 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
5354 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
5355 ; GFX6-NEXT: s_cbranch_execnz .LBB19_1
5356 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
5357 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
5358 ; GFX6-NEXT: s_setpc_b64 s[30:31]
5359 %gep = getelementptr bfloat, ptr addrspace(3) %ptr, i32 32767
5360 %unused = atomicrmw fsub ptr addrspace(3) %gep, bfloat 4.0 seq_cst, align 4
5361 ret void
5362 }
5364 ; --------------------------------------------------------------------
5365 ; <2 x half>
5366 ; --------------------------------------------------------------------
5368 define <2 x half> @local_atomic_fsub_ret_v2f16(ptr addrspace(3) %ptr, <2 x half> %val) {
5369 ; GFX12-LABEL: local_atomic_fsub_ret_v2f16:
5370 ; GFX12: ; %bb.0:
5371 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
5372 ; GFX12-NEXT: s_wait_expcnt 0x0
5373 ; GFX12-NEXT: s_wait_samplecnt 0x0
5374 ; GFX12-NEXT: s_wait_bvhcnt 0x0
5375 ; GFX12-NEXT: s_wait_kmcnt 0x0
5376 ; GFX12-NEXT: ds_load_b32 v2, v0
5377 ; GFX12-NEXT: s_mov_b32 s0, 0
5378 ; GFX12-NEXT: .LBB20_1: ; %atomicrmw.start
5379 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
5380 ; GFX12-NEXT: s_wait_dscnt 0x0
5381 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
5382 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
5383 ; GFX12-NEXT: v_pk_add_f16 v2, v3, v1 neg_lo:[0,1] neg_hi:[0,1]
5384 ; GFX12-NEXT: s_wait_storecnt 0x0
5385 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3
5386 ; GFX12-NEXT: s_wait_dscnt 0x0
5387 ; GFX12-NEXT: global_inv scope:SCOPE_SE
5388 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
5389 ; GFX12-NEXT: s_wait_alu 0xfffe
5390 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
5391 ; GFX12-NEXT: s_wait_alu 0xfffe
5392 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5393 ; GFX12-NEXT: s_cbranch_execnz .LBB20_1
5394 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
5395 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
5396 ; GFX12-NEXT: v_mov_b32_e32 v0, v2
5397 ; GFX12-NEXT: s_setpc_b64 s[30:31]
5398 ;
5399 ; GFX940-LABEL: local_atomic_fsub_ret_v2f16:
5400 ; GFX940: ; %bb.0:
5401 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5402 ; GFX940-NEXT: ds_read_b32 v2, v0
5403 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
5404 ; GFX940-NEXT: .LBB20_1: ; %atomicrmw.start
5405 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
5406 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
5407 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
5408 ; GFX940-NEXT: v_pk_add_f16 v2, v3, v1 neg_lo:[0,1] neg_hi:[0,1]
5409 ; GFX940-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
5410 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
5411 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
5412 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
5413 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
5414 ; GFX940-NEXT: s_cbranch_execnz .LBB20_1
5415 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
5416 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
5417 ; GFX940-NEXT: v_mov_b32_e32 v0, v2
5418 ; GFX940-NEXT: s_setpc_b64 s[30:31]
5419 ;
5420 ; GFX11-LABEL: local_atomic_fsub_ret_v2f16:
5421 ; GFX11: ; %bb.0:
5422 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5423 ; GFX11-NEXT: ds_load_b32 v2, v0
5424 ; GFX11-NEXT: s_mov_b32 s0, 0
5425 ; GFX11-NEXT: .LBB20_1: ; %atomicrmw.start
5426 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
5427 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
5428 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
5429 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
5430 ; GFX11-NEXT: v_pk_add_f16 v2, v3, v1 neg_lo:[0,1] neg_hi:[0,1]
5431 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
5432 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3
5433 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
5434 ; GFX11-NEXT: buffer_gl0_inv
5435 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
5436 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
5437 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
5438 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5439 ; GFX11-NEXT: s_cbranch_execnz .LBB20_1
5440 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
5441 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
5442 ; GFX11-NEXT: v_mov_b32_e32 v0, v2
5443 ; GFX11-NEXT: s_setpc_b64 s[30:31]
5444 ;
5445 ; GFX10-LABEL: local_atomic_fsub_ret_v2f16:
5446 ; GFX10: ; %bb.0:
5447 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5448 ; GFX10-NEXT: ds_read_b32 v2, v0
5449 ; GFX10-NEXT: s_mov_b32 s4, 0
5450 ; GFX10-NEXT: .LBB20_1: ; %atomicrmw.start
5451 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
5452 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
5453 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
5454 ; GFX10-NEXT: v_pk_add_f16 v2, v3, v1 neg_lo:[0,1] neg_hi:[0,1]
5455 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
5456 ; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
5457 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
5458 ; GFX10-NEXT: buffer_gl0_inv
5459 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
5460 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
5461 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
5462 ; GFX10-NEXT: s_cbranch_execnz .LBB20_1
5463 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
5464 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
5465 ; GFX10-NEXT: v_mov_b32_e32 v0, v2
5466 ; GFX10-NEXT: s_setpc_b64 s[30:31]
5467 ;
5468 ; GFX90A-LABEL: local_atomic_fsub_ret_v2f16:
5469 ; GFX90A: ; %bb.0:
5470 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5471 ; GFX90A-NEXT: ds_read_b32 v2, v0
5472 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
5473 ; GFX90A-NEXT: .LBB20_1: ; %atomicrmw.start
5474 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
5475 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
5476 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
5477 ; GFX90A-NEXT: v_pk_add_f16 v2, v3, v1 neg_lo:[0,1] neg_hi:[0,1]
5478 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
5479 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
5480 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
5481 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5482 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
5483 ; GFX90A-NEXT: s_cbranch_execnz .LBB20_1
5484 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
5485 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
5486 ; GFX90A-NEXT: v_mov_b32_e32 v0, v2
5487 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
5488 ;
5489 ; GFX908-LABEL: local_atomic_fsub_ret_v2f16:
5490 ; GFX908: ; %bb.0:
5491 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5492 ; GFX908-NEXT: ds_read_b32 v2, v0
5493 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
5494 ; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
5495 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
5496 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
5497 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
5498 ; GFX908-NEXT: v_pk_add_f16 v2, v3, v1 neg_lo:[0,1] neg_hi:[0,1]
5499 ; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
5500 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
5501 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
5502 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5503 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
5504 ; GFX908-NEXT: s_cbranch_execnz .LBB20_1
5505 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
5506 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
5507 ; GFX908-NEXT: v_mov_b32_e32 v0, v2
5508 ; GFX908-NEXT: s_setpc_b64 s[30:31]
5509 ;
5510 ; GFX8-LABEL: local_atomic_fsub_ret_v2f16:
5511 ; GFX8: ; %bb.0:
5512 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5513 ; GFX8-NEXT: s_mov_b32 m0, -1
5514 ; GFX8-NEXT: ds_read_b32 v2, v0
5515 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
5516 ; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
5517 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5518 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
5519 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
5520 ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v3
5521 ; GFX8-NEXT: v_sub_f16_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
5522 ; GFX8-NEXT: v_sub_f16_e32 v4, v3, v1
5523 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
5524 ; GFX8-NEXT: v_or_b32_e32 v2, v4, v2
5525 ; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
5526 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
5527 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
5528 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5529 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
5530 ; GFX8-NEXT: s_cbranch_execnz .LBB20_1
5531 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5532 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
5533 ; GFX8-NEXT: v_mov_b32_e32 v0, v2
5534 ; GFX8-NEXT: s_setpc_b64 s[30:31]
5535 ;
5536 ; GFX7-LABEL: local_atomic_fsub_ret_v2f16:
5537 ; GFX7: ; %bb.0:
5538 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5539 ; GFX7-NEXT: s_mov_b32 m0, -1
5540 ; GFX7-NEXT: ds_read_b32 v3, v0
5541 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v2
5542 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v1
5543 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
5544 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5545 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v3
5546 ; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v2
5547 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
5548 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v4
5549 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
5550 ; GFX7-NEXT: .LBB20_1: ; %atomicrmw.start
5551 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5552 ; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
5553 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
5554 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v1
5555 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v2
5556 ; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
5557 ; GFX7-NEXT: v_or_b32_e32 v7, v2, v1
5558 ; GFX7-NEXT: v_sub_f32_e32 v5, v5, v3
5559 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v4
5560 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
5561 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6
5562 ; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
5563 ; GFX7-NEXT: v_or_b32_e32 v1, v6, v1
5564 ; GFX7-NEXT: ds_cmpst_rtn_b32 v5, v0, v7, v1
5565 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5566 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v5
5567 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v5
5568 ; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
5569 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
5570 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5571 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
5572 ; GFX7-NEXT: s_cbranch_execnz .LBB20_1
5573 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5574 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
5575 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
5576 ; GFX7-NEXT: s_setpc_b64 s[30:31]
5577 ;
5578 ; GFX6-LABEL: local_atomic_fsub_ret_v2f16:
5579 ; GFX6: ; %bb.0:
5580 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5581 ; GFX6-NEXT: s_mov_b32 m0, -1
5582 ; GFX6-NEXT: ds_read_b32 v3, v0
5583 ; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v2
5584 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v1
5585 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
5586 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
5587 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v3
5588 ; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v2
5589 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v3
5590 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v4
5591 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v5
5592 ; GFX6-NEXT: .LBB20_1: ; %atomicrmw.start
5593 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
5594 ; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
5595 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
5596 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v1
5597 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v2
5598 ; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
5599 ; GFX6-NEXT: v_or_b32_e32 v7, v2, v1
5600 ; GFX6-NEXT: v_sub_f32_e32 v5, v5, v3
5601 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v4
5602 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
5603 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
5604 ; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v5
5605 ; GFX6-NEXT: v_or_b32_e32 v1, v6, v1
5606 ; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v0, v7, v1
5607 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
5608 ; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v5
5609 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v5
5610 ; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
5611 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
5612 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5613 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
5614 ; GFX6-NEXT: s_cbranch_execnz .LBB20_1
5615 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
5616 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
5617 ; GFX6-NEXT: v_mov_b32_e32 v0, v2
5618 ; GFX6-NEXT: s_setpc_b64 s[30:31]
5619 %gep = getelementptr <2 x half>, ptr addrspace(3) %ptr, i32 16383
5620 %result = atomicrmw fsub ptr addrspace(3) %ptr, <2 x half> %val seq_cst
5621 ret <2 x half> %result
5622 }
5624 define <2 x half> @local_atomic_fsub_ret_v2f16__offset(ptr addrspace(3) %ptr, <2 x half> %val) {
5625 ; GFX12-LABEL: local_atomic_fsub_ret_v2f16__offset:
5626 ; GFX12: ; %bb.0:
5627 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
5628 ; GFX12-NEXT: s_wait_expcnt 0x0
5629 ; GFX12-NEXT: s_wait_samplecnt 0x0
5630 ; GFX12-NEXT: s_wait_bvhcnt 0x0
5631 ; GFX12-NEXT: s_wait_kmcnt 0x0
5632 ; GFX12-NEXT: ds_load_b32 v2, v0 offset:65532
5633 ; GFX12-NEXT: s_mov_b32 s0, 0
5634 ; GFX12-NEXT: .LBB21_1: ; %atomicrmw.start
5635 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
5636 ; GFX12-NEXT: s_wait_dscnt 0x0
5637 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
5638 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
5639 ; GFX12-NEXT: v_pk_add_f16 v2, v3, v1 neg_lo:[0,1] neg_hi:[0,1]
5640 ; GFX12-NEXT: s_wait_storecnt 0x0
5641 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3 offset:65532
5642 ; GFX12-NEXT: s_wait_dscnt 0x0
5643 ; GFX12-NEXT: global_inv scope:SCOPE_SE
5644 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
5645 ; GFX12-NEXT: s_wait_alu 0xfffe
5646 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
5647 ; GFX12-NEXT: s_wait_alu 0xfffe
5648 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5649 ; GFX12-NEXT: s_cbranch_execnz .LBB21_1
5650 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
5651 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
5652 ; GFX12-NEXT: v_mov_b32_e32 v0, v2
5653 ; GFX12-NEXT: s_setpc_b64 s[30:31]
5654 ;
5655 ; GFX940-LABEL: local_atomic_fsub_ret_v2f16__offset:
5656 ; GFX940: ; %bb.0:
5657 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5658 ; GFX940-NEXT: ds_read_b32 v2, v0 offset:65532
5659 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
5660 ; GFX940-NEXT: .LBB21_1: ; %atomicrmw.start
5661 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
5662 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
5663 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
5664 ; GFX940-NEXT: v_pk_add_f16 v2, v3, v1 neg_lo:[0,1] neg_hi:[0,1]
5665 ; GFX940-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
5666 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
5667 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
5668 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
5669 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
5670 ; GFX940-NEXT: s_cbranch_execnz .LBB21_1
5671 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
5672 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
5673 ; GFX940-NEXT: v_mov_b32_e32 v0, v2
5674 ; GFX940-NEXT: s_setpc_b64 s[30:31]
5675 ;
5676 ; GFX11-LABEL: local_atomic_fsub_ret_v2f16__offset:
5677 ; GFX11: ; %bb.0:
5678 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5679 ; GFX11-NEXT: ds_load_b32 v2, v0 offset:65532
5680 ; GFX11-NEXT: s_mov_b32 s0, 0
5681 ; GFX11-NEXT: .LBB21_1: ; %atomicrmw.start
5682 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
5683 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
5684 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
5685 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
5686 ; GFX11-NEXT: v_pk_add_f16 v2, v3, v1 neg_lo:[0,1] neg_hi:[0,1]
5687 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
5688 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3 offset:65532
5689 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
5690 ; GFX11-NEXT: buffer_gl0_inv
5691 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
5692 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
5693 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
5694 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5695 ; GFX11-NEXT: s_cbranch_execnz .LBB21_1
5696 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
5697 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
5698 ; GFX11-NEXT: v_mov_b32_e32 v0, v2
5699 ; GFX11-NEXT: s_setpc_b64 s[30:31]
5700 ;
5701 ; GFX10-LABEL: local_atomic_fsub_ret_v2f16__offset:
5702 ; GFX10: ; %bb.0:
5703 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5704 ; GFX10-NEXT: ds_read_b32 v2, v0 offset:65532
5705 ; GFX10-NEXT: s_mov_b32 s4, 0
5706 ; GFX10-NEXT: .LBB21_1: ; %atomicrmw.start
5707 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
5708 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
5709 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
5710 ; GFX10-NEXT: v_pk_add_f16 v2, v3, v1 neg_lo:[0,1] neg_hi:[0,1]
5711 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
5712 ; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
5713 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
5714 ; GFX10-NEXT: buffer_gl0_inv
5715 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
5716 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
5717 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
5718 ; GFX10-NEXT: s_cbranch_execnz .LBB21_1
5719 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
5720 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
5721 ; GFX10-NEXT: v_mov_b32_e32 v0, v2
5722 ; GFX10-NEXT: s_setpc_b64 s[30:31]
5723 ;
5724 ; GFX90A-LABEL: local_atomic_fsub_ret_v2f16__offset:
5725 ; GFX90A: ; %bb.0:
5726 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5727 ; GFX90A-NEXT: ds_read_b32 v2, v0 offset:65532
5728 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
5729 ; GFX90A-NEXT: .LBB21_1: ; %atomicrmw.start
5730 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
5731 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
5732 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
5733 ; GFX90A-NEXT: v_pk_add_f16 v2, v3, v1 neg_lo:[0,1] neg_hi:[0,1]
5734 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
5735 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
5736 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
5737 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5738 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
5739 ; GFX90A-NEXT: s_cbranch_execnz .LBB21_1
5740 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
5741 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
5742 ; GFX90A-NEXT: v_mov_b32_e32 v0, v2
5743 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
5744 ;
5745 ; GFX908-LABEL: local_atomic_fsub_ret_v2f16__offset:
5746 ; GFX908: ; %bb.0:
5747 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5748 ; GFX908-NEXT: ds_read_b32 v2, v0 offset:65532
5749 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
5750 ; GFX908-NEXT: .LBB21_1: ; %atomicrmw.start
5751 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
5752 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
5753 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
5754 ; GFX908-NEXT: v_pk_add_f16 v2, v3, v1 neg_lo:[0,1] neg_hi:[0,1]
5755 ; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
5756 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
5757 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
5758 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5759 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
5760 ; GFX908-NEXT: s_cbranch_execnz .LBB21_1
5761 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
5762 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
5763 ; GFX908-NEXT: v_mov_b32_e32 v0, v2
5764 ; GFX908-NEXT: s_setpc_b64 s[30:31]
5765 ;
5766 ; GFX8-LABEL: local_atomic_fsub_ret_v2f16__offset:
5767 ; GFX8: ; %bb.0:
5768 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5769 ; GFX8-NEXT: s_mov_b32 m0, -1
5770 ; GFX8-NEXT: ds_read_b32 v2, v0 offset:65532
5771 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
5772 ; GFX8-NEXT: .LBB21_1: ; %atomicrmw.start
5773 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5774 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
5775 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
5776 ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v3
5777 ; GFX8-NEXT: v_sub_f16_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
5778 ; GFX8-NEXT: v_sub_f16_e32 v4, v3, v1
5779 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
5780 ; GFX8-NEXT: v_or_b32_e32 v2, v4, v2
5781 ; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
5782 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
5783 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
5784 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5785 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
5786 ; GFX8-NEXT: s_cbranch_execnz .LBB21_1
5787 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5788 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
5789 ; GFX8-NEXT: v_mov_b32_e32 v0, v2
5790 ; GFX8-NEXT: s_setpc_b64 s[30:31]
5791 ;
5792 ; GFX7-LABEL: local_atomic_fsub_ret_v2f16__offset:
5793 ; GFX7: ; %bb.0:
5794 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5795 ; GFX7-NEXT: s_mov_b32 m0, -1
5796 ; GFX7-NEXT: ds_read_b32 v3, v0 offset:65532
5797 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v2
5798 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v1
5799 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
5800 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5801 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v3
5802 ; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v2
5803 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
5804 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v4
5805 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
5806 ; GFX7-NEXT: .LBB21_1: ; %atomicrmw.start
5807 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5808 ; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
5809 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
5810 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v1
5811 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v2
5812 ; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
5813 ; GFX7-NEXT: v_or_b32_e32 v7, v2, v1
5814 ; GFX7-NEXT: v_sub_f32_e32 v5, v5, v3
5815 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v4
5816 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
5817 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6
5818 ; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
5819 ; GFX7-NEXT: v_or_b32_e32 v1, v6, v1
5820 ; GFX7-NEXT: ds_cmpst_rtn_b32 v5, v0, v7, v1 offset:65532
5821 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5822 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v5
5823 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v5
5824 ; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
5825 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
5826 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5827 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
5828 ; GFX7-NEXT: s_cbranch_execnz .LBB21_1
5829 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5830 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
5831 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
5832 ; GFX7-NEXT: s_setpc_b64 s[30:31]
5833 ;
5834 ; GFX6-LABEL: local_atomic_fsub_ret_v2f16__offset:
5835 ; GFX6: ; %bb.0:
5836 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5837 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, 0xfffc, v0
5838 ; GFX6-NEXT: s_mov_b32 m0, -1
5839 ; GFX6-NEXT: ds_read_b32 v4, v3
5840 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
5841 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v1
5842 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
5843 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
5844 ; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v4
5845 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v4
5846 ; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
5847 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
5848 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v5
5849 ; GFX6-NEXT: .LBB21_1: ; %atomicrmw.start
5850 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
5851 ; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
5852 ; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
5853 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v1
5854 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v0
5855 ; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
5856 ; GFX6-NEXT: v_or_b32_e32 v7, v0, v1
5857 ; GFX6-NEXT: v_sub_f32_e32 v5, v5, v2
5858 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v4
5859 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
5860 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
5861 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v5
5862 ; GFX6-NEXT: v_or_b32_e32 v0, v6, v0
5863 ; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v3, v7, v0
5864 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
5865 ; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v5
5866 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v5
5867 ; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
5868 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
5869 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5870 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
5871 ; GFX6-NEXT: s_cbranch_execnz .LBB21_1
5872 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
5873 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
5874 ; GFX6-NEXT: s_setpc_b64 s[30:31]
5875 %gep = getelementptr <2 x half>, ptr addrspace(3) %ptr, i32 16383
5876 %result = atomicrmw fsub ptr addrspace(3) %gep, <2 x half> %val seq_cst
5877 ret <2 x half> %result
5878 }
5880 define void @local_atomic_fsub_noret_v2f16(ptr addrspace(3) %ptr, <2 x half> %val) {
5881 ; GFX12-LABEL: local_atomic_fsub_noret_v2f16:
5882 ; GFX12: ; %bb.0:
5883 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
5884 ; GFX12-NEXT: s_wait_expcnt 0x0
5885 ; GFX12-NEXT: s_wait_samplecnt 0x0
5886 ; GFX12-NEXT: s_wait_bvhcnt 0x0
5887 ; GFX12-NEXT: s_wait_kmcnt 0x0
5888 ; GFX12-NEXT: ds_load_b32 v2, v0
5889 ; GFX12-NEXT: s_mov_b32 s0, 0
5890 ; GFX12-NEXT: .LBB22_1: ; %atomicrmw.start
5891 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
5892 ; GFX12-NEXT: s_wait_dscnt 0x0
5893 ; GFX12-NEXT: v_pk_add_f16 v3, v2, v1 neg_lo:[0,1] neg_hi:[0,1]
5894 ; GFX12-NEXT: s_wait_storecnt 0x0
5895 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2
5896 ; GFX12-NEXT: s_wait_dscnt 0x0
5897 ; GFX12-NEXT: global_inv scope:SCOPE_SE
5898 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
5899 ; GFX12-NEXT: v_mov_b32_e32 v2, v3
5900 ; GFX12-NEXT: s_wait_alu 0xfffe
5901 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
5902 ; GFX12-NEXT: s_wait_alu 0xfffe
5903 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5904 ; GFX12-NEXT: s_cbranch_execnz .LBB22_1
5905 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
5906 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
5907 ; GFX12-NEXT: s_setpc_b64 s[30:31]
5908 ;
5909 ; GFX940-LABEL: local_atomic_fsub_noret_v2f16:
5910 ; GFX940: ; %bb.0:
5911 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5912 ; GFX940-NEXT: ds_read_b32 v2, v0
5913 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
5914 ; GFX940-NEXT: .LBB22_1: ; %atomicrmw.start
5915 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
5916 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
5917 ; GFX940-NEXT: v_pk_add_f16 v3, v2, v1 neg_lo:[0,1] neg_hi:[0,1]
5918 ; GFX940-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
5919 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
5920 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
5921 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
5922 ; GFX940-NEXT: v_mov_b32_e32 v2, v3
5923 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
5924 ; GFX940-NEXT: s_cbranch_execnz .LBB22_1
5925 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
5926 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
5927 ; GFX940-NEXT: s_setpc_b64 s[30:31]
5928 ;
5929 ; GFX11-LABEL: local_atomic_fsub_noret_v2f16:
5930 ; GFX11: ; %bb.0:
5931 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5932 ; GFX11-NEXT: ds_load_b32 v2, v0
5933 ; GFX11-NEXT: s_mov_b32 s0, 0
5934 ; GFX11-NEXT: .LBB22_1: ; %atomicrmw.start
5935 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
5936 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
5937 ; GFX11-NEXT: v_pk_add_f16 v3, v2, v1 neg_lo:[0,1] neg_hi:[0,1]
5938 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
5939 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2
5940 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
5941 ; GFX11-NEXT: buffer_gl0_inv
5942 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
5943 ; GFX11-NEXT: v_mov_b32_e32 v2, v3
5944 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
5945 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
5946 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5947 ; GFX11-NEXT: s_cbranch_execnz .LBB22_1
5948 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
5949 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
5950 ; GFX11-NEXT: s_setpc_b64 s[30:31]
5951 ;
5952 ; GFX10-LABEL: local_atomic_fsub_noret_v2f16:
5953 ; GFX10: ; %bb.0:
5954 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5955 ; GFX10-NEXT: ds_read_b32 v2, v0
5956 ; GFX10-NEXT: s_mov_b32 s4, 0
5957 ; GFX10-NEXT: .LBB22_1: ; %atomicrmw.start
5958 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
5959 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
5960 ; GFX10-NEXT: v_pk_add_f16 v3, v2, v1 neg_lo:[0,1] neg_hi:[0,1]
5961 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
5962 ; GFX10-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
5963 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
5964 ; GFX10-NEXT: buffer_gl0_inv
5965 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
5966 ; GFX10-NEXT: v_mov_b32_e32 v2, v3
5967 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
5968 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
5969 ; GFX10-NEXT: s_cbranch_execnz .LBB22_1
5970 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
5971 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
5972 ; GFX10-NEXT: s_setpc_b64 s[30:31]
5973 ;
5974 ; GFX90A-LABEL: local_atomic_fsub_noret_v2f16:
5975 ; GFX90A: ; %bb.0:
5976 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5977 ; GFX90A-NEXT: ds_read_b32 v2, v0
5978 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
5979 ; GFX90A-NEXT: .LBB22_1: ; %atomicrmw.start
5980 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
5981 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
5982 ; GFX90A-NEXT: v_pk_add_f16 v3, v2, v1 neg_lo:[0,1] neg_hi:[0,1]
5983 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
5984 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
5985 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
5986 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5987 ; GFX90A-NEXT: v_mov_b32_e32 v2, v3
5988 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
5989 ; GFX90A-NEXT: s_cbranch_execnz .LBB22_1
5990 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
5991 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
5992 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
5993 ;
5994 ; GFX908-LABEL: local_atomic_fsub_noret_v2f16:
5995 ; GFX908: ; %bb.0:
5996 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5997 ; GFX908-NEXT: ds_read_b32 v2, v0
5998 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
5999 ; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
6000 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
6001 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
6002 ; GFX908-NEXT: v_pk_add_f16 v3, v2, v1 neg_lo:[0,1] neg_hi:[0,1]
6003 ; GFX908-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
6004 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
6005 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
6006 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6007 ; GFX908-NEXT: v_mov_b32_e32 v2, v3
6008 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
6009 ; GFX908-NEXT: s_cbranch_execnz .LBB22_1
6010 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
6011 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
6012 ; GFX908-NEXT: s_setpc_b64 s[30:31]
6013 ;
6014 ; GFX8-LABEL: local_atomic_fsub_noret_v2f16:
6015 ; GFX8: ; %bb.0:
6016 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6017 ; GFX8-NEXT: s_mov_b32 m0, -1
6018 ; GFX8-NEXT: ds_read_b32 v2, v0
6019 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
6020 ; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
6021 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6022 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
6023 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v2
6024 ; GFX8-NEXT: v_sub_f16_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
6025 ; GFX8-NEXT: v_sub_f16_e32 v4, v2, v1
6026 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
6027 ; GFX8-NEXT: v_or_b32_e32 v3, v4, v3
6028 ; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
6029 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
6030 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
6031 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6032 ; GFX8-NEXT: v_mov_b32_e32 v2, v3
6033 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
6034 ; GFX8-NEXT: s_cbranch_execnz .LBB22_1
6035 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6036 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
6037 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6038 ;
6039 ; GFX7-LABEL: local_atomic_fsub_noret_v2f16:
6040 ; GFX7: ; %bb.0:
6041 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6042 ; GFX7-NEXT: s_mov_b32 m0, -1
6043 ; GFX7-NEXT: ds_read_b32 v3, v0
6044 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
6045 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v1
6046 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
6047 ; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v2
6048 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6049 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
6050 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v4
6051 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
6052 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v5
6053 ; GFX7-NEXT: .LBB22_1: ; %atomicrmw.start
6054 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
6055 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
6056 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
6057 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v4
6058 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v3
6059 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
6060 ; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
6061 ; GFX7-NEXT: v_sub_f32_e32 v5, v5, v1
6062 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
6063 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
6064 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6
6065 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
6066 ; GFX7-NEXT: v_or_b32_e32 v3, v6, v3
6067 ; GFX7-NEXT: ds_cmpst_rtn_b32 v5, v0, v7, v3
6068 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6069 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v5
6070 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v5
6071 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v4
6072 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
6073 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6074 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
6075 ; GFX7-NEXT: s_cbranch_execnz .LBB22_1
6076 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6077 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
6078 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6079 ;
6080 ; GFX6-LABEL: local_atomic_fsub_noret_v2f16:
6081 ; GFX6: ; %bb.0:
6082 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6083 ; GFX6-NEXT: s_mov_b32 m0, -1
6084 ; GFX6-NEXT: ds_read_b32 v3, v0
6085 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
6086 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v1
6087 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
6088 ; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v2
6089 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
6090 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v3
6091 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v4
6092 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
6093 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v5
6094 ; GFX6-NEXT: .LBB22_1: ; %atomicrmw.start
6095 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
6096 ; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
6097 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
6098 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v4
6099 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v3
6100 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
6101 ; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
6102 ; GFX6-NEXT: v_sub_f32_e32 v5, v5, v1
6103 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
6104 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
6105 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
6106 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
6107 ; GFX6-NEXT: v_or_b32_e32 v3, v6, v3
6108 ; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v0, v7, v3
6109 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
6110 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v5
6111 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v5
6112 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v4
6113 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
6114 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6115 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
6116 ; GFX6-NEXT: s_cbranch_execnz .LBB22_1
6117 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
6118 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
6119 ; GFX6-NEXT: s_setpc_b64 s[30:31]
6120 %result = atomicrmw fsub ptr addrspace(3) %ptr, <2 x half> %val seq_cst
6121 ret void
6122 }
6124 define void @local_atomic_fsub_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x half> %val) {
6125 ; GFX12-LABEL: local_atomic_fsub_noret_v2f16__offset:
6126 ; GFX12: ; %bb.0:
6127 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
6128 ; GFX12-NEXT: s_wait_expcnt 0x0
6129 ; GFX12-NEXT: s_wait_samplecnt 0x0
6130 ; GFX12-NEXT: s_wait_bvhcnt 0x0
6131 ; GFX12-NEXT: s_wait_kmcnt 0x0
6132 ; GFX12-NEXT: ds_load_b32 v2, v0 offset:65532
6133 ; GFX12-NEXT: s_mov_b32 s0, 0
6134 ; GFX12-NEXT: .LBB23_1: ; %atomicrmw.start
6135 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
6136 ; GFX12-NEXT: s_wait_dscnt 0x0
6137 ; GFX12-NEXT: v_pk_add_f16 v3, v2, v1 neg_lo:[0,1] neg_hi:[0,1]
6138 ; GFX12-NEXT: s_wait_storecnt 0x0
6139 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2 offset:65532
6140 ; GFX12-NEXT: s_wait_dscnt 0x0
6141 ; GFX12-NEXT: global_inv scope:SCOPE_SE
6142 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
6143 ; GFX12-NEXT: v_mov_b32_e32 v2, v3
6144 ; GFX12-NEXT: s_wait_alu 0xfffe
6145 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
6146 ; GFX12-NEXT: s_wait_alu 0xfffe
6147 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
6148 ; GFX12-NEXT: s_cbranch_execnz .LBB23_1
6149 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
6150 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
6151 ; GFX12-NEXT: s_setpc_b64 s[30:31]
6152 ;
6153 ; GFX940-LABEL: local_atomic_fsub_noret_v2f16__offset:
6154 ; GFX940: ; %bb.0:
6155 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6156 ; GFX940-NEXT: ds_read_b32 v2, v0 offset:65532
6157 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
6158 ; GFX940-NEXT: .LBB23_1: ; %atomicrmw.start
6159 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
6160 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
6161 ; GFX940-NEXT: v_pk_add_f16 v3, v2, v1 neg_lo:[0,1] neg_hi:[0,1]
6162 ; GFX940-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
6163 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
6164 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
6165 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
6166 ; GFX940-NEXT: v_mov_b32_e32 v2, v3
6167 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
6168 ; GFX940-NEXT: s_cbranch_execnz .LBB23_1
6169 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
6170 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
6171 ; GFX940-NEXT: s_setpc_b64 s[30:31]
6172 ;
6173 ; GFX11-LABEL: local_atomic_fsub_noret_v2f16__offset:
6174 ; GFX11: ; %bb.0:
6175 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6176 ; GFX11-NEXT: ds_load_b32 v2, v0 offset:65532
6177 ; GFX11-NEXT: s_mov_b32 s0, 0
6178 ; GFX11-NEXT: .LBB23_1: ; %atomicrmw.start
6179 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
6180 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
6181 ; GFX11-NEXT: v_pk_add_f16 v3, v2, v1 neg_lo:[0,1] neg_hi:[0,1]
6182 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
6183 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2 offset:65532
6184 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
6185 ; GFX11-NEXT: buffer_gl0_inv
6186 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
6187 ; GFX11-NEXT: v_mov_b32_e32 v2, v3
6188 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
6189 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
6190 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
6191 ; GFX11-NEXT: s_cbranch_execnz .LBB23_1
6192 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
6193 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
6194 ; GFX11-NEXT: s_setpc_b64 s[30:31]
6195 ;
6196 ; GFX10-LABEL: local_atomic_fsub_noret_v2f16__offset:
6197 ; GFX10: ; %bb.0:
6198 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6199 ; GFX10-NEXT: ds_read_b32 v2, v0 offset:65532
6200 ; GFX10-NEXT: s_mov_b32 s4, 0
6201 ; GFX10-NEXT: .LBB23_1: ; %atomicrmw.start
6202 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
6203 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
6204 ; GFX10-NEXT: v_pk_add_f16 v3, v2, v1 neg_lo:[0,1] neg_hi:[0,1]
6205 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
6206 ; GFX10-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
6207 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
6208 ; GFX10-NEXT: buffer_gl0_inv
6209 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
6210 ; GFX10-NEXT: v_mov_b32_e32 v2, v3
6211 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
6212 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
6213 ; GFX10-NEXT: s_cbranch_execnz .LBB23_1
6214 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
6215 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
6216 ; GFX10-NEXT: s_setpc_b64 s[30:31]
6217 ;
6218 ; GFX90A-LABEL: local_atomic_fsub_noret_v2f16__offset:
6219 ; GFX90A: ; %bb.0:
6220 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6221 ; GFX90A-NEXT: ds_read_b32 v2, v0 offset:65532
6222 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
6223 ; GFX90A-NEXT: .LBB23_1: ; %atomicrmw.start
6224 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
6225 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
6226 ; GFX90A-NEXT: v_pk_add_f16 v3, v2, v1 neg_lo:[0,1] neg_hi:[0,1]
6227 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
6228 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
6229 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
6230 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6231 ; GFX90A-NEXT: v_mov_b32_e32 v2, v3
6232 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
6233 ; GFX90A-NEXT: s_cbranch_execnz .LBB23_1
6234 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
6235 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
6236 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
6237 ;
6238 ; GFX908-LABEL: local_atomic_fsub_noret_v2f16__offset:
6239 ; GFX908: ; %bb.0:
6240 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6241 ; GFX908-NEXT: ds_read_b32 v2, v0 offset:65532
6242 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
6243 ; GFX908-NEXT: .LBB23_1: ; %atomicrmw.start
6244 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
6245 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
6246 ; GFX908-NEXT: v_pk_add_f16 v3, v2, v1 neg_lo:[0,1] neg_hi:[0,1]
6247 ; GFX908-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
6248 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
6249 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
6250 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6251 ; GFX908-NEXT: v_mov_b32_e32 v2, v3
6252 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
6253 ; GFX908-NEXT: s_cbranch_execnz .LBB23_1
6254 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
6255 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
6256 ; GFX908-NEXT: s_setpc_b64 s[30:31]
6257 ;
6258 ; GFX8-LABEL: local_atomic_fsub_noret_v2f16__offset:
6259 ; GFX8: ; %bb.0:
6260 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6261 ; GFX8-NEXT: s_mov_b32 m0, -1
6262 ; GFX8-NEXT: ds_read_b32 v2, v0 offset:65532
6263 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
6264 ; GFX8-NEXT: .LBB23_1: ; %atomicrmw.start
6265 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6266 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
6267 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v2
6268 ; GFX8-NEXT: v_sub_f16_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
6269 ; GFX8-NEXT: v_sub_f16_e32 v4, v2, v1
6270 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
6271 ; GFX8-NEXT: v_or_b32_e32 v3, v4, v3
6272 ; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
6273 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
6274 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
6275 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6276 ; GFX8-NEXT: v_mov_b32_e32 v2, v3
6277 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
6278 ; GFX8-NEXT: s_cbranch_execnz .LBB23_1
6279 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6280 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
6281 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6282 ;
6283 ; GFX7-LABEL: local_atomic_fsub_noret_v2f16__offset:
6284 ; GFX7: ; %bb.0:
6285 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6286 ; GFX7-NEXT: s_mov_b32 m0, -1
6287 ; GFX7-NEXT: ds_read_b32 v3, v0 offset:65532
6288 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
6289 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v1
6290 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
6291 ; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v2
6292 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6293 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
6294 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v4
6295 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
6296 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v5
6297 ; GFX7-NEXT: .LBB23_1: ; %atomicrmw.start
6298 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
6299 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
6300 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
6301 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v4
6302 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v3
6303 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
6304 ; GFX7-NEXT: v_or_b32_e32 v7, v3, v4
6305 ; GFX7-NEXT: v_sub_f32_e32 v5, v5, v1
6306 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
6307 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
6308 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6
6309 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
6310 ; GFX7-NEXT: v_or_b32_e32 v3, v6, v3
6311 ; GFX7-NEXT: ds_cmpst_rtn_b32 v5, v0, v7, v3 offset:65532
6312 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6313 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v5
6314 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v5
6315 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v4
6316 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
6317 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6318 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
6319 ; GFX7-NEXT: s_cbranch_execnz .LBB23_1
6320 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6321 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
6322 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6323 ;
6324 ; GFX6-LABEL: local_atomic_fsub_noret_v2f16__offset:
6325 ; GFX6: ; %bb.0:
6326 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6327 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v0
6328 ; GFX6-NEXT: s_mov_b32 m0, -1
6329 ; GFX6-NEXT: ds_read_b32 v4, v0
6330 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
6331 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v1
6332 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
6333 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
6334 ; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v4
6335 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v4
6336 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v1
6337 ; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v2
6338 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v5
6339 ; GFX6-NEXT: .LBB23_1: ; %atomicrmw.start
6340 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
6341 ; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
6342 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
6343 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v4
6344 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v3
6345 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
6346 ; GFX6-NEXT: v_or_b32_e32 v7, v3, v4
6347 ; GFX6-NEXT: v_sub_f32_e32 v5, v5, v1
6348 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
6349 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
6350 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
6351 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
6352 ; GFX6-NEXT: v_or_b32_e32 v3, v6, v3
6353 ; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v0, v7, v3
6354 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
6355 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v5
6356 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v5
6357 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v4
6358 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
6359 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6360 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
6361 ; GFX6-NEXT: s_cbranch_execnz .LBB23_1
6362 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
6363 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
6364 ; GFX6-NEXT: s_setpc_b64 s[30:31]
6365 %gep = getelementptr <2 x half>, ptr addrspace(3) %ptr, i32 16383
6366 %result = atomicrmw fsub ptr addrspace(3) %gep, <2 x half> %val seq_cst
6367 ret void
6368 }
6370 ; --------------------------------------------------------------------
6371 ; <2 x bfloat>
6372 ; --------------------------------------------------------------------
6374 define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat> %val) {
6375 ; GFX12-LABEL: local_atomic_fsub_ret_v2bf16:
6376 ; GFX12: ; %bb.0:
6377 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
6378 ; GFX12-NEXT: s_wait_expcnt 0x0
6379 ; GFX12-NEXT: s_wait_samplecnt 0x0
6380 ; GFX12-NEXT: s_wait_bvhcnt 0x0
6381 ; GFX12-NEXT: s_wait_kmcnt 0x0
6382 ; GFX12-NEXT: ds_load_b32 v2, v0
6383 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v1
6384 ; GFX12-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
6385 ; GFX12-NEXT: s_mov_b32 s1, 0
6386 ; GFX12-NEXT: .LBB24_1: ; %atomicrmw.start
6387 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
6388 ; GFX12-NEXT: s_wait_dscnt 0x0
6389 ; GFX12-NEXT: v_mov_b32_e32 v4, v2
6390 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6391 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
6392 ; GFX12-NEXT: v_sub_f32_e32 v5, v5, v1
6393 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
6394 ; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
6395 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v5
6396 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
6397 ; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
6398 ; GFX12-NEXT: s_wait_alu 0xfffd
6399 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6400 ; GFX12-NEXT: v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
6401 ; GFX12-NEXT: v_sub_f32_e32 v2, v2, v3
6402 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
6403 ; GFX12-NEXT: v_bfe_u32 v6, v2, 16, 1
6404 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v2
6405 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v2, v2
6406 ; GFX12-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
6407 ; GFX12-NEXT: s_wait_alu 0xf1ff
6408 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6409 ; GFX12-NEXT: v_cndmask_b32_e64 v2, v6, v8, s0
6410 ; GFX12-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
6411 ; GFX12-NEXT: s_wait_storecnt 0x0
6412 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4
6413 ; GFX12-NEXT: s_wait_dscnt 0x0
6414 ; GFX12-NEXT: global_inv scope:SCOPE_SE
6415 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
6416 ; GFX12-NEXT: s_wait_alu 0xfffe
6417 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
6418 ; GFX12-NEXT: s_wait_alu 0xfffe
6419 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
6420 ; GFX12-NEXT: s_cbranch_execnz .LBB24_1
6421 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
6422 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
6423 ; GFX12-NEXT: v_mov_b32_e32 v0, v2
6424 ; GFX12-NEXT: s_setpc_b64 s[30:31]
6425 ;
6426 ; GFX940-LABEL: local_atomic_fsub_ret_v2bf16:
6427 ; GFX940: ; %bb.0:
6428 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6429 ; GFX940-NEXT: ds_read_b32 v2, v0
6430 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
6431 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 16, v1
6432 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
6433 ; GFX940-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
6434 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
6435 ; GFX940-NEXT: .LBB24_1: ; %atomicrmw.start
6436 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
6437 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
6438 ; GFX940-NEXT: v_mov_b32_e32 v4, v2
6439 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v4
6440 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
6441 ; GFX940-NEXT: v_sub_f32_e32 v2, v2, v3
6442 ; GFX940-NEXT: v_sub_f32_e32 v5, v5, v1
6443 ; GFX940-NEXT: v_bfe_u32 v6, v2, 16, 1
6444 ; GFX940-NEXT: v_bfe_u32 v8, v5, 16, 1
6445 ; GFX940-NEXT: v_or_b32_e32 v7, 0x400000, v2
6446 ; GFX940-NEXT: v_or_b32_e32 v9, 0x400000, v5
6447 ; GFX940-NEXT: v_add3_u32 v6, v6, v2, s4
6448 ; GFX940-NEXT: v_add3_u32 v8, v8, v5, s4
6449 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
6450 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
6451 ; GFX940-NEXT: s_nop 0
6452 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
6453 ; GFX940-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[0:1]
6454 ; GFX940-NEXT: v_perm_b32 v2, v5, v2, s5
6455 ; GFX940-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
6456 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
6457 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
6458 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
6459 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
6460 ; GFX940-NEXT: s_cbranch_execnz .LBB24_1
6461 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
6462 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
6463 ; GFX940-NEXT: v_mov_b32_e32 v0, v2
6464 ; GFX940-NEXT: s_setpc_b64 s[30:31]
6465 ;
6466 ; GFX11-LABEL: local_atomic_fsub_ret_v2bf16:
6467 ; GFX11: ; %bb.0:
6468 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6469 ; GFX11-NEXT: ds_load_b32 v2, v0
6470 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v1
6471 ; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
6472 ; GFX11-NEXT: s_mov_b32 s1, 0
6473 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
6474 ; GFX11-NEXT: .p2align 6
6475 ; GFX11-NEXT: .LBB24_1: ; %atomicrmw.start
6476 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
6477 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
6478 ; GFX11-NEXT: v_mov_b32_e32 v4, v2
6479 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6480 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
6481 ; GFX11-NEXT: v_sub_f32_e32 v5, v5, v1
6482 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
6483 ; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
6484 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v5
6485 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
6486 ; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
6487 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6488 ; GFX11-NEXT: v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
6489 ; GFX11-NEXT: v_sub_f32_e32 v2, v2, v3
6490 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
6491 ; GFX11-NEXT: v_bfe_u32 v6, v2, 16, 1
6492 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v2
6493 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v2, v2
6494 ; GFX11-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
6495 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6496 ; GFX11-NEXT: v_cndmask_b32_e64 v2, v6, v8, s0
6497 ; GFX11-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
6498 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
6499 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4
6500 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
6501 ; GFX11-NEXT: buffer_gl0_inv
6502 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
6503 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
6504 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
6505 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
6506 ; GFX11-NEXT: s_cbranch_execnz .LBB24_1
6507 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
6508 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
6509 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
6510 ; GFX11-NEXT: v_mov_b32_e32 v0, v2
6511 ; GFX11-NEXT: s_setpc_b64 s[30:31]
6512 ;
6513 ; GFX10-LABEL: local_atomic_fsub_ret_v2bf16:
6514 ; GFX10: ; %bb.0:
6515 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6516 ; GFX10-NEXT: ds_read_b32 v2, v0
6517 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v1
6518 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
6519 ; GFX10-NEXT: s_mov_b32 s5, 0
6520 ; GFX10-NEXT: .LBB24_1: ; %atomicrmw.start
6521 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
6522 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
6523 ; GFX10-NEXT: v_mov_b32_e32 v4, v2
6524 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v4
6525 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
6526 ; GFX10-NEXT: v_sub_f32_e32 v2, v2, v3
6527 ; GFX10-NEXT: v_sub_f32_e32 v5, v5, v1
6528 ; GFX10-NEXT: v_bfe_u32 v6, v2, 16, 1
6529 ; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
6530 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v2
6531 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
6532 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
6533 ; GFX10-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
6534 ; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
6535 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
6536 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
6537 ; GFX10-NEXT: v_cndmask_b32_e64 v2, v6, v8, s4
6538 ; GFX10-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
6539 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
6540 ; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
6541 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
6542 ; GFX10-NEXT: buffer_gl0_inv
6543 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
6544 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
6545 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
6546 ; GFX10-NEXT: s_cbranch_execnz .LBB24_1
6547 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
6548 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
6549 ; GFX10-NEXT: v_mov_b32_e32 v0, v2
6550 ; GFX10-NEXT: s_setpc_b64 s[30:31]
6551 ;
6552 ; GFX90A-LABEL: local_atomic_fsub_ret_v2bf16:
6553 ; GFX90A: ; %bb.0:
6554 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6555 ; GFX90A-NEXT: ds_read_b32 v2, v0
6556 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
6557 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
6558 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
6559 ; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
6560 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
6561 ; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
6562 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
6563 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
6564 ; GFX90A-NEXT: v_mov_b32_e32 v4, v2
6565 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
6566 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
6567 ; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v3
6568 ; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v1
6569 ; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
6570 ; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
6571 ; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
6572 ; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
6573 ; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s8
6574 ; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
6575 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
6576 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
6577 ; GFX90A-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
6578 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
6579 ; GFX90A-NEXT: v_perm_b32 v2, v5, v2, s9
6580 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
6581 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
6582 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
6583 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
6584 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
6585 ; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
6586 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
6587 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
6588 ; GFX90A-NEXT: v_mov_b32_e32 v0, v2
6589 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
6590 ;
6591 ; GFX908-LABEL: local_atomic_fsub_ret_v2bf16:
6592 ; GFX908: ; %bb.0:
6593 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6594 ; GFX908-NEXT: ds_read_b32 v2, v0
6595 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
6596 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
6597 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
6598 ; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
6599 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
6600 ; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start
6601 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
6602 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
6603 ; GFX908-NEXT: v_mov_b32_e32 v4, v2
6604 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v4
6605 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
6606 ; GFX908-NEXT: v_sub_f32_e32 v2, v2, v3
6607 ; GFX908-NEXT: v_sub_f32_e32 v5, v5, v1
6608 ; GFX908-NEXT: v_bfe_u32 v6, v2, 16, 1
6609 ; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
6610 ; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v2
6611 ; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
6612 ; GFX908-NEXT: v_add3_u32 v6, v6, v2, s8
6613 ; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
6614 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
6615 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
6616 ; GFX908-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
6617 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
6618 ; GFX908-NEXT: v_perm_b32 v2, v5, v2, s9
6619 ; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
6620 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
6621 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
6622 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
6623 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
6624 ; GFX908-NEXT: s_cbranch_execnz .LBB24_1
6625 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
6626 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
6627 ; GFX908-NEXT: v_mov_b32_e32 v0, v2
6628 ; GFX908-NEXT: s_setpc_b64 s[30:31]
6629 ;
6630 ; GFX8-LABEL: local_atomic_fsub_ret_v2bf16:
6631 ; GFX8: ; %bb.0:
6632 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6633 ; GFX8-NEXT: s_mov_b32 m0, -1
6634 ; GFX8-NEXT: ds_read_b32 v2, v0
6635 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
6636 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
6637 ; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
6638 ; GFX8-NEXT: .LBB24_1: ; %atomicrmw.start
6639 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6640 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
6641 ; GFX8-NEXT: v_mov_b32_e32 v4, v2
6642 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v4
6643 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
6644 ; GFX8-NEXT: v_sub_f32_e32 v2, v2, v3
6645 ; GFX8-NEXT: v_sub_f32_e32 v5, v5, v1
6646 ; GFX8-NEXT: v_bfe_u32 v6, v2, 16, 1
6647 ; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
6648 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v2
6649 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
6650 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
6651 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
6652 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
6653 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
6654 ; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v2
6655 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
6656 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
6657 ; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
6658 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
6659 ; GFX8-NEXT: v_alignbit_b32 v2, v5, v2, 16
6660 ; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
6661 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
6662 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
6663 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
6664 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
6665 ; GFX8-NEXT: s_cbranch_execnz .LBB24_1
6666 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6667 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
6668 ; GFX8-NEXT: v_mov_b32_e32 v0, v2
6669 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6670 ;
6671 ; GFX7-LABEL: local_atomic_fsub_ret_v2bf16:
6672 ; GFX7: ; %bb.0:
6673 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6674 ; GFX7-NEXT: s_mov_b32 m0, -1
6675 ; GFX7-NEXT: ds_read_b32 v3, v0
6676 ; GFX7-NEXT: v_mov_b32_e32 v4, v1
6677 ; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
6678 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
6679 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
6680 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6681 ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
6682 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
6683 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
6684 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
6685 ; GFX7-NEXT: .LBB24_1: ; %atomicrmw.start
6686 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
6687 ; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
6688 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
6689 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
6690 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
6691 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
6692 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
6693 ; GFX7-NEXT: v_sub_f32_e32 v5, v5, v4
6694 ; GFX7-NEXT: v_alignbit_b32 v1, v1, v3, 16
6695 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v6
6696 ; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
6697 ; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v0, v1, v3
6698 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6699 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
6700 ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
6701 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6702 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
6703 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
6704 ; GFX7-NEXT: s_cbranch_execnz .LBB24_1
6705 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6706 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
6707 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
6708 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6709 ;
6710 ; GFX6-LABEL: local_atomic_fsub_ret_v2bf16:
6711 ; GFX6: ; %bb.0:
6712 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6713 ; GFX6-NEXT: s_mov_b32 m0, -1
6714 ; GFX6-NEXT: ds_read_b32 v3, v0
6715 ; GFX6-NEXT: v_mov_b32_e32 v4, v1
6716 ; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
6717 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
6718 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
6719 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
6720 ; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
6721 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
6722 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
6723 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
6724 ; GFX6-NEXT: .LBB24_1: ; %atomicrmw.start
6725 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
6726 ; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
6727 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
6728 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
6729 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
6730 ; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
6731 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
6732 ; GFX6-NEXT: v_sub_f32_e32 v5, v5, v4
6733 ; GFX6-NEXT: v_alignbit_b32 v1, v1, v3, 16
6734 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v6
6735 ; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
6736 ; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v0, v1, v3
6737 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
6738 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
6739 ; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
6740 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6741 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
6742 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
6743 ; GFX6-NEXT: s_cbranch_execnz .LBB24_1
6744 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
6745 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
6746 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
6747 ; GFX6-NEXT: s_setpc_b64 s[30:31]
6748 %result = atomicrmw fsub ptr addrspace(3) %ptr, <2 x bfloat> %val seq_cst
6749 ret <2 x bfloat> %result
6750 }
6752 define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr, <2 x bfloat> %val) {
6753 ; GFX12-LABEL: local_atomic_fsub_ret_v2bf16__offset:
6754 ; GFX12: ; %bb.0:
6755 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
6756 ; GFX12-NEXT: s_wait_expcnt 0x0
6757 ; GFX12-NEXT: s_wait_samplecnt 0x0
6758 ; GFX12-NEXT: s_wait_bvhcnt 0x0
6759 ; GFX12-NEXT: s_wait_kmcnt 0x0
6760 ; GFX12-NEXT: ds_load_b32 v2, v0 offset:65532
6761 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v1
6762 ; GFX12-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
6763 ; GFX12-NEXT: s_mov_b32 s1, 0
6764 ; GFX12-NEXT: .LBB25_1: ; %atomicrmw.start
6765 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
6766 ; GFX12-NEXT: s_wait_dscnt 0x0
6767 ; GFX12-NEXT: v_mov_b32_e32 v4, v2
6768 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6769 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
6770 ; GFX12-NEXT: v_sub_f32_e32 v5, v5, v1
6771 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
6772 ; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
6773 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v5
6774 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
6775 ; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
6776 ; GFX12-NEXT: s_wait_alu 0xfffd
6777 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6778 ; GFX12-NEXT: v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
6779 ; GFX12-NEXT: v_sub_f32_e32 v2, v2, v3
6780 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
6781 ; GFX12-NEXT: v_bfe_u32 v6, v2, 16, 1
6782 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v2
6783 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v2, v2
6784 ; GFX12-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
6785 ; GFX12-NEXT: s_wait_alu 0xf1ff
6786 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6787 ; GFX12-NEXT: v_cndmask_b32_e64 v2, v6, v8, s0
6788 ; GFX12-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
6789 ; GFX12-NEXT: s_wait_storecnt 0x0
6790 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4 offset:65532
6791 ; GFX12-NEXT: s_wait_dscnt 0x0
6792 ; GFX12-NEXT: global_inv scope:SCOPE_SE
6793 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
6794 ; GFX12-NEXT: s_wait_alu 0xfffe
6795 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
6796 ; GFX12-NEXT: s_wait_alu 0xfffe
6797 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
6798 ; GFX12-NEXT: s_cbranch_execnz .LBB25_1
6799 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
6800 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
6801 ; GFX12-NEXT: v_mov_b32_e32 v0, v2
6802 ; GFX12-NEXT: s_setpc_b64 s[30:31]
6803 ;
6804 ; GFX940-LABEL: local_atomic_fsub_ret_v2bf16__offset:
6805 ; GFX940: ; %bb.0:
6806 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6807 ; GFX940-NEXT: ds_read_b32 v2, v0 offset:65532
6808 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
6809 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 16, v1
6810 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
6811 ; GFX940-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
6812 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
6813 ; GFX940-NEXT: .LBB25_1: ; %atomicrmw.start
6814 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
6815 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
6816 ; GFX940-NEXT: v_mov_b32_e32 v4, v2
6817 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v4
6818 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
6819 ; GFX940-NEXT: v_sub_f32_e32 v2, v2, v3
6820 ; GFX940-NEXT: v_sub_f32_e32 v5, v5, v1
6821 ; GFX940-NEXT: v_bfe_u32 v6, v2, 16, 1
6822 ; GFX940-NEXT: v_bfe_u32 v8, v5, 16, 1
6823 ; GFX940-NEXT: v_or_b32_e32 v7, 0x400000, v2
6824 ; GFX940-NEXT: v_or_b32_e32 v9, 0x400000, v5
6825 ; GFX940-NEXT: v_add3_u32 v6, v6, v2, s4
6826 ; GFX940-NEXT: v_add3_u32 v8, v8, v5, s4
6827 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
6828 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
6829 ; GFX940-NEXT: s_nop 0
6830 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
6831 ; GFX940-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[0:1]
6832 ; GFX940-NEXT: v_perm_b32 v2, v5, v2, s5
6833 ; GFX940-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
6834 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
6835 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
6836 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
6837 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
6838 ; GFX940-NEXT: s_cbranch_execnz .LBB25_1
6839 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
6840 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
6841 ; GFX940-NEXT: v_mov_b32_e32 v0, v2
6842 ; GFX940-NEXT: s_setpc_b64 s[30:31]
6843 ;
6844 ; GFX11-LABEL: local_atomic_fsub_ret_v2bf16__offset:
6845 ; GFX11: ; %bb.0:
6846 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6847 ; GFX11-NEXT: ds_load_b32 v2, v0 offset:65532
6848 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v1
6849 ; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
6850 ; GFX11-NEXT: s_mov_b32 s1, 0
6851 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
6852 ; GFX11-NEXT: .p2align 6
6853 ; GFX11-NEXT: .LBB25_1: ; %atomicrmw.start
6854 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
6855 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
6856 ; GFX11-NEXT: v_mov_b32_e32 v4, v2
6857 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6858 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
6859 ; GFX11-NEXT: v_sub_f32_e32 v5, v5, v1
6860 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
6861 ; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
6862 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v5
6863 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
6864 ; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
6865 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6866 ; GFX11-NEXT: v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
6867 ; GFX11-NEXT: v_sub_f32_e32 v2, v2, v3
6868 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
6869 ; GFX11-NEXT: v_bfe_u32 v6, v2, 16, 1
6870 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v2
6871 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v2, v2
6872 ; GFX11-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
6873 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6874 ; GFX11-NEXT: v_cndmask_b32_e64 v2, v6, v8, s0
6875 ; GFX11-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
6876 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
6877 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4 offset:65532
6878 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
6879 ; GFX11-NEXT: buffer_gl0_inv
6880 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
6881 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
6882 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
6883 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
6884 ; GFX11-NEXT: s_cbranch_execnz .LBB25_1
6885 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
6886 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
6887 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
6888 ; GFX11-NEXT: v_mov_b32_e32 v0, v2
6889 ; GFX11-NEXT: s_setpc_b64 s[30:31]
6890 ;
6891 ; GFX10-LABEL: local_atomic_fsub_ret_v2bf16__offset:
6892 ; GFX10: ; %bb.0:
6893 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6894 ; GFX10-NEXT: ds_read_b32 v2, v0 offset:65532
6895 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v1
6896 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
6897 ; GFX10-NEXT: s_mov_b32 s5, 0
6898 ; GFX10-NEXT: .LBB25_1: ; %atomicrmw.start
6899 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
6900 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
6901 ; GFX10-NEXT: v_mov_b32_e32 v4, v2
6902 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v4
6903 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
6904 ; GFX10-NEXT: v_sub_f32_e32 v2, v2, v3
6905 ; GFX10-NEXT: v_sub_f32_e32 v5, v5, v1
6906 ; GFX10-NEXT: v_bfe_u32 v6, v2, 16, 1
6907 ; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
6908 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v2
6909 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
6910 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
6911 ; GFX10-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
6912 ; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
6913 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
6914 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
6915 ; GFX10-NEXT: v_cndmask_b32_e64 v2, v6, v8, s4
6916 ; GFX10-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
6917 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
6918 ; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
6919 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
6920 ; GFX10-NEXT: buffer_gl0_inv
6921 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
6922 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
6923 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
6924 ; GFX10-NEXT: s_cbranch_execnz .LBB25_1
6925 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
6926 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
6927 ; GFX10-NEXT: v_mov_b32_e32 v0, v2
6928 ; GFX10-NEXT: s_setpc_b64 s[30:31]
6929 ;
6930 ; GFX90A-LABEL: local_atomic_fsub_ret_v2bf16__offset:
6931 ; GFX90A: ; %bb.0:
6932 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6933 ; GFX90A-NEXT: ds_read_b32 v2, v0 offset:65532
6934 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
6935 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
6936 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
6937 ; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
6938 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
6939 ; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
6940 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
6941 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
6942 ; GFX90A-NEXT: v_mov_b32_e32 v4, v2
6943 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
6944 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
6945 ; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v3
6946 ; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v1
6947 ; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
6948 ; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
6949 ; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
6950 ; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
6951 ; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s8
6952 ; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
6953 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
6954 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
6955 ; GFX90A-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
6956 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
6957 ; GFX90A-NEXT: v_perm_b32 v2, v5, v2, s9
6958 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
6959 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
6960 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
6961 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
6962 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
6963 ; GFX90A-NEXT: s_cbranch_execnz .LBB25_1
6964 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
6965 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
6966 ; GFX90A-NEXT: v_mov_b32_e32 v0, v2
6967 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
6968 ;
6969 ; GFX908-LABEL: local_atomic_fsub_ret_v2bf16__offset:
6970 ; GFX908: ; %bb.0:
6971 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6972 ; GFX908-NEXT: ds_read_b32 v2, v0 offset:65532
6973 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
6974 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
6975 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
6976 ; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
6977 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
6978 ; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start
6979 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
6980 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
6981 ; GFX908-NEXT: v_mov_b32_e32 v4, v2
6982 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v4
6983 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
6984 ; GFX908-NEXT: v_sub_f32_e32 v2, v2, v3
6985 ; GFX908-NEXT: v_sub_f32_e32 v5, v5, v1
6986 ; GFX908-NEXT: v_bfe_u32 v6, v2, 16, 1
6987 ; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
6988 ; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v2
6989 ; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
6990 ; GFX908-NEXT: v_add3_u32 v6, v6, v2, s8
6991 ; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
6992 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
6993 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
6994 ; GFX908-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
6995 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
6996 ; GFX908-NEXT: v_perm_b32 v2, v5, v2, s9
6997 ; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
6998 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
6999 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
7000 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
7001 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
7002 ; GFX908-NEXT: s_cbranch_execnz .LBB25_1
7003 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
7004 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
7005 ; GFX908-NEXT: v_mov_b32_e32 v0, v2
7006 ; GFX908-NEXT: s_setpc_b64 s[30:31]
7007 ;
7008 ; GFX8-LABEL: local_atomic_fsub_ret_v2bf16__offset:
7009 ; GFX8: ; %bb.0:
7010 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7011 ; GFX8-NEXT: s_mov_b32 m0, -1
7012 ; GFX8-NEXT: ds_read_b32 v2, v0 offset:65532
7013 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
7014 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
7015 ; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
7016 ; GFX8-NEXT: .LBB25_1: ; %atomicrmw.start
7017 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7018 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
7019 ; GFX8-NEXT: v_mov_b32_e32 v4, v2
7020 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v4
7021 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
7022 ; GFX8-NEXT: v_sub_f32_e32 v2, v2, v3
7023 ; GFX8-NEXT: v_sub_f32_e32 v5, v5, v1
7024 ; GFX8-NEXT: v_bfe_u32 v6, v2, 16, 1
7025 ; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
7026 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v2
7027 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
7028 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
7029 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
7030 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
7031 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
7032 ; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v2
7033 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
7034 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
7035 ; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
7036 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
7037 ; GFX8-NEXT: v_alignbit_b32 v2, v5, v2, 16
7038 ; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
7039 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
7040 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
7041 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
7042 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
7043 ; GFX8-NEXT: s_cbranch_execnz .LBB25_1
7044 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7045 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
7046 ; GFX8-NEXT: v_mov_b32_e32 v0, v2
7047 ; GFX8-NEXT: s_setpc_b64 s[30:31]
7048 ;
7049 ; GFX7-LABEL: local_atomic_fsub_ret_v2bf16__offset:
7050 ; GFX7: ; %bb.0:
7051 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7052 ; GFX7-NEXT: s_mov_b32 m0, -1
7053 ; GFX7-NEXT: ds_read_b32 v3, v0 offset:65532
7054 ; GFX7-NEXT: v_mov_b32_e32 v4, v1
7055 ; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
7056 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
7057 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
7058 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7059 ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
7060 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
7061 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
7062 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
7063 ; GFX7-NEXT: .LBB25_1: ; %atomicrmw.start
7064 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7065 ; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
7066 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
7067 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
7068 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
7069 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
7070 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
7071 ; GFX7-NEXT: v_sub_f32_e32 v5, v5, v4
7072 ; GFX7-NEXT: v_alignbit_b32 v1, v1, v3, 16
7073 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v6
7074 ; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
7075 ; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v0, v1, v3 offset:65532
7076 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7077 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
7078 ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
7079 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7080 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
7081 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
7082 ; GFX7-NEXT: s_cbranch_execnz .LBB25_1
7083 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
7084 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
7085 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
7086 ; GFX7-NEXT: s_setpc_b64 s[30:31]
7087 ;
7088 ; GFX6-LABEL: local_atomic_fsub_ret_v2bf16__offset:
7089 ; GFX6: ; %bb.0:
7090 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7091 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffc, v0
7092 ; GFX6-NEXT: s_mov_b32 m0, -1
7093 ; GFX6-NEXT: ds_read_b32 v0, v4
7094 ; GFX6-NEXT: v_mov_b32_e32 v3, v1
7095 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
7096 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
7097 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
7098 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
7099 ; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
7100 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
7101 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
7102 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
7103 ; GFX6-NEXT: .LBB25_1: ; %atomicrmw.start
7104 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
7105 ; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
7106 ; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
7107 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
7108 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
7109 ; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
7110 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
7111 ; GFX6-NEXT: v_sub_f32_e32 v5, v5, v3
7112 ; GFX6-NEXT: v_alignbit_b32 v0, v1, v0, 16
7113 ; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v6
7114 ; GFX6-NEXT: v_alignbit_b32 v1, v1, v5, 16
7115 ; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v4, v0, v1
7116 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
7117 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v0
7118 ; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v5
7119 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7120 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v5
7121 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
7122 ; GFX6-NEXT: s_cbranch_execnz .LBB25_1
7123 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
7124 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
7125 ; GFX6-NEXT: s_setpc_b64 s[30:31]
7126 %gep = getelementptr <2 x bfloat>, ptr addrspace(3) %ptr, i32 16383
7127 %result = atomicrmw fsub ptr addrspace(3) %gep, <2 x bfloat> %val seq_cst
7128 ret <2 x bfloat> %result
7129 }
7131 define void @local_atomic_fsub_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat> %val) {
7132 ; GFX12-LABEL: local_atomic_fsub_noret_v2bf16:
7133 ; GFX12: ; %bb.0:
7134 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
7135 ; GFX12-NEXT: s_wait_expcnt 0x0
7136 ; GFX12-NEXT: s_wait_samplecnt 0x0
7137 ; GFX12-NEXT: s_wait_bvhcnt 0x0
7138 ; GFX12-NEXT: s_wait_kmcnt 0x0
7139 ; GFX12-NEXT: ds_load_b32 v3, v0
7140 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v1
7141 ; GFX12-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
7142 ; GFX12-NEXT: s_mov_b32 s1, 0
7143 ; GFX12-NEXT: .LBB26_1: ; %atomicrmw.start
7144 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
7145 ; GFX12-NEXT: s_wait_dscnt 0x0
7146 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
7147 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7148 ; GFX12-NEXT: v_dual_sub_f32 v5, v5, v1 :: v_dual_lshlrev_b32 v4, 16, v3
7149 ; GFX12-NEXT: v_sub_f32_e32 v4, v4, v2
7150 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
7151 ; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
7152 ; GFX12-NEXT: v_bfe_u32 v6, v4, 16, 1
7153 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v4
7154 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v5
7155 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
7156 ; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
7157 ; GFX12-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
7158 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v4, v4
7159 ; GFX12-NEXT: s_wait_alu 0xfffd
7160 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
7161 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
7162 ; GFX12-NEXT: s_wait_alu 0xf1ff
7163 ; GFX12-NEXT: v_cndmask_b32_e64 v4, v6, v8, s0
7164 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
7165 ; GFX12-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
7166 ; GFX12-NEXT: s_wait_storecnt 0x0
7167 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
7168 ; GFX12-NEXT: s_wait_dscnt 0x0
7169 ; GFX12-NEXT: global_inv scope:SCOPE_SE
7170 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
7171 ; GFX12-NEXT: v_mov_b32_e32 v3, v4
7172 ; GFX12-NEXT: s_wait_alu 0xfffe
7173 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
7174 ; GFX12-NEXT: s_wait_alu 0xfffe
7175 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
7176 ; GFX12-NEXT: s_cbranch_execnz .LBB26_1
7177 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
7178 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
7179 ; GFX12-NEXT: s_setpc_b64 s[30:31]
7180 ;
7181 ; GFX940-LABEL: local_atomic_fsub_noret_v2bf16:
7182 ; GFX940: ; %bb.0:
7183 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7184 ; GFX940-NEXT: ds_read_b32 v3, v0
7185 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
7186 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v1
7187 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
7188 ; GFX940-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
7189 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
7190 ; GFX940-NEXT: .LBB26_1: ; %atomicrmw.start
7191 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
7192 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
7193 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v3
7194 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
7195 ; GFX940-NEXT: v_sub_f32_e32 v4, v4, v2
7196 ; GFX940-NEXT: v_sub_f32_e32 v5, v5, v1
7197 ; GFX940-NEXT: v_bfe_u32 v6, v4, 16, 1
7198 ; GFX940-NEXT: v_bfe_u32 v8, v5, 16, 1
7199 ; GFX940-NEXT: v_or_b32_e32 v7, 0x400000, v4
7200 ; GFX940-NEXT: v_or_b32_e32 v9, 0x400000, v5
7201 ; GFX940-NEXT: v_add3_u32 v6, v6, v4, s4
7202 ; GFX940-NEXT: v_add3_u32 v8, v8, v5, s4
7203 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
7204 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v4, v4
7205 ; GFX940-NEXT: s_nop 0
7206 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
7207 ; GFX940-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[0:1]
7208 ; GFX940-NEXT: v_perm_b32 v4, v5, v4, s5
7209 ; GFX940-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
7210 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
7211 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
7212 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
7213 ; GFX940-NEXT: v_mov_b32_e32 v3, v4
7214 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
7215 ; GFX940-NEXT: s_cbranch_execnz .LBB26_1
7216 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
7217 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
7218 ; GFX940-NEXT: s_setpc_b64 s[30:31]
7219 ;
7220 ; GFX11-LABEL: local_atomic_fsub_noret_v2bf16:
7221 ; GFX11: ; %bb.0:
7222 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7223 ; GFX11-NEXT: ds_load_b32 v3, v0
7224 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v1
7225 ; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
7226 ; GFX11-NEXT: s_mov_b32 s1, 0
7227 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
7228 ; GFX11-NEXT: .p2align 6
7229 ; GFX11-NEXT: .LBB26_1: ; %atomicrmw.start
7230 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
7231 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
7232 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
7233 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7234 ; GFX11-NEXT: v_dual_sub_f32 v5, v5, v1 :: v_dual_lshlrev_b32 v4, 16, v3
7235 ; GFX11-NEXT: v_sub_f32_e32 v4, v4, v2
7236 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
7237 ; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
7238 ; GFX11-NEXT: v_bfe_u32 v6, v4, 16, 1
7239 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v4
7240 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v5
7241 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
7242 ; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
7243 ; GFX11-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
7244 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v4, v4
7245 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
7246 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
7247 ; GFX11-NEXT: v_cndmask_b32_e64 v4, v6, v8, s0
7248 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
7249 ; GFX11-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
7250 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
7251 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
7252 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
7253 ; GFX11-NEXT: buffer_gl0_inv
7254 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
7255 ; GFX11-NEXT: v_mov_b32_e32 v3, v4
7256 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
7257 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
7258 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
7259 ; GFX11-NEXT: s_cbranch_execnz .LBB26_1
7260 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
7261 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
7262 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
7263 ; GFX11-NEXT: s_setpc_b64 s[30:31]
7264 ;
7265 ; GFX10-LABEL: local_atomic_fsub_noret_v2bf16:
7266 ; GFX10: ; %bb.0:
7267 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7268 ; GFX10-NEXT: ds_read_b32 v3, v0
7269 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
7270 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
7271 ; GFX10-NEXT: s_mov_b32 s5, 0
7272 ; GFX10-NEXT: .LBB26_1: ; %atomicrmw.start
7273 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
7274 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
7275 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v3
7276 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
7277 ; GFX10-NEXT: v_sub_f32_e32 v4, v4, v2
7278 ; GFX10-NEXT: v_sub_f32_e32 v5, v5, v1
7279 ; GFX10-NEXT: v_bfe_u32 v6, v4, 16, 1
7280 ; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
7281 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v4
7282 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
7283 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
7284 ; GFX10-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
7285 ; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
7286 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v4, v4
7287 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
7288 ; GFX10-NEXT: v_cndmask_b32_e64 v4, v6, v8, s4
7289 ; GFX10-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
7290 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
7291 ; GFX10-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
7292 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
7293 ; GFX10-NEXT: buffer_gl0_inv
7294 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
7295 ; GFX10-NEXT: v_mov_b32_e32 v3, v4
7296 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
7297 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
7298 ; GFX10-NEXT: s_cbranch_execnz .LBB26_1
7299 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
7300 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
7301 ; GFX10-NEXT: s_setpc_b64 s[30:31]
7302 ;
7303 ; GFX90A-LABEL: local_atomic_fsub_noret_v2bf16:
7304 ; GFX90A: ; %bb.0:
7305 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7306 ; GFX90A-NEXT: ds_read_b32 v3, v0
7307 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
7308 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
7309 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
7310 ; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
7311 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
7312 ; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
7313 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
7314 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
7315 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v3
7316 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
7317 ; GFX90A-NEXT: v_sub_f32_e32 v4, v4, v2
7318 ; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v1
7319 ; GFX90A-NEXT: v_bfe_u32 v6, v4, 16, 1
7320 ; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
7321 ; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v4
7322 ; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
7323 ; GFX90A-NEXT: v_add3_u32 v6, v6, v4, s8
7324 ; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
7325 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
7326 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
7327 ; GFX90A-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
7328 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
7329 ; GFX90A-NEXT: v_perm_b32 v4, v5, v4, s9
7330 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
7331 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
7332 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
7333 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
7334 ; GFX90A-NEXT: v_mov_b32_e32 v3, v4
7335 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
7336 ; GFX90A-NEXT: s_cbranch_execnz .LBB26_1
7337 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
7338 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
7339 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
7340 ;
7341 ; GFX908-LABEL: local_atomic_fsub_noret_v2bf16:
7342 ; GFX908: ; %bb.0:
7343 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7344 ; GFX908-NEXT: ds_read_b32 v3, v0
7345 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
7346 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v1
7347 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
7348 ; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
7349 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
7350 ; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
7351 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
7352 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
7353 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v3
7354 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
7355 ; GFX908-NEXT: v_sub_f32_e32 v4, v4, v2
7356 ; GFX908-NEXT: v_sub_f32_e32 v5, v5, v1
7357 ; GFX908-NEXT: v_bfe_u32 v6, v4, 16, 1
7358 ; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
7359 ; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v4
7360 ; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
7361 ; GFX908-NEXT: v_add3_u32 v6, v6, v4, s8
7362 ; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
7363 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
7364 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
7365 ; GFX908-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
7366 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
7367 ; GFX908-NEXT: v_perm_b32 v4, v5, v4, s9
7368 ; GFX908-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
7369 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
7370 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
7371 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
7372 ; GFX908-NEXT: v_mov_b32_e32 v3, v4
7373 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
7374 ; GFX908-NEXT: s_cbranch_execnz .LBB26_1
7375 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
7376 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
7377 ; GFX908-NEXT: s_setpc_b64 s[30:31]
7378 ;
7379 ; GFX8-LABEL: local_atomic_fsub_noret_v2bf16:
7380 ; GFX8: ; %bb.0:
7381 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7382 ; GFX8-NEXT: s_mov_b32 m0, -1
7383 ; GFX8-NEXT: ds_read_b32 v3, v0
7384 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
7385 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
7386 ; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
7387 ; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start
7388 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7389 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
7390 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v3
7391 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
7392 ; GFX8-NEXT: v_sub_f32_e32 v4, v4, v2
7393 ; GFX8-NEXT: v_sub_f32_e32 v5, v5, v1
7394 ; GFX8-NEXT: v_bfe_u32 v6, v4, 16, 1
7395 ; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
7396 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v4
7397 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
7398 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
7399 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
7400 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
7401 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
7402 ; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v4
7403 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
7404 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
7405 ; GFX8-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
7406 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
7407 ; GFX8-NEXT: v_alignbit_b32 v4, v5, v4, 16
7408 ; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
7409 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
7410 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
7411 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
7412 ; GFX8-NEXT: v_mov_b32_e32 v3, v4
7413 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
7414 ; GFX8-NEXT: s_cbranch_execnz .LBB26_1
7415 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7416 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
7417 ; GFX8-NEXT: s_setpc_b64 s[30:31]
7418 ;
7419 ; GFX7-LABEL: local_atomic_fsub_noret_v2bf16:
7420 ; GFX7: ; %bb.0:
7421 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7422 ; GFX7-NEXT: s_mov_b32 m0, -1
7423 ; GFX7-NEXT: ds_read_b32 v4, v0
7424 ; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
7425 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
7426 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
7427 ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
7428 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7429 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
7430 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
7431 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
7432 ; GFX7-NEXT: .LBB26_1: ; %atomicrmw.start
7433 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7434 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
7435 ; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
7436 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
7437 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
7438 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
7439 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
7440 ; GFX7-NEXT: v_sub_f32_e32 v5, v5, v1
7441 ; GFX7-NEXT: v_alignbit_b32 v3, v3, v4, 16
7442 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
7443 ; GFX7-NEXT: v_alignbit_b32 v4, v4, v5, 16
7444 ; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
7445 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7446 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
7447 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
7448 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7449 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
7450 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
7451 ; GFX7-NEXT: s_cbranch_execnz .LBB26_1
7452 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
7453 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
7454 ; GFX7-NEXT: s_setpc_b64 s[30:31]
7455 ;
7456 ; GFX6-LABEL: local_atomic_fsub_noret_v2bf16:
7457 ; GFX6: ; %bb.0:
7458 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7459 ; GFX6-NEXT: s_mov_b32 m0, -1
7460 ; GFX6-NEXT: ds_read_b32 v4, v0
7461 ; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
7462 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
7463 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
7464 ; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
7465 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
7466 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
7467 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
7468 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
7469 ; GFX6-NEXT: .LBB26_1: ; %atomicrmw.start
7470 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
7471 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
7472 ; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
7473 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
7474 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
7475 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
7476 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
7477 ; GFX6-NEXT: v_sub_f32_e32 v5, v5, v1
7478 ; GFX6-NEXT: v_alignbit_b32 v3, v3, v4, 16
7479 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
7480 ; GFX6-NEXT: v_alignbit_b32 v4, v4, v5, 16
7481 ; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
7482 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
7483 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
7484 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
7485 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7486 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
7487 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
7488 ; GFX6-NEXT: s_cbranch_execnz .LBB26_1
7489 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
7490 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
7491 ; GFX6-NEXT: s_setpc_b64 s[30:31]
7492 %result = atomicrmw fsub ptr addrspace(3) %ptr, <2 x bfloat> %val seq_cst
7493 ret void
7494 }
7496 define void @local_atomic_fsub_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x bfloat> %val) {
7497 ; GFX12-LABEL: local_atomic_fsub_noret_v2bf16__ofset:
7498 ; GFX12: ; %bb.0:
7499 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
7500 ; GFX12-NEXT: s_wait_expcnt 0x0
7501 ; GFX12-NEXT: s_wait_samplecnt 0x0
7502 ; GFX12-NEXT: s_wait_bvhcnt 0x0
7503 ; GFX12-NEXT: s_wait_kmcnt 0x0
7504 ; GFX12-NEXT: ds_load_b32 v3, v0 offset:65532
7505 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v1
7506 ; GFX12-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
7507 ; GFX12-NEXT: s_mov_b32 s1, 0
7508 ; GFX12-NEXT: .LBB27_1: ; %atomicrmw.start
7509 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
7510 ; GFX12-NEXT: s_wait_dscnt 0x0
7511 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
7512 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7513 ; GFX12-NEXT: v_dual_sub_f32 v5, v5, v1 :: v_dual_lshlrev_b32 v4, 16, v3
7514 ; GFX12-NEXT: v_sub_f32_e32 v4, v4, v2
7515 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
7516 ; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
7517 ; GFX12-NEXT: v_bfe_u32 v6, v4, 16, 1
7518 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v4
7519 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v5
7520 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
7521 ; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
7522 ; GFX12-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
7523 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v4, v4
7524 ; GFX12-NEXT: s_wait_alu 0xfffd
7525 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
7526 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
7527 ; GFX12-NEXT: s_wait_alu 0xf1ff
7528 ; GFX12-NEXT: v_cndmask_b32_e64 v4, v6, v8, s0
7529 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
7530 ; GFX12-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
7531 ; GFX12-NEXT: s_wait_storecnt 0x0
7532 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3 offset:65532
7533 ; GFX12-NEXT: s_wait_dscnt 0x0
7534 ; GFX12-NEXT: global_inv scope:SCOPE_SE
7535 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
7536 ; GFX12-NEXT: v_mov_b32_e32 v3, v4
7537 ; GFX12-NEXT: s_wait_alu 0xfffe
7538 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
7539 ; GFX12-NEXT: s_wait_alu 0xfffe
7540 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
7541 ; GFX12-NEXT: s_cbranch_execnz .LBB27_1
7542 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
7543 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
7544 ; GFX12-NEXT: s_setpc_b64 s[30:31]
7545 ;
7546 ; GFX940-LABEL: local_atomic_fsub_noret_v2bf16__ofset:
7547 ; GFX940: ; %bb.0:
7548 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7549 ; GFX940-NEXT: ds_read_b32 v3, v0 offset:65532
7550 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
7551 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v1
7552 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
7553 ; GFX940-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
7554 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
7555 ; GFX940-NEXT: .LBB27_1: ; %atomicrmw.start
7556 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
7557 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
7558 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v3
7559 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
7560 ; GFX940-NEXT: v_sub_f32_e32 v4, v4, v2
7561 ; GFX940-NEXT: v_sub_f32_e32 v5, v5, v1
7562 ; GFX940-NEXT: v_bfe_u32 v6, v4, 16, 1
7563 ; GFX940-NEXT: v_bfe_u32 v8, v5, 16, 1
7564 ; GFX940-NEXT: v_or_b32_e32 v7, 0x400000, v4
7565 ; GFX940-NEXT: v_or_b32_e32 v9, 0x400000, v5
7566 ; GFX940-NEXT: v_add3_u32 v6, v6, v4, s4
7567 ; GFX940-NEXT: v_add3_u32 v8, v8, v5, s4
7568 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
7569 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v4, v4
7570 ; GFX940-NEXT: s_nop 0
7571 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
7572 ; GFX940-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[0:1]
7573 ; GFX940-NEXT: v_perm_b32 v4, v5, v4, s5
7574 ; GFX940-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
7575 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
7576 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
7577 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
7578 ; GFX940-NEXT: v_mov_b32_e32 v3, v4
7579 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
7580 ; GFX940-NEXT: s_cbranch_execnz .LBB27_1
7581 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
7582 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
7583 ; GFX940-NEXT: s_setpc_b64 s[30:31]
7584 ;
7585 ; GFX11-LABEL: local_atomic_fsub_noret_v2bf16__ofset:
7586 ; GFX11: ; %bb.0:
7587 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7588 ; GFX11-NEXT: ds_load_b32 v3, v0 offset:65532
7589 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v1
7590 ; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
7591 ; GFX11-NEXT: s_mov_b32 s1, 0
7592 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
7593 ; GFX11-NEXT: .p2align 6
7594 ; GFX11-NEXT: .LBB27_1: ; %atomicrmw.start
7595 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
7596 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
7597 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
7598 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7599 ; GFX11-NEXT: v_dual_sub_f32 v5, v5, v1 :: v_dual_lshlrev_b32 v4, 16, v3
7600 ; GFX11-NEXT: v_sub_f32_e32 v4, v4, v2
7601 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
7602 ; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
7603 ; GFX11-NEXT: v_bfe_u32 v6, v4, 16, 1
7604 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v4
7605 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v5
7606 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
7607 ; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
7608 ; GFX11-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
7609 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v4, v4
7610 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
7611 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
7612 ; GFX11-NEXT: v_cndmask_b32_e64 v4, v6, v8, s0
7613 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
7614 ; GFX11-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
7615 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
7616 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3 offset:65532
7617 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
7618 ; GFX11-NEXT: buffer_gl0_inv
7619 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
7620 ; GFX11-NEXT: v_mov_b32_e32 v3, v4
7621 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
7622 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
7623 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
7624 ; GFX11-NEXT: s_cbranch_execnz .LBB27_1
7625 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
7626 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
7627 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
7628 ; GFX11-NEXT: s_setpc_b64 s[30:31]
7629 ;
7630 ; GFX10-LABEL: local_atomic_fsub_noret_v2bf16__ofset:
7631 ; GFX10: ; %bb.0:
7632 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7633 ; GFX10-NEXT: ds_read_b32 v3, v0 offset:65532
7634 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
7635 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
7636 ; GFX10-NEXT: s_mov_b32 s5, 0
7637 ; GFX10-NEXT: .LBB27_1: ; %atomicrmw.start
7638 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
7639 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
7640 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v3
7641 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
7642 ; GFX10-NEXT: v_sub_f32_e32 v4, v4, v2
7643 ; GFX10-NEXT: v_sub_f32_e32 v5, v5, v1
7644 ; GFX10-NEXT: v_bfe_u32 v6, v4, 16, 1
7645 ; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
7646 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v4
7647 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
7648 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
7649 ; GFX10-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
7650 ; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
7651 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v4, v4
7652 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
7653 ; GFX10-NEXT: v_cndmask_b32_e64 v4, v6, v8, s4
7654 ; GFX10-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
7655 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
7656 ; GFX10-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
7657 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
7658 ; GFX10-NEXT: buffer_gl0_inv
7659 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
7660 ; GFX10-NEXT: v_mov_b32_e32 v3, v4
7661 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
7662 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
7663 ; GFX10-NEXT: s_cbranch_execnz .LBB27_1
7664 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
7665 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
7666 ; GFX10-NEXT: s_setpc_b64 s[30:31]
7667 ;
7668 ; GFX90A-LABEL: local_atomic_fsub_noret_v2bf16__ofset:
7669 ; GFX90A: ; %bb.0:
7670 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7671 ; GFX90A-NEXT: ds_read_b32 v3, v0 offset:65532
7672 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
7673 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
7674 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
7675 ; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
7676 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
7677 ; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
7678 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
7679 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
7680 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v3
7681 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
7682 ; GFX90A-NEXT: v_sub_f32_e32 v4, v4, v2
7683 ; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v1
7684 ; GFX90A-NEXT: v_bfe_u32 v6, v4, 16, 1
7685 ; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
7686 ; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v4
7687 ; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
7688 ; GFX90A-NEXT: v_add3_u32 v6, v6, v4, s8
7689 ; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
7690 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
7691 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
7692 ; GFX90A-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
7693 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
7694 ; GFX90A-NEXT: v_perm_b32 v4, v5, v4, s9
7695 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
7696 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
7697 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
7698 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
7699 ; GFX90A-NEXT: v_mov_b32_e32 v3, v4
7700 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
7701 ; GFX90A-NEXT: s_cbranch_execnz .LBB27_1
7702 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
7703 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
7704 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
7705 ;
7706 ; GFX908-LABEL: local_atomic_fsub_noret_v2bf16__ofset:
7707 ; GFX908: ; %bb.0:
7708 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7709 ; GFX908-NEXT: ds_read_b32 v3, v0 offset:65532
7710 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
7711 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v1
7712 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
7713 ; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
7714 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
7715 ; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
7716 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
7717 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
7718 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v3
7719 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
7720 ; GFX908-NEXT: v_sub_f32_e32 v4, v4, v2
7721 ; GFX908-NEXT: v_sub_f32_e32 v5, v5, v1
7722 ; GFX908-NEXT: v_bfe_u32 v6, v4, 16, 1
7723 ; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
7724 ; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v4
7725 ; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
7726 ; GFX908-NEXT: v_add3_u32 v6, v6, v4, s8
7727 ; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
7728 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
7729 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
7730 ; GFX908-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
7731 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
7732 ; GFX908-NEXT: v_perm_b32 v4, v5, v4, s9
7733 ; GFX908-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
7734 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
7735 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
7736 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
7737 ; GFX908-NEXT: v_mov_b32_e32 v3, v4
7738 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
7739 ; GFX908-NEXT: s_cbranch_execnz .LBB27_1
7740 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
7741 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
7742 ; GFX908-NEXT: s_setpc_b64 s[30:31]
7743 ;
7744 ; GFX8-LABEL: local_atomic_fsub_noret_v2bf16__ofset:
7745 ; GFX8: ; %bb.0:
7746 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7747 ; GFX8-NEXT: s_mov_b32 m0, -1
7748 ; GFX8-NEXT: ds_read_b32 v3, v0 offset:65532
7749 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
7750 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
7751 ; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
7752 ; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start
7753 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7754 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
7755 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v3
7756 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
7757 ; GFX8-NEXT: v_sub_f32_e32 v4, v4, v2
7758 ; GFX8-NEXT: v_sub_f32_e32 v5, v5, v1
7759 ; GFX8-NEXT: v_bfe_u32 v6, v4, 16, 1
7760 ; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
7761 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v4
7762 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
7763 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
7764 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
7765 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
7766 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
7767 ; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v4
7768 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
7769 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
7770 ; GFX8-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
7771 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
7772 ; GFX8-NEXT: v_alignbit_b32 v4, v5, v4, 16
7773 ; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
7774 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
7775 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
7776 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
7777 ; GFX8-NEXT: v_mov_b32_e32 v3, v4
7778 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
7779 ; GFX8-NEXT: s_cbranch_execnz .LBB27_1
7780 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7781 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
7782 ; GFX8-NEXT: s_setpc_b64 s[30:31]
7783 ;
7784 ; GFX7-LABEL: local_atomic_fsub_noret_v2bf16__ofset:
7785 ; GFX7: ; %bb.0:
7786 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7787 ; GFX7-NEXT: s_mov_b32 m0, -1
7788 ; GFX7-NEXT: ds_read_b32 v4, v0 offset:65532
7789 ; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
7790 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
7791 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
7792 ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
7793 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7794 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
7795 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
7796 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
7797 ; GFX7-NEXT: .LBB27_1: ; %atomicrmw.start
7798 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7799 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
7800 ; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
7801 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
7802 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
7803 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
7804 ; GFX7-NEXT: v_sub_f32_e32 v6, v6, v2
7805 ; GFX7-NEXT: v_sub_f32_e32 v5, v5, v1
7806 ; GFX7-NEXT: v_alignbit_b32 v3, v3, v4, 16
7807 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
7808 ; GFX7-NEXT: v_alignbit_b32 v4, v4, v5, 16
7809 ; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
7810 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7811 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
7812 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
7813 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7814 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
7815 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
7816 ; GFX7-NEXT: s_cbranch_execnz .LBB27_1
7817 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
7818 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
7819 ; GFX7-NEXT: s_setpc_b64 s[30:31]
7820 ;
7821 ; GFX6-LABEL: local_atomic_fsub_noret_v2bf16__ofset:
7822 ; GFX6: ; %bb.0:
7823 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7824 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v0
7825 ; GFX6-NEXT: s_mov_b32 m0, -1
7826 ; GFX6-NEXT: ds_read_b32 v4, v0
7827 ; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
7828 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
7829 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
7830 ; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
7831 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
7832 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
7833 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
7834 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
7835 ; GFX6-NEXT: .LBB27_1: ; %atomicrmw.start
7836 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
7837 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
7838 ; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
7839 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
7840 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
7841 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
7842 ; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
7843 ; GFX6-NEXT: v_sub_f32_e32 v5, v5, v1
7844 ; GFX6-NEXT: v_alignbit_b32 v3, v3, v4, 16
7845 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
7846 ; GFX6-NEXT: v_alignbit_b32 v4, v4, v5, 16
7847 ; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
7848 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
7849 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
7850 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
7851 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7852 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
7853 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
7854 ; GFX6-NEXT: s_cbranch_execnz .LBB27_1
7855 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
7856 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
7857 ; GFX6-NEXT: s_setpc_b64 s[30:31]
7858 %gep = getelementptr <2 x bfloat>, ptr addrspace(3) %ptr, i32 16383
7859 %result = atomicrmw fsub ptr addrspace(3) %gep, <2 x bfloat> %val seq_cst
7860 ret void
7861 }
7863 ; --------------------------------------------------------------------
7864 ; misc
7865 ; --------------------------------------------------------------------
7867 define float @local_atomic_fsub_ret_f32__amdgpu_ignore_denormal_mode(ptr addrspace(3) %ptr) {
7868 ; GFX12-LABEL: local_atomic_fsub_ret_f32__amdgpu_ignore_denormal_mode:
7869 ; GFX12: ; %bb.0:
7870 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
7871 ; GFX12-NEXT: s_wait_expcnt 0x0
7872 ; GFX12-NEXT: s_wait_samplecnt 0x0
7873 ; GFX12-NEXT: s_wait_bvhcnt 0x0
7874 ; GFX12-NEXT: s_wait_kmcnt 0x0
7875 ; GFX12-NEXT: ds_load_b32 v1, v0
7876 ; GFX12-NEXT: s_mov_b32 s0, 0
7877 ; GFX12-NEXT: .LBB28_1: ; %atomicrmw.start
7878 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
7879 ; GFX12-NEXT: s_wait_dscnt 0x0
7880 ; GFX12-NEXT: v_mov_b32_e32 v2, v1
7881 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
7882 ; GFX12-NEXT: v_add_f32_e32 v1, -4.0, v2
7883 ; GFX12-NEXT: s_wait_storecnt 0x0
7884 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2
7885 ; GFX12-NEXT: s_wait_dscnt 0x0
7886 ; GFX12-NEXT: global_inv scope:SCOPE_SE
7887 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
7888 ; GFX12-NEXT: s_wait_alu 0xfffe
7889 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
7890 ; GFX12-NEXT: s_wait_alu 0xfffe
7891 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
7892 ; GFX12-NEXT: s_cbranch_execnz .LBB28_1
7893 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
7894 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
7895 ; GFX12-NEXT: v_mov_b32_e32 v0, v1
7896 ; GFX12-NEXT: s_setpc_b64 s[30:31]
7897 ;
7898 ; GFX940-LABEL: local_atomic_fsub_ret_f32__amdgpu_ignore_denormal_mode:
7899 ; GFX940: ; %bb.0:
7900 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7901 ; GFX940-NEXT: ds_read_b32 v1, v0
7902 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
7903 ; GFX940-NEXT: .LBB28_1: ; %atomicrmw.start
7904 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
7905 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
7906 ; GFX940-NEXT: v_mov_b32_e32 v2, v1
7907 ; GFX940-NEXT: v_add_f32_e32 v1, -4.0, v2
7908 ; GFX940-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1
7909 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
7910 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
7911 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
7912 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
7913 ; GFX940-NEXT: s_cbranch_execnz .LBB28_1
7914 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
7915 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
7916 ; GFX940-NEXT: v_mov_b32_e32 v0, v1
7917 ; GFX940-NEXT: s_setpc_b64 s[30:31]
7918 ;
7919 ; GFX11-LABEL: local_atomic_fsub_ret_f32__amdgpu_ignore_denormal_mode:
7920 ; GFX11: ; %bb.0:
7921 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7922 ; GFX11-NEXT: ds_load_b32 v1, v0
7923 ; GFX11-NEXT: s_mov_b32 s0, 0
7924 ; GFX11-NEXT: .LBB28_1: ; %atomicrmw.start
7925 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
7926 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
7927 ; GFX11-NEXT: v_mov_b32_e32 v2, v1
7928 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
7929 ; GFX11-NEXT: v_add_f32_e32 v1, -4.0, v2
7930 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
7931 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2
7932 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
7933 ; GFX11-NEXT: buffer_gl0_inv
7934 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
7935 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
7936 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
7937 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
7938 ; GFX11-NEXT: s_cbranch_execnz .LBB28_1
7939 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
7940 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
7941 ; GFX11-NEXT: v_mov_b32_e32 v0, v1
7942 ; GFX11-NEXT: s_setpc_b64 s[30:31]
7943 ;
7944 ; GFX10-LABEL: local_atomic_fsub_ret_f32__amdgpu_ignore_denormal_mode:
7945 ; GFX10: ; %bb.0:
7946 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7947 ; GFX10-NEXT: ds_read_b32 v1, v0
7948 ; GFX10-NEXT: s_mov_b32 s4, 0
7949 ; GFX10-NEXT: .LBB28_1: ; %atomicrmw.start
7950 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
7951 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
7952 ; GFX10-NEXT: v_mov_b32_e32 v2, v1
7953 ; GFX10-NEXT: v_add_f32_e32 v1, -4.0, v2
7954 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
7955 ; GFX10-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1
7956 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
7957 ; GFX10-NEXT: buffer_gl0_inv
7958 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
7959 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
7960 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
7961 ; GFX10-NEXT: s_cbranch_execnz .LBB28_1
7962 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
7963 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
7964 ; GFX10-NEXT: v_mov_b32_e32 v0, v1
7965 ; GFX10-NEXT: s_setpc_b64 s[30:31]
7966 ;
7967 ; GFX90A-LABEL: local_atomic_fsub_ret_f32__amdgpu_ignore_denormal_mode:
7968 ; GFX90A: ; %bb.0:
7969 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7970 ; GFX90A-NEXT: ds_read_b32 v1, v0
7971 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
7972 ; GFX90A-NEXT: .LBB28_1: ; %atomicrmw.start
7973 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
7974 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
7975 ; GFX90A-NEXT: v_mov_b32_e32 v2, v1
7976 ; GFX90A-NEXT: v_add_f32_e32 v1, -4.0, v2
7977 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1
7978 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
7979 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
7980 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7981 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
7982 ; GFX90A-NEXT: s_cbranch_execnz .LBB28_1
7983 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
7984 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
7985 ; GFX90A-NEXT: v_mov_b32_e32 v0, v1
7986 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
7987 ;
7988 ; GFX908-LABEL: local_atomic_fsub_ret_f32__amdgpu_ignore_denormal_mode:
7989 ; GFX908: ; %bb.0:
7990 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7991 ; GFX908-NEXT: ds_read_b32 v1, v0
7992 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
7993 ; GFX908-NEXT: .LBB28_1: ; %atomicrmw.start
7994 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
7995 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
7996 ; GFX908-NEXT: v_mov_b32_e32 v2, v1
7997 ; GFX908-NEXT: v_add_f32_e32 v1, -4.0, v2
7998 ; GFX908-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1
7999 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
8000 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
8001 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8002 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
8003 ; GFX908-NEXT: s_cbranch_execnz .LBB28_1
8004 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
8005 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
8006 ; GFX908-NEXT: v_mov_b32_e32 v0, v1
8007 ; GFX908-NEXT: s_setpc_b64 s[30:31]
8008 ;
8009 ; GFX8-LABEL: local_atomic_fsub_ret_f32__amdgpu_ignore_denormal_mode:
8010 ; GFX8: ; %bb.0:
8011 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8012 ; GFX8-NEXT: s_mov_b32 m0, -1
8013 ; GFX8-NEXT: ds_read_b32 v1, v0
8014 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
8015 ; GFX8-NEXT: .LBB28_1: ; %atomicrmw.start
8016 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
8017 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
8018 ; GFX8-NEXT: v_mov_b32_e32 v2, v1
8019 ; GFX8-NEXT: v_add_f32_e32 v1, -4.0, v2
8020 ; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1
8021 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
8022 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
8023 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8024 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
8025 ; GFX8-NEXT: s_cbranch_execnz .LBB28_1
8026 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
8027 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
8028 ; GFX8-NEXT: v_mov_b32_e32 v0, v1
8029 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8030 ;
8031 ; GFX7-LABEL: local_atomic_fsub_ret_f32__amdgpu_ignore_denormal_mode:
8032 ; GFX7: ; %bb.0:
8033 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8034 ; GFX7-NEXT: s_mov_b32 m0, -1
8035 ; GFX7-NEXT: ds_read_b32 v1, v0
8036 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
8037 ; GFX7-NEXT: .LBB28_1: ; %atomicrmw.start
8038 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
8039 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8040 ; GFX7-NEXT: v_mov_b32_e32 v2, v1
8041 ; GFX7-NEXT: v_add_f32_e32 v1, -4.0, v2
8042 ; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1
8043 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8044 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
8045 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8046 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
8047 ; GFX7-NEXT: s_cbranch_execnz .LBB28_1
8048 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
8049 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
8050 ; GFX7-NEXT: v_mov_b32_e32 v0, v1
8051 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8052 ;
8053 ; GFX6-LABEL: local_atomic_fsub_ret_f32__amdgpu_ignore_denormal_mode:
8054 ; GFX6: ; %bb.0:
8055 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8056 ; GFX6-NEXT: s_mov_b32 m0, -1
8057 ; GFX6-NEXT: ds_read_b32 v1, v0
8058 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
8059 ; GFX6-NEXT: .LBB28_1: ; %atomicrmw.start
8060 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
8061 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
8062 ; GFX6-NEXT: v_mov_b32_e32 v2, v1
8063 ; GFX6-NEXT: v_add_f32_e32 v1, -4.0, v2
8064 ; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1
8065 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
8066 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
8067 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8068 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
8069 ; GFX6-NEXT: s_cbranch_execnz .LBB28_1
8070 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
8071 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
8072 ; GFX6-NEXT: v_mov_b32_e32 v0, v1
8073 ; GFX6-NEXT: s_setpc_b64 s[30:31]
8074 %result = atomicrmw fsub ptr addrspace(3) %ptr, float 4.0 seq_cst, !amdgpu.ignore.denormal.mode !0
8075 ret float %result
8076 }
8078 define void @local_atomic_fsub_noret_f32__amdgpu_ignore_denormal_mode(ptr addrspace(3) %ptr) {
8079 ; GFX12-LABEL: local_atomic_fsub_noret_f32__amdgpu_ignore_denormal_mode:
8080 ; GFX12: ; %bb.0:
8081 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
8082 ; GFX12-NEXT: s_wait_expcnt 0x0
8083 ; GFX12-NEXT: s_wait_samplecnt 0x0
8084 ; GFX12-NEXT: s_wait_bvhcnt 0x0
8085 ; GFX12-NEXT: s_wait_kmcnt 0x0
8086 ; GFX12-NEXT: ds_load_b32 v1, v0
8087 ; GFX12-NEXT: s_mov_b32 s0, 0
8088 ; GFX12-NEXT: .LBB29_1: ; %atomicrmw.start
8089 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
8090 ; GFX12-NEXT: s_wait_dscnt 0x0
8091 ; GFX12-NEXT: v_add_f32_e32 v2, -4.0, v1
8092 ; GFX12-NEXT: s_wait_storecnt 0x0
8093 ; GFX12-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1
8094 ; GFX12-NEXT: s_wait_dscnt 0x0
8095 ; GFX12-NEXT: global_inv scope:SCOPE_SE
8096 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
8097 ; GFX12-NEXT: v_mov_b32_e32 v1, v2
8098 ; GFX12-NEXT: s_wait_alu 0xfffe
8099 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
8100 ; GFX12-NEXT: s_wait_alu 0xfffe
8101 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
8102 ; GFX12-NEXT: s_cbranch_execnz .LBB29_1
8103 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
8104 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
8105 ; GFX12-NEXT: s_setpc_b64 s[30:31]
8106 ;
8107 ; GFX940-LABEL: local_atomic_fsub_noret_f32__amdgpu_ignore_denormal_mode:
8108 ; GFX940: ; %bb.0:
8109 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8110 ; GFX940-NEXT: ds_read_b32 v1, v0
8111 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
8112 ; GFX940-NEXT: .LBB29_1: ; %atomicrmw.start
8113 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
8114 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
8115 ; GFX940-NEXT: v_add_f32_e32 v2, -4.0, v1
8116 ; GFX940-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
8117 ; GFX940-NEXT: s_waitcnt lgkmcnt(0)
8118 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
8119 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
8120 ; GFX940-NEXT: v_mov_b32_e32 v1, v2
8121 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
8122 ; GFX940-NEXT: s_cbranch_execnz .LBB29_1
8123 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
8124 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
8125 ; GFX940-NEXT: s_setpc_b64 s[30:31]
8126 ;
8127 ; GFX11-LABEL: local_atomic_fsub_noret_f32__amdgpu_ignore_denormal_mode:
8128 ; GFX11: ; %bb.0:
8129 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8130 ; GFX11-NEXT: ds_load_b32 v1, v0
8131 ; GFX11-NEXT: s_mov_b32 s0, 0
8132 ; GFX11-NEXT: .LBB29_1: ; %atomicrmw.start
8133 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
8134 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
8135 ; GFX11-NEXT: v_add_f32_e32 v2, -4.0, v1
8136 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
8137 ; GFX11-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1
8138 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
8139 ; GFX11-NEXT: buffer_gl0_inv
8140 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
8141 ; GFX11-NEXT: v_mov_b32_e32 v1, v2
8142 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
8143 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
8144 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
8145 ; GFX11-NEXT: s_cbranch_execnz .LBB29_1
8146 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
8147 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
8148 ; GFX11-NEXT: s_setpc_b64 s[30:31]
8149 ;
8150 ; GFX10-LABEL: local_atomic_fsub_noret_f32__amdgpu_ignore_denormal_mode:
8151 ; GFX10: ; %bb.0:
8152 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8153 ; GFX10-NEXT: ds_read_b32 v1, v0
8154 ; GFX10-NEXT: s_mov_b32 s4, 0
8155 ; GFX10-NEXT: .LBB29_1: ; %atomicrmw.start
8156 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
8157 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
8158 ; GFX10-NEXT: v_add_f32_e32 v2, -4.0, v1
8159 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
8160 ; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
8161 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
8162 ; GFX10-NEXT: buffer_gl0_inv
8163 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
8164 ; GFX10-NEXT: v_mov_b32_e32 v1, v2
8165 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
8166 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
8167 ; GFX10-NEXT: s_cbranch_execnz .LBB29_1
8168 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
8169 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
8170 ; GFX10-NEXT: s_setpc_b64 s[30:31]
8171 ;
8172 ; GFX90A-LABEL: local_atomic_fsub_noret_f32__amdgpu_ignore_denormal_mode:
8173 ; GFX90A: ; %bb.0:
8174 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8175 ; GFX90A-NEXT: ds_read_b32 v1, v0
8176 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
8177 ; GFX90A-NEXT: .LBB29_1: ; %atomicrmw.start
8178 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
8179 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
8180 ; GFX90A-NEXT: v_add_f32_e32 v2, -4.0, v1
8181 ; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
8182 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
8183 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
8184 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8185 ; GFX90A-NEXT: v_mov_b32_e32 v1, v2
8186 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
8187 ; GFX90A-NEXT: s_cbranch_execnz .LBB29_1
8188 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
8189 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
8190 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
8191 ;
8192 ; GFX908-LABEL: local_atomic_fsub_noret_f32__amdgpu_ignore_denormal_mode:
8193 ; GFX908: ; %bb.0:
8194 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8195 ; GFX908-NEXT: ds_read_b32 v1, v0
8196 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
8197 ; GFX908-NEXT: .LBB29_1: ; %atomicrmw.start
8198 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
8199 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
8200 ; GFX908-NEXT: v_add_f32_e32 v2, -4.0, v1
8201 ; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
8202 ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
8203 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
8204 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8205 ; GFX908-NEXT: v_mov_b32_e32 v1, v2
8206 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
8207 ; GFX908-NEXT: s_cbranch_execnz .LBB29_1
8208 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
8209 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
8210 ; GFX908-NEXT: s_setpc_b64 s[30:31]
8211 ;
8212 ; GFX8-LABEL: local_atomic_fsub_noret_f32__amdgpu_ignore_denormal_mode:
8213 ; GFX8: ; %bb.0:
8214 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8215 ; GFX8-NEXT: s_mov_b32 m0, -1
8216 ; GFX8-NEXT: ds_read_b32 v1, v0
8217 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
8218 ; GFX8-NEXT: .LBB29_1: ; %atomicrmw.start
8219 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
8220 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
8221 ; GFX8-NEXT: v_add_f32_e32 v2, -4.0, v1
8222 ; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
8223 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
8224 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
8225 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8226 ; GFX8-NEXT: v_mov_b32_e32 v1, v2
8227 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
8228 ; GFX8-NEXT: s_cbranch_execnz .LBB29_1
8229 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
8230 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
8231 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8232 ;
8233 ; GFX7-LABEL: local_atomic_fsub_noret_f32__amdgpu_ignore_denormal_mode:
8234 ; GFX7: ; %bb.0:
8235 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8236 ; GFX7-NEXT: s_mov_b32 m0, -1
8237 ; GFX7-NEXT: ds_read_b32 v1, v0
8238 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
8239 ; GFX7-NEXT: .LBB29_1: ; %atomicrmw.start
8240 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
8241 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8242 ; GFX7-NEXT: v_add_f32_e32 v2, -4.0, v1
8243 ; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
8244 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8245 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
8246 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8247 ; GFX7-NEXT: v_mov_b32_e32 v1, v2
8248 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
8249 ; GFX7-NEXT: s_cbranch_execnz .LBB29_1
8250 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
8251 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
8252 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8253 ;
8254 ; GFX6-LABEL: local_atomic_fsub_noret_f32__amdgpu_ignore_denormal_mode:
8255 ; GFX6: ; %bb.0:
8256 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8257 ; GFX6-NEXT: s_mov_b32 m0, -1
8258 ; GFX6-NEXT: ds_read_b32 v1, v0
8259 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
8260 ; GFX6-NEXT: .LBB29_1: ; %atomicrmw.start
8261 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
8262 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
8263 ; GFX6-NEXT: v_add_f32_e32 v2, -4.0, v1
8264 ; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
8265 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
8266 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
8267 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8268 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
8269 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
8270 ; GFX6-NEXT: s_cbranch_execnz .LBB29_1
8271 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
8272 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
8273 ; GFX6-NEXT: s_setpc_b64 s[30:31]
8274 %result = atomicrmw fsub ptr addrspace(3) %ptr, float 4.0 seq_cst, !amdgpu.ignore.denormal.mode !0
8275 ret void
8276 }
8278 !0 = !{}