| blob | e2fde562d36b11a550964936cef5d004582d9b39 |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
2 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
3 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx940 < %s | FileCheck -check-prefix=GFX940 %s
4 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 < %s | FileCheck -check-prefix=GFX11 %s
5 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
6 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx90a < %s | FileCheck -check-prefix=GFX90A %s
7 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx908 < %s | FileCheck -check-prefix=GFX908 %s
8 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=tonga < %s | FileCheck -check-prefix=GFX8 %s
9 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=hawaii < %s | FileCheck -check-prefix=GFX7 %s
10 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=tahiti < %s | FileCheck -check-prefix=GFX6 %s
12 ; --------------------------------------------------------------------
13 ; float
14 ; --------------------------------------------------------------------
16 define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
17 ; GFX12-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
18 ; GFX12: ; %bb.0:
19 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
20 ; GFX12-NEXT: s_wait_expcnt 0x0
21 ; GFX12-NEXT: s_wait_samplecnt 0x0
22 ; GFX12-NEXT: s_wait_bvhcnt 0x0
23 ; GFX12-NEXT: s_wait_kmcnt 0x0
24 ; GFX12-NEXT: s_wait_storecnt 0x0
25 ; GFX12-NEXT: global_atomic_max_num_f32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
26 ; GFX12-NEXT: s_wait_loadcnt 0x0
27 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
28 ; GFX12-NEXT: s_setpc_b64 s[30:31]
29 ;
30 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
31 ; GFX940: ; %bb.0:
32 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
33 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
34 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
35 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
36 ; GFX940-NEXT: .LBB0_1: ; %atomicrmw.start
37 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
38 ; GFX940-NEXT: s_waitcnt vmcnt(0)
39 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
40 ; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
41 ; GFX940-NEXT: v_max_f32_e32 v4, v3, v2
42 ; GFX940-NEXT: buffer_wbl2 sc1
43 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
44 ; GFX940-NEXT: s_waitcnt vmcnt(0)
45 ; GFX940-NEXT: buffer_inv sc1
46 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
47 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
48 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
49 ; GFX940-NEXT: s_cbranch_execnz .LBB0_1
50 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
51 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
52 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
53 ; GFX940-NEXT: s_setpc_b64 s[30:31]
54 ;
55 ; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
56 ; GFX11: ; %bb.0:
57 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
58 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
59 ; GFX11-NEXT: global_atomic_max_f32 v0, v[0:1], v2, off glc
60 ; GFX11-NEXT: s_waitcnt vmcnt(0)
61 ; GFX11-NEXT: buffer_gl1_inv
62 ; GFX11-NEXT: buffer_gl0_inv
63 ; GFX11-NEXT: s_setpc_b64 s[30:31]
64 ;
65 ; GFX10-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
66 ; GFX10: ; %bb.0:
67 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
68 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
69 ; GFX10-NEXT: global_atomic_fmax v0, v[0:1], v2, off glc
70 ; GFX10-NEXT: s_waitcnt vmcnt(0)
71 ; GFX10-NEXT: buffer_gl1_inv
72 ; GFX10-NEXT: buffer_gl0_inv
73 ; GFX10-NEXT: s_setpc_b64 s[30:31]
74 ;
75 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
76 ; GFX90A: ; %bb.0:
77 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
78 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
79 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
80 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
81 ; GFX90A-NEXT: .LBB0_1: ; %atomicrmw.start
82 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
83 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
84 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
85 ; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
86 ; GFX90A-NEXT: v_max_f32_e32 v4, v3, v2
87 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
88 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
89 ; GFX90A-NEXT: buffer_wbinvl1
90 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
91 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
92 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
93 ; GFX90A-NEXT: s_cbranch_execnz .LBB0_1
94 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
95 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
96 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
97 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
98 ;
99 ; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
100 ; GFX908: ; %bb.0:
101 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
102 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
103 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
104 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
105 ; GFX908-NEXT: .LBB0_1: ; %atomicrmw.start
106 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
107 ; GFX908-NEXT: s_waitcnt vmcnt(0)
108 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
109 ; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
110 ; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
111 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
112 ; GFX908-NEXT: s_waitcnt vmcnt(0)
113 ; GFX908-NEXT: buffer_wbinvl1
114 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
115 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
116 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
117 ; GFX908-NEXT: s_cbranch_execnz .LBB0_1
118 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
119 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
120 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
121 ; GFX908-NEXT: s_setpc_b64 s[30:31]
122 ;
123 ; GFX8-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
124 ; GFX8: ; %bb.0:
125 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
126 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
127 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
128 ; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v2
129 ; GFX8-NEXT: .LBB0_1: ; %atomicrmw.start
130 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
131 ; GFX8-NEXT: s_waitcnt vmcnt(0)
132 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
133 ; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v4
134 ; GFX8-NEXT: v_max_f32_e32 v3, v3, v2
135 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
136 ; GFX8-NEXT: s_waitcnt vmcnt(0)
137 ; GFX8-NEXT: buffer_wbinvl1
138 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
139 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
140 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
141 ; GFX8-NEXT: s_cbranch_execnz .LBB0_1
142 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
143 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
144 ; GFX8-NEXT: v_mov_b32_e32 v0, v3
145 ; GFX8-NEXT: s_setpc_b64 s[30:31]
146 ;
147 ; GFX7-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
148 ; GFX7: ; %bb.0:
149 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
150 ; GFX7-NEXT: s_mov_b32 s6, 0
151 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
152 ; GFX7-NEXT: s_mov_b32 s4, s6
153 ; GFX7-NEXT: s_mov_b32 s5, s6
154 ; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 glc
155 ; GFX7-NEXT: s_waitcnt vmcnt(0)
156 ; GFX7-NEXT: buffer_wbinvl1
157 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
158 ; GFX7-NEXT: s_setpc_b64 s[30:31]
159 ;
160 ; GFX6-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
161 ; GFX6: ; %bb.0:
162 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
163 ; GFX6-NEXT: s_mov_b32 s6, 0
164 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
165 ; GFX6-NEXT: s_mov_b32 s4, s6
166 ; GFX6-NEXT: s_mov_b32 s5, s6
167 ; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 glc
168 ; GFX6-NEXT: s_waitcnt vmcnt(0)
169 ; GFX6-NEXT: buffer_wbinvl1
170 ; GFX6-NEXT: v_mov_b32_e32 v0, v2
171 ; GFX6-NEXT: s_waitcnt expcnt(0)
172 ; GFX6-NEXT: s_setpc_b64 s[30:31]
173 %result = atomicrmw fmax ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
174 ret float %result
175 }
177 define float @global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
178 ; GFX12-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
179 ; GFX12: ; %bb.0:
180 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
181 ; GFX12-NEXT: s_wait_expcnt 0x0
182 ; GFX12-NEXT: s_wait_samplecnt 0x0
183 ; GFX12-NEXT: s_wait_bvhcnt 0x0
184 ; GFX12-NEXT: s_wait_kmcnt 0x0
185 ; GFX12-NEXT: s_wait_storecnt 0x0
186 ; GFX12-NEXT: global_atomic_max_num_f32 v0, v[0:1], v2, off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
187 ; GFX12-NEXT: s_wait_loadcnt 0x0
188 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
189 ; GFX12-NEXT: s_setpc_b64 s[30:31]
190 ;
191 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
192 ; GFX940: ; %bb.0:
193 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
194 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
195 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
196 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
197 ; GFX940-NEXT: .LBB1_1: ; %atomicrmw.start
198 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
199 ; GFX940-NEXT: s_waitcnt vmcnt(0)
200 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
201 ; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
202 ; GFX940-NEXT: v_max_f32_e32 v4, v3, v2
203 ; GFX940-NEXT: buffer_wbl2 sc1
204 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
205 ; GFX940-NEXT: s_waitcnt vmcnt(0)
206 ; GFX940-NEXT: buffer_inv sc1
207 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
208 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
209 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
210 ; GFX940-NEXT: s_cbranch_execnz .LBB1_1
211 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
212 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
213 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
214 ; GFX940-NEXT: s_setpc_b64 s[30:31]
215 ;
216 ; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
217 ; GFX11: ; %bb.0:
218 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
219 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
220 ; GFX11-NEXT: global_atomic_max_f32 v0, v[0:1], v2, off offset:2044 glc
221 ; GFX11-NEXT: s_waitcnt vmcnt(0)
222 ; GFX11-NEXT: buffer_gl1_inv
223 ; GFX11-NEXT: buffer_gl0_inv
224 ; GFX11-NEXT: s_setpc_b64 s[30:31]
225 ;
226 ; GFX10-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
227 ; GFX10: ; %bb.0:
228 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
229 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
230 ; GFX10-NEXT: global_atomic_fmax v0, v[0:1], v2, off offset:2044 glc
231 ; GFX10-NEXT: s_waitcnt vmcnt(0)
232 ; GFX10-NEXT: buffer_gl1_inv
233 ; GFX10-NEXT: buffer_gl0_inv
234 ; GFX10-NEXT: s_setpc_b64 s[30:31]
235 ;
236 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
237 ; GFX90A: ; %bb.0:
238 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
239 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
240 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
241 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
242 ; GFX90A-NEXT: .LBB1_1: ; %atomicrmw.start
243 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
244 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
245 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
246 ; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
247 ; GFX90A-NEXT: v_max_f32_e32 v4, v3, v2
248 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
249 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
250 ; GFX90A-NEXT: buffer_wbinvl1
251 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
252 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
253 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
254 ; GFX90A-NEXT: s_cbranch_execnz .LBB1_1
255 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
256 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
257 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
258 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
259 ;
260 ; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
261 ; GFX908: ; %bb.0:
262 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
263 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
264 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
265 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
266 ; GFX908-NEXT: .LBB1_1: ; %atomicrmw.start
267 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
268 ; GFX908-NEXT: s_waitcnt vmcnt(0)
269 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
270 ; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
271 ; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
272 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
273 ; GFX908-NEXT: s_waitcnt vmcnt(0)
274 ; GFX908-NEXT: buffer_wbinvl1
275 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
276 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
277 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
278 ; GFX908-NEXT: s_cbranch_execnz .LBB1_1
279 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
280 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
281 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
282 ; GFX908-NEXT: s_setpc_b64 s[30:31]
283 ;
284 ; GFX8-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
285 ; GFX8: ; %bb.0:
286 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
287 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
288 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
289 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
290 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
291 ; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
292 ; GFX8-NEXT: .LBB1_1: ; %atomicrmw.start
293 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
294 ; GFX8-NEXT: s_waitcnt vmcnt(0)
295 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
296 ; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
297 ; GFX8-NEXT: v_max_f32_e32 v5, v0, v1
298 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
299 ; GFX8-NEXT: s_waitcnt vmcnt(0)
300 ; GFX8-NEXT: buffer_wbinvl1
301 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
302 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
303 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
304 ; GFX8-NEXT: s_cbranch_execnz .LBB1_1
305 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
306 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
307 ; GFX8-NEXT: s_setpc_b64 s[30:31]
308 ;
309 ; GFX7-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
310 ; GFX7: ; %bb.0:
311 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
312 ; GFX7-NEXT: s_mov_b32 s6, 0
313 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
314 ; GFX7-NEXT: s_mov_b32 s4, s6
315 ; GFX7-NEXT: s_mov_b32 s5, s6
316 ; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044 glc
317 ; GFX7-NEXT: s_waitcnt vmcnt(0)
318 ; GFX7-NEXT: buffer_wbinvl1
319 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
320 ; GFX7-NEXT: s_setpc_b64 s[30:31]
321 ;
322 ; GFX6-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
323 ; GFX6: ; %bb.0:
324 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
325 ; GFX6-NEXT: s_mov_b32 s6, 0
326 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
327 ; GFX6-NEXT: s_mov_b32 s4, s6
328 ; GFX6-NEXT: s_mov_b32 s5, s6
329 ; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044 glc
330 ; GFX6-NEXT: s_waitcnt vmcnt(0)
331 ; GFX6-NEXT: buffer_wbinvl1
332 ; GFX6-NEXT: v_mov_b32_e32 v0, v2
333 ; GFX6-NEXT: s_waitcnt expcnt(0)
334 ; GFX6-NEXT: s_setpc_b64 s[30:31]
335 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
336 %result = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
337 ret float %result
338 }
340 define float @global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
341 ; GFX12-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
342 ; GFX12: ; %bb.0:
343 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
344 ; GFX12-NEXT: s_wait_expcnt 0x0
345 ; GFX12-NEXT: s_wait_samplecnt 0x0
346 ; GFX12-NEXT: s_wait_bvhcnt 0x0
347 ; GFX12-NEXT: s_wait_kmcnt 0x0
348 ; GFX12-NEXT: s_wait_storecnt 0x0
349 ; GFX12-NEXT: global_atomic_max_num_f32 v0, v[0:1], v2, off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
350 ; GFX12-NEXT: s_wait_loadcnt 0x0
351 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
352 ; GFX12-NEXT: s_setpc_b64 s[30:31]
353 ;
354 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
355 ; GFX940: ; %bb.0:
356 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
357 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
358 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
359 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
360 ; GFX940-NEXT: .LBB2_1: ; %atomicrmw.start
361 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
362 ; GFX940-NEXT: s_waitcnt vmcnt(0)
363 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
364 ; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
365 ; GFX940-NEXT: v_max_f32_e32 v4, v3, v2
366 ; GFX940-NEXT: buffer_wbl2 sc1
367 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
368 ; GFX940-NEXT: s_waitcnt vmcnt(0)
369 ; GFX940-NEXT: buffer_inv sc1
370 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
371 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
372 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
373 ; GFX940-NEXT: s_cbranch_execnz .LBB2_1
374 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
375 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
376 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
377 ; GFX940-NEXT: s_setpc_b64 s[30:31]
378 ;
379 ; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
380 ; GFX11: ; %bb.0:
381 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
382 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
383 ; GFX11-NEXT: global_atomic_max_f32 v0, v[0:1], v2, off offset:-2048 glc
384 ; GFX11-NEXT: s_waitcnt vmcnt(0)
385 ; GFX11-NEXT: buffer_gl1_inv
386 ; GFX11-NEXT: buffer_gl0_inv
387 ; GFX11-NEXT: s_setpc_b64 s[30:31]
388 ;
389 ; GFX10-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
390 ; GFX10: ; %bb.0:
391 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
392 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
393 ; GFX10-NEXT: global_atomic_fmax v0, v[0:1], v2, off offset:-2048 glc
394 ; GFX10-NEXT: s_waitcnt vmcnt(0)
395 ; GFX10-NEXT: buffer_gl1_inv
396 ; GFX10-NEXT: buffer_gl0_inv
397 ; GFX10-NEXT: s_setpc_b64 s[30:31]
398 ;
399 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
400 ; GFX90A: ; %bb.0:
401 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
402 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
403 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
404 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
405 ; GFX90A-NEXT: .LBB2_1: ; %atomicrmw.start
406 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
407 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
408 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
409 ; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
410 ; GFX90A-NEXT: v_max_f32_e32 v4, v3, v2
411 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
412 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
413 ; GFX90A-NEXT: buffer_wbinvl1
414 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
415 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
416 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
417 ; GFX90A-NEXT: s_cbranch_execnz .LBB2_1
418 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
419 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
420 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
421 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
422 ;
423 ; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
424 ; GFX908: ; %bb.0:
425 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
426 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
427 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
428 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
429 ; GFX908-NEXT: .LBB2_1: ; %atomicrmw.start
430 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
431 ; GFX908-NEXT: s_waitcnt vmcnt(0)
432 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
433 ; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
434 ; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
435 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
436 ; GFX908-NEXT: s_waitcnt vmcnt(0)
437 ; GFX908-NEXT: buffer_wbinvl1
438 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
439 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
440 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
441 ; GFX908-NEXT: s_cbranch_execnz .LBB2_1
442 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
443 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
444 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
445 ; GFX908-NEXT: s_setpc_b64 s[30:31]
446 ;
447 ; GFX8-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
448 ; GFX8: ; %bb.0:
449 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
450 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
451 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
452 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
453 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
454 ; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
455 ; GFX8-NEXT: .LBB2_1: ; %atomicrmw.start
456 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
457 ; GFX8-NEXT: s_waitcnt vmcnt(0)
458 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
459 ; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
460 ; GFX8-NEXT: v_max_f32_e32 v5, v0, v1
461 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
462 ; GFX8-NEXT: s_waitcnt vmcnt(0)
463 ; GFX8-NEXT: buffer_wbinvl1
464 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
465 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
466 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
467 ; GFX8-NEXT: s_cbranch_execnz .LBB2_1
468 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
469 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
470 ; GFX8-NEXT: s_setpc_b64 s[30:31]
471 ;
472 ; GFX7-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
473 ; GFX7: ; %bb.0:
474 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
475 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
476 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
477 ; GFX7-NEXT: s_mov_b32 s6, 0
478 ; GFX7-NEXT: s_mov_b32 s5, -1
479 ; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 glc
480 ; GFX7-NEXT: s_waitcnt vmcnt(0)
481 ; GFX7-NEXT: buffer_wbinvl1
482 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
483 ; GFX7-NEXT: s_setpc_b64 s[30:31]
484 ;
485 ; GFX6-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
486 ; GFX6: ; %bb.0:
487 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
488 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
489 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
490 ; GFX6-NEXT: s_mov_b32 s6, 0
491 ; GFX6-NEXT: s_mov_b32 s5, -1
492 ; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 glc
493 ; GFX6-NEXT: s_waitcnt vmcnt(0)
494 ; GFX6-NEXT: buffer_wbinvl1
495 ; GFX6-NEXT: v_mov_b32_e32 v0, v2
496 ; GFX6-NEXT: s_waitcnt expcnt(0)
497 ; GFX6-NEXT: s_setpc_b64 s[30:31]
498 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
499 %result = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
500 ret float %result
501 }
503 define void @global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
504 ; GFX12-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
505 ; GFX12: ; %bb.0:
506 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
507 ; GFX12-NEXT: s_wait_expcnt 0x0
508 ; GFX12-NEXT: s_wait_samplecnt 0x0
509 ; GFX12-NEXT: s_wait_bvhcnt 0x0
510 ; GFX12-NEXT: s_wait_kmcnt 0x0
511 ; GFX12-NEXT: s_wait_storecnt 0x0
512 ; GFX12-NEXT: global_atomic_max_num_f32 v[0:1], v2, off scope:SCOPE_DEV
513 ; GFX12-NEXT: s_wait_storecnt 0x0
514 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
515 ; GFX12-NEXT: s_setpc_b64 s[30:31]
516 ;
517 ; GFX940-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
518 ; GFX940: ; %bb.0:
519 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
520 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
521 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
522 ; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
523 ; GFX940-NEXT: .LBB3_1: ; %atomicrmw.start
524 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
525 ; GFX940-NEXT: s_waitcnt vmcnt(0)
526 ; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
527 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
528 ; GFX940-NEXT: buffer_wbl2 sc1
529 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
530 ; GFX940-NEXT: s_waitcnt vmcnt(0)
531 ; GFX940-NEXT: buffer_inv sc1
532 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
533 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
534 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
535 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
536 ; GFX940-NEXT: s_cbranch_execnz .LBB3_1
537 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
538 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
539 ; GFX940-NEXT: s_setpc_b64 s[30:31]
540 ;
541 ; GFX11-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
542 ; GFX11: ; %bb.0:
543 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
544 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
545 ; GFX11-NEXT: global_atomic_max_f32 v[0:1], v2, off
546 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
547 ; GFX11-NEXT: buffer_gl1_inv
548 ; GFX11-NEXT: buffer_gl0_inv
549 ; GFX11-NEXT: s_setpc_b64 s[30:31]
550 ;
551 ; GFX10-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
552 ; GFX10: ; %bb.0:
553 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
554 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
555 ; GFX10-NEXT: global_atomic_fmax v[0:1], v2, off
556 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
557 ; GFX10-NEXT: buffer_gl1_inv
558 ; GFX10-NEXT: buffer_gl0_inv
559 ; GFX10-NEXT: s_setpc_b64 s[30:31]
560 ;
561 ; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
562 ; GFX90A: ; %bb.0:
563 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
564 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
565 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
566 ; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
567 ; GFX90A-NEXT: .LBB3_1: ; %atomicrmw.start
568 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
569 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
570 ; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
571 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
572 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
573 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
574 ; GFX90A-NEXT: buffer_wbinvl1
575 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
576 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
577 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
578 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
579 ; GFX90A-NEXT: s_cbranch_execnz .LBB3_1
580 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
581 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
582 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
583 ;
584 ; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
585 ; GFX908: ; %bb.0:
586 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
587 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
588 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
589 ; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
590 ; GFX908-NEXT: .LBB3_1: ; %atomicrmw.start
591 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
592 ; GFX908-NEXT: s_waitcnt vmcnt(0)
593 ; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
594 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
595 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
596 ; GFX908-NEXT: s_waitcnt vmcnt(0)
597 ; GFX908-NEXT: buffer_wbinvl1
598 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
599 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
600 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
601 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
602 ; GFX908-NEXT: s_cbranch_execnz .LBB3_1
603 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
604 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
605 ; GFX908-NEXT: s_setpc_b64 s[30:31]
606 ;
607 ; GFX8-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
608 ; GFX8: ; %bb.0:
609 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
610 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
611 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
612 ; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
613 ; GFX8-NEXT: .LBB3_1: ; %atomicrmw.start
614 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
615 ; GFX8-NEXT: s_waitcnt vmcnt(0)
616 ; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
617 ; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
618 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
619 ; GFX8-NEXT: s_waitcnt vmcnt(0)
620 ; GFX8-NEXT: buffer_wbinvl1
621 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
622 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
623 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
624 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
625 ; GFX8-NEXT: s_cbranch_execnz .LBB3_1
626 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
627 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
628 ; GFX8-NEXT: s_setpc_b64 s[30:31]
629 ;
630 ; GFX7-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
631 ; GFX7: ; %bb.0:
632 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
633 ; GFX7-NEXT: s_mov_b32 s6, 0
634 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
635 ; GFX7-NEXT: s_mov_b32 s4, s6
636 ; GFX7-NEXT: s_mov_b32 s5, s6
637 ; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64
638 ; GFX7-NEXT: s_waitcnt vmcnt(0)
639 ; GFX7-NEXT: buffer_wbinvl1
640 ; GFX7-NEXT: s_setpc_b64 s[30:31]
641 ;
642 ; GFX6-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
643 ; GFX6: ; %bb.0:
644 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
645 ; GFX6-NEXT: s_mov_b32 s6, 0
646 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
647 ; GFX6-NEXT: s_mov_b32 s4, s6
648 ; GFX6-NEXT: s_mov_b32 s5, s6
649 ; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64
650 ; GFX6-NEXT: s_waitcnt vmcnt(0)
651 ; GFX6-NEXT: buffer_wbinvl1
652 ; GFX6-NEXT: s_waitcnt expcnt(0)
653 ; GFX6-NEXT: s_setpc_b64 s[30:31]
654 %unused = atomicrmw fmax ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
655 ret void
656 }
658 define void @global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
659 ; GFX12-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
660 ; GFX12: ; %bb.0:
661 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
662 ; GFX12-NEXT: s_wait_expcnt 0x0
663 ; GFX12-NEXT: s_wait_samplecnt 0x0
664 ; GFX12-NEXT: s_wait_bvhcnt 0x0
665 ; GFX12-NEXT: s_wait_kmcnt 0x0
666 ; GFX12-NEXT: s_wait_storecnt 0x0
667 ; GFX12-NEXT: global_atomic_max_num_f32 v[0:1], v2, off offset:2044 scope:SCOPE_DEV
668 ; GFX12-NEXT: s_wait_storecnt 0x0
669 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
670 ; GFX12-NEXT: s_setpc_b64 s[30:31]
671 ;
672 ; GFX940-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
673 ; GFX940: ; %bb.0:
674 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
675 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
676 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
677 ; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
678 ; GFX940-NEXT: .LBB4_1: ; %atomicrmw.start
679 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
680 ; GFX940-NEXT: s_waitcnt vmcnt(0)
681 ; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
682 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
683 ; GFX940-NEXT: buffer_wbl2 sc1
684 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
685 ; GFX940-NEXT: s_waitcnt vmcnt(0)
686 ; GFX940-NEXT: buffer_inv sc1
687 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
688 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
689 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
690 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
691 ; GFX940-NEXT: s_cbranch_execnz .LBB4_1
692 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
693 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
694 ; GFX940-NEXT: s_setpc_b64 s[30:31]
695 ;
696 ; GFX11-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
697 ; GFX11: ; %bb.0:
698 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
699 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
700 ; GFX11-NEXT: global_atomic_max_f32 v[0:1], v2, off offset:2044
701 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
702 ; GFX11-NEXT: buffer_gl1_inv
703 ; GFX11-NEXT: buffer_gl0_inv
704 ; GFX11-NEXT: s_setpc_b64 s[30:31]
705 ;
706 ; GFX10-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
707 ; GFX10: ; %bb.0:
708 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
709 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
710 ; GFX10-NEXT: global_atomic_fmax v[0:1], v2, off offset:2044
711 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
712 ; GFX10-NEXT: buffer_gl1_inv
713 ; GFX10-NEXT: buffer_gl0_inv
714 ; GFX10-NEXT: s_setpc_b64 s[30:31]
715 ;
716 ; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
717 ; GFX90A: ; %bb.0:
718 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
719 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
720 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
721 ; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
722 ; GFX90A-NEXT: .LBB4_1: ; %atomicrmw.start
723 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
724 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
725 ; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
726 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
727 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
728 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
729 ; GFX90A-NEXT: buffer_wbinvl1
730 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
731 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
732 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
733 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
734 ; GFX90A-NEXT: s_cbranch_execnz .LBB4_1
735 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
736 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
737 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
738 ;
739 ; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
740 ; GFX908: ; %bb.0:
741 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
742 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
743 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
744 ; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
745 ; GFX908-NEXT: .LBB4_1: ; %atomicrmw.start
746 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
747 ; GFX908-NEXT: s_waitcnt vmcnt(0)
748 ; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
749 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
750 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
751 ; GFX908-NEXT: s_waitcnt vmcnt(0)
752 ; GFX908-NEXT: buffer_wbinvl1
753 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
754 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
755 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
756 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
757 ; GFX908-NEXT: s_cbranch_execnz .LBB4_1
758 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
759 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
760 ; GFX908-NEXT: s_setpc_b64 s[30:31]
761 ;
762 ; GFX8-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
763 ; GFX8: ; %bb.0:
764 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
765 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
766 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
767 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
768 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
769 ; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
770 ; GFX8-NEXT: .LBB4_1: ; %atomicrmw.start
771 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
772 ; GFX8-NEXT: s_waitcnt vmcnt(0)
773 ; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
774 ; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
775 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
776 ; GFX8-NEXT: s_waitcnt vmcnt(0)
777 ; GFX8-NEXT: buffer_wbinvl1
778 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
779 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
780 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
781 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
782 ; GFX8-NEXT: s_cbranch_execnz .LBB4_1
783 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
784 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
785 ; GFX8-NEXT: s_setpc_b64 s[30:31]
786 ;
787 ; GFX7-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
788 ; GFX7: ; %bb.0:
789 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
790 ; GFX7-NEXT: s_mov_b32 s6, 0
791 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
792 ; GFX7-NEXT: s_mov_b32 s4, s6
793 ; GFX7-NEXT: s_mov_b32 s5, s6
794 ; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044
795 ; GFX7-NEXT: s_waitcnt vmcnt(0)
796 ; GFX7-NEXT: buffer_wbinvl1
797 ; GFX7-NEXT: s_setpc_b64 s[30:31]
798 ;
799 ; GFX6-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
800 ; GFX6: ; %bb.0:
801 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
802 ; GFX6-NEXT: s_mov_b32 s6, 0
803 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
804 ; GFX6-NEXT: s_mov_b32 s4, s6
805 ; GFX6-NEXT: s_mov_b32 s5, s6
806 ; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044
807 ; GFX6-NEXT: s_waitcnt vmcnt(0)
808 ; GFX6-NEXT: buffer_wbinvl1
809 ; GFX6-NEXT: s_waitcnt expcnt(0)
810 ; GFX6-NEXT: s_setpc_b64 s[30:31]
811 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
812 %unused = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
813 ret void
814 }
816 define void @global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
817 ; GFX12-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
818 ; GFX12: ; %bb.0:
819 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
820 ; GFX12-NEXT: s_wait_expcnt 0x0
821 ; GFX12-NEXT: s_wait_samplecnt 0x0
822 ; GFX12-NEXT: s_wait_bvhcnt 0x0
823 ; GFX12-NEXT: s_wait_kmcnt 0x0
824 ; GFX12-NEXT: s_wait_storecnt 0x0
825 ; GFX12-NEXT: global_atomic_max_num_f32 v[0:1], v2, off offset:-2048 scope:SCOPE_DEV
826 ; GFX12-NEXT: s_wait_storecnt 0x0
827 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
828 ; GFX12-NEXT: s_setpc_b64 s[30:31]
829 ;
830 ; GFX940-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
831 ; GFX940: ; %bb.0:
832 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
833 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
834 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
835 ; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
836 ; GFX940-NEXT: .LBB5_1: ; %atomicrmw.start
837 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
838 ; GFX940-NEXT: s_waitcnt vmcnt(0)
839 ; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
840 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
841 ; GFX940-NEXT: buffer_wbl2 sc1
842 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
843 ; GFX940-NEXT: s_waitcnt vmcnt(0)
844 ; GFX940-NEXT: buffer_inv sc1
845 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
846 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
847 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
848 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
849 ; GFX940-NEXT: s_cbranch_execnz .LBB5_1
850 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
851 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
852 ; GFX940-NEXT: s_setpc_b64 s[30:31]
853 ;
854 ; GFX11-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
855 ; GFX11: ; %bb.0:
856 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
857 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
858 ; GFX11-NEXT: global_atomic_max_f32 v[0:1], v2, off offset:-2048
859 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
860 ; GFX11-NEXT: buffer_gl1_inv
861 ; GFX11-NEXT: buffer_gl0_inv
862 ; GFX11-NEXT: s_setpc_b64 s[30:31]
863 ;
864 ; GFX10-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
865 ; GFX10: ; %bb.0:
866 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
867 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
868 ; GFX10-NEXT: global_atomic_fmax v[0:1], v2, off offset:-2048
869 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
870 ; GFX10-NEXT: buffer_gl1_inv
871 ; GFX10-NEXT: buffer_gl0_inv
872 ; GFX10-NEXT: s_setpc_b64 s[30:31]
873 ;
874 ; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
875 ; GFX90A: ; %bb.0:
876 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
877 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
878 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
879 ; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
880 ; GFX90A-NEXT: .LBB5_1: ; %atomicrmw.start
881 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
882 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
883 ; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
884 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
885 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
886 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
887 ; GFX90A-NEXT: buffer_wbinvl1
888 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
889 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
890 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
891 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
892 ; GFX90A-NEXT: s_cbranch_execnz .LBB5_1
893 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
894 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
895 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
896 ;
897 ; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
898 ; GFX908: ; %bb.0:
899 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
900 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
901 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
902 ; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
903 ; GFX908-NEXT: .LBB5_1: ; %atomicrmw.start
904 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
905 ; GFX908-NEXT: s_waitcnt vmcnt(0)
906 ; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
907 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
908 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
909 ; GFX908-NEXT: s_waitcnt vmcnt(0)
910 ; GFX908-NEXT: buffer_wbinvl1
911 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
912 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
913 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
914 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
915 ; GFX908-NEXT: s_cbranch_execnz .LBB5_1
916 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
917 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
918 ; GFX908-NEXT: s_setpc_b64 s[30:31]
919 ;
920 ; GFX8-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
921 ; GFX8: ; %bb.0:
922 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
923 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
924 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
925 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
926 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
927 ; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
928 ; GFX8-NEXT: .LBB5_1: ; %atomicrmw.start
929 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
930 ; GFX8-NEXT: s_waitcnt vmcnt(0)
931 ; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
932 ; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
933 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
934 ; GFX8-NEXT: s_waitcnt vmcnt(0)
935 ; GFX8-NEXT: buffer_wbinvl1
936 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
937 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
938 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
939 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
940 ; GFX8-NEXT: s_cbranch_execnz .LBB5_1
941 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
942 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
943 ; GFX8-NEXT: s_setpc_b64 s[30:31]
944 ;
945 ; GFX7-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
946 ; GFX7: ; %bb.0:
947 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
948 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
949 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
950 ; GFX7-NEXT: s_mov_b32 s6, 0
951 ; GFX7-NEXT: s_mov_b32 s5, -1
952 ; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64
953 ; GFX7-NEXT: s_waitcnt vmcnt(0)
954 ; GFX7-NEXT: buffer_wbinvl1
955 ; GFX7-NEXT: s_setpc_b64 s[30:31]
956 ;
957 ; GFX6-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
958 ; GFX6: ; %bb.0:
959 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
960 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
961 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
962 ; GFX6-NEXT: s_mov_b32 s6, 0
963 ; GFX6-NEXT: s_mov_b32 s5, -1
964 ; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64
965 ; GFX6-NEXT: s_waitcnt vmcnt(0)
966 ; GFX6-NEXT: buffer_wbinvl1
967 ; GFX6-NEXT: s_waitcnt expcnt(0)
968 ; GFX6-NEXT: s_setpc_b64 s[30:31]
969 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
970 %unused = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
971 ret void
972 }
974 define float @global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
975 ; GFX12-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
976 ; GFX12: ; %bb.0:
977 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
978 ; GFX12-NEXT: s_wait_expcnt 0x0
979 ; GFX12-NEXT: s_wait_samplecnt 0x0
980 ; GFX12-NEXT: s_wait_bvhcnt 0x0
981 ; GFX12-NEXT: s_wait_kmcnt 0x0
982 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
983 ; GFX12-NEXT: s_wait_storecnt 0x0
984 ; GFX12-NEXT: global_atomic_max_num_f32 v0, v[0:1], v2, off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
985 ; GFX12-NEXT: s_wait_loadcnt 0x0
986 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
987 ; GFX12-NEXT: s_setpc_b64 s[30:31]
988 ;
989 ; GFX940-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
990 ; GFX940: ; %bb.0:
991 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
992 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
993 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
994 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
995 ; GFX940-NEXT: .LBB6_1: ; %atomicrmw.start
996 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
997 ; GFX940-NEXT: s_waitcnt vmcnt(0)
998 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
999 ; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
1000 ; GFX940-NEXT: v_max_f32_e32 v4, v3, v2
1001 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
1002 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
1003 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1004 ; GFX940-NEXT: buffer_inv sc0 sc1
1005 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1006 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
1007 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
1008 ; GFX940-NEXT: s_cbranch_execnz .LBB6_1
1009 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
1010 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
1011 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
1012 ; GFX940-NEXT: s_setpc_b64 s[30:31]
1013 ;
1014 ; GFX11-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
1015 ; GFX11: ; %bb.0:
1016 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1017 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1018 ; GFX11-NEXT: global_atomic_max_f32 v0, v[0:1], v2, off offset:2044 glc
1019 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1020 ; GFX11-NEXT: buffer_gl1_inv
1021 ; GFX11-NEXT: buffer_gl0_inv
1022 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1023 ;
1024 ; GFX10-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
1025 ; GFX10: ; %bb.0:
1026 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1027 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1028 ; GFX10-NEXT: global_atomic_fmax v0, v[0:1], v2, off offset:2044 glc
1029 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1030 ; GFX10-NEXT: buffer_gl1_inv
1031 ; GFX10-NEXT: buffer_gl0_inv
1032 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1033 ;
1034 ; GFX90A-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
1035 ; GFX90A: ; %bb.0:
1036 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1037 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
1038 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
1039 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
1040 ; GFX90A-NEXT: .LBB6_1: ; %atomicrmw.start
1041 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
1042 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1043 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
1044 ; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
1045 ; GFX90A-NEXT: v_max_f32_e32 v4, v3, v2
1046 ; GFX90A-NEXT: buffer_wbl2
1047 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
1048 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1049 ; GFX90A-NEXT: buffer_invl2
1050 ; GFX90A-NEXT: buffer_wbinvl1
1051 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1052 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1053 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
1054 ; GFX90A-NEXT: s_cbranch_execnz .LBB6_1
1055 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
1056 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
1057 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
1058 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1059 ;
1060 ; GFX908-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
1061 ; GFX908: ; %bb.0:
1062 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1063 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
1064 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
1065 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
1066 ; GFX908-NEXT: .LBB6_1: ; %atomicrmw.start
1067 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
1068 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1069 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
1070 ; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
1071 ; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
1072 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
1073 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1074 ; GFX908-NEXT: buffer_wbinvl1
1075 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1076 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1077 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
1078 ; GFX908-NEXT: s_cbranch_execnz .LBB6_1
1079 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
1080 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
1081 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
1082 ; GFX908-NEXT: s_setpc_b64 s[30:31]
1083 ;
1084 ; GFX8-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
1085 ; GFX8: ; %bb.0:
1086 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1087 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
1088 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
1089 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
1090 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
1091 ; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
1092 ; GFX8-NEXT: .LBB6_1: ; %atomicrmw.start
1093 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
1094 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1095 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
1096 ; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
1097 ; GFX8-NEXT: v_max_f32_e32 v5, v0, v1
1098 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
1099 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1100 ; GFX8-NEXT: buffer_wbinvl1
1101 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
1102 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1103 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1104 ; GFX8-NEXT: s_cbranch_execnz .LBB6_1
1105 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1106 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1107 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1108 ;
1109 ; GFX7-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
1110 ; GFX7: ; %bb.0:
1111 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1112 ; GFX7-NEXT: s_mov_b32 s6, 0
1113 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
1114 ; GFX7-NEXT: s_mov_b32 s4, s6
1115 ; GFX7-NEXT: s_mov_b32 s5, s6
1116 ; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044 glc
1117 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1118 ; GFX7-NEXT: buffer_wbinvl1
1119 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
1120 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1121 ;
1122 ; GFX6-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
1123 ; GFX6: ; %bb.0:
1124 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1125 ; GFX6-NEXT: s_mov_b32 s6, 0
1126 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
1127 ; GFX6-NEXT: s_mov_b32 s4, s6
1128 ; GFX6-NEXT: s_mov_b32 s5, s6
1129 ; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044 glc
1130 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1131 ; GFX6-NEXT: buffer_wbinvl1
1132 ; GFX6-NEXT: v_mov_b32_e32 v0, v2
1133 ; GFX6-NEXT: s_waitcnt expcnt(0)
1134 ; GFX6-NEXT: s_setpc_b64 s[30:31]
1135 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
1136 %result = atomicrmw fmax ptr addrspace(1) %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
1137 ret float %result
1138 }
1140 define void @global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #0 {
1141 ; GFX12-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
1142 ; GFX12: ; %bb.0:
1143 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
1144 ; GFX12-NEXT: s_wait_expcnt 0x0
1145 ; GFX12-NEXT: s_wait_samplecnt 0x0
1146 ; GFX12-NEXT: s_wait_bvhcnt 0x0
1147 ; GFX12-NEXT: s_wait_kmcnt 0x0
1148 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
1149 ; GFX12-NEXT: s_wait_storecnt 0x0
1150 ; GFX12-NEXT: global_atomic_max_num_f32 v[0:1], v2, off offset:2044 scope:SCOPE_SYS
1151 ; GFX12-NEXT: s_wait_storecnt 0x0
1152 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
1153 ; GFX12-NEXT: s_setpc_b64 s[30:31]
1154 ;
1155 ; GFX940-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
1156 ; GFX940: ; %bb.0:
1157 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1158 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
1159 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
1160 ; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
1161 ; GFX940-NEXT: .LBB7_1: ; %atomicrmw.start
1162 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
1163 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1164 ; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
1165 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
1166 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
1167 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
1168 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1169 ; GFX940-NEXT: buffer_inv sc0 sc1
1170 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
1171 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
1172 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
1173 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
1174 ; GFX940-NEXT: s_cbranch_execnz .LBB7_1
1175 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
1176 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
1177 ; GFX940-NEXT: s_setpc_b64 s[30:31]
1178 ;
1179 ; GFX11-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
1180 ; GFX11: ; %bb.0:
1181 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1182 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1183 ; GFX11-NEXT: global_atomic_max_f32 v[0:1], v2, off offset:2044
1184 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1185 ; GFX11-NEXT: buffer_gl1_inv
1186 ; GFX11-NEXT: buffer_gl0_inv
1187 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1188 ;
1189 ; GFX10-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
1190 ; GFX10: ; %bb.0:
1191 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1192 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1193 ; GFX10-NEXT: global_atomic_fmax v[0:1], v2, off offset:2044
1194 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1195 ; GFX10-NEXT: buffer_gl1_inv
1196 ; GFX10-NEXT: buffer_gl0_inv
1197 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1198 ;
1199 ; GFX90A-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
1200 ; GFX90A: ; %bb.0:
1201 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1202 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
1203 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
1204 ; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
1205 ; GFX90A-NEXT: .LBB7_1: ; %atomicrmw.start
1206 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
1207 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1208 ; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
1209 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
1210 ; GFX90A-NEXT: buffer_wbl2
1211 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
1212 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1213 ; GFX90A-NEXT: buffer_invl2
1214 ; GFX90A-NEXT: buffer_wbinvl1
1215 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
1216 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1217 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
1218 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
1219 ; GFX90A-NEXT: s_cbranch_execnz .LBB7_1
1220 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
1221 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
1222 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1223 ;
1224 ; GFX908-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
1225 ; GFX908: ; %bb.0:
1226 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1227 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
1228 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
1229 ; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
1230 ; GFX908-NEXT: .LBB7_1: ; %atomicrmw.start
1231 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
1232 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1233 ; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
1234 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
1235 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
1236 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1237 ; GFX908-NEXT: buffer_wbinvl1
1238 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
1239 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1240 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
1241 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
1242 ; GFX908-NEXT: s_cbranch_execnz .LBB7_1
1243 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
1244 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
1245 ; GFX908-NEXT: s_setpc_b64 s[30:31]
1246 ;
1247 ; GFX8-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
1248 ; GFX8: ; %bb.0:
1249 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1250 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
1251 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1252 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
1253 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
1254 ; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
1255 ; GFX8-NEXT: .LBB7_1: ; %atomicrmw.start
1256 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
1257 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1258 ; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
1259 ; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
1260 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
1261 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1262 ; GFX8-NEXT: buffer_wbinvl1
1263 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
1264 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1265 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
1266 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1267 ; GFX8-NEXT: s_cbranch_execnz .LBB7_1
1268 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1269 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1270 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1271 ;
1272 ; GFX7-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
1273 ; GFX7: ; %bb.0:
1274 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1275 ; GFX7-NEXT: s_mov_b32 s6, 0
1276 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
1277 ; GFX7-NEXT: s_mov_b32 s4, s6
1278 ; GFX7-NEXT: s_mov_b32 s5, s6
1279 ; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044
1280 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1281 ; GFX7-NEXT: buffer_wbinvl1
1282 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1283 ;
1284 ; GFX6-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
1285 ; GFX6: ; %bb.0:
1286 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1287 ; GFX6-NEXT: s_mov_b32 s6, 0
1288 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
1289 ; GFX6-NEXT: s_mov_b32 s4, s6
1290 ; GFX6-NEXT: s_mov_b32 s5, s6
1291 ; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044
1292 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1293 ; GFX6-NEXT: buffer_wbinvl1
1294 ; GFX6-NEXT: s_waitcnt expcnt(0)
1295 ; GFX6-NEXT: s_setpc_b64 s[30:31]
1296 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
1297 %unused = atomicrmw fmax ptr addrspace(1) %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
1298 ret void
1299 }
1301 define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, float %val) #0 {
1302 ; GFX12-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
1303 ; GFX12: ; %bb.0:
1304 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
1305 ; GFX12-NEXT: s_wait_expcnt 0x0
1306 ; GFX12-NEXT: s_wait_samplecnt 0x0
1307 ; GFX12-NEXT: s_wait_bvhcnt 0x0
1308 ; GFX12-NEXT: s_wait_kmcnt 0x0
1309 ; GFX12-NEXT: s_wait_storecnt 0x0
1310 ; GFX12-NEXT: global_atomic_max_num_f32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
1311 ; GFX12-NEXT: s_wait_loadcnt 0x0
1312 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
1313 ; GFX12-NEXT: s_setpc_b64 s[30:31]
1314 ;
1315 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
1316 ; GFX940: ; %bb.0:
1317 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1318 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
1319 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
1320 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
1321 ; GFX940-NEXT: .LBB8_1: ; %atomicrmw.start
1322 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
1323 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1324 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
1325 ; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
1326 ; GFX940-NEXT: v_max_f32_e32 v4, v3, v2
1327 ; GFX940-NEXT: buffer_wbl2 sc1
1328 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
1329 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1330 ; GFX940-NEXT: buffer_inv sc1
1331 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1332 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
1333 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
1334 ; GFX940-NEXT: s_cbranch_execnz .LBB8_1
1335 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
1336 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
1337 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
1338 ; GFX940-NEXT: s_setpc_b64 s[30:31]
1339 ;
1340 ; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
1341 ; GFX11: ; %bb.0:
1342 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1343 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
1344 ; GFX11-NEXT: v_max_f32_e32 v2, v2, v2
1345 ; GFX11-NEXT: s_mov_b32 s0, 0
1346 ; GFX11-NEXT: .LBB8_1: ; %atomicrmw.start
1347 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
1348 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1349 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
1350 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
1351 ; GFX11-NEXT: v_max_f32_e32 v3, v4, v4
1352 ; GFX11-NEXT: v_max_f32_e32 v3, v3, v2
1353 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1354 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
1355 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1356 ; GFX11-NEXT: buffer_gl1_inv
1357 ; GFX11-NEXT: buffer_gl0_inv
1358 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1359 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
1360 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1361 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
1362 ; GFX11-NEXT: s_cbranch_execnz .LBB8_1
1363 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
1364 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
1365 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
1366 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1367 ;
1368 ; GFX10-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
1369 ; GFX10: ; %bb.0:
1370 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1371 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
1372 ; GFX10-NEXT: v_max_f32_e32 v2, v2, v2
1373 ; GFX10-NEXT: s_mov_b32 s4, 0
1374 ; GFX10-NEXT: .LBB8_1: ; %atomicrmw.start
1375 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
1376 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1377 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
1378 ; GFX10-NEXT: v_max_f32_e32 v3, v4, v4
1379 ; GFX10-NEXT: v_max_f32_e32 v3, v3, v2
1380 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1381 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
1382 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1383 ; GFX10-NEXT: buffer_gl1_inv
1384 ; GFX10-NEXT: buffer_gl0_inv
1385 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
1386 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
1387 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
1388 ; GFX10-NEXT: s_cbranch_execnz .LBB8_1
1389 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
1390 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
1391 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
1392 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1393 ;
1394 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
1395 ; GFX90A: ; %bb.0:
1396 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1397 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
1398 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
1399 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
1400 ; GFX90A-NEXT: .LBB8_1: ; %atomicrmw.start
1401 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
1402 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1403 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
1404 ; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
1405 ; GFX90A-NEXT: v_max_f32_e32 v4, v3, v2
1406 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
1407 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1408 ; GFX90A-NEXT: buffer_wbinvl1
1409 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1410 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1411 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
1412 ; GFX90A-NEXT: s_cbranch_execnz .LBB8_1
1413 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
1414 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
1415 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
1416 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1417 ;
1418 ; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
1419 ; GFX908: ; %bb.0:
1420 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1421 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
1422 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
1423 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
1424 ; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start
1425 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
1426 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1427 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
1428 ; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
1429 ; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
1430 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
1431 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1432 ; GFX908-NEXT: buffer_wbinvl1
1433 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1434 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1435 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
1436 ; GFX908-NEXT: s_cbranch_execnz .LBB8_1
1437 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
1438 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
1439 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
1440 ; GFX908-NEXT: s_setpc_b64 s[30:31]
1441 ;
1442 ; GFX8-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
1443 ; GFX8: ; %bb.0:
1444 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1445 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
1446 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
1447 ; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v2
1448 ; GFX8-NEXT: .LBB8_1: ; %atomicrmw.start
1449 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
1450 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1451 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
1452 ; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v4
1453 ; GFX8-NEXT: v_max_f32_e32 v3, v3, v2
1454 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
1455 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1456 ; GFX8-NEXT: buffer_wbinvl1
1457 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1458 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1459 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1460 ; GFX8-NEXT: s_cbranch_execnz .LBB8_1
1461 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1462 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1463 ; GFX8-NEXT: v_mov_b32_e32 v0, v3
1464 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1465 ;
1466 ; GFX7-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
1467 ; GFX7: ; %bb.0:
1468 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1469 ; GFX7-NEXT: s_mov_b32 s6, 0
1470 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
1471 ; GFX7-NEXT: s_mov_b32 s4, s6
1472 ; GFX7-NEXT: s_mov_b32 s5, s6
1473 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
1474 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
1475 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
1476 ; GFX7-NEXT: .LBB8_1: ; %atomicrmw.start
1477 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
1478 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1479 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
1480 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v5
1481 ; GFX7-NEXT: v_max_f32_e32 v4, v3, v2
1482 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
1483 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
1484 ; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
1485 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1486 ; GFX7-NEXT: buffer_wbinvl1
1487 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1488 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
1489 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
1490 ; GFX7-NEXT: s_cbranch_execnz .LBB8_1
1491 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
1492 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
1493 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
1494 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1495 ;
1496 ; GFX6-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
1497 ; GFX6: ; %bb.0:
1498 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1499 ; GFX6-NEXT: s_mov_b32 s6, 0
1500 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
1501 ; GFX6-NEXT: s_mov_b32 s4, s6
1502 ; GFX6-NEXT: s_mov_b32 s5, s6
1503 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
1504 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
1505 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
1506 ; GFX6-NEXT: .LBB8_1: ; %atomicrmw.start
1507 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
1508 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1509 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
1510 ; GFX6-NEXT: s_waitcnt expcnt(0)
1511 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v5
1512 ; GFX6-NEXT: v_max_f32_e32 v4, v3, v2
1513 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
1514 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
1515 ; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
1516 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1517 ; GFX6-NEXT: buffer_wbinvl1
1518 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1519 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
1520 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
1521 ; GFX6-NEXT: s_cbranch_execnz .LBB8_1
1522 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
1523 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
1524 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
1525 ; GFX6-NEXT: s_waitcnt expcnt(0)
1526 ; GFX6-NEXT: s_setpc_b64 s[30:31]
1527 %result = atomicrmw fmax ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
1528 ret float %result
1529 }
1531 define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, float %val) #0 {
1532 ; GFX12-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
1533 ; GFX12: ; %bb.0:
1534 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
1535 ; GFX12-NEXT: s_wait_expcnt 0x0
1536 ; GFX12-NEXT: s_wait_samplecnt 0x0
1537 ; GFX12-NEXT: s_wait_bvhcnt 0x0
1538 ; GFX12-NEXT: s_wait_kmcnt 0x0
1539 ; GFX12-NEXT: s_wait_storecnt 0x0
1540 ; GFX12-NEXT: global_atomic_max_num_f32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
1541 ; GFX12-NEXT: s_wait_loadcnt 0x0
1542 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
1543 ; GFX12-NEXT: s_setpc_b64 s[30:31]
1544 ;
1545 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
1546 ; GFX940: ; %bb.0:
1547 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1548 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
1549 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
1550 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
1551 ; GFX940-NEXT: .LBB9_1: ; %atomicrmw.start
1552 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
1553 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1554 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
1555 ; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
1556 ; GFX940-NEXT: v_max_f32_e32 v4, v3, v2
1557 ; GFX940-NEXT: buffer_wbl2 sc1
1558 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
1559 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1560 ; GFX940-NEXT: buffer_inv sc1
1561 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1562 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
1563 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
1564 ; GFX940-NEXT: s_cbranch_execnz .LBB9_1
1565 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
1566 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
1567 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
1568 ; GFX940-NEXT: s_setpc_b64 s[30:31]
1569 ;
1570 ; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
1571 ; GFX11: ; %bb.0:
1572 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1573 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1574 ; GFX11-NEXT: global_atomic_max_f32 v0, v[0:1], v2, off glc
1575 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1576 ; GFX11-NEXT: buffer_gl1_inv
1577 ; GFX11-NEXT: buffer_gl0_inv
1578 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1579 ;
1580 ; GFX10-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
1581 ; GFX10: ; %bb.0:
1582 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1583 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1584 ; GFX10-NEXT: global_atomic_fmax v0, v[0:1], v2, off glc
1585 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1586 ; GFX10-NEXT: buffer_gl1_inv
1587 ; GFX10-NEXT: buffer_gl0_inv
1588 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1589 ;
1590 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
1591 ; GFX90A: ; %bb.0:
1592 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1593 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
1594 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
1595 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
1596 ; GFX90A-NEXT: .LBB9_1: ; %atomicrmw.start
1597 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
1598 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1599 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
1600 ; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
1601 ; GFX90A-NEXT: v_max_f32_e32 v4, v3, v2
1602 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
1603 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1604 ; GFX90A-NEXT: buffer_wbinvl1
1605 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1606 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1607 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
1608 ; GFX90A-NEXT: s_cbranch_execnz .LBB9_1
1609 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
1610 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
1611 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
1612 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1613 ;
1614 ; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
1615 ; GFX908: ; %bb.0:
1616 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1617 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
1618 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
1619 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
1620 ; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
1621 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
1622 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1623 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
1624 ; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
1625 ; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
1626 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
1627 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1628 ; GFX908-NEXT: buffer_wbinvl1
1629 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1630 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1631 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
1632 ; GFX908-NEXT: s_cbranch_execnz .LBB9_1
1633 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
1634 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
1635 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
1636 ; GFX908-NEXT: s_setpc_b64 s[30:31]
1637 ;
1638 ; GFX8-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
1639 ; GFX8: ; %bb.0:
1640 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1641 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
1642 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
1643 ; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v2
1644 ; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start
1645 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
1646 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1647 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
1648 ; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v4
1649 ; GFX8-NEXT: v_max_f32_e32 v3, v3, v2
1650 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
1651 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1652 ; GFX8-NEXT: buffer_wbinvl1
1653 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1654 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1655 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1656 ; GFX8-NEXT: s_cbranch_execnz .LBB9_1
1657 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1658 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1659 ; GFX8-NEXT: v_mov_b32_e32 v0, v3
1660 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1661 ;
1662 ; GFX7-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
1663 ; GFX7: ; %bb.0:
1664 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1665 ; GFX7-NEXT: s_mov_b32 s6, 0
1666 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
1667 ; GFX7-NEXT: s_mov_b32 s4, s6
1668 ; GFX7-NEXT: s_mov_b32 s5, s6
1669 ; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 glc
1670 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1671 ; GFX7-NEXT: buffer_wbinvl1
1672 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
1673 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1674 ;
1675 ; GFX6-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
1676 ; GFX6: ; %bb.0:
1677 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1678 ; GFX6-NEXT: s_mov_b32 s6, 0
1679 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
1680 ; GFX6-NEXT: s_mov_b32 s4, s6
1681 ; GFX6-NEXT: s_mov_b32 s5, s6
1682 ; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 glc
1683 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1684 ; GFX6-NEXT: buffer_wbinvl1
1685 ; GFX6-NEXT: v_mov_b32_e32 v0, v2
1686 ; GFX6-NEXT: s_waitcnt expcnt(0)
1687 ; GFX6-NEXT: s_setpc_b64 s[30:31]
1688 %result = atomicrmw fmax ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
1689 ret float %result
1690 }
1692 ; --------------------------------------------------------------------
1693 ; float with ftz/daz
1694 ; --------------------------------------------------------------------
1696 define float @global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
1697 ; GFX12-LABEL: global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
1698 ; GFX12: ; %bb.0:
1699 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
1700 ; GFX12-NEXT: s_wait_expcnt 0x0
1701 ; GFX12-NEXT: s_wait_samplecnt 0x0
1702 ; GFX12-NEXT: s_wait_bvhcnt 0x0
1703 ; GFX12-NEXT: s_wait_kmcnt 0x0
1704 ; GFX12-NEXT: s_wait_storecnt 0x0
1705 ; GFX12-NEXT: global_atomic_max_num_f32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
1706 ; GFX12-NEXT: s_wait_loadcnt 0x0
1707 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
1708 ; GFX12-NEXT: s_setpc_b64 s[30:31]
1709 ;
1710 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
1711 ; GFX940: ; %bb.0:
1712 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1713 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
1714 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
1715 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
1716 ; GFX940-NEXT: .LBB10_1: ; %atomicrmw.start
1717 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
1718 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1719 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
1720 ; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
1721 ; GFX940-NEXT: v_max_f32_e32 v4, v3, v2
1722 ; GFX940-NEXT: buffer_wbl2 sc1
1723 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
1724 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1725 ; GFX940-NEXT: buffer_inv sc1
1726 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1727 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
1728 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
1729 ; GFX940-NEXT: s_cbranch_execnz .LBB10_1
1730 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
1731 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
1732 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
1733 ; GFX940-NEXT: s_setpc_b64 s[30:31]
1734 ;
1735 ; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
1736 ; GFX11: ; %bb.0:
1737 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1738 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1739 ; GFX11-NEXT: global_atomic_max_f32 v0, v[0:1], v2, off glc
1740 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1741 ; GFX11-NEXT: buffer_gl1_inv
1742 ; GFX11-NEXT: buffer_gl0_inv
1743 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1744 ;
1745 ; GFX10-LABEL: global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
1746 ; GFX10: ; %bb.0:
1747 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1748 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1749 ; GFX10-NEXT: global_atomic_fmax v0, v[0:1], v2, off glc
1750 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1751 ; GFX10-NEXT: buffer_gl1_inv
1752 ; GFX10-NEXT: buffer_gl0_inv
1753 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1754 ;
1755 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
1756 ; GFX90A: ; %bb.0:
1757 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1758 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
1759 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
1760 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
1761 ; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start
1762 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
1763 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1764 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
1765 ; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
1766 ; GFX90A-NEXT: v_max_f32_e32 v4, v3, v2
1767 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
1768 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1769 ; GFX90A-NEXT: buffer_wbinvl1
1770 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1771 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1772 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
1773 ; GFX90A-NEXT: s_cbranch_execnz .LBB10_1
1774 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
1775 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
1776 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
1777 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1778 ;
1779 ; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
1780 ; GFX908: ; %bb.0:
1781 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1782 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
1783 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
1784 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
1785 ; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
1786 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
1787 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1788 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
1789 ; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
1790 ; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
1791 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
1792 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1793 ; GFX908-NEXT: buffer_wbinvl1
1794 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1795 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1796 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
1797 ; GFX908-NEXT: s_cbranch_execnz .LBB10_1
1798 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
1799 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
1800 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
1801 ; GFX908-NEXT: s_setpc_b64 s[30:31]
1802 ;
1803 ; GFX8-LABEL: global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
1804 ; GFX8: ; %bb.0:
1805 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1806 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
1807 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
1808 ; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v2
1809 ; GFX8-NEXT: .LBB10_1: ; %atomicrmw.start
1810 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
1811 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1812 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
1813 ; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v4
1814 ; GFX8-NEXT: v_max_f32_e32 v3, v3, v2
1815 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
1816 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1817 ; GFX8-NEXT: buffer_wbinvl1
1818 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1819 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1820 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1821 ; GFX8-NEXT: s_cbranch_execnz .LBB10_1
1822 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1823 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1824 ; GFX8-NEXT: v_mov_b32_e32 v0, v3
1825 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1826 ;
1827 ; GFX7-LABEL: global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
1828 ; GFX7: ; %bb.0:
1829 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1830 ; GFX7-NEXT: s_mov_b32 s6, 0
1831 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
1832 ; GFX7-NEXT: s_mov_b32 s4, s6
1833 ; GFX7-NEXT: s_mov_b32 s5, s6
1834 ; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 glc
1835 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1836 ; GFX7-NEXT: buffer_wbinvl1
1837 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
1838 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1839 ;
1840 ; GFX6-LABEL: global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
1841 ; GFX6: ; %bb.0:
1842 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1843 ; GFX6-NEXT: s_mov_b32 s6, 0
1844 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
1845 ; GFX6-NEXT: s_mov_b32 s4, s6
1846 ; GFX6-NEXT: s_mov_b32 s5, s6
1847 ; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 glc
1848 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1849 ; GFX6-NEXT: buffer_wbinvl1
1850 ; GFX6-NEXT: v_mov_b32_e32 v0, v2
1851 ; GFX6-NEXT: s_waitcnt expcnt(0)
1852 ; GFX6-NEXT: s_setpc_b64 s[30:31]
1853 %result = atomicrmw fmax ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
1854 ret float %result
1855 }
1857 define float @global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
1858 ; GFX12-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
1859 ; GFX12: ; %bb.0:
1860 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
1861 ; GFX12-NEXT: s_wait_expcnt 0x0
1862 ; GFX12-NEXT: s_wait_samplecnt 0x0
1863 ; GFX12-NEXT: s_wait_bvhcnt 0x0
1864 ; GFX12-NEXT: s_wait_kmcnt 0x0
1865 ; GFX12-NEXT: s_wait_storecnt 0x0
1866 ; GFX12-NEXT: global_atomic_max_num_f32 v0, v[0:1], v2, off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
1867 ; GFX12-NEXT: s_wait_loadcnt 0x0
1868 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
1869 ; GFX12-NEXT: s_setpc_b64 s[30:31]
1870 ;
1871 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
1872 ; GFX940: ; %bb.0:
1873 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1874 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
1875 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
1876 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
1877 ; GFX940-NEXT: .LBB11_1: ; %atomicrmw.start
1878 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
1879 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1880 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
1881 ; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
1882 ; GFX940-NEXT: v_max_f32_e32 v4, v3, v2
1883 ; GFX940-NEXT: buffer_wbl2 sc1
1884 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
1885 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1886 ; GFX940-NEXT: buffer_inv sc1
1887 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1888 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
1889 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
1890 ; GFX940-NEXT: s_cbranch_execnz .LBB11_1
1891 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
1892 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
1893 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
1894 ; GFX940-NEXT: s_setpc_b64 s[30:31]
1895 ;
1896 ; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
1897 ; GFX11: ; %bb.0:
1898 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1899 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1900 ; GFX11-NEXT: global_atomic_max_f32 v0, v[0:1], v2, off offset:2044 glc
1901 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1902 ; GFX11-NEXT: buffer_gl1_inv
1903 ; GFX11-NEXT: buffer_gl0_inv
1904 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1905 ;
1906 ; GFX10-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
1907 ; GFX10: ; %bb.0:
1908 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1909 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1910 ; GFX10-NEXT: global_atomic_fmax v0, v[0:1], v2, off offset:2044 glc
1911 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1912 ; GFX10-NEXT: buffer_gl1_inv
1913 ; GFX10-NEXT: buffer_gl0_inv
1914 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1915 ;
1916 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
1917 ; GFX90A: ; %bb.0:
1918 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1919 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
1920 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
1921 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
1922 ; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start
1923 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
1924 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1925 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
1926 ; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
1927 ; GFX90A-NEXT: v_max_f32_e32 v4, v3, v2
1928 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
1929 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1930 ; GFX90A-NEXT: buffer_wbinvl1
1931 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
1932 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1933 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
1934 ; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
1935 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
1936 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
1937 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
1938 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1939 ;
1940 ; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
1941 ; GFX908: ; %bb.0:
1942 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1943 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
1944 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
1945 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
1946 ; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
1947 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
1948 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1949 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
1950 ; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
1951 ; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
1952 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
1953 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1954 ; GFX908-NEXT: buffer_wbinvl1
1955 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
1956 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1957 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
1958 ; GFX908-NEXT: s_cbranch_execnz .LBB11_1
1959 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
1960 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
1961 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
1962 ; GFX908-NEXT: s_setpc_b64 s[30:31]
1963 ;
1964 ; GFX8-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
1965 ; GFX8: ; %bb.0:
1966 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1967 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
1968 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
1969 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
1970 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
1971 ; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
1972 ; GFX8-NEXT: .LBB11_1: ; %atomicrmw.start
1973 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
1974 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1975 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
1976 ; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
1977 ; GFX8-NEXT: v_max_f32_e32 v5, v0, v1
1978 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
1979 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1980 ; GFX8-NEXT: buffer_wbinvl1
1981 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
1982 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1983 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1984 ; GFX8-NEXT: s_cbranch_execnz .LBB11_1
1985 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1986 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1987 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1988 ;
1989 ; GFX7-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
1990 ; GFX7: ; %bb.0:
1991 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1992 ; GFX7-NEXT: s_mov_b32 s6, 0
1993 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
1994 ; GFX7-NEXT: s_mov_b32 s4, s6
1995 ; GFX7-NEXT: s_mov_b32 s5, s6
1996 ; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044 glc
1997 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1998 ; GFX7-NEXT: buffer_wbinvl1
1999 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
2000 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2001 ;
2002 ; GFX6-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2003 ; GFX6: ; %bb.0:
2004 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2005 ; GFX6-NEXT: s_mov_b32 s6, 0
2006 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
2007 ; GFX6-NEXT: s_mov_b32 s4, s6
2008 ; GFX6-NEXT: s_mov_b32 s5, s6
2009 ; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044 glc
2010 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2011 ; GFX6-NEXT: buffer_wbinvl1
2012 ; GFX6-NEXT: v_mov_b32_e32 v0, v2
2013 ; GFX6-NEXT: s_waitcnt expcnt(0)
2014 ; GFX6-NEXT: s_setpc_b64 s[30:31]
2015 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
2016 %result = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
2017 ret float %result
2018 }
2020 define float @global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
2021 ; GFX12-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
2022 ; GFX12: ; %bb.0:
2023 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
2024 ; GFX12-NEXT: s_wait_expcnt 0x0
2025 ; GFX12-NEXT: s_wait_samplecnt 0x0
2026 ; GFX12-NEXT: s_wait_bvhcnt 0x0
2027 ; GFX12-NEXT: s_wait_kmcnt 0x0
2028 ; GFX12-NEXT: s_wait_storecnt 0x0
2029 ; GFX12-NEXT: global_atomic_max_num_f32 v0, v[0:1], v2, off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
2030 ; GFX12-NEXT: s_wait_loadcnt 0x0
2031 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
2032 ; GFX12-NEXT: s_setpc_b64 s[30:31]
2033 ;
2034 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
2035 ; GFX940: ; %bb.0:
2036 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2037 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
2038 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
2039 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
2040 ; GFX940-NEXT: .LBB12_1: ; %atomicrmw.start
2041 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
2042 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2043 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
2044 ; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
2045 ; GFX940-NEXT: v_max_f32_e32 v4, v3, v2
2046 ; GFX940-NEXT: buffer_wbl2 sc1
2047 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
2048 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2049 ; GFX940-NEXT: buffer_inv sc1
2050 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2051 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
2052 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
2053 ; GFX940-NEXT: s_cbranch_execnz .LBB12_1
2054 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
2055 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
2056 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
2057 ; GFX940-NEXT: s_setpc_b64 s[30:31]
2058 ;
2059 ; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
2060 ; GFX11: ; %bb.0:
2061 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2062 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2063 ; GFX11-NEXT: global_atomic_max_f32 v0, v[0:1], v2, off offset:-2048 glc
2064 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2065 ; GFX11-NEXT: buffer_gl1_inv
2066 ; GFX11-NEXT: buffer_gl0_inv
2067 ; GFX11-NEXT: s_setpc_b64 s[30:31]
2068 ;
2069 ; GFX10-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
2070 ; GFX10: ; %bb.0:
2071 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2072 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2073 ; GFX10-NEXT: global_atomic_fmax v0, v[0:1], v2, off offset:-2048 glc
2074 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2075 ; GFX10-NEXT: buffer_gl1_inv
2076 ; GFX10-NEXT: buffer_gl0_inv
2077 ; GFX10-NEXT: s_setpc_b64 s[30:31]
2078 ;
2079 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
2080 ; GFX90A: ; %bb.0:
2081 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2082 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
2083 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
2084 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
2085 ; GFX90A-NEXT: .LBB12_1: ; %atomicrmw.start
2086 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
2087 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2088 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
2089 ; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
2090 ; GFX90A-NEXT: v_max_f32_e32 v4, v3, v2
2091 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
2092 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2093 ; GFX90A-NEXT: buffer_wbinvl1
2094 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2095 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2096 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
2097 ; GFX90A-NEXT: s_cbranch_execnz .LBB12_1
2098 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
2099 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
2100 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
2101 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
2102 ;
2103 ; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
2104 ; GFX908: ; %bb.0:
2105 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2106 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
2107 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
2108 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
2109 ; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start
2110 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
2111 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2112 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
2113 ; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
2114 ; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
2115 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
2116 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2117 ; GFX908-NEXT: buffer_wbinvl1
2118 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
2119 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2120 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
2121 ; GFX908-NEXT: s_cbranch_execnz .LBB12_1
2122 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
2123 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
2124 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
2125 ; GFX908-NEXT: s_setpc_b64 s[30:31]
2126 ;
2127 ; GFX8-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
2128 ; GFX8: ; %bb.0:
2129 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2130 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
2131 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
2132 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
2133 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2134 ; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
2135 ; GFX8-NEXT: .LBB12_1: ; %atomicrmw.start
2136 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2137 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2138 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
2139 ; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
2140 ; GFX8-NEXT: v_max_f32_e32 v5, v0, v1
2141 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
2142 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2143 ; GFX8-NEXT: buffer_wbinvl1
2144 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
2145 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2146 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2147 ; GFX8-NEXT: s_cbranch_execnz .LBB12_1
2148 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2149 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2150 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2151 ;
2152 ; GFX7-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
2153 ; GFX7: ; %bb.0:
2154 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2155 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
2156 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
2157 ; GFX7-NEXT: s_mov_b32 s6, 0
2158 ; GFX7-NEXT: s_mov_b32 s5, -1
2159 ; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 glc
2160 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2161 ; GFX7-NEXT: buffer_wbinvl1
2162 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
2163 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2164 ;
2165 ; GFX6-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
2166 ; GFX6: ; %bb.0:
2167 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2168 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
2169 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
2170 ; GFX6-NEXT: s_mov_b32 s6, 0
2171 ; GFX6-NEXT: s_mov_b32 s5, -1
2172 ; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 glc
2173 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2174 ; GFX6-NEXT: buffer_wbinvl1
2175 ; GFX6-NEXT: v_mov_b32_e32 v0, v2
2176 ; GFX6-NEXT: s_waitcnt expcnt(0)
2177 ; GFX6-NEXT: s_setpc_b64 s[30:31]
2178 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
2179 %result = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
2180 ret float %result
2181 }
2183 define void @global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
2184 ; GFX12-LABEL: global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
2185 ; GFX12: ; %bb.0:
2186 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
2187 ; GFX12-NEXT: s_wait_expcnt 0x0
2188 ; GFX12-NEXT: s_wait_samplecnt 0x0
2189 ; GFX12-NEXT: s_wait_bvhcnt 0x0
2190 ; GFX12-NEXT: s_wait_kmcnt 0x0
2191 ; GFX12-NEXT: s_wait_storecnt 0x0
2192 ; GFX12-NEXT: global_atomic_max_num_f32 v[0:1], v2, off scope:SCOPE_DEV
2193 ; GFX12-NEXT: s_wait_storecnt 0x0
2194 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
2195 ; GFX12-NEXT: s_setpc_b64 s[30:31]
2196 ;
2197 ; GFX940-LABEL: global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
2198 ; GFX940: ; %bb.0:
2199 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2200 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
2201 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
2202 ; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
2203 ; GFX940-NEXT: .LBB13_1: ; %atomicrmw.start
2204 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
2205 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2206 ; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
2207 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
2208 ; GFX940-NEXT: buffer_wbl2 sc1
2209 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
2210 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2211 ; GFX940-NEXT: buffer_inv sc1
2212 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
2213 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
2214 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
2215 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
2216 ; GFX940-NEXT: s_cbranch_execnz .LBB13_1
2217 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
2218 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
2219 ; GFX940-NEXT: s_setpc_b64 s[30:31]
2220 ;
2221 ; GFX11-LABEL: global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
2222 ; GFX11: ; %bb.0:
2223 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2224 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2225 ; GFX11-NEXT: global_atomic_max_f32 v[0:1], v2, off
2226 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2227 ; GFX11-NEXT: buffer_gl1_inv
2228 ; GFX11-NEXT: buffer_gl0_inv
2229 ; GFX11-NEXT: s_setpc_b64 s[30:31]
2230 ;
2231 ; GFX10-LABEL: global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
2232 ; GFX10: ; %bb.0:
2233 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2234 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2235 ; GFX10-NEXT: global_atomic_fmax v[0:1], v2, off
2236 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2237 ; GFX10-NEXT: buffer_gl1_inv
2238 ; GFX10-NEXT: buffer_gl0_inv
2239 ; GFX10-NEXT: s_setpc_b64 s[30:31]
2240 ;
2241 ; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
2242 ; GFX90A: ; %bb.0:
2243 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2244 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
2245 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
2246 ; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
2247 ; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
2248 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
2249 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2250 ; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
2251 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
2252 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
2253 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2254 ; GFX90A-NEXT: buffer_wbinvl1
2255 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
2256 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2257 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
2258 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
2259 ; GFX90A-NEXT: s_cbranch_execnz .LBB13_1
2260 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
2261 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
2262 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
2263 ;
2264 ; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
2265 ; GFX908: ; %bb.0:
2266 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2267 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
2268 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
2269 ; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
2270 ; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
2271 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
2272 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2273 ; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
2274 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
2275 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
2276 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2277 ; GFX908-NEXT: buffer_wbinvl1
2278 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
2279 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2280 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
2281 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
2282 ; GFX908-NEXT: s_cbranch_execnz .LBB13_1
2283 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
2284 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
2285 ; GFX908-NEXT: s_setpc_b64 s[30:31]
2286 ;
2287 ; GFX8-LABEL: global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
2288 ; GFX8: ; %bb.0:
2289 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2290 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
2291 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2292 ; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
2293 ; GFX8-NEXT: .LBB13_1: ; %atomicrmw.start
2294 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2295 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2296 ; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
2297 ; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
2298 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2299 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2300 ; GFX8-NEXT: buffer_wbinvl1
2301 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
2302 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2303 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
2304 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2305 ; GFX8-NEXT: s_cbranch_execnz .LBB13_1
2306 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2307 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2308 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2309 ;
2310 ; GFX7-LABEL: global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
2311 ; GFX7: ; %bb.0:
2312 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2313 ; GFX7-NEXT: s_mov_b32 s6, 0
2314 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
2315 ; GFX7-NEXT: s_mov_b32 s4, s6
2316 ; GFX7-NEXT: s_mov_b32 s5, s6
2317 ; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64
2318 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2319 ; GFX7-NEXT: buffer_wbinvl1
2320 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2321 ;
2322 ; GFX6-LABEL: global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
2323 ; GFX6: ; %bb.0:
2324 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2325 ; GFX6-NEXT: s_mov_b32 s6, 0
2326 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
2327 ; GFX6-NEXT: s_mov_b32 s4, s6
2328 ; GFX6-NEXT: s_mov_b32 s5, s6
2329 ; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64
2330 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2331 ; GFX6-NEXT: buffer_wbinvl1
2332 ; GFX6-NEXT: s_waitcnt expcnt(0)
2333 ; GFX6-NEXT: s_setpc_b64 s[30:31]
2334 %unused = atomicrmw fmax ptr addrspace(1) %ptr, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
2335 ret void
2336 }
2338 define void @global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
2339 ; GFX12-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2340 ; GFX12: ; %bb.0:
2341 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
2342 ; GFX12-NEXT: s_wait_expcnt 0x0
2343 ; GFX12-NEXT: s_wait_samplecnt 0x0
2344 ; GFX12-NEXT: s_wait_bvhcnt 0x0
2345 ; GFX12-NEXT: s_wait_kmcnt 0x0
2346 ; GFX12-NEXT: s_wait_storecnt 0x0
2347 ; GFX12-NEXT: global_atomic_max_num_f32 v[0:1], v2, off offset:2044 scope:SCOPE_DEV
2348 ; GFX12-NEXT: s_wait_storecnt 0x0
2349 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
2350 ; GFX12-NEXT: s_setpc_b64 s[30:31]
2351 ;
2352 ; GFX940-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2353 ; GFX940: ; %bb.0:
2354 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2355 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
2356 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
2357 ; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
2358 ; GFX940-NEXT: .LBB14_1: ; %atomicrmw.start
2359 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
2360 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2361 ; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
2362 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
2363 ; GFX940-NEXT: buffer_wbl2 sc1
2364 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
2365 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2366 ; GFX940-NEXT: buffer_inv sc1
2367 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
2368 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
2369 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
2370 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
2371 ; GFX940-NEXT: s_cbranch_execnz .LBB14_1
2372 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
2373 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
2374 ; GFX940-NEXT: s_setpc_b64 s[30:31]
2375 ;
2376 ; GFX11-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2377 ; GFX11: ; %bb.0:
2378 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2379 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2380 ; GFX11-NEXT: global_atomic_max_f32 v[0:1], v2, off offset:2044
2381 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2382 ; GFX11-NEXT: buffer_gl1_inv
2383 ; GFX11-NEXT: buffer_gl0_inv
2384 ; GFX11-NEXT: s_setpc_b64 s[30:31]
2385 ;
2386 ; GFX10-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2387 ; GFX10: ; %bb.0:
2388 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2389 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2390 ; GFX10-NEXT: global_atomic_fmax v[0:1], v2, off offset:2044
2391 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2392 ; GFX10-NEXT: buffer_gl1_inv
2393 ; GFX10-NEXT: buffer_gl0_inv
2394 ; GFX10-NEXT: s_setpc_b64 s[30:31]
2395 ;
2396 ; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2397 ; GFX90A: ; %bb.0:
2398 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2399 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
2400 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
2401 ; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
2402 ; GFX90A-NEXT: .LBB14_1: ; %atomicrmw.start
2403 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
2404 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2405 ; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
2406 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
2407 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
2408 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2409 ; GFX90A-NEXT: buffer_wbinvl1
2410 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
2411 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2412 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
2413 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
2414 ; GFX90A-NEXT: s_cbranch_execnz .LBB14_1
2415 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
2416 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
2417 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
2418 ;
2419 ; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2420 ; GFX908: ; %bb.0:
2421 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2422 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
2423 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
2424 ; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
2425 ; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
2426 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
2427 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2428 ; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
2429 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
2430 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
2431 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2432 ; GFX908-NEXT: buffer_wbinvl1
2433 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
2434 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2435 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
2436 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
2437 ; GFX908-NEXT: s_cbranch_execnz .LBB14_1
2438 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
2439 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
2440 ; GFX908-NEXT: s_setpc_b64 s[30:31]
2441 ;
2442 ; GFX8-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2443 ; GFX8: ; %bb.0:
2444 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2445 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
2446 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2447 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
2448 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2449 ; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
2450 ; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
2451 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2452 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2453 ; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
2454 ; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
2455 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2456 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2457 ; GFX8-NEXT: buffer_wbinvl1
2458 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
2459 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2460 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
2461 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2462 ; GFX8-NEXT: s_cbranch_execnz .LBB14_1
2463 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2464 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2465 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2466 ;
2467 ; GFX7-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2468 ; GFX7: ; %bb.0:
2469 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2470 ; GFX7-NEXT: s_mov_b32 s6, 0
2471 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
2472 ; GFX7-NEXT: s_mov_b32 s4, s6
2473 ; GFX7-NEXT: s_mov_b32 s5, s6
2474 ; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044
2475 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2476 ; GFX7-NEXT: buffer_wbinvl1
2477 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2478 ;
2479 ; GFX6-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2480 ; GFX6: ; %bb.0:
2481 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2482 ; GFX6-NEXT: s_mov_b32 s6, 0
2483 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
2484 ; GFX6-NEXT: s_mov_b32 s4, s6
2485 ; GFX6-NEXT: s_mov_b32 s5, s6
2486 ; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044
2487 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2488 ; GFX6-NEXT: buffer_wbinvl1
2489 ; GFX6-NEXT: s_waitcnt expcnt(0)
2490 ; GFX6-NEXT: s_setpc_b64 s[30:31]
2491 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
2492 %unused = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
2493 ret void
2494 }
2496 define void @global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
2497 ; GFX12-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
2498 ; GFX12: ; %bb.0:
2499 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
2500 ; GFX12-NEXT: s_wait_expcnt 0x0
2501 ; GFX12-NEXT: s_wait_samplecnt 0x0
2502 ; GFX12-NEXT: s_wait_bvhcnt 0x0
2503 ; GFX12-NEXT: s_wait_kmcnt 0x0
2504 ; GFX12-NEXT: s_wait_storecnt 0x0
2505 ; GFX12-NEXT: global_atomic_max_num_f32 v[0:1], v2, off offset:-2048 scope:SCOPE_DEV
2506 ; GFX12-NEXT: s_wait_storecnt 0x0
2507 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
2508 ; GFX12-NEXT: s_setpc_b64 s[30:31]
2509 ;
2510 ; GFX940-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
2511 ; GFX940: ; %bb.0:
2512 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2513 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
2514 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
2515 ; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
2516 ; GFX940-NEXT: .LBB15_1: ; %atomicrmw.start
2517 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
2518 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2519 ; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
2520 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
2521 ; GFX940-NEXT: buffer_wbl2 sc1
2522 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
2523 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2524 ; GFX940-NEXT: buffer_inv sc1
2525 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
2526 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
2527 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
2528 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
2529 ; GFX940-NEXT: s_cbranch_execnz .LBB15_1
2530 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
2531 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
2532 ; GFX940-NEXT: s_setpc_b64 s[30:31]
2533 ;
2534 ; GFX11-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
2535 ; GFX11: ; %bb.0:
2536 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2537 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2538 ; GFX11-NEXT: global_atomic_max_f32 v[0:1], v2, off offset:-2048
2539 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2540 ; GFX11-NEXT: buffer_gl1_inv
2541 ; GFX11-NEXT: buffer_gl0_inv
2542 ; GFX11-NEXT: s_setpc_b64 s[30:31]
2543 ;
2544 ; GFX10-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
2545 ; GFX10: ; %bb.0:
2546 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2547 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2548 ; GFX10-NEXT: global_atomic_fmax v[0:1], v2, off offset:-2048
2549 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2550 ; GFX10-NEXT: buffer_gl1_inv
2551 ; GFX10-NEXT: buffer_gl0_inv
2552 ; GFX10-NEXT: s_setpc_b64 s[30:31]
2553 ;
2554 ; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
2555 ; GFX90A: ; %bb.0:
2556 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2557 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
2558 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
2559 ; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
2560 ; GFX90A-NEXT: .LBB15_1: ; %atomicrmw.start
2561 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
2562 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2563 ; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
2564 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
2565 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
2566 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2567 ; GFX90A-NEXT: buffer_wbinvl1
2568 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
2569 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2570 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
2571 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
2572 ; GFX90A-NEXT: s_cbranch_execnz .LBB15_1
2573 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
2574 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
2575 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
2576 ;
2577 ; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
2578 ; GFX908: ; %bb.0:
2579 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2580 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
2581 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
2582 ; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
2583 ; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start
2584 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
2585 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2586 ; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
2587 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
2588 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
2589 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2590 ; GFX908-NEXT: buffer_wbinvl1
2591 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
2592 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2593 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
2594 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
2595 ; GFX908-NEXT: s_cbranch_execnz .LBB15_1
2596 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
2597 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
2598 ; GFX908-NEXT: s_setpc_b64 s[30:31]
2599 ;
2600 ; GFX8-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
2601 ; GFX8: ; %bb.0:
2602 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2603 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
2604 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
2605 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
2606 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2607 ; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
2608 ; GFX8-NEXT: .LBB15_1: ; %atomicrmw.start
2609 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2610 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2611 ; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
2612 ; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
2613 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2614 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2615 ; GFX8-NEXT: buffer_wbinvl1
2616 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
2617 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2618 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
2619 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2620 ; GFX8-NEXT: s_cbranch_execnz .LBB15_1
2621 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2622 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2623 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2624 ;
2625 ; GFX7-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
2626 ; GFX7: ; %bb.0:
2627 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2628 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
2629 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
2630 ; GFX7-NEXT: s_mov_b32 s6, 0
2631 ; GFX7-NEXT: s_mov_b32 s5, -1
2632 ; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64
2633 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2634 ; GFX7-NEXT: buffer_wbinvl1
2635 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2636 ;
2637 ; GFX6-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
2638 ; GFX6: ; %bb.0:
2639 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2640 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
2641 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
2642 ; GFX6-NEXT: s_mov_b32 s6, 0
2643 ; GFX6-NEXT: s_mov_b32 s5, -1
2644 ; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64
2645 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2646 ; GFX6-NEXT: buffer_wbinvl1
2647 ; GFX6-NEXT: s_waitcnt expcnt(0)
2648 ; GFX6-NEXT: s_setpc_b64 s[30:31]
2649 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 -512
2650 %unused = atomicrmw fmax ptr addrspace(1) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
2651 ret void
2652 }
2654 define float @global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
2655 ; GFX12-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2656 ; GFX12: ; %bb.0:
2657 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
2658 ; GFX12-NEXT: s_wait_expcnt 0x0
2659 ; GFX12-NEXT: s_wait_samplecnt 0x0
2660 ; GFX12-NEXT: s_wait_bvhcnt 0x0
2661 ; GFX12-NEXT: s_wait_kmcnt 0x0
2662 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
2663 ; GFX12-NEXT: s_wait_storecnt 0x0
2664 ; GFX12-NEXT: global_atomic_max_num_f32 v0, v[0:1], v2, off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
2665 ; GFX12-NEXT: s_wait_loadcnt 0x0
2666 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
2667 ; GFX12-NEXT: s_setpc_b64 s[30:31]
2668 ;
2669 ; GFX940-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2670 ; GFX940: ; %bb.0:
2671 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2672 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
2673 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
2674 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v2
2675 ; GFX940-NEXT: .LBB16_1: ; %atomicrmw.start
2676 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
2677 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2678 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
2679 ; GFX940-NEXT: v_max_f32_e32 v3, v5, v5
2680 ; GFX940-NEXT: v_max_f32_e32 v4, v3, v2
2681 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
2682 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
2683 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2684 ; GFX940-NEXT: buffer_inv sc0 sc1
2685 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2686 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
2687 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
2688 ; GFX940-NEXT: s_cbranch_execnz .LBB16_1
2689 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
2690 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
2691 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
2692 ; GFX940-NEXT: s_setpc_b64 s[30:31]
2693 ;
2694 ; GFX11-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2695 ; GFX11: ; %bb.0:
2696 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2697 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2698 ; GFX11-NEXT: global_atomic_max_f32 v0, v[0:1], v2, off offset:2044 glc
2699 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2700 ; GFX11-NEXT: buffer_gl1_inv
2701 ; GFX11-NEXT: buffer_gl0_inv
2702 ; GFX11-NEXT: s_setpc_b64 s[30:31]
2703 ;
2704 ; GFX10-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2705 ; GFX10: ; %bb.0:
2706 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2707 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2708 ; GFX10-NEXT: global_atomic_fmax v0, v[0:1], v2, off offset:2044 glc
2709 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2710 ; GFX10-NEXT: buffer_gl1_inv
2711 ; GFX10-NEXT: buffer_gl0_inv
2712 ; GFX10-NEXT: s_setpc_b64 s[30:31]
2713 ;
2714 ; GFX90A-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2715 ; GFX90A: ; %bb.0:
2716 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2717 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
2718 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
2719 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
2720 ; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
2721 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
2722 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2723 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
2724 ; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
2725 ; GFX90A-NEXT: v_max_f32_e32 v4, v3, v2
2726 ; GFX90A-NEXT: buffer_wbl2
2727 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
2728 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2729 ; GFX90A-NEXT: buffer_invl2
2730 ; GFX90A-NEXT: buffer_wbinvl1
2731 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
2732 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2733 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
2734 ; GFX90A-NEXT: s_cbranch_execnz .LBB16_1
2735 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
2736 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
2737 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
2738 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
2739 ;
2740 ; GFX908-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2741 ; GFX908: ; %bb.0:
2742 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2743 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
2744 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
2745 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
2746 ; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
2747 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
2748 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2749 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
2750 ; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
2751 ; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
2752 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
2753 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2754 ; GFX908-NEXT: buffer_wbinvl1
2755 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
2756 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2757 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
2758 ; GFX908-NEXT: s_cbranch_execnz .LBB16_1
2759 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
2760 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
2761 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
2762 ; GFX908-NEXT: s_setpc_b64 s[30:31]
2763 ;
2764 ; GFX8-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2765 ; GFX8: ; %bb.0:
2766 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2767 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
2768 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
2769 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
2770 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2771 ; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v2
2772 ; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start
2773 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2774 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2775 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
2776 ; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v6
2777 ; GFX8-NEXT: v_max_f32_e32 v5, v0, v1
2778 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
2779 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2780 ; GFX8-NEXT: buffer_wbinvl1
2781 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
2782 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2783 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2784 ; GFX8-NEXT: s_cbranch_execnz .LBB16_1
2785 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2786 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2787 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2788 ;
2789 ; GFX7-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2790 ; GFX7: ; %bb.0:
2791 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2792 ; GFX7-NEXT: s_mov_b32 s6, 0
2793 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
2794 ; GFX7-NEXT: s_mov_b32 s4, s6
2795 ; GFX7-NEXT: s_mov_b32 s5, s6
2796 ; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044 glc
2797 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2798 ; GFX7-NEXT: buffer_wbinvl1
2799 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
2800 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2801 ;
2802 ; GFX6-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2803 ; GFX6: ; %bb.0:
2804 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2805 ; GFX6-NEXT: s_mov_b32 s6, 0
2806 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
2807 ; GFX6-NEXT: s_mov_b32 s4, s6
2808 ; GFX6-NEXT: s_mov_b32 s5, s6
2809 ; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044 glc
2810 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2811 ; GFX6-NEXT: buffer_wbinvl1
2812 ; GFX6-NEXT: v_mov_b32_e32 v0, v2
2813 ; GFX6-NEXT: s_waitcnt expcnt(0)
2814 ; GFX6-NEXT: s_setpc_b64 s[30:31]
2815 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
2816 %result = atomicrmw fmax ptr addrspace(1) %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
2817 ret float %result
2818 }
2820 define void @global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %val) #1 {
2821 ; GFX12-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2822 ; GFX12: ; %bb.0:
2823 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
2824 ; GFX12-NEXT: s_wait_expcnt 0x0
2825 ; GFX12-NEXT: s_wait_samplecnt 0x0
2826 ; GFX12-NEXT: s_wait_bvhcnt 0x0
2827 ; GFX12-NEXT: s_wait_kmcnt 0x0
2828 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
2829 ; GFX12-NEXT: s_wait_storecnt 0x0
2830 ; GFX12-NEXT: global_atomic_max_num_f32 v[0:1], v2, off offset:2044 scope:SCOPE_SYS
2831 ; GFX12-NEXT: s_wait_storecnt 0x0
2832 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
2833 ; GFX12-NEXT: s_setpc_b64 s[30:31]
2834 ;
2835 ; GFX940-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2836 ; GFX940: ; %bb.0:
2837 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2838 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
2839 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
2840 ; GFX940-NEXT: v_max_f32_e32 v4, v2, v2
2841 ; GFX940-NEXT: .LBB17_1: ; %atomicrmw.start
2842 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
2843 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2844 ; GFX940-NEXT: v_max_f32_e32 v2, v3, v3
2845 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
2846 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
2847 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
2848 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2849 ; GFX940-NEXT: buffer_inv sc0 sc1
2850 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
2851 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
2852 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
2853 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
2854 ; GFX940-NEXT: s_cbranch_execnz .LBB17_1
2855 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
2856 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
2857 ; GFX940-NEXT: s_setpc_b64 s[30:31]
2858 ;
2859 ; GFX11-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2860 ; GFX11: ; %bb.0:
2861 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2862 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2863 ; GFX11-NEXT: global_atomic_max_f32 v[0:1], v2, off offset:2044
2864 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2865 ; GFX11-NEXT: buffer_gl1_inv
2866 ; GFX11-NEXT: buffer_gl0_inv
2867 ; GFX11-NEXT: s_setpc_b64 s[30:31]
2868 ;
2869 ; GFX10-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2870 ; GFX10: ; %bb.0:
2871 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2872 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2873 ; GFX10-NEXT: global_atomic_fmax v[0:1], v2, off offset:2044
2874 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2875 ; GFX10-NEXT: buffer_gl1_inv
2876 ; GFX10-NEXT: buffer_gl0_inv
2877 ; GFX10-NEXT: s_setpc_b64 s[30:31]
2878 ;
2879 ; GFX90A-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2880 ; GFX90A: ; %bb.0:
2881 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2882 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
2883 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
2884 ; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
2885 ; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
2886 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
2887 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2888 ; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
2889 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
2890 ; GFX90A-NEXT: buffer_wbl2
2891 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
2892 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2893 ; GFX90A-NEXT: buffer_invl2
2894 ; GFX90A-NEXT: buffer_wbinvl1
2895 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
2896 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2897 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
2898 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
2899 ; GFX90A-NEXT: s_cbranch_execnz .LBB17_1
2900 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
2901 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
2902 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
2903 ;
2904 ; GFX908-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2905 ; GFX908: ; %bb.0:
2906 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2907 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
2908 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
2909 ; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
2910 ; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
2911 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
2912 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2913 ; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
2914 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
2915 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
2916 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2917 ; GFX908-NEXT: buffer_wbinvl1
2918 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
2919 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2920 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
2921 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
2922 ; GFX908-NEXT: s_cbranch_execnz .LBB17_1
2923 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
2924 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
2925 ; GFX908-NEXT: s_setpc_b64 s[30:31]
2926 ;
2927 ; GFX8-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2928 ; GFX8: ; %bb.0:
2929 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2930 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
2931 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2932 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
2933 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2934 ; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v2
2935 ; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
2936 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2937 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2938 ; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v3
2939 ; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
2940 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2941 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2942 ; GFX8-NEXT: buffer_wbinvl1
2943 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
2944 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2945 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
2946 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2947 ; GFX8-NEXT: s_cbranch_execnz .LBB17_1
2948 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2949 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2950 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2951 ;
2952 ; GFX7-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2953 ; GFX7: ; %bb.0:
2954 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2955 ; GFX7-NEXT: s_mov_b32 s6, 0
2956 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
2957 ; GFX7-NEXT: s_mov_b32 s4, s6
2958 ; GFX7-NEXT: s_mov_b32 s5, s6
2959 ; GFX7-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044
2960 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2961 ; GFX7-NEXT: buffer_wbinvl1
2962 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2963 ;
2964 ; GFX6-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
2965 ; GFX6: ; %bb.0:
2966 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2967 ; GFX6-NEXT: s_mov_b32 s6, 0
2968 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
2969 ; GFX6-NEXT: s_mov_b32 s4, s6
2970 ; GFX6-NEXT: s_mov_b32 s5, s6
2971 ; GFX6-NEXT: buffer_atomic_fmax v2, v[0:1], s[4:7], 0 addr64 offset:2044
2972 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2973 ; GFX6-NEXT: buffer_wbinvl1
2974 ; GFX6-NEXT: s_waitcnt expcnt(0)
2975 ; GFX6-NEXT: s_setpc_b64 s[30:31]
2976 %gep = getelementptr float, ptr addrspace(1) %ptr, i64 511
2977 %unused = atomicrmw fmax ptr addrspace(1) %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
2978 ret void
2979 }
2981 ; --------------------------------------------------------------------
2982 ; double
2983 ; --------------------------------------------------------------------
2985 define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %val) #0 {
2986 ; GFX12-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
2987 ; GFX12: ; %bb.0:
2988 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
2989 ; GFX12-NEXT: s_wait_expcnt 0x0
2990 ; GFX12-NEXT: s_wait_samplecnt 0x0
2991 ; GFX12-NEXT: s_wait_bvhcnt 0x0
2992 ; GFX12-NEXT: s_wait_kmcnt 0x0
2993 ; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
2994 ; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
2995 ; GFX12-NEXT: s_mov_b32 s0, 0
2996 ; GFX12-NEXT: .LBB18_1: ; %atomicrmw.start
2997 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
2998 ; GFX12-NEXT: s_wait_loadcnt 0x0
2999 ; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
3000 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3001 ; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
3002 ; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
3003 ; GFX12-NEXT: s_wait_storecnt 0x0
3004 ; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
3005 ; GFX12-NEXT: s_wait_loadcnt 0x0
3006 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
3007 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
3008 ; GFX12-NEXT: s_wait_alu 0xfffe
3009 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
3010 ; GFX12-NEXT: s_wait_alu 0xfffe
3011 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3012 ; GFX12-NEXT: s_cbranch_execnz .LBB18_1
3013 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
3014 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
3015 ; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
3016 ; GFX12-NEXT: s_wait_alu 0xfffe
3017 ; GFX12-NEXT: s_setpc_b64 s[30:31]
3018 ;
3019 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
3020 ; GFX940: ; %bb.0:
3021 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3022 ; GFX940-NEXT: buffer_wbl2 sc1
3023 ; GFX940-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off sc0
3024 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3025 ; GFX940-NEXT: buffer_inv sc1
3026 ; GFX940-NEXT: s_setpc_b64 s[30:31]
3027 ;
3028 ; GFX11-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
3029 ; GFX11: ; %bb.0:
3030 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3031 ; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
3032 ; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
3033 ; GFX11-NEXT: s_mov_b32 s0, 0
3034 ; GFX11-NEXT: .LBB18_1: ; %atomicrmw.start
3035 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
3036 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3037 ; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
3038 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3039 ; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
3040 ; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
3041 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
3042 ; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
3043 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3044 ; GFX11-NEXT: buffer_gl1_inv
3045 ; GFX11-NEXT: buffer_gl0_inv
3046 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
3047 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
3048 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3049 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3050 ; GFX11-NEXT: s_cbranch_execnz .LBB18_1
3051 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
3052 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
3053 ; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
3054 ; GFX11-NEXT: s_setpc_b64 s[30:31]
3055 ;
3056 ; GFX10-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
3057 ; GFX10: ; %bb.0:
3058 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3059 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
3060 ; GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[0:1], v[2:3], off glc
3061 ; GFX10-NEXT: s_waitcnt vmcnt(0)
3062 ; GFX10-NEXT: buffer_gl1_inv
3063 ; GFX10-NEXT: buffer_gl0_inv
3064 ; GFX10-NEXT: s_setpc_b64 s[30:31]
3065 ;
3066 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
3067 ; GFX90A: ; %bb.0:
3068 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3069 ; GFX90A-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off glc
3070 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3071 ; GFX90A-NEXT: buffer_wbinvl1
3072 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
3073 ;
3074 ; GFX908-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
3075 ; GFX908: ; %bb.0:
3076 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3077 ; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
3078 ; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
3079 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
3080 ; GFX908-NEXT: .LBB18_1: ; %atomicrmw.start
3081 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
3082 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3083 ; GFX908-NEXT: v_mov_b32_e32 v7, v5
3084 ; GFX908-NEXT: v_mov_b32_e32 v6, v4
3085 ; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
3086 ; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
3087 ; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
3088 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3089 ; GFX908-NEXT: buffer_wbinvl1
3090 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
3091 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3092 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
3093 ; GFX908-NEXT: s_cbranch_execnz .LBB18_1
3094 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
3095 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
3096 ; GFX908-NEXT: v_mov_b32_e32 v0, v4
3097 ; GFX908-NEXT: v_mov_b32_e32 v1, v5
3098 ; GFX908-NEXT: s_setpc_b64 s[30:31]
3099 ;
3100 ; GFX8-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
3101 ; GFX8: ; %bb.0:
3102 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3103 ; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
3104 ; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
3105 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
3106 ; GFX8-NEXT: .LBB18_1: ; %atomicrmw.start
3107 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3108 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3109 ; GFX8-NEXT: v_mov_b32_e32 v7, v5
3110 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
3111 ; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
3112 ; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
3113 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
3114 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3115 ; GFX8-NEXT: buffer_wbinvl1
3116 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
3117 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3118 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
3119 ; GFX8-NEXT: s_cbranch_execnz .LBB18_1
3120 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
3121 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
3122 ; GFX8-NEXT: v_mov_b32_e32 v0, v4
3123 ; GFX8-NEXT: v_mov_b32_e32 v1, v5
3124 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3125 ;
3126 ; GFX7-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
3127 ; GFX7: ; %bb.0:
3128 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3129 ; GFX7-NEXT: s_mov_b32 s6, 0
3130 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
3131 ; GFX7-NEXT: s_mov_b32 s4, s6
3132 ; GFX7-NEXT: s_mov_b32 s5, s6
3133 ; GFX7-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
3134 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3135 ; GFX7-NEXT: buffer_wbinvl1
3136 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
3137 ; GFX7-NEXT: v_mov_b32_e32 v1, v3
3138 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3139 ;
3140 ; GFX6-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
3141 ; GFX6: ; %bb.0:
3142 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3143 ; GFX6-NEXT: s_mov_b32 s6, 0
3144 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
3145 ; GFX6-NEXT: s_mov_b32 s4, s6
3146 ; GFX6-NEXT: s_mov_b32 s5, s6
3147 ; GFX6-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
3148 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3149 ; GFX6-NEXT: buffer_wbinvl1
3150 ; GFX6-NEXT: v_mov_b32_e32 v0, v2
3151 ; GFX6-NEXT: v_mov_b32_e32 v1, v3
3152 ; GFX6-NEXT: s_waitcnt expcnt(0)
3153 ; GFX6-NEXT: s_setpc_b64 s[30:31]
3154 %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
3155 ret double %result
3156 }
3158 define double @global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %val) #0 {
3159 ; GFX12-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
3160 ; GFX12: ; %bb.0:
3161 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
3162 ; GFX12-NEXT: s_wait_expcnt 0x0
3163 ; GFX12-NEXT: s_wait_samplecnt 0x0
3164 ; GFX12-NEXT: s_wait_bvhcnt 0x0
3165 ; GFX12-NEXT: s_wait_kmcnt 0x0
3166 ; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
3167 ; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
3168 ; GFX12-NEXT: s_mov_b32 s0, 0
3169 ; GFX12-NEXT: .LBB19_1: ; %atomicrmw.start
3170 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
3171 ; GFX12-NEXT: s_wait_loadcnt 0x0
3172 ; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
3173 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3174 ; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
3175 ; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
3176 ; GFX12-NEXT: s_wait_storecnt 0x0
3177 ; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
3178 ; GFX12-NEXT: s_wait_loadcnt 0x0
3179 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
3180 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
3181 ; GFX12-NEXT: s_wait_alu 0xfffe
3182 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
3183 ; GFX12-NEXT: s_wait_alu 0xfffe
3184 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3185 ; GFX12-NEXT: s_cbranch_execnz .LBB19_1
3186 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
3187 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
3188 ; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
3189 ; GFX12-NEXT: s_wait_alu 0xfffe
3190 ; GFX12-NEXT: s_setpc_b64 s[30:31]
3191 ;
3192 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
3193 ; GFX940: ; %bb.0:
3194 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3195 ; GFX940-NEXT: buffer_wbl2 sc1
3196 ; GFX940-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off offset:2040 sc0
3197 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3198 ; GFX940-NEXT: buffer_inv sc1
3199 ; GFX940-NEXT: s_setpc_b64 s[30:31]
3200 ;
3201 ; GFX11-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
3202 ; GFX11: ; %bb.0:
3203 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3204 ; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
3205 ; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
3206 ; GFX11-NEXT: s_mov_b32 s0, 0
3207 ; GFX11-NEXT: .LBB19_1: ; %atomicrmw.start
3208 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
3209 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3210 ; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
3211 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3212 ; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
3213 ; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
3214 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
3215 ; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 glc
3216 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3217 ; GFX11-NEXT: buffer_gl1_inv
3218 ; GFX11-NEXT: buffer_gl0_inv
3219 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
3220 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
3221 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3222 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3223 ; GFX11-NEXT: s_cbranch_execnz .LBB19_1
3224 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
3225 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
3226 ; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
3227 ; GFX11-NEXT: s_setpc_b64 s[30:31]
3228 ;
3229 ; GFX10-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
3230 ; GFX10: ; %bb.0:
3231 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3232 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
3233 ; GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[0:1], v[2:3], off offset:2040 glc
3234 ; GFX10-NEXT: s_waitcnt vmcnt(0)
3235 ; GFX10-NEXT: buffer_gl1_inv
3236 ; GFX10-NEXT: buffer_gl0_inv
3237 ; GFX10-NEXT: s_setpc_b64 s[30:31]
3238 ;
3239 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
3240 ; GFX90A: ; %bb.0:
3241 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3242 ; GFX90A-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off offset:2040 glc
3243 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3244 ; GFX90A-NEXT: buffer_wbinvl1
3245 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
3246 ;
3247 ; GFX908-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
3248 ; GFX908: ; %bb.0:
3249 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3250 ; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:2040
3251 ; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
3252 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
3253 ; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
3254 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
3255 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3256 ; GFX908-NEXT: v_mov_b32_e32 v7, v5
3257 ; GFX908-NEXT: v_mov_b32_e32 v6, v4
3258 ; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
3259 ; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
3260 ; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 glc
3261 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3262 ; GFX908-NEXT: buffer_wbinvl1
3263 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
3264 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3265 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
3266 ; GFX908-NEXT: s_cbranch_execnz .LBB19_1
3267 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
3268 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
3269 ; GFX908-NEXT: v_mov_b32_e32 v0, v4
3270 ; GFX908-NEXT: v_mov_b32_e32 v1, v5
3271 ; GFX908-NEXT: s_setpc_b64 s[30:31]
3272 ;
3273 ; GFX8-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
3274 ; GFX8: ; %bb.0:
3275 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3276 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7f8, v0
3277 ; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
3278 ; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
3279 ; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
3280 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
3281 ; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
3282 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3283 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3284 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
3285 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
3286 ; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
3287 ; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[2:3]
3288 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
3289 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3290 ; GFX8-NEXT: buffer_wbinvl1
3291 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
3292 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3293 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
3294 ; GFX8-NEXT: s_cbranch_execnz .LBB19_1
3295 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
3296 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
3297 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3298 ;
3299 ; GFX7-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
3300 ; GFX7: ; %bb.0:
3301 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3302 ; GFX7-NEXT: s_mov_b32 s6, 0
3303 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
3304 ; GFX7-NEXT: s_mov_b32 s4, s6
3305 ; GFX7-NEXT: s_mov_b32 s5, s6
3306 ; GFX7-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:2040 glc
3307 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3308 ; GFX7-NEXT: buffer_wbinvl1
3309 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
3310 ; GFX7-NEXT: v_mov_b32_e32 v1, v3
3311 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3312 ;
3313 ; GFX6-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
3314 ; GFX6: ; %bb.0:
3315 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3316 ; GFX6-NEXT: s_mov_b32 s6, 0
3317 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
3318 ; GFX6-NEXT: s_mov_b32 s4, s6
3319 ; GFX6-NEXT: s_mov_b32 s5, s6
3320 ; GFX6-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:2040 glc
3321 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3322 ; GFX6-NEXT: buffer_wbinvl1
3323 ; GFX6-NEXT: v_mov_b32_e32 v0, v2
3324 ; GFX6-NEXT: v_mov_b32_e32 v1, v3
3325 ; GFX6-NEXT: s_waitcnt expcnt(0)
3326 ; GFX6-NEXT: s_setpc_b64 s[30:31]
3327 %gep = getelementptr double, ptr addrspace(1) %ptr, i64 255
3328 %result = atomicrmw fmax ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
3329 ret double %result
3330 }
3332 define double @global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %val) #0 {
3333 ; GFX12-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
3334 ; GFX12: ; %bb.0:
3335 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
3336 ; GFX12-NEXT: s_wait_expcnt 0x0
3337 ; GFX12-NEXT: s_wait_samplecnt 0x0
3338 ; GFX12-NEXT: s_wait_bvhcnt 0x0
3339 ; GFX12-NEXT: s_wait_kmcnt 0x0
3340 ; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
3341 ; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
3342 ; GFX12-NEXT: s_mov_b32 s0, 0
3343 ; GFX12-NEXT: .LBB20_1: ; %atomicrmw.start
3344 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
3345 ; GFX12-NEXT: s_wait_loadcnt 0x0
3346 ; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
3347 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3348 ; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
3349 ; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
3350 ; GFX12-NEXT: s_wait_storecnt 0x0
3351 ; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
3352 ; GFX12-NEXT: s_wait_loadcnt 0x0
3353 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
3354 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
3355 ; GFX12-NEXT: s_wait_alu 0xfffe
3356 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
3357 ; GFX12-NEXT: s_wait_alu 0xfffe
3358 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3359 ; GFX12-NEXT: s_cbranch_execnz .LBB20_1
3360 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
3361 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
3362 ; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
3363 ; GFX12-NEXT: s_wait_alu 0xfffe
3364 ; GFX12-NEXT: s_setpc_b64 s[30:31]
3365 ;
3366 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
3367 ; GFX940: ; %bb.0:
3368 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3369 ; GFX940-NEXT: buffer_wbl2 sc1
3370 ; GFX940-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off offset:-2048 sc0
3371 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3372 ; GFX940-NEXT: buffer_inv sc1
3373 ; GFX940-NEXT: s_setpc_b64 s[30:31]
3374 ;
3375 ; GFX11-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
3376 ; GFX11: ; %bb.0:
3377 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3378 ; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
3379 ; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
3380 ; GFX11-NEXT: s_mov_b32 s0, 0
3381 ; GFX11-NEXT: .LBB20_1: ; %atomicrmw.start
3382 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
3383 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3384 ; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
3385 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3386 ; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
3387 ; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
3388 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
3389 ; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
3390 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3391 ; GFX11-NEXT: buffer_gl1_inv
3392 ; GFX11-NEXT: buffer_gl0_inv
3393 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
3394 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
3395 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3396 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3397 ; GFX11-NEXT: s_cbranch_execnz .LBB20_1
3398 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
3399 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
3400 ; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
3401 ; GFX11-NEXT: s_setpc_b64 s[30:31]
3402 ;
3403 ; GFX10-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
3404 ; GFX10: ; %bb.0:
3405 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3406 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
3407 ; GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[0:1], v[2:3], off offset:-2048 glc
3408 ; GFX10-NEXT: s_waitcnt vmcnt(0)
3409 ; GFX10-NEXT: buffer_gl1_inv
3410 ; GFX10-NEXT: buffer_gl0_inv
3411 ; GFX10-NEXT: s_setpc_b64 s[30:31]
3412 ;
3413 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
3414 ; GFX90A: ; %bb.0:
3415 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3416 ; GFX90A-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off offset:-2048 glc
3417 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3418 ; GFX90A-NEXT: buffer_wbinvl1
3419 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
3420 ;
3421 ; GFX908-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
3422 ; GFX908: ; %bb.0:
3423 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3424 ; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:-2048
3425 ; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
3426 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
3427 ; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
3428 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
3429 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3430 ; GFX908-NEXT: v_mov_b32_e32 v7, v5
3431 ; GFX908-NEXT: v_mov_b32_e32 v6, v4
3432 ; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
3433 ; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
3434 ; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
3435 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3436 ; GFX908-NEXT: buffer_wbinvl1
3437 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
3438 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3439 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
3440 ; GFX908-NEXT: s_cbranch_execnz .LBB20_1
3441 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
3442 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
3443 ; GFX908-NEXT: v_mov_b32_e32 v0, v4
3444 ; GFX908-NEXT: v_mov_b32_e32 v1, v5
3445 ; GFX908-NEXT: s_setpc_b64 s[30:31]
3446 ;
3447 ; GFX8-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
3448 ; GFX8: ; %bb.0:
3449 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3450 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
3451 ; GFX8-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
3452 ; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
3453 ; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
3454 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
3455 ; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
3456 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3457 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3458 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
3459 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
3460 ; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
3461 ; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[2:3]
3462 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
3463 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3464 ; GFX8-NEXT: buffer_wbinvl1
3465 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
3466 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3467 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
3468 ; GFX8-NEXT: s_cbranch_execnz .LBB20_1
3469 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
3470 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
3471 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3472 ;
3473 ; GFX7-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
3474 ; GFX7: ; %bb.0:
3475 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3476 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
3477 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
3478 ; GFX7-NEXT: s_mov_b32 s6, 0
3479 ; GFX7-NEXT: s_mov_b32 s5, -1
3480 ; GFX7-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
3481 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3482 ; GFX7-NEXT: buffer_wbinvl1
3483 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
3484 ; GFX7-NEXT: v_mov_b32_e32 v1, v3
3485 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3486 ;
3487 ; GFX6-LABEL: global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
3488 ; GFX6: ; %bb.0:
3489 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3490 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
3491 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
3492 ; GFX6-NEXT: s_mov_b32 s6, 0
3493 ; GFX6-NEXT: s_mov_b32 s5, -1
3494 ; GFX6-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
3495 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3496 ; GFX6-NEXT: buffer_wbinvl1
3497 ; GFX6-NEXT: v_mov_b32_e32 v0, v2
3498 ; GFX6-NEXT: v_mov_b32_e32 v1, v3
3499 ; GFX6-NEXT: s_waitcnt expcnt(0)
3500 ; GFX6-NEXT: s_setpc_b64 s[30:31]
3501 %gep = getelementptr double, ptr addrspace(1) %ptr, i64 -256
3502 %result = atomicrmw fmax ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
3503 ret double %result
3504 }
3506 define void @global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %val) #0 {
3507 ; GFX12-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
3508 ; GFX12: ; %bb.0:
3509 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
3510 ; GFX12-NEXT: s_wait_expcnt 0x0
3511 ; GFX12-NEXT: s_wait_samplecnt 0x0
3512 ; GFX12-NEXT: s_wait_bvhcnt 0x0
3513 ; GFX12-NEXT: s_wait_kmcnt 0x0
3514 ; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
3515 ; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
3516 ; GFX12-NEXT: s_mov_b32 s0, 0
3517 ; GFX12-NEXT: .LBB21_1: ; %atomicrmw.start
3518 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
3519 ; GFX12-NEXT: s_wait_loadcnt 0x0
3520 ; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
3521 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
3522 ; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
3523 ; GFX12-NEXT: s_wait_storecnt 0x0
3524 ; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
3525 ; GFX12-NEXT: s_wait_loadcnt 0x0
3526 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
3527 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
3528 ; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
3529 ; GFX12-NEXT: s_wait_alu 0xfffe
3530 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
3531 ; GFX12-NEXT: s_wait_alu 0xfffe
3532 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3533 ; GFX12-NEXT: s_cbranch_execnz .LBB21_1
3534 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
3535 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
3536 ; GFX12-NEXT: s_wait_alu 0xfffe
3537 ; GFX12-NEXT: s_setpc_b64 s[30:31]
3538 ;
3539 ; GFX940-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
3540 ; GFX940: ; %bb.0:
3541 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3542 ; GFX940-NEXT: buffer_wbl2 sc1
3543 ; GFX940-NEXT: global_atomic_max_f64 v[0:1], v[2:3], off
3544 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3545 ; GFX940-NEXT: buffer_inv sc1
3546 ; GFX940-NEXT: s_setpc_b64 s[30:31]
3547 ;
3548 ; GFX11-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
3549 ; GFX11: ; %bb.0:
3550 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3551 ; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
3552 ; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
3553 ; GFX11-NEXT: s_mov_b32 s0, 0
3554 ; GFX11-NEXT: .LBB21_1: ; %atomicrmw.start
3555 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
3556 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3557 ; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
3558 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
3559 ; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
3560 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
3561 ; GFX11-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off glc
3562 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3563 ; GFX11-NEXT: buffer_gl1_inv
3564 ; GFX11-NEXT: buffer_gl0_inv
3565 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
3566 ; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
3567 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
3568 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3569 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3570 ; GFX11-NEXT: s_cbranch_execnz .LBB21_1
3571 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
3572 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
3573 ; GFX11-NEXT: s_setpc_b64 s[30:31]
3574 ;
3575 ; GFX10-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
3576 ; GFX10: ; %bb.0:
3577 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3578 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
3579 ; GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[2:3], off
3580 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
3581 ; GFX10-NEXT: buffer_gl1_inv
3582 ; GFX10-NEXT: buffer_gl0_inv
3583 ; GFX10-NEXT: s_setpc_b64 s[30:31]
3584 ;
3585 ; GFX90A-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
3586 ; GFX90A: ; %bb.0:
3587 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3588 ; GFX90A-NEXT: global_atomic_max_f64 v[0:1], v[2:3], off
3589 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3590 ; GFX90A-NEXT: buffer_wbinvl1
3591 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
3592 ;
3593 ; GFX908-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
3594 ; GFX908: ; %bb.0:
3595 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3596 ; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
3597 ; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
3598 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
3599 ; GFX908-NEXT: .LBB21_1: ; %atomicrmw.start
3600 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
3601 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3602 ; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
3603 ; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
3604 ; GFX908-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off glc
3605 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3606 ; GFX908-NEXT: buffer_wbinvl1
3607 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
3608 ; GFX908-NEXT: v_mov_b32_e32 v5, v3
3609 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3610 ; GFX908-NEXT: v_mov_b32_e32 v4, v2
3611 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
3612 ; GFX908-NEXT: s_cbranch_execnz .LBB21_1
3613 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
3614 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
3615 ; GFX908-NEXT: s_setpc_b64 s[30:31]
3616 ;
3617 ; GFX8-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
3618 ; GFX8: ; %bb.0:
3619 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3620 ; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
3621 ; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
3622 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
3623 ; GFX8-NEXT: .LBB21_1: ; %atomicrmw.start
3624 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3625 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3626 ; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
3627 ; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
3628 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
3629 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3630 ; GFX8-NEXT: buffer_wbinvl1
3631 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
3632 ; GFX8-NEXT: v_mov_b32_e32 v5, v3
3633 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3634 ; GFX8-NEXT: v_mov_b32_e32 v4, v2
3635 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
3636 ; GFX8-NEXT: s_cbranch_execnz .LBB21_1
3637 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
3638 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
3639 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3640 ;
3641 ; GFX7-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
3642 ; GFX7: ; %bb.0:
3643 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3644 ; GFX7-NEXT: s_mov_b32 s6, 0
3645 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
3646 ; GFX7-NEXT: s_mov_b32 s4, s6
3647 ; GFX7-NEXT: s_mov_b32 s5, s6
3648 ; GFX7-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64
3649 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3650 ; GFX7-NEXT: buffer_wbinvl1
3651 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3652 ;
3653 ; GFX6-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
3654 ; GFX6: ; %bb.0:
3655 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3656 ; GFX6-NEXT: s_mov_b32 s6, 0
3657 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
3658 ; GFX6-NEXT: s_mov_b32 s4, s6
3659 ; GFX6-NEXT: s_mov_b32 s5, s6
3660 ; GFX6-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64
3661 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3662 ; GFX6-NEXT: buffer_wbinvl1
3663 ; GFX6-NEXT: s_waitcnt expcnt(0)
3664 ; GFX6-NEXT: s_setpc_b64 s[30:31]
3665 %unused = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
3666 ret void
3667 }
3669 define void @global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %val) #0 {
3670 ; GFX12-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
3671 ; GFX12: ; %bb.0:
3672 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
3673 ; GFX12-NEXT: s_wait_expcnt 0x0
3674 ; GFX12-NEXT: s_wait_samplecnt 0x0
3675 ; GFX12-NEXT: s_wait_bvhcnt 0x0
3676 ; GFX12-NEXT: s_wait_kmcnt 0x0
3677 ; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
3678 ; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
3679 ; GFX12-NEXT: s_mov_b32 s0, 0
3680 ; GFX12-NEXT: .LBB22_1: ; %atomicrmw.start
3681 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
3682 ; GFX12-NEXT: s_wait_loadcnt 0x0
3683 ; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
3684 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
3685 ; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
3686 ; GFX12-NEXT: s_wait_storecnt 0x0
3687 ; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:2040 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
3688 ; GFX12-NEXT: s_wait_loadcnt 0x0
3689 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
3690 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
3691 ; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
3692 ; GFX12-NEXT: s_wait_alu 0xfffe
3693 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
3694 ; GFX12-NEXT: s_wait_alu 0xfffe
3695 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3696 ; GFX12-NEXT: s_cbranch_execnz .LBB22_1
3697 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
3698 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
3699 ; GFX12-NEXT: s_wait_alu 0xfffe
3700 ; GFX12-NEXT: s_setpc_b64 s[30:31]
3701 ;
3702 ; GFX940-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
3703 ; GFX940: ; %bb.0:
3704 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3705 ; GFX940-NEXT: buffer_wbl2 sc1
3706 ; GFX940-NEXT: global_atomic_max_f64 v[0:1], v[2:3], off offset:2040
3707 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3708 ; GFX940-NEXT: buffer_inv sc1
3709 ; GFX940-NEXT: s_setpc_b64 s[30:31]
3710 ;
3711 ; GFX11-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
3712 ; GFX11: ; %bb.0:
3713 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3714 ; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
3715 ; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
3716 ; GFX11-NEXT: s_mov_b32 s0, 0
3717 ; GFX11-NEXT: .LBB22_1: ; %atomicrmw.start
3718 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
3719 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3720 ; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
3721 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
3722 ; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
3723 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
3724 ; GFX11-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:2040 glc
3725 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3726 ; GFX11-NEXT: buffer_gl1_inv
3727 ; GFX11-NEXT: buffer_gl0_inv
3728 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
3729 ; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
3730 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
3731 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3732 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3733 ; GFX11-NEXT: s_cbranch_execnz .LBB22_1
3734 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
3735 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
3736 ; GFX11-NEXT: s_setpc_b64 s[30:31]
3737 ;
3738 ; GFX10-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
3739 ; GFX10: ; %bb.0:
3740 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3741 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
3742 ; GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[2:3], off offset:2040
3743 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
3744 ; GFX10-NEXT: buffer_gl1_inv
3745 ; GFX10-NEXT: buffer_gl0_inv
3746 ; GFX10-NEXT: s_setpc_b64 s[30:31]
3747 ;
3748 ; GFX90A-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
3749 ; GFX90A: ; %bb.0:
3750 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3751 ; GFX90A-NEXT: global_atomic_max_f64 v[0:1], v[2:3], off offset:2040
3752 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3753 ; GFX90A-NEXT: buffer_wbinvl1
3754 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
3755 ;
3756 ; GFX908-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
3757 ; GFX908: ; %bb.0:
3758 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3759 ; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:2040
3760 ; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
3761 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
3762 ; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
3763 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
3764 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3765 ; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
3766 ; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
3767 ; GFX908-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off offset:2040 glc
3768 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3769 ; GFX908-NEXT: buffer_wbinvl1
3770 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
3771 ; GFX908-NEXT: v_mov_b32_e32 v5, v3
3772 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3773 ; GFX908-NEXT: v_mov_b32_e32 v4, v2
3774 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
3775 ; GFX908-NEXT: s_cbranch_execnz .LBB22_1
3776 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
3777 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
3778 ; GFX908-NEXT: s_setpc_b64 s[30:31]
3779 ;
3780 ; GFX8-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
3781 ; GFX8: ; %bb.0:
3782 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3783 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7f8, v0
3784 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3785 ; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
3786 ; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
3787 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
3788 ; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
3789 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3790 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3791 ; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
3792 ; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
3793 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
3794 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3795 ; GFX8-NEXT: buffer_wbinvl1
3796 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
3797 ; GFX8-NEXT: v_mov_b32_e32 v5, v3
3798 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3799 ; GFX8-NEXT: v_mov_b32_e32 v4, v2
3800 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
3801 ; GFX8-NEXT: s_cbranch_execnz .LBB22_1
3802 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
3803 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
3804 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3805 ;
3806 ; GFX7-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
3807 ; GFX7: ; %bb.0:
3808 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3809 ; GFX7-NEXT: s_mov_b32 s6, 0
3810 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
3811 ; GFX7-NEXT: s_mov_b32 s4, s6
3812 ; GFX7-NEXT: s_mov_b32 s5, s6
3813 ; GFX7-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:2040
3814 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3815 ; GFX7-NEXT: buffer_wbinvl1
3816 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3817 ;
3818 ; GFX6-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
3819 ; GFX6: ; %bb.0:
3820 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3821 ; GFX6-NEXT: s_mov_b32 s6, 0
3822 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
3823 ; GFX6-NEXT: s_mov_b32 s4, s6
3824 ; GFX6-NEXT: s_mov_b32 s5, s6
3825 ; GFX6-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:2040
3826 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3827 ; GFX6-NEXT: buffer_wbinvl1
3828 ; GFX6-NEXT: s_waitcnt expcnt(0)
3829 ; GFX6-NEXT: s_setpc_b64 s[30:31]
3830 %gep = getelementptr double, ptr addrspace(1) %ptr, i64 255
3831 %unused = atomicrmw fmax ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
3832 ret void
3833 }
3835 define void @global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %val) #0 {
3836 ; GFX12-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
3837 ; GFX12: ; %bb.0:
3838 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
3839 ; GFX12-NEXT: s_wait_expcnt 0x0
3840 ; GFX12-NEXT: s_wait_samplecnt 0x0
3841 ; GFX12-NEXT: s_wait_bvhcnt 0x0
3842 ; GFX12-NEXT: s_wait_kmcnt 0x0
3843 ; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
3844 ; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
3845 ; GFX12-NEXT: s_mov_b32 s0, 0
3846 ; GFX12-NEXT: .LBB23_1: ; %atomicrmw.start
3847 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
3848 ; GFX12-NEXT: s_wait_loadcnt 0x0
3849 ; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
3850 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
3851 ; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
3852 ; GFX12-NEXT: s_wait_storecnt 0x0
3853 ; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
3854 ; GFX12-NEXT: s_wait_loadcnt 0x0
3855 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
3856 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
3857 ; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
3858 ; GFX12-NEXT: s_wait_alu 0xfffe
3859 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
3860 ; GFX12-NEXT: s_wait_alu 0xfffe
3861 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3862 ; GFX12-NEXT: s_cbranch_execnz .LBB23_1
3863 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
3864 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
3865 ; GFX12-NEXT: s_wait_alu 0xfffe
3866 ; GFX12-NEXT: s_setpc_b64 s[30:31]
3867 ;
3868 ; GFX940-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
3869 ; GFX940: ; %bb.0:
3870 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3871 ; GFX940-NEXT: buffer_wbl2 sc1
3872 ; GFX940-NEXT: global_atomic_max_f64 v[0:1], v[2:3], off offset:-2048
3873 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3874 ; GFX940-NEXT: buffer_inv sc1
3875 ; GFX940-NEXT: s_setpc_b64 s[30:31]
3876 ;
3877 ; GFX11-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
3878 ; GFX11: ; %bb.0:
3879 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3880 ; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
3881 ; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
3882 ; GFX11-NEXT: s_mov_b32 s0, 0
3883 ; GFX11-NEXT: .LBB23_1: ; %atomicrmw.start
3884 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
3885 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3886 ; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
3887 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
3888 ; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
3889 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
3890 ; GFX11-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:-2048 glc
3891 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3892 ; GFX11-NEXT: buffer_gl1_inv
3893 ; GFX11-NEXT: buffer_gl0_inv
3894 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
3895 ; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
3896 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
3897 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3898 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
3899 ; GFX11-NEXT: s_cbranch_execnz .LBB23_1
3900 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
3901 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
3902 ; GFX11-NEXT: s_setpc_b64 s[30:31]
3903 ;
3904 ; GFX10-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
3905 ; GFX10: ; %bb.0:
3906 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3907 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
3908 ; GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[2:3], off offset:-2048
3909 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
3910 ; GFX10-NEXT: buffer_gl1_inv
3911 ; GFX10-NEXT: buffer_gl0_inv
3912 ; GFX10-NEXT: s_setpc_b64 s[30:31]
3913 ;
3914 ; GFX90A-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
3915 ; GFX90A: ; %bb.0:
3916 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3917 ; GFX90A-NEXT: global_atomic_max_f64 v[0:1], v[2:3], off offset:-2048
3918 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3919 ; GFX90A-NEXT: buffer_wbinvl1
3920 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
3921 ;
3922 ; GFX908-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
3923 ; GFX908: ; %bb.0:
3924 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3925 ; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:-2048
3926 ; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
3927 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
3928 ; GFX908-NEXT: .LBB23_1: ; %atomicrmw.start
3929 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
3930 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3931 ; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
3932 ; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
3933 ; GFX908-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off offset:-2048 glc
3934 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3935 ; GFX908-NEXT: buffer_wbinvl1
3936 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
3937 ; GFX908-NEXT: v_mov_b32_e32 v5, v3
3938 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3939 ; GFX908-NEXT: v_mov_b32_e32 v4, v2
3940 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
3941 ; GFX908-NEXT: s_cbranch_execnz .LBB23_1
3942 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
3943 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
3944 ; GFX908-NEXT: s_setpc_b64 s[30:31]
3945 ;
3946 ; GFX8-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
3947 ; GFX8: ; %bb.0:
3948 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3949 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
3950 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
3951 ; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
3952 ; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
3953 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
3954 ; GFX8-NEXT: .LBB23_1: ; %atomicrmw.start
3955 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3956 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3957 ; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
3958 ; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
3959 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
3960 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3961 ; GFX8-NEXT: buffer_wbinvl1
3962 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
3963 ; GFX8-NEXT: v_mov_b32_e32 v5, v3
3964 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3965 ; GFX8-NEXT: v_mov_b32_e32 v4, v2
3966 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
3967 ; GFX8-NEXT: s_cbranch_execnz .LBB23_1
3968 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
3969 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
3970 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3971 ;
3972 ; GFX7-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
3973 ; GFX7: ; %bb.0:
3974 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3975 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
3976 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
3977 ; GFX7-NEXT: s_mov_b32 s6, 0
3978 ; GFX7-NEXT: s_mov_b32 s5, -1
3979 ; GFX7-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64
3980 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3981 ; GFX7-NEXT: buffer_wbinvl1
3982 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3983 ;
3984 ; GFX6-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
3985 ; GFX6: ; %bb.0:
3986 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3987 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
3988 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
3989 ; GFX6-NEXT: s_mov_b32 s6, 0
3990 ; GFX6-NEXT: s_mov_b32 s5, -1
3991 ; GFX6-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64
3992 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3993 ; GFX6-NEXT: buffer_wbinvl1
3994 ; GFX6-NEXT: s_waitcnt expcnt(0)
3995 ; GFX6-NEXT: s_setpc_b64 s[30:31]
3996 %gep = getelementptr double, ptr addrspace(1) %ptr, i64 -256
3997 %unused = atomicrmw fmax ptr addrspace(1) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
3998 ret void
3999 }
4001 define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %val) #0 {
4002 ; GFX12-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
4003 ; GFX12: ; %bb.0:
4004 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
4005 ; GFX12-NEXT: s_wait_expcnt 0x0
4006 ; GFX12-NEXT: s_wait_samplecnt 0x0
4007 ; GFX12-NEXT: s_wait_bvhcnt 0x0
4008 ; GFX12-NEXT: s_wait_kmcnt 0x0
4009 ; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
4010 ; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
4011 ; GFX12-NEXT: s_mov_b32 s0, 0
4012 ; GFX12-NEXT: .LBB24_1: ; %atomicrmw.start
4013 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
4014 ; GFX12-NEXT: s_wait_loadcnt 0x0
4015 ; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
4016 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4017 ; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
4018 ; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
4019 ; GFX12-NEXT: s_wait_storecnt 0x0
4020 ; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
4021 ; GFX12-NEXT: s_wait_loadcnt 0x0
4022 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
4023 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4024 ; GFX12-NEXT: s_wait_alu 0xfffe
4025 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
4026 ; GFX12-NEXT: s_wait_alu 0xfffe
4027 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4028 ; GFX12-NEXT: s_cbranch_execnz .LBB24_1
4029 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
4030 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
4031 ; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
4032 ; GFX12-NEXT: s_wait_alu 0xfffe
4033 ; GFX12-NEXT: s_setpc_b64 s[30:31]
4034 ;
4035 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
4036 ; GFX940: ; %bb.0:
4037 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4038 ; GFX940-NEXT: buffer_wbl2 sc1
4039 ; GFX940-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off sc0
4040 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4041 ; GFX940-NEXT: buffer_inv sc1
4042 ; GFX940-NEXT: s_setpc_b64 s[30:31]
4043 ;
4044 ; GFX11-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
4045 ; GFX11: ; %bb.0:
4046 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4047 ; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
4048 ; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
4049 ; GFX11-NEXT: s_mov_b32 s0, 0
4050 ; GFX11-NEXT: .LBB24_1: ; %atomicrmw.start
4051 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
4052 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4053 ; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
4054 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4055 ; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
4056 ; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
4057 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
4058 ; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
4059 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4060 ; GFX11-NEXT: buffer_gl1_inv
4061 ; GFX11-NEXT: buffer_gl0_inv
4062 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4063 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
4064 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4065 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4066 ; GFX11-NEXT: s_cbranch_execnz .LBB24_1
4067 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
4068 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
4069 ; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
4070 ; GFX11-NEXT: s_setpc_b64 s[30:31]
4071 ;
4072 ; GFX10-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
4073 ; GFX10: ; %bb.0:
4074 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4075 ; GFX10-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
4076 ; GFX10-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
4077 ; GFX10-NEXT: s_mov_b32 s4, 0
4078 ; GFX10-NEXT: .LBB24_1: ; %atomicrmw.start
4079 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
4080 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4081 ; GFX10-NEXT: v_mov_b32_e32 v7, v5
4082 ; GFX10-NEXT: v_mov_b32_e32 v6, v4
4083 ; GFX10-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
4084 ; GFX10-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
4085 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
4086 ; GFX10-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
4087 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4088 ; GFX10-NEXT: buffer_gl1_inv
4089 ; GFX10-NEXT: buffer_gl0_inv
4090 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4091 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
4092 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
4093 ; GFX10-NEXT: s_cbranch_execnz .LBB24_1
4094 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
4095 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
4096 ; GFX10-NEXT: v_mov_b32_e32 v0, v4
4097 ; GFX10-NEXT: v_mov_b32_e32 v1, v5
4098 ; GFX10-NEXT: s_setpc_b64 s[30:31]
4099 ;
4100 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
4101 ; GFX90A: ; %bb.0:
4102 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4103 ; GFX90A-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
4104 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
4105 ; GFX90A-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
4106 ; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
4107 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
4108 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4109 ; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1]
4110 ; GFX90A-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
4111 ; GFX90A-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
4112 ; GFX90A-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
4113 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4114 ; GFX90A-NEXT: buffer_wbinvl1
4115 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4116 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4117 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
4118 ; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
4119 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
4120 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
4121 ; GFX90A-NEXT: v_mov_b32_e32 v0, v4
4122 ; GFX90A-NEXT: v_mov_b32_e32 v1, v5
4123 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
4124 ;
4125 ; GFX908-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
4126 ; GFX908: ; %bb.0:
4127 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4128 ; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
4129 ; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
4130 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
4131 ; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start
4132 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
4133 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4134 ; GFX908-NEXT: v_mov_b32_e32 v7, v5
4135 ; GFX908-NEXT: v_mov_b32_e32 v6, v4
4136 ; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
4137 ; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
4138 ; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
4139 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4140 ; GFX908-NEXT: buffer_wbinvl1
4141 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4142 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4143 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
4144 ; GFX908-NEXT: s_cbranch_execnz .LBB24_1
4145 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
4146 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
4147 ; GFX908-NEXT: v_mov_b32_e32 v0, v4
4148 ; GFX908-NEXT: v_mov_b32_e32 v1, v5
4149 ; GFX908-NEXT: s_setpc_b64 s[30:31]
4150 ;
4151 ; GFX8-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
4152 ; GFX8: ; %bb.0:
4153 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4154 ; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
4155 ; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
4156 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
4157 ; GFX8-NEXT: .LBB24_1: ; %atomicrmw.start
4158 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4159 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4160 ; GFX8-NEXT: v_mov_b32_e32 v7, v5
4161 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
4162 ; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
4163 ; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
4164 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
4165 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4166 ; GFX8-NEXT: buffer_wbinvl1
4167 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4168 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4169 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
4170 ; GFX8-NEXT: s_cbranch_execnz .LBB24_1
4171 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4172 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
4173 ; GFX8-NEXT: v_mov_b32_e32 v0, v4
4174 ; GFX8-NEXT: v_mov_b32_e32 v1, v5
4175 ; GFX8-NEXT: s_setpc_b64 s[30:31]
4176 ;
4177 ; GFX7-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
4178 ; GFX7: ; %bb.0:
4179 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4180 ; GFX7-NEXT: s_mov_b32 s6, 0
4181 ; GFX7-NEXT: v_mov_b32_e32 v5, v1
4182 ; GFX7-NEXT: v_mov_b32_e32 v4, v0
4183 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
4184 ; GFX7-NEXT: s_mov_b32 s4, s6
4185 ; GFX7-NEXT: s_mov_b32 s5, s6
4186 ; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[4:5], s[4:7], 0 addr64
4187 ; GFX7-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
4188 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
4189 ; GFX7-NEXT: .LBB24_1: ; %atomicrmw.start
4190 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4191 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4192 ; GFX7-NEXT: v_mov_b32_e32 v11, v1
4193 ; GFX7-NEXT: v_mov_b32_e32 v10, v0
4194 ; GFX7-NEXT: v_max_f64 v[0:1], v[10:11], v[10:11]
4195 ; GFX7-NEXT: v_max_f64 v[8:9], v[0:1], v[6:7]
4196 ; GFX7-NEXT: v_mov_b32_e32 v0, v8
4197 ; GFX7-NEXT: v_mov_b32_e32 v1, v9
4198 ; GFX7-NEXT: v_mov_b32_e32 v2, v10
4199 ; GFX7-NEXT: v_mov_b32_e32 v3, v11
4200 ; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 glc
4201 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4202 ; GFX7-NEXT: buffer_wbinvl1
4203 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
4204 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
4205 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
4206 ; GFX7-NEXT: s_cbranch_execnz .LBB24_1
4207 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4208 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
4209 ; GFX7-NEXT: s_setpc_b64 s[30:31]
4210 ;
4211 ; GFX6-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
4212 ; GFX6: ; %bb.0:
4213 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4214 ; GFX6-NEXT: s_mov_b32 s6, 0
4215 ; GFX6-NEXT: v_mov_b32_e32 v5, v1
4216 ; GFX6-NEXT: v_mov_b32_e32 v4, v0
4217 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
4218 ; GFX6-NEXT: s_mov_b32 s4, s6
4219 ; GFX6-NEXT: s_mov_b32 s5, s6
4220 ; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v[4:5], s[4:7], 0 addr64
4221 ; GFX6-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
4222 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
4223 ; GFX6-NEXT: .LBB24_1: ; %atomicrmw.start
4224 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
4225 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4226 ; GFX6-NEXT: v_mov_b32_e32 v11, v1
4227 ; GFX6-NEXT: v_mov_b32_e32 v10, v0
4228 ; GFX6-NEXT: s_waitcnt expcnt(0)
4229 ; GFX6-NEXT: v_max_f64 v[0:1], v[10:11], v[10:11]
4230 ; GFX6-NEXT: v_max_f64 v[8:9], v[0:1], v[6:7]
4231 ; GFX6-NEXT: v_mov_b32_e32 v0, v8
4232 ; GFX6-NEXT: v_mov_b32_e32 v1, v9
4233 ; GFX6-NEXT: v_mov_b32_e32 v2, v10
4234 ; GFX6-NEXT: v_mov_b32_e32 v3, v11
4235 ; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 glc
4236 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4237 ; GFX6-NEXT: buffer_wbinvl1
4238 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
4239 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
4240 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
4241 ; GFX6-NEXT: s_cbranch_execnz .LBB24_1
4242 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
4243 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
4244 ; GFX6-NEXT: s_waitcnt expcnt(0)
4245 ; GFX6-NEXT: s_setpc_b64 s[30:31]
4246 %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
4247 ret double %result
4248 }
4250 define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %val) #0 {
4251 ; GFX12-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
4252 ; GFX12: ; %bb.0:
4253 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
4254 ; GFX12-NEXT: s_wait_expcnt 0x0
4255 ; GFX12-NEXT: s_wait_samplecnt 0x0
4256 ; GFX12-NEXT: s_wait_bvhcnt 0x0
4257 ; GFX12-NEXT: s_wait_kmcnt 0x0
4258 ; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
4259 ; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
4260 ; GFX12-NEXT: s_mov_b32 s0, 0
4261 ; GFX12-NEXT: .LBB25_1: ; %atomicrmw.start
4262 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
4263 ; GFX12-NEXT: s_wait_loadcnt 0x0
4264 ; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
4265 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4266 ; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
4267 ; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
4268 ; GFX12-NEXT: s_wait_storecnt 0x0
4269 ; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
4270 ; GFX12-NEXT: s_wait_loadcnt 0x0
4271 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
4272 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4273 ; GFX12-NEXT: s_wait_alu 0xfffe
4274 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
4275 ; GFX12-NEXT: s_wait_alu 0xfffe
4276 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4277 ; GFX12-NEXT: s_cbranch_execnz .LBB25_1
4278 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
4279 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
4280 ; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
4281 ; GFX12-NEXT: s_wait_alu 0xfffe
4282 ; GFX12-NEXT: s_setpc_b64 s[30:31]
4283 ;
4284 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
4285 ; GFX940: ; %bb.0:
4286 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4287 ; GFX940-NEXT: buffer_wbl2 sc1
4288 ; GFX940-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off sc0
4289 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4290 ; GFX940-NEXT: buffer_inv sc1
4291 ; GFX940-NEXT: s_setpc_b64 s[30:31]
4292 ;
4293 ; GFX11-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
4294 ; GFX11: ; %bb.0:
4295 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4296 ; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
4297 ; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
4298 ; GFX11-NEXT: s_mov_b32 s0, 0
4299 ; GFX11-NEXT: .LBB25_1: ; %atomicrmw.start
4300 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
4301 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4302 ; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
4303 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4304 ; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
4305 ; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
4306 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
4307 ; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
4308 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4309 ; GFX11-NEXT: buffer_gl1_inv
4310 ; GFX11-NEXT: buffer_gl0_inv
4311 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
4312 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
4313 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4314 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4315 ; GFX11-NEXT: s_cbranch_execnz .LBB25_1
4316 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
4317 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
4318 ; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
4319 ; GFX11-NEXT: s_setpc_b64 s[30:31]
4320 ;
4321 ; GFX10-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
4322 ; GFX10: ; %bb.0:
4323 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4324 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
4325 ; GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[0:1], v[2:3], off glc
4326 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4327 ; GFX10-NEXT: buffer_gl1_inv
4328 ; GFX10-NEXT: buffer_gl0_inv
4329 ; GFX10-NEXT: s_setpc_b64 s[30:31]
4330 ;
4331 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
4332 ; GFX90A: ; %bb.0:
4333 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4334 ; GFX90A-NEXT: global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off glc
4335 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4336 ; GFX90A-NEXT: buffer_wbinvl1
4337 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
4338 ;
4339 ; GFX908-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
4340 ; GFX908: ; %bb.0:
4341 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4342 ; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
4343 ; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
4344 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
4345 ; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start
4346 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
4347 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4348 ; GFX908-NEXT: v_mov_b32_e32 v7, v5
4349 ; GFX908-NEXT: v_mov_b32_e32 v6, v4
4350 ; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
4351 ; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
4352 ; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
4353 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4354 ; GFX908-NEXT: buffer_wbinvl1
4355 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4356 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4357 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
4358 ; GFX908-NEXT: s_cbranch_execnz .LBB25_1
4359 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
4360 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
4361 ; GFX908-NEXT: v_mov_b32_e32 v0, v4
4362 ; GFX908-NEXT: v_mov_b32_e32 v1, v5
4363 ; GFX908-NEXT: s_setpc_b64 s[30:31]
4364 ;
4365 ; GFX8-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
4366 ; GFX8: ; %bb.0:
4367 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4368 ; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
4369 ; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
4370 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
4371 ; GFX8-NEXT: .LBB25_1: ; %atomicrmw.start
4372 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4373 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4374 ; GFX8-NEXT: v_mov_b32_e32 v7, v5
4375 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
4376 ; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
4377 ; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
4378 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
4379 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4380 ; GFX8-NEXT: buffer_wbinvl1
4381 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4382 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4383 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
4384 ; GFX8-NEXT: s_cbranch_execnz .LBB25_1
4385 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4386 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
4387 ; GFX8-NEXT: v_mov_b32_e32 v0, v4
4388 ; GFX8-NEXT: v_mov_b32_e32 v1, v5
4389 ; GFX8-NEXT: s_setpc_b64 s[30:31]
4390 ;
4391 ; GFX7-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
4392 ; GFX7: ; %bb.0:
4393 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4394 ; GFX7-NEXT: s_mov_b32 s6, 0
4395 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
4396 ; GFX7-NEXT: s_mov_b32 s4, s6
4397 ; GFX7-NEXT: s_mov_b32 s5, s6
4398 ; GFX7-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
4399 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4400 ; GFX7-NEXT: buffer_wbinvl1
4401 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
4402 ; GFX7-NEXT: v_mov_b32_e32 v1, v3
4403 ; GFX7-NEXT: s_setpc_b64 s[30:31]
4404 ;
4405 ; GFX6-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
4406 ; GFX6: ; %bb.0:
4407 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4408 ; GFX6-NEXT: s_mov_b32 s6, 0
4409 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
4410 ; GFX6-NEXT: s_mov_b32 s4, s6
4411 ; GFX6-NEXT: s_mov_b32 s5, s6
4412 ; GFX6-NEXT: buffer_atomic_fmax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
4413 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4414 ; GFX6-NEXT: buffer_wbinvl1
4415 ; GFX6-NEXT: v_mov_b32_e32 v0, v2
4416 ; GFX6-NEXT: v_mov_b32_e32 v1, v3
4417 ; GFX6-NEXT: s_waitcnt expcnt(0)
4418 ; GFX6-NEXT: s_setpc_b64 s[30:31]
4419 %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
4420 ret double %result
4421 }
4423 ; --------------------------------------------------------------------
4424 ; half
4425 ; --------------------------------------------------------------------
4427 define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
4428 ; GFX12-LABEL: global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
4429 ; GFX12: ; %bb.0:
4430 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
4431 ; GFX12-NEXT: s_wait_expcnt 0x0
4432 ; GFX12-NEXT: s_wait_samplecnt 0x0
4433 ; GFX12-NEXT: s_wait_bvhcnt 0x0
4434 ; GFX12-NEXT: s_wait_kmcnt 0x0
4435 ; GFX12-NEXT: v_mov_b32_e32 v3, v0
4436 ; GFX12-NEXT: v_max_num_f16_e32 v2, v2, v2
4437 ; GFX12-NEXT: s_mov_b32 s0, 0
4438 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
4439 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
4440 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
4441 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
4442 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
4443 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
4444 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
4445 ; GFX12-NEXT: v_not_b32_e32 v4, v4
4446 ; GFX12-NEXT: .LBB26_1: ; %atomicrmw.start
4447 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
4448 ; GFX12-NEXT: s_wait_loadcnt 0x0
4449 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
4450 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4451 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
4452 ; GFX12-NEXT: v_max_num_f16_e32 v5, v5, v5
4453 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4454 ; GFX12-NEXT: v_max_num_f16_e32 v5, v5, v2
4455 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff, v5
4456 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4457 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
4458 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
4459 ; GFX12-NEXT: s_wait_storecnt 0x0
4460 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
4461 ; GFX12-NEXT: s_wait_loadcnt 0x0
4462 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
4463 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
4464 ; GFX12-NEXT: s_wait_alu 0xfffe
4465 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
4466 ; GFX12-NEXT: s_wait_alu 0xfffe
4467 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4468 ; GFX12-NEXT: s_cbranch_execnz .LBB26_1
4469 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
4470 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
4471 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
4472 ; GFX12-NEXT: s_wait_alu 0xfffe
4473 ; GFX12-NEXT: s_setpc_b64 s[30:31]
4474 ;
4475 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
4476 ; GFX940: ; %bb.0:
4477 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4478 ; GFX940-NEXT: v_mov_b32_e32 v3, v0
4479 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v3
4480 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
4481 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v3
4482 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
4483 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
4484 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
4485 ; GFX940-NEXT: v_not_b32_e32 v4, v4
4486 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
4487 ; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
4488 ; GFX940-NEXT: .LBB26_1: ; %atomicrmw.start
4489 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
4490 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4491 ; GFX940-NEXT: v_mov_b32_e32 v7, v5
4492 ; GFX940-NEXT: v_lshrrev_b32_e32 v5, v3, v7
4493 ; GFX940-NEXT: v_max_f16_e32 v5, v5, v5
4494 ; GFX940-NEXT: v_max_f16_e32 v5, v5, v2
4495 ; GFX940-NEXT: v_lshlrev_b32_e32 v5, v3, v5
4496 ; GFX940-NEXT: v_and_or_b32 v6, v7, v4, v5
4497 ; GFX940-NEXT: buffer_wbl2 sc1
4498 ; GFX940-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off sc0
4499 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4500 ; GFX940-NEXT: buffer_inv sc1
4501 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
4502 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4503 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
4504 ; GFX940-NEXT: s_cbranch_execnz .LBB26_1
4505 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
4506 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
4507 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
4508 ; GFX940-NEXT: s_setpc_b64 s[30:31]
4509 ;
4510 ; GFX11-LABEL: global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
4511 ; GFX11: ; %bb.0:
4512 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4513 ; GFX11-NEXT: v_mov_b32_e32 v3, v0
4514 ; GFX11-NEXT: v_max_f16_e32 v2, v2, v2
4515 ; GFX11-NEXT: s_mov_b32 s0, 0
4516 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
4517 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
4518 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
4519 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
4520 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
4521 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
4522 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
4523 ; GFX11-NEXT: v_not_b32_e32 v4, v4
4524 ; GFX11-NEXT: .LBB26_1: ; %atomicrmw.start
4525 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
4526 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4527 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
4528 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4529 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
4530 ; GFX11-NEXT: v_max_f16_e32 v5, v5, v5
4531 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4532 ; GFX11-NEXT: v_max_f16_e32 v5, v5, v2
4533 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
4534 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4535 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
4536 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
4537 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
4538 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
4539 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4540 ; GFX11-NEXT: buffer_gl1_inv
4541 ; GFX11-NEXT: buffer_gl0_inv
4542 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
4543 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
4544 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4545 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4546 ; GFX11-NEXT: s_cbranch_execnz .LBB26_1
4547 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
4548 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
4549 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
4550 ; GFX11-NEXT: s_setpc_b64 s[30:31]
4551 ;
4552 ; GFX10-LABEL: global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
4553 ; GFX10: ; %bb.0:
4554 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4555 ; GFX10-NEXT: v_mov_b32_e32 v3, v0
4556 ; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
4557 ; GFX10-NEXT: s_mov_b32 s4, 0
4558 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
4559 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
4560 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
4561 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
4562 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
4563 ; GFX10-NEXT: v_not_b32_e32 v4, v4
4564 ; GFX10-NEXT: .LBB26_1: ; %atomicrmw.start
4565 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
4566 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4567 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
4568 ; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
4569 ; GFX10-NEXT: v_max_f16_e32 v5, v5, v5
4570 ; GFX10-NEXT: v_max_f16_e32 v5, v5, v2
4571 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
4572 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
4573 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
4574 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
4575 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4576 ; GFX10-NEXT: buffer_gl1_inv
4577 ; GFX10-NEXT: buffer_gl0_inv
4578 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
4579 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
4580 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
4581 ; GFX10-NEXT: s_cbranch_execnz .LBB26_1
4582 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
4583 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
4584 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
4585 ; GFX10-NEXT: s_setpc_b64 s[30:31]
4586 ;
4587 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
4588 ; GFX90A: ; %bb.0:
4589 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4590 ; GFX90A-NEXT: v_mov_b32_e32 v3, v0
4591 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
4592 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
4593 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
4594 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
4595 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
4596 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
4597 ; GFX90A-NEXT: v_not_b32_e32 v4, v4
4598 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
4599 ; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
4600 ; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
4601 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
4602 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4603 ; GFX90A-NEXT: v_mov_b32_e32 v7, v5
4604 ; GFX90A-NEXT: v_lshrrev_b32_e32 v5, v3, v7
4605 ; GFX90A-NEXT: v_max_f16_e32 v5, v5, v5
4606 ; GFX90A-NEXT: v_max_f16_e32 v5, v5, v2
4607 ; GFX90A-NEXT: v_lshlrev_b32_e32 v5, v3, v5
4608 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v4, v5
4609 ; GFX90A-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off glc
4610 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4611 ; GFX90A-NEXT: buffer_wbinvl1
4612 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
4613 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4614 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
4615 ; GFX90A-NEXT: s_cbranch_execnz .LBB26_1
4616 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
4617 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
4618 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
4619 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
4620 ;
4621 ; GFX908-LABEL: global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
4622 ; GFX908: ; %bb.0:
4623 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4624 ; GFX908-NEXT: v_mov_b32_e32 v3, v0
4625 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
4626 ; GFX908-NEXT: global_load_dword v5, v[0:1], off
4627 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
4628 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
4629 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
4630 ; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
4631 ; GFX908-NEXT: v_not_b32_e32 v4, v4
4632 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
4633 ; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
4634 ; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
4635 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
4636 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4637 ; GFX908-NEXT: v_mov_b32_e32 v6, v5
4638 ; GFX908-NEXT: v_lshrrev_b32_e32 v5, v3, v6
4639 ; GFX908-NEXT: v_max_f16_e32 v5, v5, v5
4640 ; GFX908-NEXT: v_max_f16_e32 v5, v5, v2
4641 ; GFX908-NEXT: v_lshlrev_b32_e32 v5, v3, v5
4642 ; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
4643 ; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
4644 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4645 ; GFX908-NEXT: buffer_wbinvl1
4646 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
4647 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4648 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
4649 ; GFX908-NEXT: s_cbranch_execnz .LBB26_1
4650 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
4651 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
4652 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
4653 ; GFX908-NEXT: s_setpc_b64 s[30:31]
4654 ;
4655 ; GFX8-LABEL: global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
4656 ; GFX8: ; %bb.0:
4657 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4658 ; GFX8-NEXT: v_mov_b32_e32 v3, v0
4659 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
4660 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
4661 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
4662 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
4663 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
4664 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
4665 ; GFX8-NEXT: v_not_b32_e32 v4, v4
4666 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
4667 ; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
4668 ; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start
4669 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4670 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4671 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
4672 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
4673 ; GFX8-NEXT: v_max_f16_e32 v5, v5, v5
4674 ; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
4675 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
4676 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
4677 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
4678 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
4679 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4680 ; GFX8-NEXT: buffer_wbinvl1
4681 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
4682 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4683 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
4684 ; GFX8-NEXT: s_cbranch_execnz .LBB26_1
4685 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4686 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
4687 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
4688 ; GFX8-NEXT: s_setpc_b64 s[30:31]
4689 ;
4690 ; GFX7-LABEL: global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
4691 ; GFX7: ; %bb.0:
4692 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4693 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
4694 ; GFX7-NEXT: s_mov_b32 s6, 0
4695 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
4696 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
4697 ; GFX7-NEXT: s_mov_b32 s4, s6
4698 ; GFX7-NEXT: s_mov_b32 s5, s6
4699 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
4700 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
4701 ; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
4702 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 3, v3
4703 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
4704 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v2
4705 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v5
4706 ; GFX7-NEXT: v_not_b32_e32 v7, v2
4707 ; GFX7-NEXT: .LBB26_1: ; %atomicrmw.start
4708 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4709 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4710 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v5, v4
4711 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
4712 ; GFX7-NEXT: v_and_b32_e32 v3, v4, v7
4713 ; GFX7-NEXT: v_max_f32_e32 v2, v2, v6
4714 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
4715 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v5, v2
4716 ; GFX7-NEXT: v_or_b32_e32 v3, v3, v2
4717 ; GFX7-NEXT: v_mov_b32_e32 v2, v3
4718 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
4719 ; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v[0:1], s[4:7], 0 addr64 glc
4720 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4721 ; GFX7-NEXT: buffer_wbinvl1
4722 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
4723 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
4724 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
4725 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
4726 ; GFX7-NEXT: s_cbranch_execnz .LBB26_1
4727 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4728 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
4729 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v5, v2
4730 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
4731 ; GFX7-NEXT: s_setpc_b64 s[30:31]
4732 ;
4733 ; GFX6-LABEL: global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory:
4734 ; GFX6: ; %bb.0:
4735 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4736 ; GFX6-NEXT: v_mov_b32_e32 v3, v0
4737 ; GFX6-NEXT: s_mov_b32 s6, 0
4738 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v3
4739 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
4740 ; GFX6-NEXT: s_mov_b32 s4, s6
4741 ; GFX6-NEXT: s_mov_b32 s5, s6
4742 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
4743 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
4744 ; GFX6-NEXT: v_and_b32_e32 v3, 3, v3
4745 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 3, v3
4746 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
4747 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v2
4748 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v5
4749 ; GFX6-NEXT: v_not_b32_e32 v7, v2
4750 ; GFX6-NEXT: .LBB26_1: ; %atomicrmw.start
4751 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
4752 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
4753 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v5, v4
4754 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
4755 ; GFX6-NEXT: v_and_b32_e32 v3, v4, v7
4756 ; GFX6-NEXT: v_max_f32_e32 v2, v2, v6
4757 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
4758 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v5, v2
4759 ; GFX6-NEXT: v_or_b32_e32 v3, v3, v2
4760 ; GFX6-NEXT: v_mov_b32_e32 v2, v3
4761 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
4762 ; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v[0:1], s[4:7], 0 addr64 glc
4763 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4764 ; GFX6-NEXT: buffer_wbinvl1
4765 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
4766 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
4767 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
4768 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
4769 ; GFX6-NEXT: s_cbranch_execnz .LBB26_1
4770 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
4771 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
4772 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v5, v2
4773 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
4774 ; GFX6-NEXT: s_waitcnt expcnt(0)
4775 ; GFX6-NEXT: s_setpc_b64 s[30:31]
4776 %result = atomicrmw fmax ptr addrspace(1) %ptr, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
4777 ret half %result
4778 }
4780 define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
4781 ; GFX12-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
4782 ; GFX12: ; %bb.0:
4783 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
4784 ; GFX12-NEXT: s_wait_expcnt 0x0
4785 ; GFX12-NEXT: s_wait_samplecnt 0x0
4786 ; GFX12-NEXT: s_wait_bvhcnt 0x0
4787 ; GFX12-NEXT: s_wait_kmcnt 0x0
4788 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
4789 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
4790 ; GFX12-NEXT: v_max_num_f16_e32 v2, v2, v2
4791 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
4792 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
4793 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
4794 ; GFX12-NEXT: s_mov_b32 s0, 0
4795 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
4796 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
4797 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
4798 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
4799 ; GFX12-NEXT: v_not_b32_e32 v4, v4
4800 ; GFX12-NEXT: .LBB27_1: ; %atomicrmw.start
4801 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
4802 ; GFX12-NEXT: s_wait_loadcnt 0x0
4803 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
4804 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4805 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
4806 ; GFX12-NEXT: v_max_num_f16_e32 v5, v5, v5
4807 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4808 ; GFX12-NEXT: v_max_num_f16_e32 v5, v5, v2
4809 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff, v5
4810 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4811 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
4812 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
4813 ; GFX12-NEXT: s_wait_storecnt 0x0
4814 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
4815 ; GFX12-NEXT: s_wait_loadcnt 0x0
4816 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
4817 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
4818 ; GFX12-NEXT: s_wait_alu 0xfffe
4819 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
4820 ; GFX12-NEXT: s_wait_alu 0xfffe
4821 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4822 ; GFX12-NEXT: s_cbranch_execnz .LBB27_1
4823 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
4824 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
4825 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
4826 ; GFX12-NEXT: s_wait_alu 0xfffe
4827 ; GFX12-NEXT: s_setpc_b64 s[30:31]
4828 ;
4829 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
4830 ; GFX940: ; %bb.0:
4831 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4832 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
4833 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
4834 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
4835 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
4836 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
4837 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v4
4838 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
4839 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
4840 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
4841 ; GFX940-NEXT: v_not_b32_e32 v4, v4
4842 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
4843 ; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
4844 ; GFX940-NEXT: .LBB27_1: ; %atomicrmw.start
4845 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
4846 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4847 ; GFX940-NEXT: v_mov_b32_e32 v7, v5
4848 ; GFX940-NEXT: v_lshrrev_b32_e32 v5, v3, v7
4849 ; GFX940-NEXT: v_max_f16_e32 v5, v5, v5
4850 ; GFX940-NEXT: v_max_f16_e32 v5, v5, v2
4851 ; GFX940-NEXT: v_lshlrev_b32_e32 v5, v3, v5
4852 ; GFX940-NEXT: v_and_or_b32 v6, v7, v4, v5
4853 ; GFX940-NEXT: buffer_wbl2 sc1
4854 ; GFX940-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off sc0
4855 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4856 ; GFX940-NEXT: buffer_inv sc1
4857 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
4858 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4859 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
4860 ; GFX940-NEXT: s_cbranch_execnz .LBB27_1
4861 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
4862 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
4863 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
4864 ; GFX940-NEXT: s_setpc_b64 s[30:31]
4865 ;
4866 ; GFX11-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
4867 ; GFX11: ; %bb.0:
4868 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4869 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
4870 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
4871 ; GFX11-NEXT: v_max_f16_e32 v2, v2, v2
4872 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
4873 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
4874 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
4875 ; GFX11-NEXT: s_mov_b32 s0, 0
4876 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
4877 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
4878 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
4879 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
4880 ; GFX11-NEXT: v_not_b32_e32 v4, v4
4881 ; GFX11-NEXT: .LBB27_1: ; %atomicrmw.start
4882 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
4883 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4884 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
4885 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4886 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
4887 ; GFX11-NEXT: v_max_f16_e32 v5, v5, v5
4888 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4889 ; GFX11-NEXT: v_max_f16_e32 v5, v5, v2
4890 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
4891 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4892 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
4893 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
4894 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
4895 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
4896 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4897 ; GFX11-NEXT: buffer_gl1_inv
4898 ; GFX11-NEXT: buffer_gl0_inv
4899 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
4900 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
4901 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4902 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
4903 ; GFX11-NEXT: s_cbranch_execnz .LBB27_1
4904 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
4905 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
4906 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
4907 ; GFX11-NEXT: s_setpc_b64 s[30:31]
4908 ;
4909 ; GFX10-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
4910 ; GFX10: ; %bb.0:
4911 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4912 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
4913 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
4914 ; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
4915 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
4916 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
4917 ; GFX10-NEXT: s_mov_b32 s4, 0
4918 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
4919 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
4920 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
4921 ; GFX10-NEXT: v_not_b32_e32 v4, v4
4922 ; GFX10-NEXT: .LBB27_1: ; %atomicrmw.start
4923 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
4924 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4925 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
4926 ; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
4927 ; GFX10-NEXT: v_max_f16_e32 v5, v5, v5
4928 ; GFX10-NEXT: v_max_f16_e32 v5, v5, v2
4929 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
4930 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
4931 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
4932 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
4933 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4934 ; GFX10-NEXT: buffer_gl1_inv
4935 ; GFX10-NEXT: buffer_gl0_inv
4936 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
4937 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
4938 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
4939 ; GFX10-NEXT: s_cbranch_execnz .LBB27_1
4940 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
4941 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
4942 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
4943 ; GFX10-NEXT: s_setpc_b64 s[30:31]
4944 ;
4945 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
4946 ; GFX90A: ; %bb.0:
4947 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4948 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
4949 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
4950 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
4951 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
4952 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
4953 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
4954 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
4955 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
4956 ; GFX90A-NEXT: v_not_b32_e32 v4, v4
4957 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
4958 ; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
4959 ; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
4960 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
4961 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4962 ; GFX90A-NEXT: v_mov_b32_e32 v7, v5
4963 ; GFX90A-NEXT: v_lshrrev_b32_e32 v5, v3, v7
4964 ; GFX90A-NEXT: v_max_f16_e32 v5, v5, v5
4965 ; GFX90A-NEXT: v_max_f16_e32 v5, v5, v2
4966 ; GFX90A-NEXT: v_lshlrev_b32_e32 v5, v3, v5
4967 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v4, v5
4968 ; GFX90A-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off glc
4969 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4970 ; GFX90A-NEXT: buffer_wbinvl1
4971 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
4972 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4973 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
4974 ; GFX90A-NEXT: s_cbranch_execnz .LBB27_1
4975 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
4976 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
4977 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
4978 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
4979 ;
4980 ; GFX908-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
4981 ; GFX908: ; %bb.0:
4982 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4983 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
4984 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
4985 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
4986 ; GFX908-NEXT: global_load_dword v5, v[0:1], off
4987 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
4988 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
4989 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
4990 ; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
4991 ; GFX908-NEXT: v_not_b32_e32 v4, v4
4992 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
4993 ; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
4994 ; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
4995 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
4996 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4997 ; GFX908-NEXT: v_mov_b32_e32 v6, v5
4998 ; GFX908-NEXT: v_lshrrev_b32_e32 v5, v3, v6
4999 ; GFX908-NEXT: v_max_f16_e32 v5, v5, v5
5000 ; GFX908-NEXT: v_max_f16_e32 v5, v5, v2
5001 ; GFX908-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5002 ; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
5003 ; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
5004 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5005 ; GFX908-NEXT: buffer_wbinvl1
5006 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
5007 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5008 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
5009 ; GFX908-NEXT: s_cbranch_execnz .LBB27_1
5010 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
5011 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
5012 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5013 ; GFX908-NEXT: s_setpc_b64 s[30:31]
5014 ;
5015 ; GFX8-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
5016 ; GFX8: ; %bb.0:
5017 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5018 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
5019 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
5020 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
5021 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
5022 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
5023 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5024 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
5025 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
5026 ; GFX8-NEXT: v_not_b32_e32 v4, v4
5027 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
5028 ; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
5029 ; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start
5030 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5031 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5032 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
5033 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5034 ; GFX8-NEXT: v_max_f16_e32 v5, v5, v5
5035 ; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
5036 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
5037 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5038 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
5039 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
5040 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5041 ; GFX8-NEXT: buffer_wbinvl1
5042 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
5043 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5044 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
5045 ; GFX8-NEXT: s_cbranch_execnz .LBB27_1
5046 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5047 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
5048 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5049 ; GFX8-NEXT: s_setpc_b64 s[30:31]
5050 ;
5051 ; GFX7-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
5052 ; GFX7: ; %bb.0:
5053 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5054 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
5055 ; GFX7-NEXT: s_mov_b32 s6, 0
5056 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
5057 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
5058 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
5059 ; GFX7-NEXT: s_mov_b32 s4, s6
5060 ; GFX7-NEXT: s_mov_b32 s5, s6
5061 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
5062 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
5063 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
5064 ; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
5065 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
5066 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
5067 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
5068 ; GFX7-NEXT: v_not_b32_e32 v8, v2
5069 ; GFX7-NEXT: .LBB27_1: ; %atomicrmw.start
5070 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5071 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5072 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
5073 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
5074 ; GFX7-NEXT: v_and_b32_e32 v4, v3, v8
5075 ; GFX7-NEXT: v_max_f32_e32 v2, v2, v7
5076 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
5077 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v6, v2
5078 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
5079 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
5080 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
5081 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
5082 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5083 ; GFX7-NEXT: buffer_wbinvl1
5084 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
5085 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
5086 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
5087 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
5088 ; GFX7-NEXT: s_cbranch_execnz .LBB27_1
5089 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5090 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
5091 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
5092 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
5093 ; GFX7-NEXT: s_setpc_b64 s[30:31]
5094 ;
5095 ; GFX6-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
5096 ; GFX6: ; %bb.0:
5097 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5098 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
5099 ; GFX6-NEXT: s_mov_b32 s6, 0
5100 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
5101 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
5102 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
5103 ; GFX6-NEXT: s_mov_b32 s4, s6
5104 ; GFX6-NEXT: s_mov_b32 s5, s6
5105 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
5106 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
5107 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
5108 ; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
5109 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
5110 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
5111 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
5112 ; GFX6-NEXT: v_not_b32_e32 v8, v2
5113 ; GFX6-NEXT: .LBB27_1: ; %atomicrmw.start
5114 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
5115 ; GFX6-NEXT: s_waitcnt vmcnt(0)
5116 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
5117 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
5118 ; GFX6-NEXT: s_waitcnt expcnt(0)
5119 ; GFX6-NEXT: v_and_b32_e32 v4, v3, v8
5120 ; GFX6-NEXT: v_max_f32_e32 v2, v2, v7
5121 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
5122 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v6, v2
5123 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
5124 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
5125 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
5126 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
5127 ; GFX6-NEXT: s_waitcnt vmcnt(0)
5128 ; GFX6-NEXT: buffer_wbinvl1
5129 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
5130 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
5131 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
5132 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
5133 ; GFX6-NEXT: s_cbranch_execnz .LBB27_1
5134 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
5135 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
5136 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
5137 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
5138 ; GFX6-NEXT: s_waitcnt expcnt(0)
5139 ; GFX6-NEXT: s_setpc_b64 s[30:31]
5140 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
5141 %result = atomicrmw fmax ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
5142 ret half %result
5143 }
5145 define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
5146 ; GFX12-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
5147 ; GFX12: ; %bb.0:
5148 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
5149 ; GFX12-NEXT: s_wait_expcnt 0x0
5150 ; GFX12-NEXT: s_wait_samplecnt 0x0
5151 ; GFX12-NEXT: s_wait_bvhcnt 0x0
5152 ; GFX12-NEXT: s_wait_kmcnt 0x0
5153 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
5154 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
5155 ; GFX12-NEXT: v_max_num_f16_e32 v2, v2, v2
5156 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
5157 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
5158 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
5159 ; GFX12-NEXT: s_mov_b32 s0, 0
5160 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
5161 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5162 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
5163 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
5164 ; GFX12-NEXT: v_not_b32_e32 v4, v4
5165 ; GFX12-NEXT: .LBB28_1: ; %atomicrmw.start
5166 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
5167 ; GFX12-NEXT: s_wait_loadcnt 0x0
5168 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
5169 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5170 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5171 ; GFX12-NEXT: v_max_num_f16_e32 v5, v5, v5
5172 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5173 ; GFX12-NEXT: v_max_num_f16_e32 v5, v5, v2
5174 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff, v5
5175 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5176 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5177 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
5178 ; GFX12-NEXT: s_wait_storecnt 0x0
5179 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
5180 ; GFX12-NEXT: s_wait_loadcnt 0x0
5181 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
5182 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
5183 ; GFX12-NEXT: s_wait_alu 0xfffe
5184 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
5185 ; GFX12-NEXT: s_wait_alu 0xfffe
5186 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5187 ; GFX12-NEXT: s_cbranch_execnz .LBB28_1
5188 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
5189 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
5190 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5191 ; GFX12-NEXT: s_wait_alu 0xfffe
5192 ; GFX12-NEXT: s_setpc_b64 s[30:31]
5193 ;
5194 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
5195 ; GFX940: ; %bb.0:
5196 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5197 ; GFX940-NEXT: s_movk_i32 s0, 0xf800
5198 ; GFX940-NEXT: s_mov_b32 s1, -1
5199 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
5200 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
5201 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
5202 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
5203 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v4
5204 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5205 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
5206 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
5207 ; GFX940-NEXT: v_not_b32_e32 v4, v4
5208 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
5209 ; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
5210 ; GFX940-NEXT: .LBB28_1: ; %atomicrmw.start
5211 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
5212 ; GFX940-NEXT: s_waitcnt vmcnt(0)
5213 ; GFX940-NEXT: v_mov_b32_e32 v7, v5
5214 ; GFX940-NEXT: v_lshrrev_b32_e32 v5, v3, v7
5215 ; GFX940-NEXT: v_max_f16_e32 v5, v5, v5
5216 ; GFX940-NEXT: v_max_f16_e32 v5, v5, v2
5217 ; GFX940-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5218 ; GFX940-NEXT: v_and_or_b32 v6, v7, v4, v5
5219 ; GFX940-NEXT: buffer_wbl2 sc1
5220 ; GFX940-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off sc0
5221 ; GFX940-NEXT: s_waitcnt vmcnt(0)
5222 ; GFX940-NEXT: buffer_inv sc1
5223 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
5224 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
5225 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
5226 ; GFX940-NEXT: s_cbranch_execnz .LBB28_1
5227 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
5228 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
5229 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5230 ; GFX940-NEXT: s_setpc_b64 s[30:31]
5231 ;
5232 ; GFX11-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
5233 ; GFX11: ; %bb.0:
5234 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5235 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
5236 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
5237 ; GFX11-NEXT: v_max_f16_e32 v2, v2, v2
5238 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
5239 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
5240 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
5241 ; GFX11-NEXT: s_mov_b32 s0, 0
5242 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
5243 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5244 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
5245 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
5246 ; GFX11-NEXT: v_not_b32_e32 v4, v4
5247 ; GFX11-NEXT: .LBB28_1: ; %atomicrmw.start
5248 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
5249 ; GFX11-NEXT: s_waitcnt vmcnt(0)
5250 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
5251 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5252 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5253 ; GFX11-NEXT: v_max_f16_e32 v5, v5, v5
5254 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5255 ; GFX11-NEXT: v_max_f16_e32 v5, v5, v2
5256 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
5257 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5258 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5259 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
5260 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
5261 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
5262 ; GFX11-NEXT: s_waitcnt vmcnt(0)
5263 ; GFX11-NEXT: buffer_gl1_inv
5264 ; GFX11-NEXT: buffer_gl0_inv
5265 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
5266 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
5267 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
5268 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5269 ; GFX11-NEXT: s_cbranch_execnz .LBB28_1
5270 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
5271 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
5272 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5273 ; GFX11-NEXT: s_setpc_b64 s[30:31]
5274 ;
5275 ; GFX10-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
5276 ; GFX10: ; %bb.0:
5277 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5278 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
5279 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
5280 ; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
5281 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
5282 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
5283 ; GFX10-NEXT: s_mov_b32 s4, 0
5284 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
5285 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5286 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
5287 ; GFX10-NEXT: v_not_b32_e32 v4, v4
5288 ; GFX10-NEXT: .LBB28_1: ; %atomicrmw.start
5289 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
5290 ; GFX10-NEXT: s_waitcnt vmcnt(0)
5291 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
5292 ; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5293 ; GFX10-NEXT: v_max_f16_e32 v5, v5, v5
5294 ; GFX10-NEXT: v_max_f16_e32 v5, v5, v2
5295 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
5296 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
5297 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
5298 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
5299 ; GFX10-NEXT: s_waitcnt vmcnt(0)
5300 ; GFX10-NEXT: buffer_gl1_inv
5301 ; GFX10-NEXT: buffer_gl0_inv
5302 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
5303 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
5304 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
5305 ; GFX10-NEXT: s_cbranch_execnz .LBB28_1
5306 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
5307 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
5308 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5309 ; GFX10-NEXT: s_setpc_b64 s[30:31]
5310 ;
5311 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
5312 ; GFX90A: ; %bb.0:
5313 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5314 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
5315 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
5316 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
5317 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
5318 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
5319 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5320 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
5321 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
5322 ; GFX90A-NEXT: v_not_b32_e32 v4, v4
5323 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
5324 ; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
5325 ; GFX90A-NEXT: .LBB28_1: ; %atomicrmw.start
5326 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
5327 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
5328 ; GFX90A-NEXT: v_mov_b32_e32 v7, v5
5329 ; GFX90A-NEXT: v_lshrrev_b32_e32 v5, v3, v7
5330 ; GFX90A-NEXT: v_max_f16_e32 v5, v5, v5
5331 ; GFX90A-NEXT: v_max_f16_e32 v5, v5, v2
5332 ; GFX90A-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5333 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v4, v5
5334 ; GFX90A-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off glc
5335 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
5336 ; GFX90A-NEXT: buffer_wbinvl1
5337 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
5338 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5339 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
5340 ; GFX90A-NEXT: s_cbranch_execnz .LBB28_1
5341 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
5342 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
5343 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5344 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
5345 ;
5346 ; GFX908-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
5347 ; GFX908: ; %bb.0:
5348 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5349 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
5350 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
5351 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
5352 ; GFX908-NEXT: global_load_dword v5, v[0:1], off
5353 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
5354 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5355 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
5356 ; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
5357 ; GFX908-NEXT: v_not_b32_e32 v4, v4
5358 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
5359 ; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
5360 ; GFX908-NEXT: .LBB28_1: ; %atomicrmw.start
5361 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
5362 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5363 ; GFX908-NEXT: v_mov_b32_e32 v6, v5
5364 ; GFX908-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5365 ; GFX908-NEXT: v_max_f16_e32 v5, v5, v5
5366 ; GFX908-NEXT: v_max_f16_e32 v5, v5, v2
5367 ; GFX908-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5368 ; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
5369 ; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
5370 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5371 ; GFX908-NEXT: buffer_wbinvl1
5372 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
5373 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5374 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
5375 ; GFX908-NEXT: s_cbranch_execnz .LBB28_1
5376 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
5377 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
5378 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5379 ; GFX908-NEXT: s_setpc_b64 s[30:31]
5380 ;
5381 ; GFX8-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
5382 ; GFX8: ; %bb.0:
5383 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5384 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
5385 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
5386 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
5387 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
5388 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
5389 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5390 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
5391 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
5392 ; GFX8-NEXT: v_not_b32_e32 v4, v4
5393 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
5394 ; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
5395 ; GFX8-NEXT: .LBB28_1: ; %atomicrmw.start
5396 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5397 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5398 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
5399 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
5400 ; GFX8-NEXT: v_max_f16_e32 v5, v5, v5
5401 ; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
5402 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
5403 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
5404 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
5405 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
5406 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5407 ; GFX8-NEXT: buffer_wbinvl1
5408 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
5409 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5410 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
5411 ; GFX8-NEXT: s_cbranch_execnz .LBB28_1
5412 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5413 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
5414 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
5415 ; GFX8-NEXT: s_setpc_b64 s[30:31]
5416 ;
5417 ; GFX7-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
5418 ; GFX7: ; %bb.0:
5419 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5420 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
5421 ; GFX7-NEXT: s_mov_b32 s6, 0
5422 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
5423 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
5424 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
5425 ; GFX7-NEXT: s_mov_b32 s4, s6
5426 ; GFX7-NEXT: s_mov_b32 s5, s6
5427 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
5428 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
5429 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
5430 ; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
5431 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
5432 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
5433 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
5434 ; GFX7-NEXT: v_not_b32_e32 v8, v2
5435 ; GFX7-NEXT: .LBB28_1: ; %atomicrmw.start
5436 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5437 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5438 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
5439 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
5440 ; GFX7-NEXT: v_and_b32_e32 v4, v3, v8
5441 ; GFX7-NEXT: v_max_f32_e32 v2, v2, v7
5442 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
5443 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v6, v2
5444 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
5445 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
5446 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
5447 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
5448 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5449 ; GFX7-NEXT: buffer_wbinvl1
5450 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
5451 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
5452 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
5453 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
5454 ; GFX7-NEXT: s_cbranch_execnz .LBB28_1
5455 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5456 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
5457 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
5458 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
5459 ; GFX7-NEXT: s_setpc_b64 s[30:31]
5460 ;
5461 ; GFX6-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
5462 ; GFX6: ; %bb.0:
5463 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5464 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
5465 ; GFX6-NEXT: s_mov_b32 s6, 0
5466 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
5467 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
5468 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
5469 ; GFX6-NEXT: s_mov_b32 s4, s6
5470 ; GFX6-NEXT: s_mov_b32 s5, s6
5471 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
5472 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
5473 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
5474 ; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
5475 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
5476 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
5477 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
5478 ; GFX6-NEXT: v_not_b32_e32 v8, v2
5479 ; GFX6-NEXT: .LBB28_1: ; %atomicrmw.start
5480 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
5481 ; GFX6-NEXT: s_waitcnt vmcnt(0)
5482 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
5483 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
5484 ; GFX6-NEXT: s_waitcnt expcnt(0)
5485 ; GFX6-NEXT: v_and_b32_e32 v4, v3, v8
5486 ; GFX6-NEXT: v_max_f32_e32 v2, v2, v7
5487 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
5488 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v6, v2
5489 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
5490 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
5491 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
5492 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
5493 ; GFX6-NEXT: s_waitcnt vmcnt(0)
5494 ; GFX6-NEXT: buffer_wbinvl1
5495 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
5496 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
5497 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
5498 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
5499 ; GFX6-NEXT: s_cbranch_execnz .LBB28_1
5500 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
5501 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
5502 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
5503 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
5504 ; GFX6-NEXT: s_waitcnt expcnt(0)
5505 ; GFX6-NEXT: s_setpc_b64 s[30:31]
5506 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 -1024
5507 %result = atomicrmw fmax ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
5508 ret half %result
5509 }
5511 define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
5512 ; GFX12-LABEL: global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
5513 ; GFX12: ; %bb.0:
5514 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
5515 ; GFX12-NEXT: s_wait_expcnt 0x0
5516 ; GFX12-NEXT: s_wait_samplecnt 0x0
5517 ; GFX12-NEXT: s_wait_bvhcnt 0x0
5518 ; GFX12-NEXT: s_wait_kmcnt 0x0
5519 ; GFX12-NEXT: v_mov_b32_e32 v3, v0
5520 ; GFX12-NEXT: v_max_num_f16_e32 v2, v2, v2
5521 ; GFX12-NEXT: s_mov_b32 s0, 0
5522 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
5523 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
5524 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
5525 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off
5526 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
5527 ; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
5528 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
5529 ; GFX12-NEXT: v_not_b32_e32 v6, v3
5530 ; GFX12-NEXT: .LBB29_1: ; %atomicrmw.start
5531 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
5532 ; GFX12-NEXT: s_wait_loadcnt 0x0
5533 ; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
5534 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5535 ; GFX12-NEXT: v_max_num_f16_e32 v3, v3, v3
5536 ; GFX12-NEXT: v_max_num_f16_e32 v3, v3, v2
5537 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5538 ; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
5539 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
5540 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
5541 ; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
5542 ; GFX12-NEXT: s_wait_storecnt 0x0
5543 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
5544 ; GFX12-NEXT: s_wait_loadcnt 0x0
5545 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
5546 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
5547 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
5548 ; GFX12-NEXT: s_wait_alu 0xfffe
5549 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
5550 ; GFX12-NEXT: s_wait_alu 0xfffe
5551 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5552 ; GFX12-NEXT: s_cbranch_execnz .LBB29_1
5553 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
5554 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
5555 ; GFX12-NEXT: s_wait_alu 0xfffe
5556 ; GFX12-NEXT: s_setpc_b64 s[30:31]
5557 ;
5558 ; GFX940-LABEL: global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
5559 ; GFX940: ; %bb.0:
5560 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5561 ; GFX940-NEXT: v_mov_b32_e32 v3, v0
5562 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v3
5563 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
5564 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v3
5565 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5566 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
5567 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
5568 ; GFX940-NEXT: v_not_b32_e32 v6, v4
5569 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
5570 ; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
5571 ; GFX940-NEXT: .LBB29_1: ; %atomicrmw.start
5572 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
5573 ; GFX940-NEXT: s_waitcnt vmcnt(0)
5574 ; GFX940-NEXT: v_lshrrev_b32_e32 v4, v3, v5
5575 ; GFX940-NEXT: v_max_f16_e32 v4, v4, v4
5576 ; GFX940-NEXT: v_max_f16_e32 v4, v4, v2
5577 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, v3, v4
5578 ; GFX940-NEXT: v_and_or_b32 v4, v5, v6, v4
5579 ; GFX940-NEXT: buffer_wbl2 sc1
5580 ; GFX940-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0
5581 ; GFX940-NEXT: s_waitcnt vmcnt(0)
5582 ; GFX940-NEXT: buffer_inv sc1
5583 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
5584 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
5585 ; GFX940-NEXT: v_mov_b32_e32 v5, v4
5586 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
5587 ; GFX940-NEXT: s_cbranch_execnz .LBB29_1
5588 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
5589 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
5590 ; GFX940-NEXT: s_setpc_b64 s[30:31]
5591 ;
5592 ; GFX11-LABEL: global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
5593 ; GFX11: ; %bb.0:
5594 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5595 ; GFX11-NEXT: v_mov_b32_e32 v3, v0
5596 ; GFX11-NEXT: v_max_f16_e32 v2, v2, v2
5597 ; GFX11-NEXT: s_mov_b32 s0, 0
5598 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
5599 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
5600 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
5601 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off
5602 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
5603 ; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
5604 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
5605 ; GFX11-NEXT: v_not_b32_e32 v6, v3
5606 ; GFX11-NEXT: .LBB29_1: ; %atomicrmw.start
5607 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
5608 ; GFX11-NEXT: s_waitcnt vmcnt(0)
5609 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
5610 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5611 ; GFX11-NEXT: v_max_f16_e32 v3, v3, v3
5612 ; GFX11-NEXT: v_max_f16_e32 v3, v3, v2
5613 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5614 ; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
5615 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
5616 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
5617 ; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
5618 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
5619 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
5620 ; GFX11-NEXT: s_waitcnt vmcnt(0)
5621 ; GFX11-NEXT: buffer_gl1_inv
5622 ; GFX11-NEXT: buffer_gl0_inv
5623 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
5624 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
5625 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
5626 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
5627 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5628 ; GFX11-NEXT: s_cbranch_execnz .LBB29_1
5629 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
5630 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
5631 ; GFX11-NEXT: s_setpc_b64 s[30:31]
5632 ;
5633 ; GFX10-LABEL: global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
5634 ; GFX10: ; %bb.0:
5635 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5636 ; GFX10-NEXT: v_mov_b32_e32 v3, v0
5637 ; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
5638 ; GFX10-NEXT: s_mov_b32 s4, 0
5639 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
5640 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
5641 ; GFX10-NEXT: global_load_dword v4, v[0:1], off
5642 ; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
5643 ; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
5644 ; GFX10-NEXT: v_not_b32_e32 v6, v3
5645 ; GFX10-NEXT: .LBB29_1: ; %atomicrmw.start
5646 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
5647 ; GFX10-NEXT: s_waitcnt vmcnt(0)
5648 ; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
5649 ; GFX10-NEXT: v_max_f16_e32 v3, v3, v3
5650 ; GFX10-NEXT: v_max_f16_e32 v3, v3, v2
5651 ; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
5652 ; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
5653 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
5654 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
5655 ; GFX10-NEXT: s_waitcnt vmcnt(0)
5656 ; GFX10-NEXT: buffer_gl1_inv
5657 ; GFX10-NEXT: buffer_gl0_inv
5658 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
5659 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
5660 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
5661 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
5662 ; GFX10-NEXT: s_cbranch_execnz .LBB29_1
5663 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
5664 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
5665 ; GFX10-NEXT: s_setpc_b64 s[30:31]
5666 ;
5667 ; GFX90A-LABEL: global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
5668 ; GFX90A: ; %bb.0:
5669 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5670 ; GFX90A-NEXT: v_mov_b32_e32 v3, v0
5671 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
5672 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
5673 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
5674 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
5675 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
5676 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
5677 ; GFX90A-NEXT: v_not_b32_e32 v6, v4
5678 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
5679 ; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
5680 ; GFX90A-NEXT: .LBB29_1: ; %atomicrmw.start
5681 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
5682 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
5683 ; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
5684 ; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
5685 ; GFX90A-NEXT: v_max_f16_e32 v4, v4, v2
5686 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
5687 ; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
5688 ; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
5689 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
5690 ; GFX90A-NEXT: buffer_wbinvl1
5691 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
5692 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5693 ; GFX90A-NEXT: v_mov_b32_e32 v5, v4
5694 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
5695 ; GFX90A-NEXT: s_cbranch_execnz .LBB29_1
5696 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
5697 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
5698 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
5699 ;
5700 ; GFX908-LABEL: global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
5701 ; GFX908: ; %bb.0:
5702 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5703 ; GFX908-NEXT: v_mov_b32_e32 v3, v0
5704 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
5705 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
5706 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
5707 ; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
5708 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
5709 ; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
5710 ; GFX908-NEXT: v_not_b32_e32 v6, v3
5711 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
5712 ; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
5713 ; GFX908-NEXT: .LBB29_1: ; %atomicrmw.start
5714 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
5715 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5716 ; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
5717 ; GFX908-NEXT: v_max_f16_e32 v3, v3, v3
5718 ; GFX908-NEXT: v_max_f16_e32 v3, v3, v2
5719 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
5720 ; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
5721 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
5722 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5723 ; GFX908-NEXT: buffer_wbinvl1
5724 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
5725 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5726 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
5727 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
5728 ; GFX908-NEXT: s_cbranch_execnz .LBB29_1
5729 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
5730 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
5731 ; GFX908-NEXT: s_setpc_b64 s[30:31]
5732 ;
5733 ; GFX8-LABEL: global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
5734 ; GFX8: ; %bb.0:
5735 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5736 ; GFX8-NEXT: v_mov_b32_e32 v3, v0
5737 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
5738 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
5739 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
5740 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
5741 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
5742 ; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
5743 ; GFX8-NEXT: v_not_b32_e32 v6, v3
5744 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
5745 ; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
5746 ; GFX8-NEXT: .LBB29_1: ; %atomicrmw.start
5747 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5748 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5749 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
5750 ; GFX8-NEXT: v_max_f16_e32 v3, v3, v3
5751 ; GFX8-NEXT: v_max_f16_e32 v3, v3, v2
5752 ; GFX8-NEXT: v_and_b32_e32 v7, v4, v6
5753 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
5754 ; GFX8-NEXT: v_or_b32_e32 v3, v7, v3
5755 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
5756 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5757 ; GFX8-NEXT: buffer_wbinvl1
5758 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
5759 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5760 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
5761 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
5762 ; GFX8-NEXT: s_cbranch_execnz .LBB29_1
5763 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5764 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
5765 ; GFX8-NEXT: s_setpc_b64 s[30:31]
5766 ;
5767 ; GFX7-LABEL: global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
5768 ; GFX7: ; %bb.0:
5769 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5770 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
5771 ; GFX7-NEXT: s_mov_b32 s6, 0
5772 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
5773 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
5774 ; GFX7-NEXT: s_mov_b32 s4, s6
5775 ; GFX7-NEXT: s_mov_b32 s5, s6
5776 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
5777 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v2
5778 ; GFX7-NEXT: v_and_b32_e32 v2, 3, v3
5779 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
5780 ; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
5781 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
5782 ; GFX7-NEXT: v_not_b32_e32 v6, v3
5783 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
5784 ; GFX7-NEXT: .LBB29_1: ; %atomicrmw.start
5785 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5786 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5787 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, v2, v4
5788 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
5789 ; GFX7-NEXT: v_and_b32_e32 v7, v4, v6
5790 ; GFX7-NEXT: v_max_f32_e32 v3, v3, v5
5791 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
5792 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, v2, v3
5793 ; GFX7-NEXT: v_or_b32_e32 v3, v7, v3
5794 ; GFX7-NEXT: v_mov_b32_e32 v8, v4
5795 ; GFX7-NEXT: v_mov_b32_e32 v7, v3
5796 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
5797 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5798 ; GFX7-NEXT: buffer_wbinvl1
5799 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v4
5800 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
5801 ; GFX7-NEXT: v_mov_b32_e32 v4, v7
5802 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
5803 ; GFX7-NEXT: s_cbranch_execnz .LBB29_1
5804 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5805 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
5806 ; GFX7-NEXT: s_setpc_b64 s[30:31]
5807 ;
5808 ; GFX6-LABEL: global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory:
5809 ; GFX6: ; %bb.0:
5810 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5811 ; GFX6-NEXT: v_mov_b32_e32 v3, v0
5812 ; GFX6-NEXT: s_mov_b32 s6, 0
5813 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v3
5814 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
5815 ; GFX6-NEXT: s_mov_b32 s4, s6
5816 ; GFX6-NEXT: s_mov_b32 s5, s6
5817 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
5818 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v2
5819 ; GFX6-NEXT: v_and_b32_e32 v2, 3, v3
5820 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 3, v2
5821 ; GFX6-NEXT: v_lshl_b32_e32 v3, 0xffff, v2
5822 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
5823 ; GFX6-NEXT: v_not_b32_e32 v6, v3
5824 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
5825 ; GFX6-NEXT: .LBB29_1: ; %atomicrmw.start
5826 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
5827 ; GFX6-NEXT: s_waitcnt vmcnt(0)
5828 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, v2, v4
5829 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
5830 ; GFX6-NEXT: s_waitcnt expcnt(0)
5831 ; GFX6-NEXT: v_and_b32_e32 v7, v4, v6
5832 ; GFX6-NEXT: v_max_f32_e32 v3, v3, v5
5833 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
5834 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, v2, v3
5835 ; GFX6-NEXT: v_or_b32_e32 v3, v7, v3
5836 ; GFX6-NEXT: v_mov_b32_e32 v8, v4
5837 ; GFX6-NEXT: v_mov_b32_e32 v7, v3
5838 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
5839 ; GFX6-NEXT: s_waitcnt vmcnt(0)
5840 ; GFX6-NEXT: buffer_wbinvl1
5841 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v4
5842 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
5843 ; GFX6-NEXT: v_mov_b32_e32 v4, v7
5844 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
5845 ; GFX6-NEXT: s_cbranch_execnz .LBB29_1
5846 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
5847 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
5848 ; GFX6-NEXT: s_waitcnt expcnt(0)
5849 ; GFX6-NEXT: s_setpc_b64 s[30:31]
5850 %unused = atomicrmw fmax ptr addrspace(1) %ptr, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
5851 ret void
5852 }
5854 define void @global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
5855 ; GFX12-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
5856 ; GFX12: ; %bb.0:
5857 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
5858 ; GFX12-NEXT: s_wait_expcnt 0x0
5859 ; GFX12-NEXT: s_wait_samplecnt 0x0
5860 ; GFX12-NEXT: s_wait_bvhcnt 0x0
5861 ; GFX12-NEXT: s_wait_kmcnt 0x0
5862 ; GFX12-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
5863 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
5864 ; GFX12-NEXT: v_max_num_f16_e32 v6, v2, v2
5865 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
5866 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v4
5867 ; GFX12-NEXT: v_and_b32_e32 v4, 3, v4
5868 ; GFX12-NEXT: s_mov_b32 s0, 0
5869 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
5870 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 3, v4
5871 ; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
5872 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
5873 ; GFX12-NEXT: v_not_b32_e32 v5, v5
5874 ; GFX12-NEXT: .LBB30_1: ; %atomicrmw.start
5875 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
5876 ; GFX12-NEXT: s_wait_loadcnt 0x0
5877 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
5878 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5879 ; GFX12-NEXT: v_max_num_f16_e32 v2, v2, v2
5880 ; GFX12-NEXT: v_max_num_f16_e32 v2, v2, v6
5881 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5882 ; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
5883 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, v4, v2
5884 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
5885 ; GFX12-NEXT: v_and_or_b32 v2, v3, v5, v2
5886 ; GFX12-NEXT: s_wait_storecnt 0x0
5887 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
5888 ; GFX12-NEXT: s_wait_loadcnt 0x0
5889 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
5890 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
5891 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
5892 ; GFX12-NEXT: s_wait_alu 0xfffe
5893 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
5894 ; GFX12-NEXT: s_wait_alu 0xfffe
5895 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5896 ; GFX12-NEXT: s_cbranch_execnz .LBB30_1
5897 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
5898 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
5899 ; GFX12-NEXT: s_wait_alu 0xfffe
5900 ; GFX12-NEXT: s_setpc_b64 s[30:31]
5901 ;
5902 ; GFX940-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
5903 ; GFX940: ; %bb.0:
5904 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5905 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
5906 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
5907 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
5908 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
5909 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
5910 ; GFX940-NEXT: v_and_b32_e32 v4, 3, v4
5911 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 3, v4
5912 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
5913 ; GFX940-NEXT: v_lshlrev_b32_e64 v5, v4, s0
5914 ; GFX940-NEXT: v_not_b32_e32 v5, v5
5915 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
5916 ; GFX940-NEXT: v_max_f16_e32 v6, v2, v2
5917 ; GFX940-NEXT: .LBB30_1: ; %atomicrmw.start
5918 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
5919 ; GFX940-NEXT: s_waitcnt vmcnt(0)
5920 ; GFX940-NEXT: v_lshrrev_b32_e32 v2, v4, v3
5921 ; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
5922 ; GFX940-NEXT: v_max_f16_e32 v2, v2, v6
5923 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, v4, v2
5924 ; GFX940-NEXT: v_and_or_b32 v2, v3, v5, v2
5925 ; GFX940-NEXT: buffer_wbl2 sc1
5926 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
5927 ; GFX940-NEXT: s_waitcnt vmcnt(0)
5928 ; GFX940-NEXT: buffer_inv sc1
5929 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
5930 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
5931 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
5932 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
5933 ; GFX940-NEXT: s_cbranch_execnz .LBB30_1
5934 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
5935 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
5936 ; GFX940-NEXT: s_setpc_b64 s[30:31]
5937 ;
5938 ; GFX11-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
5939 ; GFX11: ; %bb.0:
5940 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5941 ; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
5942 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
5943 ; GFX11-NEXT: v_max_f16_e32 v6, v2, v2
5944 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
5945 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v4
5946 ; GFX11-NEXT: v_and_b32_e32 v4, 3, v4
5947 ; GFX11-NEXT: s_mov_b32 s0, 0
5948 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
5949 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v4
5950 ; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
5951 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
5952 ; GFX11-NEXT: v_not_b32_e32 v5, v5
5953 ; GFX11-NEXT: .LBB30_1: ; %atomicrmw.start
5954 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
5955 ; GFX11-NEXT: s_waitcnt vmcnt(0)
5956 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
5957 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5958 ; GFX11-NEXT: v_max_f16_e32 v2, v2, v2
5959 ; GFX11-NEXT: v_max_f16_e32 v2, v2, v6
5960 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5961 ; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
5962 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, v4, v2
5963 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
5964 ; GFX11-NEXT: v_and_or_b32 v2, v3, v5, v2
5965 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
5966 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
5967 ; GFX11-NEXT: s_waitcnt vmcnt(0)
5968 ; GFX11-NEXT: buffer_gl1_inv
5969 ; GFX11-NEXT: buffer_gl0_inv
5970 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
5971 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
5972 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
5973 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
5974 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
5975 ; GFX11-NEXT: s_cbranch_execnz .LBB30_1
5976 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
5977 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
5978 ; GFX11-NEXT: s_setpc_b64 s[30:31]
5979 ;
5980 ; GFX10-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
5981 ; GFX10: ; %bb.0:
5982 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5983 ; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
5984 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
5985 ; GFX10-NEXT: v_max_f16_e32 v6, v2, v2
5986 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
5987 ; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
5988 ; GFX10-NEXT: s_mov_b32 s4, 0
5989 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
5990 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
5991 ; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
5992 ; GFX10-NEXT: v_not_b32_e32 v5, v5
5993 ; GFX10-NEXT: .LBB30_1: ; %atomicrmw.start
5994 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
5995 ; GFX10-NEXT: s_waitcnt vmcnt(0)
5996 ; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
5997 ; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
5998 ; GFX10-NEXT: v_max_f16_e32 v2, v2, v6
5999 ; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
6000 ; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
6001 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
6002 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
6003 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6004 ; GFX10-NEXT: buffer_gl1_inv
6005 ; GFX10-NEXT: buffer_gl0_inv
6006 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
6007 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
6008 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
6009 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
6010 ; GFX10-NEXT: s_cbranch_execnz .LBB30_1
6011 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
6012 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
6013 ; GFX10-NEXT: s_setpc_b64 s[30:31]
6014 ;
6015 ; GFX90A-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
6016 ; GFX90A: ; %bb.0:
6017 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6018 ; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
6019 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
6020 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
6021 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
6022 ; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
6023 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
6024 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
6025 ; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
6026 ; GFX90A-NEXT: v_not_b32_e32 v5, v5
6027 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
6028 ; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
6029 ; GFX90A-NEXT: .LBB30_1: ; %atomicrmw.start
6030 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
6031 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6032 ; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
6033 ; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
6034 ; GFX90A-NEXT: v_max_f16_e32 v2, v2, v6
6035 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
6036 ; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
6037 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
6038 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6039 ; GFX90A-NEXT: buffer_wbinvl1
6040 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
6041 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6042 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
6043 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
6044 ; GFX90A-NEXT: s_cbranch_execnz .LBB30_1
6045 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
6046 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
6047 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
6048 ;
6049 ; GFX908-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
6050 ; GFX908: ; %bb.0:
6051 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6052 ; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
6053 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
6054 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
6055 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
6056 ; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
6057 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
6058 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
6059 ; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
6060 ; GFX908-NEXT: v_not_b32_e32 v5, v5
6061 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
6062 ; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
6063 ; GFX908-NEXT: .LBB30_1: ; %atomicrmw.start
6064 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
6065 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6066 ; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
6067 ; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
6068 ; GFX908-NEXT: v_max_f16_e32 v2, v2, v6
6069 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, v4, v2
6070 ; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
6071 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
6072 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6073 ; GFX908-NEXT: buffer_wbinvl1
6074 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
6075 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6076 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
6077 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
6078 ; GFX908-NEXT: s_cbranch_execnz .LBB30_1
6079 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
6080 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
6081 ; GFX908-NEXT: s_setpc_b64 s[30:31]
6082 ;
6083 ; GFX8-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
6084 ; GFX8: ; %bb.0:
6085 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6086 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
6087 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
6088 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
6089 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
6090 ; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
6091 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
6092 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
6093 ; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
6094 ; GFX8-NEXT: v_not_b32_e32 v5, v5
6095 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
6096 ; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
6097 ; GFX8-NEXT: .LBB30_1: ; %atomicrmw.start
6098 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6099 ; GFX8-NEXT: s_waitcnt vmcnt(0)
6100 ; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
6101 ; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
6102 ; GFX8-NEXT: v_max_f16_e32 v2, v2, v6
6103 ; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
6104 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, v4, v2
6105 ; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
6106 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6107 ; GFX8-NEXT: s_waitcnt vmcnt(0)
6108 ; GFX8-NEXT: buffer_wbinvl1
6109 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
6110 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6111 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
6112 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
6113 ; GFX8-NEXT: s_cbranch_execnz .LBB30_1
6114 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6115 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
6116 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6117 ;
6118 ; GFX7-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
6119 ; GFX7: ; %bb.0:
6120 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6121 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
6122 ; GFX7-NEXT: s_mov_b32 s6, 0
6123 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
6124 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
6125 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
6126 ; GFX7-NEXT: s_mov_b32 s4, s6
6127 ; GFX7-NEXT: s_mov_b32 s5, s6
6128 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
6129 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
6130 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
6131 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
6132 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
6133 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
6134 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
6135 ; GFX7-NEXT: v_not_b32_e32 v6, v2
6136 ; GFX7-NEXT: .LBB30_1: ; %atomicrmw.start
6137 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
6138 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6139 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
6140 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
6141 ; GFX7-NEXT: v_and_b32_e32 v7, v3, v6
6142 ; GFX7-NEXT: v_max_f32_e32 v2, v2, v5
6143 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
6144 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v4, v2
6145 ; GFX7-NEXT: v_or_b32_e32 v2, v7, v2
6146 ; GFX7-NEXT: v_mov_b32_e32 v8, v3
6147 ; GFX7-NEXT: v_mov_b32_e32 v7, v2
6148 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
6149 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6150 ; GFX7-NEXT: buffer_wbinvl1
6151 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
6152 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
6153 ; GFX7-NEXT: v_mov_b32_e32 v3, v7
6154 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
6155 ; GFX7-NEXT: s_cbranch_execnz .LBB30_1
6156 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6157 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
6158 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6159 ;
6160 ; GFX6-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
6161 ; GFX6: ; %bb.0:
6162 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6163 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
6164 ; GFX6-NEXT: s_mov_b32 s6, 0
6165 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
6166 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
6167 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
6168 ; GFX6-NEXT: s_mov_b32 s4, s6
6169 ; GFX6-NEXT: s_mov_b32 s5, s6
6170 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
6171 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
6172 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
6173 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
6174 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
6175 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
6176 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
6177 ; GFX6-NEXT: v_not_b32_e32 v6, v2
6178 ; GFX6-NEXT: .LBB30_1: ; %atomicrmw.start
6179 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
6180 ; GFX6-NEXT: s_waitcnt vmcnt(0)
6181 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
6182 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
6183 ; GFX6-NEXT: s_waitcnt expcnt(0)
6184 ; GFX6-NEXT: v_and_b32_e32 v7, v3, v6
6185 ; GFX6-NEXT: v_max_f32_e32 v2, v2, v5
6186 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
6187 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v4, v2
6188 ; GFX6-NEXT: v_or_b32_e32 v2, v7, v2
6189 ; GFX6-NEXT: v_mov_b32_e32 v8, v3
6190 ; GFX6-NEXT: v_mov_b32_e32 v7, v2
6191 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
6192 ; GFX6-NEXT: s_waitcnt vmcnt(0)
6193 ; GFX6-NEXT: buffer_wbinvl1
6194 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
6195 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
6196 ; GFX6-NEXT: v_mov_b32_e32 v3, v7
6197 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
6198 ; GFX6-NEXT: s_cbranch_execnz .LBB30_1
6199 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
6200 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
6201 ; GFX6-NEXT: s_waitcnt expcnt(0)
6202 ; GFX6-NEXT: s_setpc_b64 s[30:31]
6203 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
6204 %unused = atomicrmw fmax ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
6205 ret void
6206 }
6208 define void @global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
6209 ; GFX12-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
6210 ; GFX12: ; %bb.0:
6211 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
6212 ; GFX12-NEXT: s_wait_expcnt 0x0
6213 ; GFX12-NEXT: s_wait_samplecnt 0x0
6214 ; GFX12-NEXT: s_wait_bvhcnt 0x0
6215 ; GFX12-NEXT: s_wait_kmcnt 0x0
6216 ; GFX12-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
6217 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
6218 ; GFX12-NEXT: v_max_num_f16_e32 v6, v2, v2
6219 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
6220 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v4
6221 ; GFX12-NEXT: v_and_b32_e32 v4, 3, v4
6222 ; GFX12-NEXT: s_mov_b32 s0, 0
6223 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
6224 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 3, v4
6225 ; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
6226 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
6227 ; GFX12-NEXT: v_not_b32_e32 v5, v5
6228 ; GFX12-NEXT: .LBB31_1: ; %atomicrmw.start
6229 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
6230 ; GFX12-NEXT: s_wait_loadcnt 0x0
6231 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
6232 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6233 ; GFX12-NEXT: v_max_num_f16_e32 v2, v2, v2
6234 ; GFX12-NEXT: v_max_num_f16_e32 v2, v2, v6
6235 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6236 ; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
6237 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, v4, v2
6238 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
6239 ; GFX12-NEXT: v_and_or_b32 v2, v3, v5, v2
6240 ; GFX12-NEXT: s_wait_storecnt 0x0
6241 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
6242 ; GFX12-NEXT: s_wait_loadcnt 0x0
6243 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
6244 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
6245 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
6246 ; GFX12-NEXT: s_wait_alu 0xfffe
6247 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
6248 ; GFX12-NEXT: s_wait_alu 0xfffe
6249 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
6250 ; GFX12-NEXT: s_cbranch_execnz .LBB31_1
6251 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
6252 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
6253 ; GFX12-NEXT: s_wait_alu 0xfffe
6254 ; GFX12-NEXT: s_setpc_b64 s[30:31]
6255 ;
6256 ; GFX940-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
6257 ; GFX940: ; %bb.0:
6258 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6259 ; GFX940-NEXT: s_movk_i32 s0, 0xf800
6260 ; GFX940-NEXT: s_mov_b32 s1, -1
6261 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
6262 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
6263 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
6264 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
6265 ; GFX940-NEXT: v_and_b32_e32 v4, 3, v4
6266 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 3, v4
6267 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
6268 ; GFX940-NEXT: v_lshlrev_b32_e64 v5, v4, s0
6269 ; GFX940-NEXT: v_not_b32_e32 v5, v5
6270 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
6271 ; GFX940-NEXT: v_max_f16_e32 v6, v2, v2
6272 ; GFX940-NEXT: .LBB31_1: ; %atomicrmw.start
6273 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
6274 ; GFX940-NEXT: s_waitcnt vmcnt(0)
6275 ; GFX940-NEXT: v_lshrrev_b32_e32 v2, v4, v3
6276 ; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
6277 ; GFX940-NEXT: v_max_f16_e32 v2, v2, v6
6278 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, v4, v2
6279 ; GFX940-NEXT: v_and_or_b32 v2, v3, v5, v2
6280 ; GFX940-NEXT: buffer_wbl2 sc1
6281 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
6282 ; GFX940-NEXT: s_waitcnt vmcnt(0)
6283 ; GFX940-NEXT: buffer_inv sc1
6284 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
6285 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
6286 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
6287 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
6288 ; GFX940-NEXT: s_cbranch_execnz .LBB31_1
6289 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
6290 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
6291 ; GFX940-NEXT: s_setpc_b64 s[30:31]
6292 ;
6293 ; GFX11-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
6294 ; GFX11: ; %bb.0:
6295 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6296 ; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
6297 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
6298 ; GFX11-NEXT: v_max_f16_e32 v6, v2, v2
6299 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
6300 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v4
6301 ; GFX11-NEXT: v_and_b32_e32 v4, 3, v4
6302 ; GFX11-NEXT: s_mov_b32 s0, 0
6303 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
6304 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v4
6305 ; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
6306 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
6307 ; GFX11-NEXT: v_not_b32_e32 v5, v5
6308 ; GFX11-NEXT: .LBB31_1: ; %atomicrmw.start
6309 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
6310 ; GFX11-NEXT: s_waitcnt vmcnt(0)
6311 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
6312 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6313 ; GFX11-NEXT: v_max_f16_e32 v2, v2, v2
6314 ; GFX11-NEXT: v_max_f16_e32 v2, v2, v6
6315 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6316 ; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
6317 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, v4, v2
6318 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
6319 ; GFX11-NEXT: v_and_or_b32 v2, v3, v5, v2
6320 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
6321 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
6322 ; GFX11-NEXT: s_waitcnt vmcnt(0)
6323 ; GFX11-NEXT: buffer_gl1_inv
6324 ; GFX11-NEXT: buffer_gl0_inv
6325 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
6326 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
6327 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
6328 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
6329 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
6330 ; GFX11-NEXT: s_cbranch_execnz .LBB31_1
6331 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
6332 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
6333 ; GFX11-NEXT: s_setpc_b64 s[30:31]
6334 ;
6335 ; GFX10-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
6336 ; GFX10: ; %bb.0:
6337 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6338 ; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
6339 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
6340 ; GFX10-NEXT: v_max_f16_e32 v6, v2, v2
6341 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
6342 ; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
6343 ; GFX10-NEXT: s_mov_b32 s4, 0
6344 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
6345 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
6346 ; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
6347 ; GFX10-NEXT: v_not_b32_e32 v5, v5
6348 ; GFX10-NEXT: .LBB31_1: ; %atomicrmw.start
6349 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
6350 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6351 ; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
6352 ; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
6353 ; GFX10-NEXT: v_max_f16_e32 v2, v2, v6
6354 ; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
6355 ; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
6356 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
6357 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
6358 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6359 ; GFX10-NEXT: buffer_gl1_inv
6360 ; GFX10-NEXT: buffer_gl0_inv
6361 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
6362 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
6363 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
6364 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
6365 ; GFX10-NEXT: s_cbranch_execnz .LBB31_1
6366 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
6367 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
6368 ; GFX10-NEXT: s_setpc_b64 s[30:31]
6369 ;
6370 ; GFX90A-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
6371 ; GFX90A: ; %bb.0:
6372 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6373 ; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
6374 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
6375 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
6376 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
6377 ; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
6378 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
6379 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
6380 ; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
6381 ; GFX90A-NEXT: v_not_b32_e32 v5, v5
6382 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
6383 ; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
6384 ; GFX90A-NEXT: .LBB31_1: ; %atomicrmw.start
6385 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
6386 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6387 ; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
6388 ; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
6389 ; GFX90A-NEXT: v_max_f16_e32 v2, v2, v6
6390 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
6391 ; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
6392 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
6393 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6394 ; GFX90A-NEXT: buffer_wbinvl1
6395 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
6396 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6397 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
6398 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
6399 ; GFX90A-NEXT: s_cbranch_execnz .LBB31_1
6400 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
6401 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
6402 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
6403 ;
6404 ; GFX908-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
6405 ; GFX908: ; %bb.0:
6406 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6407 ; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
6408 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
6409 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
6410 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
6411 ; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
6412 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
6413 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
6414 ; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
6415 ; GFX908-NEXT: v_not_b32_e32 v5, v5
6416 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
6417 ; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
6418 ; GFX908-NEXT: .LBB31_1: ; %atomicrmw.start
6419 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
6420 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6421 ; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
6422 ; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
6423 ; GFX908-NEXT: v_max_f16_e32 v2, v2, v6
6424 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, v4, v2
6425 ; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
6426 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
6427 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6428 ; GFX908-NEXT: buffer_wbinvl1
6429 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
6430 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6431 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
6432 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
6433 ; GFX908-NEXT: s_cbranch_execnz .LBB31_1
6434 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
6435 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
6436 ; GFX908-NEXT: s_setpc_b64 s[30:31]
6437 ;
6438 ; GFX8-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
6439 ; GFX8: ; %bb.0:
6440 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6441 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
6442 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
6443 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
6444 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
6445 ; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
6446 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
6447 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
6448 ; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
6449 ; GFX8-NEXT: v_not_b32_e32 v5, v5
6450 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
6451 ; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
6452 ; GFX8-NEXT: .LBB31_1: ; %atomicrmw.start
6453 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6454 ; GFX8-NEXT: s_waitcnt vmcnt(0)
6455 ; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
6456 ; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
6457 ; GFX8-NEXT: v_max_f16_e32 v2, v2, v6
6458 ; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
6459 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, v4, v2
6460 ; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
6461 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6462 ; GFX8-NEXT: s_waitcnt vmcnt(0)
6463 ; GFX8-NEXT: buffer_wbinvl1
6464 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
6465 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6466 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
6467 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
6468 ; GFX8-NEXT: s_cbranch_execnz .LBB31_1
6469 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6470 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
6471 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6472 ;
6473 ; GFX7-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
6474 ; GFX7: ; %bb.0:
6475 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6476 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
6477 ; GFX7-NEXT: s_mov_b32 s6, 0
6478 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
6479 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
6480 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
6481 ; GFX7-NEXT: s_mov_b32 s4, s6
6482 ; GFX7-NEXT: s_mov_b32 s5, s6
6483 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
6484 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
6485 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
6486 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
6487 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
6488 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
6489 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
6490 ; GFX7-NEXT: v_not_b32_e32 v6, v2
6491 ; GFX7-NEXT: .LBB31_1: ; %atomicrmw.start
6492 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
6493 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6494 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
6495 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
6496 ; GFX7-NEXT: v_and_b32_e32 v7, v3, v6
6497 ; GFX7-NEXT: v_max_f32_e32 v2, v2, v5
6498 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
6499 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v4, v2
6500 ; GFX7-NEXT: v_or_b32_e32 v2, v7, v2
6501 ; GFX7-NEXT: v_mov_b32_e32 v8, v3
6502 ; GFX7-NEXT: v_mov_b32_e32 v7, v2
6503 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
6504 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6505 ; GFX7-NEXT: buffer_wbinvl1
6506 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
6507 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
6508 ; GFX7-NEXT: v_mov_b32_e32 v3, v7
6509 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
6510 ; GFX7-NEXT: s_cbranch_execnz .LBB31_1
6511 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6512 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
6513 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6514 ;
6515 ; GFX6-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
6516 ; GFX6: ; %bb.0:
6517 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6518 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
6519 ; GFX6-NEXT: s_mov_b32 s6, 0
6520 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
6521 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
6522 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
6523 ; GFX6-NEXT: s_mov_b32 s4, s6
6524 ; GFX6-NEXT: s_mov_b32 s5, s6
6525 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
6526 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
6527 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
6528 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
6529 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
6530 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
6531 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
6532 ; GFX6-NEXT: v_not_b32_e32 v6, v2
6533 ; GFX6-NEXT: .LBB31_1: ; %atomicrmw.start
6534 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
6535 ; GFX6-NEXT: s_waitcnt vmcnt(0)
6536 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
6537 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
6538 ; GFX6-NEXT: s_waitcnt expcnt(0)
6539 ; GFX6-NEXT: v_and_b32_e32 v7, v3, v6
6540 ; GFX6-NEXT: v_max_f32_e32 v2, v2, v5
6541 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
6542 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v4, v2
6543 ; GFX6-NEXT: v_or_b32_e32 v2, v7, v2
6544 ; GFX6-NEXT: v_mov_b32_e32 v8, v3
6545 ; GFX6-NEXT: v_mov_b32_e32 v7, v2
6546 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
6547 ; GFX6-NEXT: s_waitcnt vmcnt(0)
6548 ; GFX6-NEXT: buffer_wbinvl1
6549 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
6550 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
6551 ; GFX6-NEXT: v_mov_b32_e32 v3, v7
6552 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
6553 ; GFX6-NEXT: s_cbranch_execnz .LBB31_1
6554 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
6555 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
6556 ; GFX6-NEXT: s_waitcnt expcnt(0)
6557 ; GFX6-NEXT: s_setpc_b64 s[30:31]
6558 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 -1024
6559 %unused = atomicrmw fmax ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
6560 ret void
6561 }
6563 define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
6564 ; GFX12-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
6565 ; GFX12: ; %bb.0:
6566 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
6567 ; GFX12-NEXT: s_wait_expcnt 0x0
6568 ; GFX12-NEXT: s_wait_samplecnt 0x0
6569 ; GFX12-NEXT: s_wait_bvhcnt 0x0
6570 ; GFX12-NEXT: s_wait_kmcnt 0x0
6571 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2046
6572 ; GFX12-NEXT: v_max_num_f16_e32 v2, v2, v2
6573 ; GFX12-NEXT: s_mov_b32 s0, 0
6574 ; GFX12-NEXT: .LBB32_1: ; %atomicrmw.start
6575 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
6576 ; GFX12-NEXT: s_wait_loadcnt 0x0
6577 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
6578 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6579 ; GFX12-NEXT: v_max_num_f16_e32 v3, v4, v4
6580 ; GFX12-NEXT: v_max_num_f16_e32 v3, v3, v2
6581 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6582 ; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
6583 ; GFX12-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
6584 ; GFX12-NEXT: s_wait_storecnt 0x0
6585 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
6586 ; GFX12-NEXT: s_wait_loadcnt 0x0
6587 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
6588 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
6589 ; GFX12-NEXT: s_wait_alu 0xfffe
6590 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
6591 ; GFX12-NEXT: s_wait_alu 0xfffe
6592 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
6593 ; GFX12-NEXT: s_cbranch_execnz .LBB32_1
6594 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
6595 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
6596 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
6597 ; GFX12-NEXT: s_wait_alu 0xfffe
6598 ; GFX12-NEXT: s_setpc_b64 s[30:31]
6599 ;
6600 ; GFX940-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
6601 ; GFX940: ; %bb.0:
6602 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6603 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2046
6604 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
6605 ; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
6606 ; GFX940-NEXT: s_mov_b32 s2, 0xffff0000
6607 ; GFX940-NEXT: .LBB32_1: ; %atomicrmw.start
6608 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
6609 ; GFX940-NEXT: s_waitcnt vmcnt(0)
6610 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
6611 ; GFX940-NEXT: v_max_f16_e32 v3, v5, v5
6612 ; GFX940-NEXT: v_max_f16_e32 v3, v3, v2
6613 ; GFX940-NEXT: v_and_or_b32 v4, v5, s2, v3
6614 ; GFX940-NEXT: buffer_wbl2 sc1
6615 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 sc0
6616 ; GFX940-NEXT: s_waitcnt vmcnt(0)
6617 ; GFX940-NEXT: buffer_inv sc1
6618 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
6619 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
6620 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
6621 ; GFX940-NEXT: s_cbranch_execnz .LBB32_1
6622 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
6623 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
6624 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
6625 ; GFX940-NEXT: s_setpc_b64 s[30:31]
6626 ;
6627 ; GFX11-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
6628 ; GFX11: ; %bb.0:
6629 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6630 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2046
6631 ; GFX11-NEXT: v_max_f16_e32 v2, v2, v2
6632 ; GFX11-NEXT: s_mov_b32 s0, 0
6633 ; GFX11-NEXT: .LBB32_1: ; %atomicrmw.start
6634 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
6635 ; GFX11-NEXT: s_waitcnt vmcnt(0)
6636 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
6637 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6638 ; GFX11-NEXT: v_max_f16_e32 v3, v4, v4
6639 ; GFX11-NEXT: v_max_f16_e32 v3, v3, v2
6640 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6641 ; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
6642 ; GFX11-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
6643 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
6644 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
6645 ; GFX11-NEXT: s_waitcnt vmcnt(0)
6646 ; GFX11-NEXT: buffer_gl1_inv
6647 ; GFX11-NEXT: buffer_gl0_inv
6648 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
6649 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
6650 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
6651 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
6652 ; GFX11-NEXT: s_cbranch_execnz .LBB32_1
6653 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
6654 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
6655 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
6656 ; GFX11-NEXT: s_setpc_b64 s[30:31]
6657 ;
6658 ; GFX10-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
6659 ; GFX10: ; %bb.0:
6660 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6661 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2046
6662 ; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
6663 ; GFX10-NEXT: s_mov_b32 s4, 0
6664 ; GFX10-NEXT: .LBB32_1: ; %atomicrmw.start
6665 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
6666 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6667 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
6668 ; GFX10-NEXT: v_max_f16_e32 v3, v4, v4
6669 ; GFX10-NEXT: v_max_f16_e32 v3, v3, v2
6670 ; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
6671 ; GFX10-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
6672 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
6673 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2046 glc
6674 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6675 ; GFX10-NEXT: buffer_gl1_inv
6676 ; GFX10-NEXT: buffer_gl0_inv
6677 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
6678 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
6679 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
6680 ; GFX10-NEXT: s_cbranch_execnz .LBB32_1
6681 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
6682 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
6683 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
6684 ; GFX10-NEXT: s_setpc_b64 s[30:31]
6685 ;
6686 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
6687 ; GFX90A: ; %bb.0:
6688 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6689 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2046
6690 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
6691 ; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
6692 ; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
6693 ; GFX90A-NEXT: .LBB32_1: ; %atomicrmw.start
6694 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
6695 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6696 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
6697 ; GFX90A-NEXT: v_max_f16_e32 v3, v5, v5
6698 ; GFX90A-NEXT: v_max_f16_e32 v3, v3, v2
6699 ; GFX90A-NEXT: v_and_or_b32 v4, v5, s6, v3
6700 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 glc
6701 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6702 ; GFX90A-NEXT: buffer_wbinvl1
6703 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
6704 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6705 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
6706 ; GFX90A-NEXT: s_cbranch_execnz .LBB32_1
6707 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
6708 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
6709 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
6710 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
6711 ;
6712 ; GFX908-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
6713 ; GFX908: ; %bb.0:
6714 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6715 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2046
6716 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
6717 ; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
6718 ; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
6719 ; GFX908-NEXT: .LBB32_1: ; %atomicrmw.start
6720 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
6721 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6722 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
6723 ; GFX908-NEXT: v_max_f16_e32 v3, v4, v4
6724 ; GFX908-NEXT: v_max_f16_e32 v3, v3, v2
6725 ; GFX908-NEXT: v_and_or_b32 v3, v4, s6, v3
6726 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2046 glc
6727 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6728 ; GFX908-NEXT: buffer_wbinvl1
6729 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
6730 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6731 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
6732 ; GFX908-NEXT: s_cbranch_execnz .LBB32_1
6733 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
6734 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
6735 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
6736 ; GFX908-NEXT: s_setpc_b64 s[30:31]
6737 ;
6738 ; GFX8-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
6739 ; GFX8: ; %bb.0:
6740 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6741 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
6742 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
6743 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
6744 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
6745 ; GFX8-NEXT: v_max_f16_e32 v1, v2, v2
6746 ; GFX8-NEXT: .LBB32_1: ; %atomicrmw.start
6747 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6748 ; GFX8-NEXT: s_waitcnt vmcnt(0)
6749 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
6750 ; GFX8-NEXT: v_max_f16_e32 v0, v6, v6
6751 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
6752 ; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
6753 ; GFX8-NEXT: v_or_b32_e32 v5, v2, v0
6754 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
6755 ; GFX8-NEXT: s_waitcnt vmcnt(0)
6756 ; GFX8-NEXT: buffer_wbinvl1
6757 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
6758 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6759 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
6760 ; GFX8-NEXT: s_cbranch_execnz .LBB32_1
6761 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6762 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
6763 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6764 ;
6765 ; GFX7-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
6766 ; GFX7: ; %bb.0:
6767 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6768 ; GFX7-NEXT: s_mov_b32 s6, 0
6769 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
6770 ; GFX7-NEXT: s_mov_b32 s4, s6
6771 ; GFX7-NEXT: s_mov_b32 s5, s6
6772 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
6773 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
6774 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
6775 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v2
6776 ; GFX7-NEXT: .LBB32_1: ; %atomicrmw.start
6777 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
6778 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6779 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
6780 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
6781 ; GFX7-NEXT: v_max_f32_e32 v2, v2, v6
6782 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
6783 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
6784 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
6785 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
6786 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 offset:2046 glc
6787 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6788 ; GFX7-NEXT: buffer_wbinvl1
6789 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
6790 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
6791 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
6792 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
6793 ; GFX7-NEXT: s_cbranch_execnz .LBB32_1
6794 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6795 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
6796 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v4
6797 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6798 ;
6799 ; GFX6-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
6800 ; GFX6: ; %bb.0:
6801 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6802 ; GFX6-NEXT: s_mov_b32 s6, 0
6803 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
6804 ; GFX6-NEXT: s_mov_b32 s4, s6
6805 ; GFX6-NEXT: s_mov_b32 s5, s6
6806 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
6807 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
6808 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
6809 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v2
6810 ; GFX6-NEXT: .LBB32_1: ; %atomicrmw.start
6811 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
6812 ; GFX6-NEXT: s_waitcnt vmcnt(0)
6813 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v3
6814 ; GFX6-NEXT: s_waitcnt expcnt(0)
6815 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
6816 ; GFX6-NEXT: v_max_f32_e32 v2, v2, v6
6817 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
6818 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
6819 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
6820 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
6821 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 offset:2046 glc
6822 ; GFX6-NEXT: s_waitcnt vmcnt(0)
6823 ; GFX6-NEXT: buffer_wbinvl1
6824 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
6825 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
6826 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
6827 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
6828 ; GFX6-NEXT: s_cbranch_execnz .LBB32_1
6829 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
6830 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
6831 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v4
6832 ; GFX6-NEXT: s_waitcnt expcnt(0)
6833 ; GFX6-NEXT: s_setpc_b64 s[30:31]
6834 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
6835 %result = atomicrmw fmax ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
6836 ret half %result
6837 }
6839 define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
6840 ; GFX12-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
6841 ; GFX12: ; %bb.0:
6842 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
6843 ; GFX12-NEXT: s_wait_expcnt 0x0
6844 ; GFX12-NEXT: s_wait_samplecnt 0x0
6845 ; GFX12-NEXT: s_wait_bvhcnt 0x0
6846 ; GFX12-NEXT: s_wait_kmcnt 0x0
6847 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2046
6848 ; GFX12-NEXT: v_max_num_f16_e32 v4, v2, v2
6849 ; GFX12-NEXT: s_mov_b32 s0, 0
6850 ; GFX12-NEXT: .LBB33_1: ; %atomicrmw.start
6851 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
6852 ; GFX12-NEXT: s_wait_loadcnt 0x0
6853 ; GFX12-NEXT: v_max_num_f16_e32 v2, v3, v3
6854 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6855 ; GFX12-NEXT: v_max_num_f16_e32 v2, v2, v4
6856 ; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
6857 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
6858 ; GFX12-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
6859 ; GFX12-NEXT: s_wait_storecnt 0x0
6860 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
6861 ; GFX12-NEXT: s_wait_loadcnt 0x0
6862 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
6863 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
6864 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
6865 ; GFX12-NEXT: s_wait_alu 0xfffe
6866 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
6867 ; GFX12-NEXT: s_wait_alu 0xfffe
6868 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
6869 ; GFX12-NEXT: s_cbranch_execnz .LBB33_1
6870 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
6871 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
6872 ; GFX12-NEXT: s_wait_alu 0xfffe
6873 ; GFX12-NEXT: s_setpc_b64 s[30:31]
6874 ;
6875 ; GFX940-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
6876 ; GFX940: ; %bb.0:
6877 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6878 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2046
6879 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
6880 ; GFX940-NEXT: v_max_f16_e32 v4, v2, v2
6881 ; GFX940-NEXT: s_mov_b32 s2, 0xffff0000
6882 ; GFX940-NEXT: .LBB33_1: ; %atomicrmw.start
6883 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
6884 ; GFX940-NEXT: s_waitcnt vmcnt(0)
6885 ; GFX940-NEXT: v_max_f16_e32 v2, v3, v3
6886 ; GFX940-NEXT: v_max_f16_e32 v2, v2, v4
6887 ; GFX940-NEXT: v_and_or_b32 v2, v3, s2, v2
6888 ; GFX940-NEXT: buffer_wbl2 sc1
6889 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 sc0
6890 ; GFX940-NEXT: s_waitcnt vmcnt(0)
6891 ; GFX940-NEXT: buffer_inv sc1
6892 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
6893 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
6894 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
6895 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
6896 ; GFX940-NEXT: s_cbranch_execnz .LBB33_1
6897 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
6898 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
6899 ; GFX940-NEXT: s_setpc_b64 s[30:31]
6900 ;
6901 ; GFX11-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
6902 ; GFX11: ; %bb.0:
6903 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6904 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2046
6905 ; GFX11-NEXT: v_max_f16_e32 v4, v2, v2
6906 ; GFX11-NEXT: s_mov_b32 s0, 0
6907 ; GFX11-NEXT: .LBB33_1: ; %atomicrmw.start
6908 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
6909 ; GFX11-NEXT: s_waitcnt vmcnt(0)
6910 ; GFX11-NEXT: v_max_f16_e32 v2, v3, v3
6911 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6912 ; GFX11-NEXT: v_max_f16_e32 v2, v2, v4
6913 ; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
6914 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
6915 ; GFX11-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
6916 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
6917 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 glc
6918 ; GFX11-NEXT: s_waitcnt vmcnt(0)
6919 ; GFX11-NEXT: buffer_gl1_inv
6920 ; GFX11-NEXT: buffer_gl0_inv
6921 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
6922 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
6923 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
6924 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
6925 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
6926 ; GFX11-NEXT: s_cbranch_execnz .LBB33_1
6927 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
6928 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
6929 ; GFX11-NEXT: s_setpc_b64 s[30:31]
6930 ;
6931 ; GFX10-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
6932 ; GFX10: ; %bb.0:
6933 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6934 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2046
6935 ; GFX10-NEXT: v_max_f16_e32 v4, v2, v2
6936 ; GFX10-NEXT: s_mov_b32 s4, 0
6937 ; GFX10-NEXT: .LBB33_1: ; %atomicrmw.start
6938 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
6939 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6940 ; GFX10-NEXT: v_max_f16_e32 v2, v3, v3
6941 ; GFX10-NEXT: v_max_f16_e32 v2, v2, v4
6942 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
6943 ; GFX10-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
6944 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
6945 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 glc
6946 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6947 ; GFX10-NEXT: buffer_gl1_inv
6948 ; GFX10-NEXT: buffer_gl0_inv
6949 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
6950 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
6951 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
6952 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
6953 ; GFX10-NEXT: s_cbranch_execnz .LBB33_1
6954 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
6955 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
6956 ; GFX10-NEXT: s_setpc_b64 s[30:31]
6957 ;
6958 ; GFX90A-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
6959 ; GFX90A: ; %bb.0:
6960 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6961 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2046
6962 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
6963 ; GFX90A-NEXT: v_max_f16_e32 v4, v2, v2
6964 ; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
6965 ; GFX90A-NEXT: .LBB33_1: ; %atomicrmw.start
6966 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
6967 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6968 ; GFX90A-NEXT: v_max_f16_e32 v2, v3, v3
6969 ; GFX90A-NEXT: v_max_f16_e32 v2, v2, v4
6970 ; GFX90A-NEXT: v_and_or_b32 v2, v3, s6, v2
6971 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 glc
6972 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6973 ; GFX90A-NEXT: buffer_wbinvl1
6974 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
6975 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6976 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
6977 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
6978 ; GFX90A-NEXT: s_cbranch_execnz .LBB33_1
6979 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
6980 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
6981 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
6982 ;
6983 ; GFX908-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
6984 ; GFX908: ; %bb.0:
6985 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6986 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2046
6987 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
6988 ; GFX908-NEXT: v_max_f16_e32 v4, v2, v2
6989 ; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
6990 ; GFX908-NEXT: .LBB33_1: ; %atomicrmw.start
6991 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
6992 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6993 ; GFX908-NEXT: v_max_f16_e32 v2, v3, v3
6994 ; GFX908-NEXT: v_max_f16_e32 v2, v2, v4
6995 ; GFX908-NEXT: v_and_or_b32 v2, v3, s6, v2
6996 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 glc
6997 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6998 ; GFX908-NEXT: buffer_wbinvl1
6999 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
7000 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7001 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
7002 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
7003 ; GFX908-NEXT: s_cbranch_execnz .LBB33_1
7004 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
7005 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
7006 ; GFX908-NEXT: s_setpc_b64 s[30:31]
7007 ;
7008 ; GFX8-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
7009 ; GFX8: ; %bb.0:
7010 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7011 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fe, v0
7012 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
7013 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
7014 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
7015 ; GFX8-NEXT: v_max_f16_e32 v4, v2, v2
7016 ; GFX8-NEXT: .LBB33_1: ; %atomicrmw.start
7017 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7018 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7019 ; GFX8-NEXT: v_max_f16_e32 v2, v3, v3
7020 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
7021 ; GFX8-NEXT: v_max_f16_e32 v2, v2, v4
7022 ; GFX8-NEXT: v_or_b32_e32 v2, v5, v2
7023 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7024 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7025 ; GFX8-NEXT: buffer_wbinvl1
7026 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
7027 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7028 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
7029 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
7030 ; GFX8-NEXT: s_cbranch_execnz .LBB33_1
7031 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7032 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
7033 ; GFX8-NEXT: s_setpc_b64 s[30:31]
7034 ;
7035 ; GFX7-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
7036 ; GFX7: ; %bb.0:
7037 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7038 ; GFX7-NEXT: s_mov_b32 s6, 0
7039 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
7040 ; GFX7-NEXT: s_mov_b32 s4, s6
7041 ; GFX7-NEXT: s_mov_b32 s5, s6
7042 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
7043 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
7044 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
7045 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v2
7046 ; GFX7-NEXT: .LBB33_1: ; %atomicrmw.start
7047 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7048 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7049 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
7050 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
7051 ; GFX7-NEXT: v_max_f32_e32 v2, v2, v4
7052 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
7053 ; GFX7-NEXT: v_or_b32_e32 v2, v5, v2
7054 ; GFX7-NEXT: v_mov_b32_e32 v6, v3
7055 ; GFX7-NEXT: v_mov_b32_e32 v5, v2
7056 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2046 glc
7057 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7058 ; GFX7-NEXT: buffer_wbinvl1
7059 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
7060 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
7061 ; GFX7-NEXT: v_mov_b32_e32 v3, v5
7062 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
7063 ; GFX7-NEXT: s_cbranch_execnz .LBB33_1
7064 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
7065 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
7066 ; GFX7-NEXT: s_setpc_b64 s[30:31]
7067 ;
7068 ; GFX6-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
7069 ; GFX6: ; %bb.0:
7070 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7071 ; GFX6-NEXT: s_mov_b32 s6, 0
7072 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
7073 ; GFX6-NEXT: s_mov_b32 s4, s6
7074 ; GFX6-NEXT: s_mov_b32 s5, s6
7075 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
7076 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
7077 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
7078 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v2
7079 ; GFX6-NEXT: .LBB33_1: ; %atomicrmw.start
7080 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
7081 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7082 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v3
7083 ; GFX6-NEXT: s_waitcnt expcnt(0)
7084 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
7085 ; GFX6-NEXT: v_max_f32_e32 v2, v2, v4
7086 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
7087 ; GFX6-NEXT: v_or_b32_e32 v2, v5, v2
7088 ; GFX6-NEXT: v_mov_b32_e32 v6, v3
7089 ; GFX6-NEXT: v_mov_b32_e32 v5, v2
7090 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2046 glc
7091 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7092 ; GFX6-NEXT: buffer_wbinvl1
7093 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
7094 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
7095 ; GFX6-NEXT: v_mov_b32_e32 v3, v5
7096 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
7097 ; GFX6-NEXT: s_cbranch_execnz .LBB33_1
7098 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
7099 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
7100 ; GFX6-NEXT: s_waitcnt expcnt(0)
7101 ; GFX6-NEXT: s_setpc_b64 s[30:31]
7102 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
7103 %unused = atomicrmw fmax ptr addrspace(1) %gep, half %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
7104 ret void
7105 }
7107 define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
7108 ; GFX12-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
7109 ; GFX12: ; %bb.0:
7110 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
7111 ; GFX12-NEXT: s_wait_expcnt 0x0
7112 ; GFX12-NEXT: s_wait_samplecnt 0x0
7113 ; GFX12-NEXT: s_wait_bvhcnt 0x0
7114 ; GFX12-NEXT: s_wait_kmcnt 0x0
7115 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
7116 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
7117 ; GFX12-NEXT: v_max_num_f16_e32 v2, v2, v2
7118 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
7119 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
7120 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
7121 ; GFX12-NEXT: s_mov_b32 s0, 0
7122 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
7123 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7124 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
7125 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
7126 ; GFX12-NEXT: v_not_b32_e32 v4, v4
7127 ; GFX12-NEXT: .LBB34_1: ; %atomicrmw.start
7128 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
7129 ; GFX12-NEXT: s_wait_loadcnt 0x0
7130 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
7131 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7132 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
7133 ; GFX12-NEXT: v_max_num_f16_e32 v5, v5, v5
7134 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7135 ; GFX12-NEXT: v_max_num_f16_e32 v5, v5, v2
7136 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff, v5
7137 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7138 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
7139 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
7140 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
7141 ; GFX12-NEXT: s_wait_storecnt 0x0
7142 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
7143 ; GFX12-NEXT: s_wait_loadcnt 0x0
7144 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
7145 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
7146 ; GFX12-NEXT: s_wait_alu 0xfffe
7147 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
7148 ; GFX12-NEXT: s_wait_alu 0xfffe
7149 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
7150 ; GFX12-NEXT: s_cbranch_execnz .LBB34_1
7151 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
7152 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
7153 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
7154 ; GFX12-NEXT: s_wait_alu 0xfffe
7155 ; GFX12-NEXT: s_setpc_b64 s[30:31]
7156 ;
7157 ; GFX940-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
7158 ; GFX940: ; %bb.0:
7159 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7160 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
7161 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
7162 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
7163 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
7164 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
7165 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v4
7166 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7167 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
7168 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
7169 ; GFX940-NEXT: v_not_b32_e32 v4, v4
7170 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
7171 ; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
7172 ; GFX940-NEXT: .LBB34_1: ; %atomicrmw.start
7173 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
7174 ; GFX940-NEXT: s_waitcnt vmcnt(0)
7175 ; GFX940-NEXT: v_mov_b32_e32 v7, v5
7176 ; GFX940-NEXT: v_lshrrev_b32_e32 v5, v3, v7
7177 ; GFX940-NEXT: v_max_f16_e32 v5, v5, v5
7178 ; GFX940-NEXT: v_max_f16_e32 v5, v5, v2
7179 ; GFX940-NEXT: v_lshlrev_b32_e32 v5, v3, v5
7180 ; GFX940-NEXT: v_and_or_b32 v6, v7, v4, v5
7181 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
7182 ; GFX940-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off sc0 sc1
7183 ; GFX940-NEXT: s_waitcnt vmcnt(0)
7184 ; GFX940-NEXT: buffer_inv sc0 sc1
7185 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
7186 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
7187 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
7188 ; GFX940-NEXT: s_cbranch_execnz .LBB34_1
7189 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
7190 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
7191 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
7192 ; GFX940-NEXT: s_setpc_b64 s[30:31]
7193 ;
7194 ; GFX11-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
7195 ; GFX11: ; %bb.0:
7196 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7197 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
7198 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
7199 ; GFX11-NEXT: v_max_f16_e32 v2, v2, v2
7200 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
7201 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
7202 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
7203 ; GFX11-NEXT: s_mov_b32 s0, 0
7204 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
7205 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7206 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
7207 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
7208 ; GFX11-NEXT: v_not_b32_e32 v4, v4
7209 ; GFX11-NEXT: .LBB34_1: ; %atomicrmw.start
7210 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
7211 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7212 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
7213 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7214 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
7215 ; GFX11-NEXT: v_max_f16_e32 v5, v5, v5
7216 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7217 ; GFX11-NEXT: v_max_f16_e32 v5, v5, v2
7218 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
7219 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7220 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
7221 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
7222 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
7223 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
7224 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7225 ; GFX11-NEXT: buffer_gl1_inv
7226 ; GFX11-NEXT: buffer_gl0_inv
7227 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
7228 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
7229 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
7230 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
7231 ; GFX11-NEXT: s_cbranch_execnz .LBB34_1
7232 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
7233 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
7234 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
7235 ; GFX11-NEXT: s_setpc_b64 s[30:31]
7236 ;
7237 ; GFX10-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
7238 ; GFX10: ; %bb.0:
7239 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7240 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
7241 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
7242 ; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
7243 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
7244 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
7245 ; GFX10-NEXT: s_mov_b32 s4, 0
7246 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
7247 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7248 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
7249 ; GFX10-NEXT: v_not_b32_e32 v4, v4
7250 ; GFX10-NEXT: .LBB34_1: ; %atomicrmw.start
7251 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
7252 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7253 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
7254 ; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
7255 ; GFX10-NEXT: v_max_f16_e32 v5, v5, v5
7256 ; GFX10-NEXT: v_max_f16_e32 v5, v5, v2
7257 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
7258 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
7259 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
7260 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
7261 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7262 ; GFX10-NEXT: buffer_gl1_inv
7263 ; GFX10-NEXT: buffer_gl0_inv
7264 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
7265 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
7266 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
7267 ; GFX10-NEXT: s_cbranch_execnz .LBB34_1
7268 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
7269 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
7270 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
7271 ; GFX10-NEXT: s_setpc_b64 s[30:31]
7272 ;
7273 ; GFX90A-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
7274 ; GFX90A: ; %bb.0:
7275 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7276 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
7277 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
7278 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
7279 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
7280 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
7281 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7282 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
7283 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
7284 ; GFX90A-NEXT: v_not_b32_e32 v4, v4
7285 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
7286 ; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
7287 ; GFX90A-NEXT: .LBB34_1: ; %atomicrmw.start
7288 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
7289 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7290 ; GFX90A-NEXT: v_mov_b32_e32 v7, v5
7291 ; GFX90A-NEXT: v_lshrrev_b32_e32 v5, v3, v7
7292 ; GFX90A-NEXT: v_max_f16_e32 v5, v5, v5
7293 ; GFX90A-NEXT: v_max_f16_e32 v5, v5, v2
7294 ; GFX90A-NEXT: v_lshlrev_b32_e32 v5, v3, v5
7295 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v4, v5
7296 ; GFX90A-NEXT: buffer_wbl2
7297 ; GFX90A-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off glc
7298 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7299 ; GFX90A-NEXT: buffer_invl2
7300 ; GFX90A-NEXT: buffer_wbinvl1
7301 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
7302 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7303 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
7304 ; GFX90A-NEXT: s_cbranch_execnz .LBB34_1
7305 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
7306 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
7307 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
7308 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
7309 ;
7310 ; GFX908-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
7311 ; GFX908: ; %bb.0:
7312 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7313 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
7314 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
7315 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
7316 ; GFX908-NEXT: global_load_dword v5, v[0:1], off
7317 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
7318 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7319 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
7320 ; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
7321 ; GFX908-NEXT: v_not_b32_e32 v4, v4
7322 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
7323 ; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
7324 ; GFX908-NEXT: .LBB34_1: ; %atomicrmw.start
7325 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
7326 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7327 ; GFX908-NEXT: v_mov_b32_e32 v6, v5
7328 ; GFX908-NEXT: v_lshrrev_b32_e32 v5, v3, v6
7329 ; GFX908-NEXT: v_max_f16_e32 v5, v5, v5
7330 ; GFX908-NEXT: v_max_f16_e32 v5, v5, v2
7331 ; GFX908-NEXT: v_lshlrev_b32_e32 v5, v3, v5
7332 ; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
7333 ; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
7334 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7335 ; GFX908-NEXT: buffer_wbinvl1
7336 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
7337 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7338 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
7339 ; GFX908-NEXT: s_cbranch_execnz .LBB34_1
7340 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
7341 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
7342 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
7343 ; GFX908-NEXT: s_setpc_b64 s[30:31]
7344 ;
7345 ; GFX8-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
7346 ; GFX8: ; %bb.0:
7347 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7348 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
7349 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
7350 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
7351 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
7352 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
7353 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7354 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
7355 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
7356 ; GFX8-NEXT: v_not_b32_e32 v4, v4
7357 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
7358 ; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
7359 ; GFX8-NEXT: .LBB34_1: ; %atomicrmw.start
7360 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7361 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7362 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
7363 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
7364 ; GFX8-NEXT: v_max_f16_e32 v5, v5, v5
7365 ; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
7366 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
7367 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
7368 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
7369 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
7370 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7371 ; GFX8-NEXT: buffer_wbinvl1
7372 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
7373 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7374 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
7375 ; GFX8-NEXT: s_cbranch_execnz .LBB34_1
7376 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7377 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
7378 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
7379 ; GFX8-NEXT: s_setpc_b64 s[30:31]
7380 ;
7381 ; GFX7-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
7382 ; GFX7: ; %bb.0:
7383 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7384 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
7385 ; GFX7-NEXT: s_mov_b32 s6, 0
7386 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
7387 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
7388 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
7389 ; GFX7-NEXT: s_mov_b32 s4, s6
7390 ; GFX7-NEXT: s_mov_b32 s5, s6
7391 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
7392 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
7393 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
7394 ; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
7395 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
7396 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
7397 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
7398 ; GFX7-NEXT: v_not_b32_e32 v8, v2
7399 ; GFX7-NEXT: .LBB34_1: ; %atomicrmw.start
7400 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7401 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7402 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
7403 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
7404 ; GFX7-NEXT: v_and_b32_e32 v4, v3, v8
7405 ; GFX7-NEXT: v_max_f32_e32 v2, v2, v7
7406 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
7407 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v6, v2
7408 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
7409 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
7410 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
7411 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
7412 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7413 ; GFX7-NEXT: buffer_wbinvl1
7414 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
7415 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
7416 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
7417 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
7418 ; GFX7-NEXT: s_cbranch_execnz .LBB34_1
7419 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
7420 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
7421 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
7422 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
7423 ; GFX7-NEXT: s_setpc_b64 s[30:31]
7424 ;
7425 ; GFX6-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
7426 ; GFX6: ; %bb.0:
7427 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7428 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
7429 ; GFX6-NEXT: s_mov_b32 s6, 0
7430 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
7431 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
7432 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
7433 ; GFX6-NEXT: s_mov_b32 s4, s6
7434 ; GFX6-NEXT: s_mov_b32 s5, s6
7435 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
7436 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
7437 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
7438 ; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
7439 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
7440 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
7441 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v6
7442 ; GFX6-NEXT: v_not_b32_e32 v8, v2
7443 ; GFX6-NEXT: .LBB34_1: ; %atomicrmw.start
7444 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
7445 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7446 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
7447 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
7448 ; GFX6-NEXT: s_waitcnt expcnt(0)
7449 ; GFX6-NEXT: v_and_b32_e32 v4, v3, v8
7450 ; GFX6-NEXT: v_max_f32_e32 v2, v2, v7
7451 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
7452 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v6, v2
7453 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
7454 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
7455 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
7456 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
7457 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7458 ; GFX6-NEXT: buffer_wbinvl1
7459 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
7460 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
7461 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
7462 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
7463 ; GFX6-NEXT: s_cbranch_execnz .LBB34_1
7464 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
7465 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
7466 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
7467 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
7468 ; GFX6-NEXT: s_waitcnt expcnt(0)
7469 ; GFX6-NEXT: s_setpc_b64 s[30:31]
7470 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
7471 %result = atomicrmw fmax ptr addrspace(1) %gep, half %val seq_cst, !amdgpu.no.fine.grained.memory !0
7472 ret half %result
7473 }
7475 define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, half %val) #0 {
7476 ; GFX12-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
7477 ; GFX12: ; %bb.0:
7478 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
7479 ; GFX12-NEXT: s_wait_expcnt 0x0
7480 ; GFX12-NEXT: s_wait_samplecnt 0x0
7481 ; GFX12-NEXT: s_wait_bvhcnt 0x0
7482 ; GFX12-NEXT: s_wait_kmcnt 0x0
7483 ; GFX12-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
7484 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
7485 ; GFX12-NEXT: v_max_num_f16_e32 v6, v2, v2
7486 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
7487 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v4
7488 ; GFX12-NEXT: v_and_b32_e32 v4, 3, v4
7489 ; GFX12-NEXT: s_mov_b32 s0, 0
7490 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
7491 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 3, v4
7492 ; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
7493 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
7494 ; GFX12-NEXT: v_not_b32_e32 v5, v5
7495 ; GFX12-NEXT: .LBB35_1: ; %atomicrmw.start
7496 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
7497 ; GFX12-NEXT: s_wait_loadcnt 0x0
7498 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
7499 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7500 ; GFX12-NEXT: v_max_num_f16_e32 v2, v2, v2
7501 ; GFX12-NEXT: v_max_num_f16_e32 v2, v2, v6
7502 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7503 ; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
7504 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, v4, v2
7505 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
7506 ; GFX12-NEXT: v_and_or_b32 v2, v3, v5, v2
7507 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
7508 ; GFX12-NEXT: s_wait_storecnt 0x0
7509 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
7510 ; GFX12-NEXT: s_wait_loadcnt 0x0
7511 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
7512 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
7513 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
7514 ; GFX12-NEXT: s_wait_alu 0xfffe
7515 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
7516 ; GFX12-NEXT: s_wait_alu 0xfffe
7517 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
7518 ; GFX12-NEXT: s_cbranch_execnz .LBB35_1
7519 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
7520 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
7521 ; GFX12-NEXT: s_wait_alu 0xfffe
7522 ; GFX12-NEXT: s_setpc_b64 s[30:31]
7523 ;
7524 ; GFX940-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
7525 ; GFX940: ; %bb.0:
7526 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7527 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
7528 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
7529 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
7530 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
7531 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
7532 ; GFX940-NEXT: v_and_b32_e32 v4, 3, v4
7533 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 3, v4
7534 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
7535 ; GFX940-NEXT: v_lshlrev_b32_e64 v5, v4, s0
7536 ; GFX940-NEXT: v_not_b32_e32 v5, v5
7537 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
7538 ; GFX940-NEXT: v_max_f16_e32 v6, v2, v2
7539 ; GFX940-NEXT: .LBB35_1: ; %atomicrmw.start
7540 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
7541 ; GFX940-NEXT: s_waitcnt vmcnt(0)
7542 ; GFX940-NEXT: v_lshrrev_b32_e32 v2, v4, v3
7543 ; GFX940-NEXT: v_max_f16_e32 v2, v2, v2
7544 ; GFX940-NEXT: v_max_f16_e32 v2, v2, v6
7545 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, v4, v2
7546 ; GFX940-NEXT: v_and_or_b32 v2, v3, v5, v2
7547 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
7548 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0 sc1
7549 ; GFX940-NEXT: s_waitcnt vmcnt(0)
7550 ; GFX940-NEXT: buffer_inv sc0 sc1
7551 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
7552 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
7553 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
7554 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
7555 ; GFX940-NEXT: s_cbranch_execnz .LBB35_1
7556 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
7557 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
7558 ; GFX940-NEXT: s_setpc_b64 s[30:31]
7559 ;
7560 ; GFX11-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
7561 ; GFX11: ; %bb.0:
7562 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7563 ; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
7564 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
7565 ; GFX11-NEXT: v_max_f16_e32 v6, v2, v2
7566 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
7567 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v4
7568 ; GFX11-NEXT: v_and_b32_e32 v4, 3, v4
7569 ; GFX11-NEXT: s_mov_b32 s0, 0
7570 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
7571 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v4
7572 ; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
7573 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
7574 ; GFX11-NEXT: v_not_b32_e32 v5, v5
7575 ; GFX11-NEXT: .LBB35_1: ; %atomicrmw.start
7576 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
7577 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7578 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
7579 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7580 ; GFX11-NEXT: v_max_f16_e32 v2, v2, v2
7581 ; GFX11-NEXT: v_max_f16_e32 v2, v2, v6
7582 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7583 ; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
7584 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, v4, v2
7585 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
7586 ; GFX11-NEXT: v_and_or_b32 v2, v3, v5, v2
7587 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
7588 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
7589 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7590 ; GFX11-NEXT: buffer_gl1_inv
7591 ; GFX11-NEXT: buffer_gl0_inv
7592 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
7593 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
7594 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
7595 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
7596 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
7597 ; GFX11-NEXT: s_cbranch_execnz .LBB35_1
7598 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
7599 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
7600 ; GFX11-NEXT: s_setpc_b64 s[30:31]
7601 ;
7602 ; GFX10-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
7603 ; GFX10: ; %bb.0:
7604 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7605 ; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
7606 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
7607 ; GFX10-NEXT: v_max_f16_e32 v6, v2, v2
7608 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
7609 ; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
7610 ; GFX10-NEXT: s_mov_b32 s4, 0
7611 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
7612 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
7613 ; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
7614 ; GFX10-NEXT: v_not_b32_e32 v5, v5
7615 ; GFX10-NEXT: .LBB35_1: ; %atomicrmw.start
7616 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
7617 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7618 ; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
7619 ; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
7620 ; GFX10-NEXT: v_max_f16_e32 v2, v2, v6
7621 ; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
7622 ; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
7623 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
7624 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
7625 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7626 ; GFX10-NEXT: buffer_gl1_inv
7627 ; GFX10-NEXT: buffer_gl0_inv
7628 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
7629 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
7630 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
7631 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
7632 ; GFX10-NEXT: s_cbranch_execnz .LBB35_1
7633 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
7634 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
7635 ; GFX10-NEXT: s_setpc_b64 s[30:31]
7636 ;
7637 ; GFX90A-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
7638 ; GFX90A: ; %bb.0:
7639 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7640 ; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
7641 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
7642 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
7643 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
7644 ; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
7645 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
7646 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
7647 ; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
7648 ; GFX90A-NEXT: v_not_b32_e32 v5, v5
7649 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
7650 ; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
7651 ; GFX90A-NEXT: .LBB35_1: ; %atomicrmw.start
7652 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
7653 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7654 ; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
7655 ; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
7656 ; GFX90A-NEXT: v_max_f16_e32 v2, v2, v6
7657 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
7658 ; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
7659 ; GFX90A-NEXT: buffer_wbl2
7660 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
7661 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7662 ; GFX90A-NEXT: buffer_invl2
7663 ; GFX90A-NEXT: buffer_wbinvl1
7664 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
7665 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7666 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
7667 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
7668 ; GFX90A-NEXT: s_cbranch_execnz .LBB35_1
7669 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
7670 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
7671 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
7672 ;
7673 ; GFX908-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
7674 ; GFX908: ; %bb.0:
7675 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7676 ; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
7677 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
7678 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
7679 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
7680 ; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
7681 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
7682 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
7683 ; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
7684 ; GFX908-NEXT: v_not_b32_e32 v5, v5
7685 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
7686 ; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
7687 ; GFX908-NEXT: .LBB35_1: ; %atomicrmw.start
7688 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
7689 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7690 ; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
7691 ; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
7692 ; GFX908-NEXT: v_max_f16_e32 v2, v2, v6
7693 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, v4, v2
7694 ; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
7695 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
7696 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7697 ; GFX908-NEXT: buffer_wbinvl1
7698 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
7699 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7700 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
7701 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
7702 ; GFX908-NEXT: s_cbranch_execnz .LBB35_1
7703 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
7704 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
7705 ; GFX908-NEXT: s_setpc_b64 s[30:31]
7706 ;
7707 ; GFX8-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
7708 ; GFX8: ; %bb.0:
7709 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7710 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
7711 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
7712 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
7713 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
7714 ; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
7715 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
7716 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
7717 ; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
7718 ; GFX8-NEXT: v_not_b32_e32 v5, v5
7719 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
7720 ; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
7721 ; GFX8-NEXT: .LBB35_1: ; %atomicrmw.start
7722 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7723 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7724 ; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
7725 ; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
7726 ; GFX8-NEXT: v_max_f16_e32 v2, v2, v6
7727 ; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
7728 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, v4, v2
7729 ; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
7730 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7731 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7732 ; GFX8-NEXT: buffer_wbinvl1
7733 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
7734 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7735 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
7736 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
7737 ; GFX8-NEXT: s_cbranch_execnz .LBB35_1
7738 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7739 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
7740 ; GFX8-NEXT: s_setpc_b64 s[30:31]
7741 ;
7742 ; GFX7-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
7743 ; GFX7: ; %bb.0:
7744 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7745 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
7746 ; GFX7-NEXT: s_mov_b32 s6, 0
7747 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
7748 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
7749 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
7750 ; GFX7-NEXT: s_mov_b32 s4, s6
7751 ; GFX7-NEXT: s_mov_b32 s5, s6
7752 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
7753 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
7754 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
7755 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
7756 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
7757 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
7758 ; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
7759 ; GFX7-NEXT: v_not_b32_e32 v6, v2
7760 ; GFX7-NEXT: .LBB35_1: ; %atomicrmw.start
7761 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7762 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7763 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
7764 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
7765 ; GFX7-NEXT: v_and_b32_e32 v7, v3, v6
7766 ; GFX7-NEXT: v_max_f32_e32 v2, v2, v5
7767 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
7768 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v4, v2
7769 ; GFX7-NEXT: v_or_b32_e32 v2, v7, v2
7770 ; GFX7-NEXT: v_mov_b32_e32 v8, v3
7771 ; GFX7-NEXT: v_mov_b32_e32 v7, v2
7772 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
7773 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7774 ; GFX7-NEXT: buffer_wbinvl1
7775 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
7776 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
7777 ; GFX7-NEXT: v_mov_b32_e32 v3, v7
7778 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
7779 ; GFX7-NEXT: s_cbranch_execnz .LBB35_1
7780 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
7781 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
7782 ; GFX7-NEXT: s_setpc_b64 s[30:31]
7783 ;
7784 ; GFX6-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
7785 ; GFX6: ; %bb.0:
7786 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7787 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
7788 ; GFX6-NEXT: s_mov_b32 s6, 0
7789 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
7790 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
7791 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
7792 ; GFX6-NEXT: s_mov_b32 s4, s6
7793 ; GFX6-NEXT: s_mov_b32 s5, s6
7794 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
7795 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
7796 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
7797 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
7798 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
7799 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
7800 ; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v4
7801 ; GFX6-NEXT: v_not_b32_e32 v6, v2
7802 ; GFX6-NEXT: .LBB35_1: ; %atomicrmw.start
7803 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
7804 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7805 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
7806 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
7807 ; GFX6-NEXT: s_waitcnt expcnt(0)
7808 ; GFX6-NEXT: v_and_b32_e32 v7, v3, v6
7809 ; GFX6-NEXT: v_max_f32_e32 v2, v2, v5
7810 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
7811 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v4, v2
7812 ; GFX6-NEXT: v_or_b32_e32 v2, v7, v2
7813 ; GFX6-NEXT: v_mov_b32_e32 v8, v3
7814 ; GFX6-NEXT: v_mov_b32_e32 v7, v2
7815 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
7816 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7817 ; GFX6-NEXT: buffer_wbinvl1
7818 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
7819 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
7820 ; GFX6-NEXT: v_mov_b32_e32 v3, v7
7821 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
7822 ; GFX6-NEXT: s_cbranch_execnz .LBB35_1
7823 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
7824 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
7825 ; GFX6-NEXT: s_waitcnt expcnt(0)
7826 ; GFX6-NEXT: s_setpc_b64 s[30:31]
7827 %gep = getelementptr half, ptr addrspace(1) %ptr, i64 1023
7828 %unused = atomicrmw fmax ptr addrspace(1) %gep, half %val seq_cst, !amdgpu.no.fine.grained.memory !0
7829 ret void
7830 }
7832 ; --------------------------------------------------------------------
7833 ; bfloat
7834 ; --------------------------------------------------------------------
7836 define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
7837 ; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
7838 ; GFX12: ; %bb.0:
7839 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
7840 ; GFX12-NEXT: s_wait_expcnt 0x0
7841 ; GFX12-NEXT: s_wait_samplecnt 0x0
7842 ; GFX12-NEXT: s_wait_bvhcnt 0x0
7843 ; GFX12-NEXT: s_wait_kmcnt 0x0
7844 ; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
7845 ; GFX12-NEXT: s_mov_b32 s0, 0
7846 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
7847 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
7848 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
7849 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
7850 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7851 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
7852 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
7853 ; GFX12-NEXT: v_not_b32_e32 v4, v4
7854 ; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
7855 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
7856 ; GFX12-NEXT: s_wait_loadcnt 0x0
7857 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
7858 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7859 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
7860 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
7861 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7862 ; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
7863 ; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
7864 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
7865 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
7866 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
7867 ; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
7868 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
7869 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7870 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
7871 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
7872 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
7873 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
7874 ; GFX12-NEXT: s_wait_storecnt 0x0
7875 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
7876 ; GFX12-NEXT: s_wait_loadcnt 0x0
7877 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
7878 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
7879 ; GFX12-NEXT: s_wait_alu 0xfffe
7880 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
7881 ; GFX12-NEXT: s_wait_alu 0xfffe
7882 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
7883 ; GFX12-NEXT: s_cbranch_execnz .LBB36_1
7884 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
7885 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
7886 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
7887 ; GFX12-NEXT: s_wait_alu 0xfffe
7888 ; GFX12-NEXT: s_setpc_b64 s[30:31]
7889 ;
7890 ; GFX940-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
7891 ; GFX940: ; %bb.0:
7892 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7893 ; GFX940-NEXT: v_mov_b32_e32 v3, v0
7894 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v3
7895 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
7896 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v3
7897 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7898 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
7899 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
7900 ; GFX940-NEXT: v_not_b32_e32 v4, v4
7901 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
7902 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
7903 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
7904 ; GFX940-NEXT: .LBB36_1: ; %atomicrmw.start
7905 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
7906 ; GFX940-NEXT: s_waitcnt vmcnt(0)
7907 ; GFX940-NEXT: v_mov_b32_e32 v7, v5
7908 ; GFX940-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
7909 ; GFX940-NEXT: s_nop 0
7910 ; GFX940-NEXT: v_max_f32_e32 v5, v5, v2
7911 ; GFX940-NEXT: v_bfe_u32 v6, v5, 16, 1
7912 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v5
7913 ; GFX940-NEXT: v_add3_u32 v6, v6, v5, s2
7914 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
7915 ; GFX940-NEXT: s_nop 1
7916 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
7917 ; GFX940-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
7918 ; GFX940-NEXT: v_and_or_b32 v6, v7, v4, v5
7919 ; GFX940-NEXT: buffer_wbl2 sc1
7920 ; GFX940-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off sc0
7921 ; GFX940-NEXT: s_waitcnt vmcnt(0)
7922 ; GFX940-NEXT: buffer_inv sc1
7923 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
7924 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
7925 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
7926 ; GFX940-NEXT: s_cbranch_execnz .LBB36_1
7927 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
7928 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
7929 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
7930 ; GFX940-NEXT: s_setpc_b64 s[30:31]
7931 ;
7932 ; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
7933 ; GFX11: ; %bb.0:
7934 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7935 ; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
7936 ; GFX11-NEXT: s_mov_b32 s0, 0
7937 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
7938 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
7939 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
7940 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
7941 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7942 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
7943 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
7944 ; GFX11-NEXT: v_not_b32_e32 v4, v4
7945 ; GFX11-NEXT: .p2align 6
7946 ; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
7947 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
7948 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7949 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
7950 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7951 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
7952 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
7953 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7954 ; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
7955 ; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
7956 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
7957 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
7958 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
7959 ; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
7960 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
7961 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7962 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
7963 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
7964 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
7965 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
7966 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
7967 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
7968 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7969 ; GFX11-NEXT: buffer_gl1_inv
7970 ; GFX11-NEXT: buffer_gl0_inv
7971 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
7972 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
7973 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
7974 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
7975 ; GFX11-NEXT: s_cbranch_execnz .LBB36_1
7976 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
7977 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
7978 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
7979 ; GFX11-NEXT: s_setpc_b64 s[30:31]
7980 ;
7981 ; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
7982 ; GFX10: ; %bb.0:
7983 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7984 ; GFX10-NEXT: v_mov_b32_e32 v3, v0
7985 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
7986 ; GFX10-NEXT: s_mov_b32 s4, 0
7987 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
7988 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
7989 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
7990 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
7991 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
7992 ; GFX10-NEXT: v_not_b32_e32 v4, v4
7993 ; GFX10-NEXT: .LBB36_1: ; %atomicrmw.start
7994 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
7995 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7996 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
7997 ; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
7998 ; GFX10-NEXT: v_max_f32_e32 v5, v5, v2
7999 ; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
8000 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v5
8001 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
8002 ; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
8003 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
8004 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8005 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
8006 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
8007 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
8008 ; GFX10-NEXT: s_waitcnt vmcnt(0)
8009 ; GFX10-NEXT: buffer_gl1_inv
8010 ; GFX10-NEXT: buffer_gl0_inv
8011 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
8012 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
8013 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
8014 ; GFX10-NEXT: s_cbranch_execnz .LBB36_1
8015 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
8016 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
8017 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8018 ; GFX10-NEXT: s_setpc_b64 s[30:31]
8019 ;
8020 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
8021 ; GFX90A: ; %bb.0:
8022 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8023 ; GFX90A-NEXT: v_mov_b32_e32 v3, v0
8024 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
8025 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
8026 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
8027 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8028 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
8029 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
8030 ; GFX90A-NEXT: v_not_b32_e32 v4, v4
8031 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
8032 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8033 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
8034 ; GFX90A-NEXT: .LBB36_1: ; %atomicrmw.start
8035 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
8036 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
8037 ; GFX90A-NEXT: v_mov_b32_e32 v7, v5
8038 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8039 ; GFX90A-NEXT: v_max_f32_e32 v5, v5, v2
8040 ; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
8041 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v5
8042 ; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
8043 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
8044 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
8045 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8046 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v4, v5
8047 ; GFX90A-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off glc
8048 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
8049 ; GFX90A-NEXT: buffer_wbinvl1
8050 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
8051 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8052 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
8053 ; GFX90A-NEXT: s_cbranch_execnz .LBB36_1
8054 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
8055 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
8056 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8057 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
8058 ;
8059 ; GFX908-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
8060 ; GFX908: ; %bb.0:
8061 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8062 ; GFX908-NEXT: v_mov_b32_e32 v3, v0
8063 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
8064 ; GFX908-NEXT: global_load_dword v5, v[0:1], off
8065 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
8066 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8067 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
8068 ; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
8069 ; GFX908-NEXT: v_not_b32_e32 v4, v4
8070 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
8071 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8072 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
8073 ; GFX908-NEXT: .LBB36_1: ; %atomicrmw.start
8074 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
8075 ; GFX908-NEXT: s_waitcnt vmcnt(0)
8076 ; GFX908-NEXT: v_mov_b32_e32 v6, v5
8077 ; GFX908-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8078 ; GFX908-NEXT: v_max_f32_e32 v5, v5, v2
8079 ; GFX908-NEXT: v_bfe_u32 v7, v5, 16, 1
8080 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v5
8081 ; GFX908-NEXT: v_add3_u32 v7, v7, v5, s6
8082 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
8083 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc
8084 ; GFX908-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8085 ; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
8086 ; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
8087 ; GFX908-NEXT: s_waitcnt vmcnt(0)
8088 ; GFX908-NEXT: buffer_wbinvl1
8089 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
8090 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8091 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
8092 ; GFX908-NEXT: s_cbranch_execnz .LBB36_1
8093 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
8094 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
8095 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8096 ; GFX908-NEXT: s_setpc_b64 s[30:31]
8097 ;
8098 ; GFX8-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
8099 ; GFX8: ; %bb.0:
8100 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8101 ; GFX8-NEXT: v_mov_b32_e32 v3, v0
8102 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
8103 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
8104 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
8105 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8106 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
8107 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
8108 ; GFX8-NEXT: v_not_b32_e32 v4, v4
8109 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
8110 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8111 ; GFX8-NEXT: .LBB36_1: ; %atomicrmw.start
8112 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
8113 ; GFX8-NEXT: s_waitcnt vmcnt(0)
8114 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
8115 ; GFX8-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8116 ; GFX8-NEXT: v_max_f32_e32 v5, v5, v2
8117 ; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
8118 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
8119 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
8120 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
8121 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
8122 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
8123 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
8124 ; GFX8-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8125 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
8126 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
8127 ; GFX8-NEXT: s_waitcnt vmcnt(0)
8128 ; GFX8-NEXT: buffer_wbinvl1
8129 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
8130 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8131 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
8132 ; GFX8-NEXT: s_cbranch_execnz .LBB36_1
8133 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
8134 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
8135 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8136 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8137 ;
8138 ; GFX7-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
8139 ; GFX7: ; %bb.0:
8140 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8141 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
8142 ; GFX7-NEXT: s_mov_b32 s6, 0
8143 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
8144 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
8145 ; GFX7-NEXT: s_mov_b32 s4, s6
8146 ; GFX7-NEXT: s_mov_b32 s5, s6
8147 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
8148 ; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
8149 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 3, v3
8150 ; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
8151 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
8152 ; GFX7-NEXT: v_not_b32_e32 v6, v3
8153 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
8154 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
8155 ; GFX7-NEXT: .LBB36_1: ; %atomicrmw.start
8156 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
8157 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8158 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v5, v4
8159 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8160 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
8161 ; GFX7-NEXT: v_max_f32_e32 v2, v2, v7
8162 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
8163 ; GFX7-NEXT: v_and_b32_e32 v3, v4, v6
8164 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v5, v2
8165 ; GFX7-NEXT: v_or_b32_e32 v3, v3, v2
8166 ; GFX7-NEXT: v_mov_b32_e32 v2, v3
8167 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
8168 ; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v[0:1], s[4:7], 0 addr64 glc
8169 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8170 ; GFX7-NEXT: buffer_wbinvl1
8171 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
8172 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
8173 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
8174 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
8175 ; GFX7-NEXT: s_cbranch_execnz .LBB36_1
8176 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
8177 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
8178 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v5, v2
8179 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
8180 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8181 ;
8182 ; GFX6-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
8183 ; GFX6: ; %bb.0:
8184 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8185 ; GFX6-NEXT: v_mov_b32_e32 v3, v0
8186 ; GFX6-NEXT: s_mov_b32 s6, 0
8187 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v3
8188 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
8189 ; GFX6-NEXT: s_mov_b32 s4, s6
8190 ; GFX6-NEXT: s_mov_b32 s5, s6
8191 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
8192 ; GFX6-NEXT: v_and_b32_e32 v3, 3, v3
8193 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 3, v3
8194 ; GFX6-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
8195 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
8196 ; GFX6-NEXT: v_not_b32_e32 v6, v3
8197 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
8198 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
8199 ; GFX6-NEXT: .LBB36_1: ; %atomicrmw.start
8200 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
8201 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
8202 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v5, v4
8203 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8204 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
8205 ; GFX6-NEXT: v_max_f32_e32 v2, v2, v7
8206 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
8207 ; GFX6-NEXT: v_and_b32_e32 v3, v4, v6
8208 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v5, v2
8209 ; GFX6-NEXT: v_or_b32_e32 v3, v3, v2
8210 ; GFX6-NEXT: v_mov_b32_e32 v2, v3
8211 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
8212 ; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v[0:1], s[4:7], 0 addr64 glc
8213 ; GFX6-NEXT: s_waitcnt vmcnt(0)
8214 ; GFX6-NEXT: buffer_wbinvl1
8215 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
8216 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
8217 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
8218 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
8219 ; GFX6-NEXT: s_cbranch_execnz .LBB36_1
8220 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
8221 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
8222 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v5, v2
8223 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
8224 ; GFX6-NEXT: s_waitcnt expcnt(0)
8225 ; GFX6-NEXT: s_setpc_b64 s[30:31]
8226 %result = atomicrmw fmax ptr addrspace(1) %ptr, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
8227 ret bfloat %result
8228 }
8230 define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
8231 ; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
8232 ; GFX12: ; %bb.0:
8233 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
8234 ; GFX12-NEXT: s_wait_expcnt 0x0
8235 ; GFX12-NEXT: s_wait_samplecnt 0x0
8236 ; GFX12-NEXT: s_wait_bvhcnt 0x0
8237 ; GFX12-NEXT: s_wait_kmcnt 0x0
8238 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
8239 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
8240 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8241 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
8242 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
8243 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
8244 ; GFX12-NEXT: s_mov_b32 s0, 0
8245 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
8246 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8247 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
8248 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
8249 ; GFX12-NEXT: v_not_b32_e32 v4, v4
8250 ; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
8251 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
8252 ; GFX12-NEXT: s_wait_loadcnt 0x0
8253 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
8254 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8255 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
8256 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
8257 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8258 ; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
8259 ; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
8260 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
8261 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
8262 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
8263 ; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
8264 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
8265 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8266 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
8267 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
8268 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
8269 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
8270 ; GFX12-NEXT: s_wait_storecnt 0x0
8271 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
8272 ; GFX12-NEXT: s_wait_loadcnt 0x0
8273 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
8274 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
8275 ; GFX12-NEXT: s_wait_alu 0xfffe
8276 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
8277 ; GFX12-NEXT: s_wait_alu 0xfffe
8278 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
8279 ; GFX12-NEXT: s_cbranch_execnz .LBB37_1
8280 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
8281 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
8282 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8283 ; GFX12-NEXT: s_wait_alu 0xfffe
8284 ; GFX12-NEXT: s_setpc_b64 s[30:31]
8285 ;
8286 ; GFX940-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
8287 ; GFX940: ; %bb.0:
8288 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8289 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
8290 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
8291 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
8292 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
8293 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
8294 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v4
8295 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8296 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
8297 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
8298 ; GFX940-NEXT: v_not_b32_e32 v4, v4
8299 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
8300 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8301 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
8302 ; GFX940-NEXT: .LBB37_1: ; %atomicrmw.start
8303 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
8304 ; GFX940-NEXT: s_waitcnt vmcnt(0)
8305 ; GFX940-NEXT: v_mov_b32_e32 v7, v5
8306 ; GFX940-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8307 ; GFX940-NEXT: s_nop 0
8308 ; GFX940-NEXT: v_max_f32_e32 v5, v5, v2
8309 ; GFX940-NEXT: v_bfe_u32 v6, v5, 16, 1
8310 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v5
8311 ; GFX940-NEXT: v_add3_u32 v6, v6, v5, s2
8312 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
8313 ; GFX940-NEXT: s_nop 1
8314 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
8315 ; GFX940-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8316 ; GFX940-NEXT: v_and_or_b32 v6, v7, v4, v5
8317 ; GFX940-NEXT: buffer_wbl2 sc1
8318 ; GFX940-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off sc0
8319 ; GFX940-NEXT: s_waitcnt vmcnt(0)
8320 ; GFX940-NEXT: buffer_inv sc1
8321 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
8322 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
8323 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
8324 ; GFX940-NEXT: s_cbranch_execnz .LBB37_1
8325 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
8326 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
8327 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8328 ; GFX940-NEXT: s_setpc_b64 s[30:31]
8329 ;
8330 ; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
8331 ; GFX11: ; %bb.0:
8332 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8333 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
8334 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
8335 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8336 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
8337 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
8338 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
8339 ; GFX11-NEXT: s_mov_b32 s0, 0
8340 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
8341 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8342 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
8343 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
8344 ; GFX11-NEXT: v_not_b32_e32 v4, v4
8345 ; GFX11-NEXT: .p2align 6
8346 ; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
8347 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
8348 ; GFX11-NEXT: s_waitcnt vmcnt(0)
8349 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
8350 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8351 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
8352 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
8353 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8354 ; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
8355 ; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
8356 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
8357 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
8358 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
8359 ; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
8360 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
8361 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8362 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
8363 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
8364 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
8365 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
8366 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
8367 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
8368 ; GFX11-NEXT: s_waitcnt vmcnt(0)
8369 ; GFX11-NEXT: buffer_gl1_inv
8370 ; GFX11-NEXT: buffer_gl0_inv
8371 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
8372 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
8373 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
8374 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
8375 ; GFX11-NEXT: s_cbranch_execnz .LBB37_1
8376 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
8377 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
8378 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8379 ; GFX11-NEXT: s_setpc_b64 s[30:31]
8380 ;
8381 ; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
8382 ; GFX10: ; %bb.0:
8383 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8384 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
8385 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
8386 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8387 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
8388 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
8389 ; GFX10-NEXT: s_mov_b32 s4, 0
8390 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
8391 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8392 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
8393 ; GFX10-NEXT: v_not_b32_e32 v4, v4
8394 ; GFX10-NEXT: .LBB37_1: ; %atomicrmw.start
8395 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
8396 ; GFX10-NEXT: s_waitcnt vmcnt(0)
8397 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
8398 ; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8399 ; GFX10-NEXT: v_max_f32_e32 v5, v5, v2
8400 ; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
8401 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v5
8402 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
8403 ; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
8404 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
8405 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8406 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
8407 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
8408 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
8409 ; GFX10-NEXT: s_waitcnt vmcnt(0)
8410 ; GFX10-NEXT: buffer_gl1_inv
8411 ; GFX10-NEXT: buffer_gl0_inv
8412 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
8413 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
8414 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
8415 ; GFX10-NEXT: s_cbranch_execnz .LBB37_1
8416 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
8417 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
8418 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8419 ; GFX10-NEXT: s_setpc_b64 s[30:31]
8420 ;
8421 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
8422 ; GFX90A: ; %bb.0:
8423 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8424 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
8425 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
8426 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
8427 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
8428 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
8429 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8430 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
8431 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
8432 ; GFX90A-NEXT: v_not_b32_e32 v4, v4
8433 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
8434 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8435 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
8436 ; GFX90A-NEXT: .LBB37_1: ; %atomicrmw.start
8437 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
8438 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
8439 ; GFX90A-NEXT: v_mov_b32_e32 v7, v5
8440 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8441 ; GFX90A-NEXT: v_max_f32_e32 v5, v5, v2
8442 ; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
8443 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v5
8444 ; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
8445 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
8446 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
8447 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8448 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v4, v5
8449 ; GFX90A-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off glc
8450 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
8451 ; GFX90A-NEXT: buffer_wbinvl1
8452 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
8453 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8454 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
8455 ; GFX90A-NEXT: s_cbranch_execnz .LBB37_1
8456 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
8457 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
8458 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8459 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
8460 ;
8461 ; GFX908-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
8462 ; GFX908: ; %bb.0:
8463 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8464 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
8465 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
8466 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
8467 ; GFX908-NEXT: global_load_dword v5, v[0:1], off
8468 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
8469 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8470 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
8471 ; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
8472 ; GFX908-NEXT: v_not_b32_e32 v4, v4
8473 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
8474 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8475 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
8476 ; GFX908-NEXT: .LBB37_1: ; %atomicrmw.start
8477 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
8478 ; GFX908-NEXT: s_waitcnt vmcnt(0)
8479 ; GFX908-NEXT: v_mov_b32_e32 v6, v5
8480 ; GFX908-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8481 ; GFX908-NEXT: v_max_f32_e32 v5, v5, v2
8482 ; GFX908-NEXT: v_bfe_u32 v7, v5, 16, 1
8483 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v5
8484 ; GFX908-NEXT: v_add3_u32 v7, v7, v5, s6
8485 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
8486 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc
8487 ; GFX908-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8488 ; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
8489 ; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
8490 ; GFX908-NEXT: s_waitcnt vmcnt(0)
8491 ; GFX908-NEXT: buffer_wbinvl1
8492 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
8493 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8494 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
8495 ; GFX908-NEXT: s_cbranch_execnz .LBB37_1
8496 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
8497 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
8498 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8499 ; GFX908-NEXT: s_setpc_b64 s[30:31]
8500 ;
8501 ; GFX8-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
8502 ; GFX8: ; %bb.0:
8503 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8504 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
8505 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8506 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
8507 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
8508 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
8509 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8510 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
8511 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
8512 ; GFX8-NEXT: v_not_b32_e32 v4, v4
8513 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
8514 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8515 ; GFX8-NEXT: .LBB37_1: ; %atomicrmw.start
8516 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
8517 ; GFX8-NEXT: s_waitcnt vmcnt(0)
8518 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
8519 ; GFX8-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8520 ; GFX8-NEXT: v_max_f32_e32 v5, v5, v2
8521 ; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
8522 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
8523 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
8524 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
8525 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
8526 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
8527 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
8528 ; GFX8-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8529 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
8530 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
8531 ; GFX8-NEXT: s_waitcnt vmcnt(0)
8532 ; GFX8-NEXT: buffer_wbinvl1
8533 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
8534 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8535 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
8536 ; GFX8-NEXT: s_cbranch_execnz .LBB37_1
8537 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
8538 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
8539 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8540 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8541 ;
8542 ; GFX7-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
8543 ; GFX7: ; %bb.0:
8544 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8545 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
8546 ; GFX7-NEXT: s_mov_b32 s6, 0
8547 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8548 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
8549 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
8550 ; GFX7-NEXT: s_mov_b32 s4, s6
8551 ; GFX7-NEXT: s_mov_b32 s5, s6
8552 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
8553 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
8554 ; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
8555 ; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
8556 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
8557 ; GFX7-NEXT: v_not_b32_e32 v7, v4
8558 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
8559 ; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
8560 ; GFX7-NEXT: .LBB37_1: ; %atomicrmw.start
8561 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
8562 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8563 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
8564 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8565 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
8566 ; GFX7-NEXT: v_max_f32_e32 v2, v2, v8
8567 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
8568 ; GFX7-NEXT: v_and_b32_e32 v4, v3, v7
8569 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v6, v2
8570 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
8571 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
8572 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
8573 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
8574 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8575 ; GFX7-NEXT: buffer_wbinvl1
8576 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
8577 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
8578 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
8579 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
8580 ; GFX7-NEXT: s_cbranch_execnz .LBB37_1
8581 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
8582 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
8583 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
8584 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
8585 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8586 ;
8587 ; GFX6-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
8588 ; GFX6: ; %bb.0:
8589 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8590 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
8591 ; GFX6-NEXT: s_mov_b32 s6, 0
8592 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8593 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
8594 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
8595 ; GFX6-NEXT: s_mov_b32 s4, s6
8596 ; GFX6-NEXT: s_mov_b32 s5, s6
8597 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
8598 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
8599 ; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
8600 ; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
8601 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
8602 ; GFX6-NEXT: v_not_b32_e32 v7, v4
8603 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
8604 ; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
8605 ; GFX6-NEXT: .LBB37_1: ; %atomicrmw.start
8606 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
8607 ; GFX6-NEXT: s_waitcnt vmcnt(0)
8608 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
8609 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8610 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
8611 ; GFX6-NEXT: v_max_f32_e32 v2, v2, v8
8612 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
8613 ; GFX6-NEXT: s_waitcnt expcnt(0)
8614 ; GFX6-NEXT: v_and_b32_e32 v4, v3, v7
8615 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v6, v2
8616 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
8617 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
8618 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
8619 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
8620 ; GFX6-NEXT: s_waitcnt vmcnt(0)
8621 ; GFX6-NEXT: buffer_wbinvl1
8622 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
8623 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
8624 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
8625 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
8626 ; GFX6-NEXT: s_cbranch_execnz .LBB37_1
8627 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
8628 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
8629 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
8630 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
8631 ; GFX6-NEXT: s_waitcnt expcnt(0)
8632 ; GFX6-NEXT: s_setpc_b64 s[30:31]
8633 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
8634 %result = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
8635 ret bfloat %result
8636 }
8638 define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
8639 ; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
8640 ; GFX12: ; %bb.0:
8641 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
8642 ; GFX12-NEXT: s_wait_expcnt 0x0
8643 ; GFX12-NEXT: s_wait_samplecnt 0x0
8644 ; GFX12-NEXT: s_wait_bvhcnt 0x0
8645 ; GFX12-NEXT: s_wait_kmcnt 0x0
8646 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
8647 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
8648 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8649 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
8650 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
8651 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
8652 ; GFX12-NEXT: s_mov_b32 s0, 0
8653 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
8654 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8655 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
8656 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
8657 ; GFX12-NEXT: v_not_b32_e32 v4, v4
8658 ; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
8659 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
8660 ; GFX12-NEXT: s_wait_loadcnt 0x0
8661 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
8662 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8663 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
8664 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
8665 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8666 ; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
8667 ; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
8668 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
8669 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
8670 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
8671 ; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
8672 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
8673 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8674 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
8675 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
8676 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
8677 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
8678 ; GFX12-NEXT: s_wait_storecnt 0x0
8679 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
8680 ; GFX12-NEXT: s_wait_loadcnt 0x0
8681 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
8682 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
8683 ; GFX12-NEXT: s_wait_alu 0xfffe
8684 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
8685 ; GFX12-NEXT: s_wait_alu 0xfffe
8686 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
8687 ; GFX12-NEXT: s_cbranch_execnz .LBB38_1
8688 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
8689 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
8690 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8691 ; GFX12-NEXT: s_wait_alu 0xfffe
8692 ; GFX12-NEXT: s_setpc_b64 s[30:31]
8693 ;
8694 ; GFX940-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
8695 ; GFX940: ; %bb.0:
8696 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8697 ; GFX940-NEXT: s_movk_i32 s0, 0xf800
8698 ; GFX940-NEXT: s_mov_b32 s1, -1
8699 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
8700 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
8701 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
8702 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
8703 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v4
8704 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8705 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
8706 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
8707 ; GFX940-NEXT: v_not_b32_e32 v4, v4
8708 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
8709 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8710 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
8711 ; GFX940-NEXT: .LBB38_1: ; %atomicrmw.start
8712 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
8713 ; GFX940-NEXT: s_waitcnt vmcnt(0)
8714 ; GFX940-NEXT: v_mov_b32_e32 v7, v5
8715 ; GFX940-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8716 ; GFX940-NEXT: s_nop 0
8717 ; GFX940-NEXT: v_max_f32_e32 v5, v5, v2
8718 ; GFX940-NEXT: v_bfe_u32 v6, v5, 16, 1
8719 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v5
8720 ; GFX940-NEXT: v_add3_u32 v6, v6, v5, s2
8721 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
8722 ; GFX940-NEXT: s_nop 1
8723 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
8724 ; GFX940-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8725 ; GFX940-NEXT: v_and_or_b32 v6, v7, v4, v5
8726 ; GFX940-NEXT: buffer_wbl2 sc1
8727 ; GFX940-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off sc0
8728 ; GFX940-NEXT: s_waitcnt vmcnt(0)
8729 ; GFX940-NEXT: buffer_inv sc1
8730 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
8731 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
8732 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
8733 ; GFX940-NEXT: s_cbranch_execnz .LBB38_1
8734 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
8735 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
8736 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8737 ; GFX940-NEXT: s_setpc_b64 s[30:31]
8738 ;
8739 ; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
8740 ; GFX11: ; %bb.0:
8741 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8742 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
8743 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
8744 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8745 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
8746 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
8747 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
8748 ; GFX11-NEXT: s_mov_b32 s0, 0
8749 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
8750 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8751 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
8752 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
8753 ; GFX11-NEXT: v_not_b32_e32 v4, v4
8754 ; GFX11-NEXT: .p2align 6
8755 ; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
8756 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
8757 ; GFX11-NEXT: s_waitcnt vmcnt(0)
8758 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
8759 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8760 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
8761 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
8762 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8763 ; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
8764 ; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
8765 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
8766 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
8767 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
8768 ; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
8769 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
8770 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
8771 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
8772 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
8773 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
8774 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
8775 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
8776 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
8777 ; GFX11-NEXT: s_waitcnt vmcnt(0)
8778 ; GFX11-NEXT: buffer_gl1_inv
8779 ; GFX11-NEXT: buffer_gl0_inv
8780 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
8781 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
8782 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
8783 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
8784 ; GFX11-NEXT: s_cbranch_execnz .LBB38_1
8785 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
8786 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
8787 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8788 ; GFX11-NEXT: s_setpc_b64 s[30:31]
8789 ;
8790 ; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
8791 ; GFX10: ; %bb.0:
8792 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8793 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
8794 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
8795 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8796 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
8797 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
8798 ; GFX10-NEXT: s_mov_b32 s4, 0
8799 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
8800 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8801 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
8802 ; GFX10-NEXT: v_not_b32_e32 v4, v4
8803 ; GFX10-NEXT: .LBB38_1: ; %atomicrmw.start
8804 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
8805 ; GFX10-NEXT: s_waitcnt vmcnt(0)
8806 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
8807 ; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8808 ; GFX10-NEXT: v_max_f32_e32 v5, v5, v2
8809 ; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
8810 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v5
8811 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
8812 ; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
8813 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
8814 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8815 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
8816 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
8817 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
8818 ; GFX10-NEXT: s_waitcnt vmcnt(0)
8819 ; GFX10-NEXT: buffer_gl1_inv
8820 ; GFX10-NEXT: buffer_gl0_inv
8821 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
8822 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
8823 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
8824 ; GFX10-NEXT: s_cbranch_execnz .LBB38_1
8825 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
8826 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
8827 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8828 ; GFX10-NEXT: s_setpc_b64 s[30:31]
8829 ;
8830 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
8831 ; GFX90A: ; %bb.0:
8832 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8833 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
8834 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
8835 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
8836 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
8837 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
8838 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8839 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
8840 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
8841 ; GFX90A-NEXT: v_not_b32_e32 v4, v4
8842 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
8843 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8844 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
8845 ; GFX90A-NEXT: .LBB38_1: ; %atomicrmw.start
8846 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
8847 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
8848 ; GFX90A-NEXT: v_mov_b32_e32 v7, v5
8849 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8850 ; GFX90A-NEXT: v_max_f32_e32 v5, v5, v2
8851 ; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
8852 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v5
8853 ; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
8854 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
8855 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
8856 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8857 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v4, v5
8858 ; GFX90A-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off glc
8859 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
8860 ; GFX90A-NEXT: buffer_wbinvl1
8861 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
8862 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8863 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
8864 ; GFX90A-NEXT: s_cbranch_execnz .LBB38_1
8865 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
8866 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
8867 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8868 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
8869 ;
8870 ; GFX908-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
8871 ; GFX908: ; %bb.0:
8872 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8873 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
8874 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
8875 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
8876 ; GFX908-NEXT: global_load_dword v5, v[0:1], off
8877 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
8878 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8879 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
8880 ; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
8881 ; GFX908-NEXT: v_not_b32_e32 v4, v4
8882 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
8883 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8884 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
8885 ; GFX908-NEXT: .LBB38_1: ; %atomicrmw.start
8886 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
8887 ; GFX908-NEXT: s_waitcnt vmcnt(0)
8888 ; GFX908-NEXT: v_mov_b32_e32 v6, v5
8889 ; GFX908-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8890 ; GFX908-NEXT: v_max_f32_e32 v5, v5, v2
8891 ; GFX908-NEXT: v_bfe_u32 v7, v5, 16, 1
8892 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v5
8893 ; GFX908-NEXT: v_add3_u32 v7, v7, v5, s6
8894 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
8895 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc
8896 ; GFX908-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8897 ; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
8898 ; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
8899 ; GFX908-NEXT: s_waitcnt vmcnt(0)
8900 ; GFX908-NEXT: buffer_wbinvl1
8901 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
8902 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8903 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
8904 ; GFX908-NEXT: s_cbranch_execnz .LBB38_1
8905 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
8906 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
8907 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8908 ; GFX908-NEXT: s_setpc_b64 s[30:31]
8909 ;
8910 ; GFX8-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
8911 ; GFX8: ; %bb.0:
8912 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8913 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
8914 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
8915 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
8916 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
8917 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
8918 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
8919 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
8920 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
8921 ; GFX8-NEXT: v_not_b32_e32 v4, v4
8922 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
8923 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8924 ; GFX8-NEXT: .LBB38_1: ; %atomicrmw.start
8925 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
8926 ; GFX8-NEXT: s_waitcnt vmcnt(0)
8927 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
8928 ; GFX8-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
8929 ; GFX8-NEXT: v_max_f32_e32 v5, v5, v2
8930 ; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
8931 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
8932 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
8933 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
8934 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
8935 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
8936 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
8937 ; GFX8-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
8938 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
8939 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
8940 ; GFX8-NEXT: s_waitcnt vmcnt(0)
8941 ; GFX8-NEXT: buffer_wbinvl1
8942 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
8943 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8944 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
8945 ; GFX8-NEXT: s_cbranch_execnz .LBB38_1
8946 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
8947 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
8948 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
8949 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8950 ;
8951 ; GFX7-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
8952 ; GFX7: ; %bb.0:
8953 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8954 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
8955 ; GFX7-NEXT: s_mov_b32 s6, 0
8956 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
8957 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
8958 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
8959 ; GFX7-NEXT: s_mov_b32 s4, s6
8960 ; GFX7-NEXT: s_mov_b32 s5, s6
8961 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
8962 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
8963 ; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
8964 ; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
8965 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
8966 ; GFX7-NEXT: v_not_b32_e32 v7, v4
8967 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
8968 ; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
8969 ; GFX7-NEXT: .LBB38_1: ; %atomicrmw.start
8970 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
8971 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8972 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
8973 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
8974 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
8975 ; GFX7-NEXT: v_max_f32_e32 v2, v2, v8
8976 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
8977 ; GFX7-NEXT: v_and_b32_e32 v4, v3, v7
8978 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v6, v2
8979 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
8980 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
8981 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
8982 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
8983 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8984 ; GFX7-NEXT: buffer_wbinvl1
8985 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
8986 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
8987 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
8988 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
8989 ; GFX7-NEXT: s_cbranch_execnz .LBB38_1
8990 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
8991 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
8992 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
8993 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
8994 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8995 ;
8996 ; GFX6-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
8997 ; GFX6: ; %bb.0:
8998 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8999 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
9000 ; GFX6-NEXT: s_mov_b32 s6, 0
9001 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
9002 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
9003 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
9004 ; GFX6-NEXT: s_mov_b32 s4, s6
9005 ; GFX6-NEXT: s_mov_b32 s5, s6
9006 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
9007 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
9008 ; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
9009 ; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
9010 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
9011 ; GFX6-NEXT: v_not_b32_e32 v7, v4
9012 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
9013 ; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
9014 ; GFX6-NEXT: .LBB38_1: ; %atomicrmw.start
9015 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
9016 ; GFX6-NEXT: s_waitcnt vmcnt(0)
9017 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
9018 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9019 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
9020 ; GFX6-NEXT: v_max_f32_e32 v2, v2, v8
9021 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
9022 ; GFX6-NEXT: s_waitcnt expcnt(0)
9023 ; GFX6-NEXT: v_and_b32_e32 v4, v3, v7
9024 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v6, v2
9025 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
9026 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
9027 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
9028 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
9029 ; GFX6-NEXT: s_waitcnt vmcnt(0)
9030 ; GFX6-NEXT: buffer_wbinvl1
9031 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
9032 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
9033 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
9034 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
9035 ; GFX6-NEXT: s_cbranch_execnz .LBB38_1
9036 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
9037 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
9038 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
9039 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
9040 ; GFX6-NEXT: s_waitcnt expcnt(0)
9041 ; GFX6-NEXT: s_setpc_b64 s[30:31]
9042 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 -1024
9043 %result = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
9044 ret bfloat %result
9045 }
9047 define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
9048 ; GFX12-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
9049 ; GFX12: ; %bb.0:
9050 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
9051 ; GFX12-NEXT: s_wait_expcnt 0x0
9052 ; GFX12-NEXT: s_wait_samplecnt 0x0
9053 ; GFX12-NEXT: s_wait_bvhcnt 0x0
9054 ; GFX12-NEXT: s_wait_kmcnt 0x0
9055 ; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
9056 ; GFX12-NEXT: s_mov_b32 s0, 0
9057 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
9058 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
9059 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
9060 ; GFX12-NEXT: global_load_b32 v4, v[0:1], off
9061 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
9062 ; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
9063 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
9064 ; GFX12-NEXT: v_not_b32_e32 v6, v3
9065 ; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
9066 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
9067 ; GFX12-NEXT: s_wait_loadcnt 0x0
9068 ; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
9069 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9070 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
9071 ; GFX12-NEXT: v_max_num_f32_e32 v3, v3, v2
9072 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
9073 ; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
9074 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
9075 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
9076 ; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
9077 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9078 ; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
9079 ; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
9080 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9081 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
9082 ; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
9083 ; GFX12-NEXT: s_wait_storecnt 0x0
9084 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
9085 ; GFX12-NEXT: s_wait_loadcnt 0x0
9086 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
9087 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
9088 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
9089 ; GFX12-NEXT: s_wait_alu 0xfffe
9090 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
9091 ; GFX12-NEXT: s_wait_alu 0xfffe
9092 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
9093 ; GFX12-NEXT: s_cbranch_execnz .LBB39_1
9094 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
9095 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
9096 ; GFX12-NEXT: s_wait_alu 0xfffe
9097 ; GFX12-NEXT: s_setpc_b64 s[30:31]
9098 ;
9099 ; GFX940-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
9100 ; GFX940: ; %bb.0:
9101 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9102 ; GFX940-NEXT: v_mov_b32_e32 v3, v0
9103 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v3
9104 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
9105 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v3
9106 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9107 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
9108 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
9109 ; GFX940-NEXT: v_not_b32_e32 v6, v4
9110 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
9111 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9112 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
9113 ; GFX940-NEXT: .LBB39_1: ; %atomicrmw.start
9114 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
9115 ; GFX940-NEXT: s_waitcnt vmcnt(0)
9116 ; GFX940-NEXT: v_lshrrev_b32_sdwa v4, v3, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9117 ; GFX940-NEXT: s_nop 0
9118 ; GFX940-NEXT: v_max_f32_e32 v4, v4, v2
9119 ; GFX940-NEXT: v_bfe_u32 v7, v4, 16, 1
9120 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v4
9121 ; GFX940-NEXT: v_add3_u32 v7, v7, v4, s2
9122 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
9123 ; GFX940-NEXT: s_nop 1
9124 ; GFX940-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
9125 ; GFX940-NEXT: v_lshlrev_b32_sdwa v4, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9126 ; GFX940-NEXT: v_and_or_b32 v4, v5, v6, v4
9127 ; GFX940-NEXT: buffer_wbl2 sc1
9128 ; GFX940-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0
9129 ; GFX940-NEXT: s_waitcnt vmcnt(0)
9130 ; GFX940-NEXT: buffer_inv sc1
9131 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
9132 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
9133 ; GFX940-NEXT: v_mov_b32_e32 v5, v4
9134 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
9135 ; GFX940-NEXT: s_cbranch_execnz .LBB39_1
9136 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
9137 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
9138 ; GFX940-NEXT: s_setpc_b64 s[30:31]
9139 ;
9140 ; GFX11-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
9141 ; GFX11: ; %bb.0:
9142 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9143 ; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
9144 ; GFX11-NEXT: s_mov_b32 s0, 0
9145 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
9146 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
9147 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
9148 ; GFX11-NEXT: global_load_b32 v4, v[0:1], off
9149 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
9150 ; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
9151 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
9152 ; GFX11-NEXT: v_not_b32_e32 v6, v3
9153 ; GFX11-NEXT: .p2align 6
9154 ; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
9155 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
9156 ; GFX11-NEXT: s_waitcnt vmcnt(0)
9157 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
9158 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9159 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
9160 ; GFX11-NEXT: v_max_f32_e32 v3, v3, v2
9161 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
9162 ; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
9163 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
9164 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
9165 ; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
9166 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9167 ; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
9168 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
9169 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9170 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
9171 ; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
9172 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
9173 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
9174 ; GFX11-NEXT: s_waitcnt vmcnt(0)
9175 ; GFX11-NEXT: buffer_gl1_inv
9176 ; GFX11-NEXT: buffer_gl0_inv
9177 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
9178 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
9179 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
9180 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
9181 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
9182 ; GFX11-NEXT: s_cbranch_execnz .LBB39_1
9183 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
9184 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
9185 ; GFX11-NEXT: s_setpc_b64 s[30:31]
9186 ;
9187 ; GFX10-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
9188 ; GFX10: ; %bb.0:
9189 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9190 ; GFX10-NEXT: v_mov_b32_e32 v3, v0
9191 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9192 ; GFX10-NEXT: s_mov_b32 s4, 0
9193 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
9194 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
9195 ; GFX10-NEXT: global_load_dword v4, v[0:1], off
9196 ; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
9197 ; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
9198 ; GFX10-NEXT: v_not_b32_e32 v6, v3
9199 ; GFX10-NEXT: .LBB39_1: ; %atomicrmw.start
9200 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
9201 ; GFX10-NEXT: s_waitcnt vmcnt(0)
9202 ; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9203 ; GFX10-NEXT: v_max_f32_e32 v3, v3, v2
9204 ; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
9205 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
9206 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
9207 ; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
9208 ; GFX10-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
9209 ; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9210 ; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
9211 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
9212 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
9213 ; GFX10-NEXT: s_waitcnt vmcnt(0)
9214 ; GFX10-NEXT: buffer_gl1_inv
9215 ; GFX10-NEXT: buffer_gl0_inv
9216 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
9217 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
9218 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
9219 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
9220 ; GFX10-NEXT: s_cbranch_execnz .LBB39_1
9221 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
9222 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
9223 ; GFX10-NEXT: s_setpc_b64 s[30:31]
9224 ;
9225 ; GFX90A-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
9226 ; GFX90A: ; %bb.0:
9227 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9228 ; GFX90A-NEXT: v_mov_b32_e32 v3, v0
9229 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
9230 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
9231 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
9232 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
9233 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
9234 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
9235 ; GFX90A-NEXT: v_not_b32_e32 v6, v4
9236 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
9237 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9238 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
9239 ; GFX90A-NEXT: .LBB39_1: ; %atomicrmw.start
9240 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
9241 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
9242 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v4, v3, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9243 ; GFX90A-NEXT: v_max_f32_e32 v4, v4, v2
9244 ; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
9245 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
9246 ; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s6
9247 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
9248 ; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
9249 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v4, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9250 ; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
9251 ; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
9252 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
9253 ; GFX90A-NEXT: buffer_wbinvl1
9254 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
9255 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9256 ; GFX90A-NEXT: v_mov_b32_e32 v5, v4
9257 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
9258 ; GFX90A-NEXT: s_cbranch_execnz .LBB39_1
9259 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
9260 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
9261 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
9262 ;
9263 ; GFX908-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
9264 ; GFX908: ; %bb.0:
9265 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9266 ; GFX908-NEXT: v_mov_b32_e32 v3, v0
9267 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
9268 ; GFX908-NEXT: global_load_dword v4, v[0:1], off
9269 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
9270 ; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
9271 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
9272 ; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
9273 ; GFX908-NEXT: v_not_b32_e32 v6, v3
9274 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
9275 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9276 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
9277 ; GFX908-NEXT: .LBB39_1: ; %atomicrmw.start
9278 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
9279 ; GFX908-NEXT: s_waitcnt vmcnt(0)
9280 ; GFX908-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9281 ; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
9282 ; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
9283 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
9284 ; GFX908-NEXT: v_add3_u32 v7, v7, v3, s6
9285 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
9286 ; GFX908-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
9287 ; GFX908-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9288 ; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
9289 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
9290 ; GFX908-NEXT: s_waitcnt vmcnt(0)
9291 ; GFX908-NEXT: buffer_wbinvl1
9292 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
9293 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9294 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
9295 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
9296 ; GFX908-NEXT: s_cbranch_execnz .LBB39_1
9297 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
9298 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
9299 ; GFX908-NEXT: s_setpc_b64 s[30:31]
9300 ;
9301 ; GFX8-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
9302 ; GFX8: ; %bb.0:
9303 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9304 ; GFX8-NEXT: v_mov_b32_e32 v3, v0
9305 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
9306 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
9307 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
9308 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
9309 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
9310 ; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
9311 ; GFX8-NEXT: v_not_b32_e32 v6, v3
9312 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
9313 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9314 ; GFX8-NEXT: .LBB39_1: ; %atomicrmw.start
9315 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
9316 ; GFX8-NEXT: s_waitcnt vmcnt(0)
9317 ; GFX8-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9318 ; GFX8-NEXT: v_max_f32_e32 v3, v3, v2
9319 ; GFX8-NEXT: v_bfe_u32 v8, v3, 16, 1
9320 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v3
9321 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
9322 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v3
9323 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
9324 ; GFX8-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc
9325 ; GFX8-NEXT: v_and_b32_e32 v7, v4, v6
9326 ; GFX8-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9327 ; GFX8-NEXT: v_or_b32_e32 v3, v7, v3
9328 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
9329 ; GFX8-NEXT: s_waitcnt vmcnt(0)
9330 ; GFX8-NEXT: buffer_wbinvl1
9331 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
9332 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9333 ; GFX8-NEXT: v_mov_b32_e32 v4, v3
9334 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
9335 ; GFX8-NEXT: s_cbranch_execnz .LBB39_1
9336 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
9337 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
9338 ; GFX8-NEXT: s_setpc_b64 s[30:31]
9339 ;
9340 ; GFX7-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
9341 ; GFX7: ; %bb.0:
9342 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9343 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
9344 ; GFX7-NEXT: s_mov_b32 s6, 0
9345 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v3
9346 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
9347 ; GFX7-NEXT: s_mov_b32 s4, s6
9348 ; GFX7-NEXT: s_mov_b32 s5, s6
9349 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
9350 ; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
9351 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 3, v3
9352 ; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
9353 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
9354 ; GFX7-NEXT: v_not_b32_e32 v6, v3
9355 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
9356 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
9357 ; GFX7-NEXT: .LBB39_1: ; %atomicrmw.start
9358 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
9359 ; GFX7-NEXT: s_waitcnt vmcnt(0)
9360 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, v5, v4
9361 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
9362 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
9363 ; GFX7-NEXT: v_max_f32_e32 v3, v3, v2
9364 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
9365 ; GFX7-NEXT: v_and_b32_e32 v7, v4, v6
9366 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, v5, v3
9367 ; GFX7-NEXT: v_or_b32_e32 v3, v7, v3
9368 ; GFX7-NEXT: v_mov_b32_e32 v8, v4
9369 ; GFX7-NEXT: v_mov_b32_e32 v7, v3
9370 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
9371 ; GFX7-NEXT: s_waitcnt vmcnt(0)
9372 ; GFX7-NEXT: buffer_wbinvl1
9373 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v4
9374 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
9375 ; GFX7-NEXT: v_mov_b32_e32 v4, v7
9376 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
9377 ; GFX7-NEXT: s_cbranch_execnz .LBB39_1
9378 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
9379 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
9380 ; GFX7-NEXT: s_setpc_b64 s[30:31]
9381 ;
9382 ; GFX6-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
9383 ; GFX6: ; %bb.0:
9384 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9385 ; GFX6-NEXT: v_mov_b32_e32 v3, v0
9386 ; GFX6-NEXT: s_mov_b32 s6, 0
9387 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v3
9388 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
9389 ; GFX6-NEXT: s_mov_b32 s4, s6
9390 ; GFX6-NEXT: s_mov_b32 s5, s6
9391 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
9392 ; GFX6-NEXT: v_and_b32_e32 v3, 3, v3
9393 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 3, v3
9394 ; GFX6-NEXT: v_lshl_b32_e32 v3, 0xffff, v5
9395 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
9396 ; GFX6-NEXT: v_not_b32_e32 v6, v3
9397 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
9398 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
9399 ; GFX6-NEXT: .LBB39_1: ; %atomicrmw.start
9400 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
9401 ; GFX6-NEXT: s_waitcnt vmcnt(0)
9402 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, v5, v4
9403 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
9404 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
9405 ; GFX6-NEXT: v_max_f32_e32 v3, v3, v2
9406 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
9407 ; GFX6-NEXT: s_waitcnt expcnt(0)
9408 ; GFX6-NEXT: v_and_b32_e32 v7, v4, v6
9409 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, v5, v3
9410 ; GFX6-NEXT: v_or_b32_e32 v3, v7, v3
9411 ; GFX6-NEXT: v_mov_b32_e32 v8, v4
9412 ; GFX6-NEXT: v_mov_b32_e32 v7, v3
9413 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
9414 ; GFX6-NEXT: s_waitcnt vmcnt(0)
9415 ; GFX6-NEXT: buffer_wbinvl1
9416 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v4
9417 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
9418 ; GFX6-NEXT: v_mov_b32_e32 v4, v7
9419 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
9420 ; GFX6-NEXT: s_cbranch_execnz .LBB39_1
9421 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
9422 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
9423 ; GFX6-NEXT: s_waitcnt expcnt(0)
9424 ; GFX6-NEXT: s_setpc_b64 s[30:31]
9425 %unused = atomicrmw fmax ptr addrspace(1) %ptr, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
9426 ret void
9427 }
9429 define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
9430 ; GFX12-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
9431 ; GFX12: ; %bb.0:
9432 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
9433 ; GFX12-NEXT: s_wait_expcnt 0x0
9434 ; GFX12-NEXT: s_wait_samplecnt 0x0
9435 ; GFX12-NEXT: s_wait_bvhcnt 0x0
9436 ; GFX12-NEXT: s_wait_kmcnt 0x0
9437 ; GFX12-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
9438 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
9439 ; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v2
9440 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
9441 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v4
9442 ; GFX12-NEXT: v_and_b32_e32 v4, 3, v4
9443 ; GFX12-NEXT: s_mov_b32 s0, 0
9444 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
9445 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 3, v4
9446 ; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
9447 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
9448 ; GFX12-NEXT: v_not_b32_e32 v5, v5
9449 ; GFX12-NEXT: .LBB40_1: ; %atomicrmw.start
9450 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
9451 ; GFX12-NEXT: s_wait_loadcnt 0x0
9452 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
9453 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9454 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9455 ; GFX12-NEXT: v_max_num_f32_e32 v2, v2, v6
9456 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
9457 ; GFX12-NEXT: v_bfe_u32 v7, v2, 16, 1
9458 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v2
9459 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
9460 ; GFX12-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
9461 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9462 ; GFX12-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
9463 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, 16, v2
9464 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9465 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, v4, v2
9466 ; GFX12-NEXT: v_and_or_b32 v2, v3, v5, v2
9467 ; GFX12-NEXT: s_wait_storecnt 0x0
9468 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
9469 ; GFX12-NEXT: s_wait_loadcnt 0x0
9470 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
9471 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
9472 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
9473 ; GFX12-NEXT: s_wait_alu 0xfffe
9474 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
9475 ; GFX12-NEXT: s_wait_alu 0xfffe
9476 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
9477 ; GFX12-NEXT: s_cbranch_execnz .LBB40_1
9478 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
9479 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
9480 ; GFX12-NEXT: s_wait_alu 0xfffe
9481 ; GFX12-NEXT: s_setpc_b64 s[30:31]
9482 ;
9483 ; GFX940-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
9484 ; GFX940: ; %bb.0:
9485 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9486 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
9487 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
9488 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
9489 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
9490 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
9491 ; GFX940-NEXT: v_and_b32_e32 v4, 3, v4
9492 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 3, v4
9493 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
9494 ; GFX940-NEXT: v_lshlrev_b32_e64 v5, v4, s0
9495 ; GFX940-NEXT: v_not_b32_e32 v5, v5
9496 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
9497 ; GFX940-NEXT: v_lshlrev_b32_e32 v6, 16, v2
9498 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
9499 ; GFX940-NEXT: .LBB40_1: ; %atomicrmw.start
9500 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
9501 ; GFX940-NEXT: s_waitcnt vmcnt(0)
9502 ; GFX940-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9503 ; GFX940-NEXT: s_nop 0
9504 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v6
9505 ; GFX940-NEXT: v_bfe_u32 v7, v2, 16, 1
9506 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v2
9507 ; GFX940-NEXT: v_add3_u32 v7, v7, v2, s2
9508 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
9509 ; GFX940-NEXT: s_nop 1
9510 ; GFX940-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
9511 ; GFX940-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9512 ; GFX940-NEXT: v_and_or_b32 v2, v3, v5, v2
9513 ; GFX940-NEXT: buffer_wbl2 sc1
9514 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
9515 ; GFX940-NEXT: s_waitcnt vmcnt(0)
9516 ; GFX940-NEXT: buffer_inv sc1
9517 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
9518 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
9519 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
9520 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
9521 ; GFX940-NEXT: s_cbranch_execnz .LBB40_1
9522 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
9523 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
9524 ; GFX940-NEXT: s_setpc_b64 s[30:31]
9525 ;
9526 ; GFX11-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
9527 ; GFX11: ; %bb.0:
9528 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9529 ; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
9530 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
9531 ; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v2
9532 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
9533 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v4
9534 ; GFX11-NEXT: v_and_b32_e32 v4, 3, v4
9535 ; GFX11-NEXT: s_mov_b32 s0, 0
9536 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
9537 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v4
9538 ; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
9539 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
9540 ; GFX11-NEXT: v_not_b32_e32 v5, v5
9541 ; GFX11-NEXT: .p2align 6
9542 ; GFX11-NEXT: .LBB40_1: ; %atomicrmw.start
9543 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
9544 ; GFX11-NEXT: s_waitcnt vmcnt(0)
9545 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
9546 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9547 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9548 ; GFX11-NEXT: v_max_f32_e32 v2, v2, v6
9549 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
9550 ; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
9551 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v2
9552 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
9553 ; GFX11-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
9554 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9555 ; GFX11-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
9556 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v2
9557 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9558 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, v4, v2
9559 ; GFX11-NEXT: v_and_or_b32 v2, v3, v5, v2
9560 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
9561 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
9562 ; GFX11-NEXT: s_waitcnt vmcnt(0)
9563 ; GFX11-NEXT: buffer_gl1_inv
9564 ; GFX11-NEXT: buffer_gl0_inv
9565 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
9566 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
9567 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
9568 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
9569 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
9570 ; GFX11-NEXT: s_cbranch_execnz .LBB40_1
9571 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
9572 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
9573 ; GFX11-NEXT: s_setpc_b64 s[30:31]
9574 ;
9575 ; GFX10-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
9576 ; GFX10: ; %bb.0:
9577 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9578 ; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
9579 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
9580 ; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v2
9581 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
9582 ; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
9583 ; GFX10-NEXT: s_mov_b32 s4, 0
9584 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
9585 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
9586 ; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
9587 ; GFX10-NEXT: v_not_b32_e32 v5, v5
9588 ; GFX10-NEXT: .LBB40_1: ; %atomicrmw.start
9589 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
9590 ; GFX10-NEXT: s_waitcnt vmcnt(0)
9591 ; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9592 ; GFX10-NEXT: v_max_f32_e32 v2, v2, v6
9593 ; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
9594 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v2
9595 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
9596 ; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
9597 ; GFX10-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
9598 ; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9599 ; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
9600 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
9601 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
9602 ; GFX10-NEXT: s_waitcnt vmcnt(0)
9603 ; GFX10-NEXT: buffer_gl1_inv
9604 ; GFX10-NEXT: buffer_gl0_inv
9605 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
9606 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
9607 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
9608 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
9609 ; GFX10-NEXT: s_cbranch_execnz .LBB40_1
9610 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
9611 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
9612 ; GFX10-NEXT: s_setpc_b64 s[30:31]
9613 ;
9614 ; GFX90A-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
9615 ; GFX90A: ; %bb.0:
9616 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9617 ; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
9618 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
9619 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
9620 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
9621 ; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
9622 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
9623 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
9624 ; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
9625 ; GFX90A-NEXT: v_not_b32_e32 v5, v5
9626 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
9627 ; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v2
9628 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
9629 ; GFX90A-NEXT: .LBB40_1: ; %atomicrmw.start
9630 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
9631 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
9632 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9633 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v6
9634 ; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
9635 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
9636 ; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s6
9637 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
9638 ; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
9639 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9640 ; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
9641 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
9642 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
9643 ; GFX90A-NEXT: buffer_wbinvl1
9644 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
9645 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9646 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
9647 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
9648 ; GFX90A-NEXT: s_cbranch_execnz .LBB40_1
9649 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
9650 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
9651 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
9652 ;
9653 ; GFX908-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
9654 ; GFX908: ; %bb.0:
9655 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9656 ; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
9657 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
9658 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
9659 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
9660 ; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
9661 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
9662 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
9663 ; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
9664 ; GFX908-NEXT: v_not_b32_e32 v5, v5
9665 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
9666 ; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v2
9667 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
9668 ; GFX908-NEXT: .LBB40_1: ; %atomicrmw.start
9669 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
9670 ; GFX908-NEXT: s_waitcnt vmcnt(0)
9671 ; GFX908-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9672 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v6
9673 ; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
9674 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
9675 ; GFX908-NEXT: v_add3_u32 v7, v7, v2, s6
9676 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
9677 ; GFX908-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
9678 ; GFX908-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9679 ; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
9680 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
9681 ; GFX908-NEXT: s_waitcnt vmcnt(0)
9682 ; GFX908-NEXT: buffer_wbinvl1
9683 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
9684 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9685 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
9686 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
9687 ; GFX908-NEXT: s_cbranch_execnz .LBB40_1
9688 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
9689 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
9690 ; GFX908-NEXT: s_setpc_b64 s[30:31]
9691 ;
9692 ; GFX8-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
9693 ; GFX8: ; %bb.0:
9694 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9695 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
9696 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
9697 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
9698 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
9699 ; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
9700 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
9701 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
9702 ; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
9703 ; GFX8-NEXT: v_not_b32_e32 v5, v5
9704 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
9705 ; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v2
9706 ; GFX8-NEXT: .LBB40_1: ; %atomicrmw.start
9707 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
9708 ; GFX8-NEXT: s_waitcnt vmcnt(0)
9709 ; GFX8-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9710 ; GFX8-NEXT: v_max_f32_e32 v2, v2, v6
9711 ; GFX8-NEXT: v_bfe_u32 v8, v2, 16, 1
9712 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v2
9713 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
9714 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v2
9715 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
9716 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v8, v9, vcc
9717 ; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
9718 ; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9719 ; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
9720 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9721 ; GFX8-NEXT: s_waitcnt vmcnt(0)
9722 ; GFX8-NEXT: buffer_wbinvl1
9723 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
9724 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
9725 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
9726 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
9727 ; GFX8-NEXT: s_cbranch_execnz .LBB40_1
9728 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
9729 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
9730 ; GFX8-NEXT: s_setpc_b64 s[30:31]
9731 ;
9732 ; GFX7-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
9733 ; GFX7: ; %bb.0:
9734 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9735 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
9736 ; GFX7-NEXT: s_mov_b32 s6, 0
9737 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
9738 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
9739 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
9740 ; GFX7-NEXT: s_mov_b32 s4, s6
9741 ; GFX7-NEXT: s_mov_b32 s5, s6
9742 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
9743 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
9744 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
9745 ; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
9746 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
9747 ; GFX7-NEXT: v_not_b32_e32 v5, v5
9748 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
9749 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
9750 ; GFX7-NEXT: .LBB40_1: ; %atomicrmw.start
9751 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
9752 ; GFX7-NEXT: s_waitcnt vmcnt(0)
9753 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
9754 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9755 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
9756 ; GFX7-NEXT: v_max_f32_e32 v2, v2, v6
9757 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
9758 ; GFX7-NEXT: v_and_b32_e32 v7, v3, v5
9759 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v4, v2
9760 ; GFX7-NEXT: v_or_b32_e32 v2, v7, v2
9761 ; GFX7-NEXT: v_mov_b32_e32 v8, v3
9762 ; GFX7-NEXT: v_mov_b32_e32 v7, v2
9763 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
9764 ; GFX7-NEXT: s_waitcnt vmcnt(0)
9765 ; GFX7-NEXT: buffer_wbinvl1
9766 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
9767 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
9768 ; GFX7-NEXT: v_mov_b32_e32 v3, v7
9769 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
9770 ; GFX7-NEXT: s_cbranch_execnz .LBB40_1
9771 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
9772 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
9773 ; GFX7-NEXT: s_setpc_b64 s[30:31]
9774 ;
9775 ; GFX6-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
9776 ; GFX6: ; %bb.0:
9777 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9778 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
9779 ; GFX6-NEXT: s_mov_b32 s6, 0
9780 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
9781 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
9782 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
9783 ; GFX6-NEXT: s_mov_b32 s4, s6
9784 ; GFX6-NEXT: s_mov_b32 s5, s6
9785 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
9786 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
9787 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
9788 ; GFX6-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
9789 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
9790 ; GFX6-NEXT: v_not_b32_e32 v5, v5
9791 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
9792 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
9793 ; GFX6-NEXT: .LBB40_1: ; %atomicrmw.start
9794 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
9795 ; GFX6-NEXT: s_waitcnt vmcnt(0)
9796 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
9797 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9798 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
9799 ; GFX6-NEXT: v_max_f32_e32 v2, v2, v6
9800 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
9801 ; GFX6-NEXT: s_waitcnt expcnt(0)
9802 ; GFX6-NEXT: v_and_b32_e32 v7, v3, v5
9803 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v4, v2
9804 ; GFX6-NEXT: v_or_b32_e32 v2, v7, v2
9805 ; GFX6-NEXT: v_mov_b32_e32 v8, v3
9806 ; GFX6-NEXT: v_mov_b32_e32 v7, v2
9807 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
9808 ; GFX6-NEXT: s_waitcnt vmcnt(0)
9809 ; GFX6-NEXT: buffer_wbinvl1
9810 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
9811 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
9812 ; GFX6-NEXT: v_mov_b32_e32 v3, v7
9813 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
9814 ; GFX6-NEXT: s_cbranch_execnz .LBB40_1
9815 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
9816 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
9817 ; GFX6-NEXT: s_waitcnt expcnt(0)
9818 ; GFX6-NEXT: s_setpc_b64 s[30:31]
9819 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
9820 %unused = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
9821 ret void
9822 }
9824 define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
9825 ; GFX12-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
9826 ; GFX12: ; %bb.0:
9827 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
9828 ; GFX12-NEXT: s_wait_expcnt 0x0
9829 ; GFX12-NEXT: s_wait_samplecnt 0x0
9830 ; GFX12-NEXT: s_wait_bvhcnt 0x0
9831 ; GFX12-NEXT: s_wait_kmcnt 0x0
9832 ; GFX12-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
9833 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
9834 ; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v2
9835 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
9836 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v4
9837 ; GFX12-NEXT: v_and_b32_e32 v4, 3, v4
9838 ; GFX12-NEXT: s_mov_b32 s0, 0
9839 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
9840 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 3, v4
9841 ; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
9842 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
9843 ; GFX12-NEXT: v_not_b32_e32 v5, v5
9844 ; GFX12-NEXT: .LBB41_1: ; %atomicrmw.start
9845 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
9846 ; GFX12-NEXT: s_wait_loadcnt 0x0
9847 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
9848 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9849 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9850 ; GFX12-NEXT: v_max_num_f32_e32 v2, v2, v6
9851 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
9852 ; GFX12-NEXT: v_bfe_u32 v7, v2, 16, 1
9853 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v2
9854 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
9855 ; GFX12-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
9856 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9857 ; GFX12-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
9858 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, 16, v2
9859 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9860 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, v4, v2
9861 ; GFX12-NEXT: v_and_or_b32 v2, v3, v5, v2
9862 ; GFX12-NEXT: s_wait_storecnt 0x0
9863 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
9864 ; GFX12-NEXT: s_wait_loadcnt 0x0
9865 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
9866 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
9867 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
9868 ; GFX12-NEXT: s_wait_alu 0xfffe
9869 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
9870 ; GFX12-NEXT: s_wait_alu 0xfffe
9871 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
9872 ; GFX12-NEXT: s_cbranch_execnz .LBB41_1
9873 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
9874 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
9875 ; GFX12-NEXT: s_wait_alu 0xfffe
9876 ; GFX12-NEXT: s_setpc_b64 s[30:31]
9877 ;
9878 ; GFX940-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
9879 ; GFX940: ; %bb.0:
9880 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9881 ; GFX940-NEXT: s_movk_i32 s0, 0xf800
9882 ; GFX940-NEXT: s_mov_b32 s1, -1
9883 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
9884 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
9885 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
9886 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
9887 ; GFX940-NEXT: v_and_b32_e32 v4, 3, v4
9888 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 3, v4
9889 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
9890 ; GFX940-NEXT: v_lshlrev_b32_e64 v5, v4, s0
9891 ; GFX940-NEXT: v_not_b32_e32 v5, v5
9892 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
9893 ; GFX940-NEXT: v_lshlrev_b32_e32 v6, 16, v2
9894 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
9895 ; GFX940-NEXT: .LBB41_1: ; %atomicrmw.start
9896 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
9897 ; GFX940-NEXT: s_waitcnt vmcnt(0)
9898 ; GFX940-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9899 ; GFX940-NEXT: s_nop 0
9900 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v6
9901 ; GFX940-NEXT: v_bfe_u32 v7, v2, 16, 1
9902 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v2
9903 ; GFX940-NEXT: v_add3_u32 v7, v7, v2, s2
9904 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
9905 ; GFX940-NEXT: s_nop 1
9906 ; GFX940-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
9907 ; GFX940-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9908 ; GFX940-NEXT: v_and_or_b32 v2, v3, v5, v2
9909 ; GFX940-NEXT: buffer_wbl2 sc1
9910 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
9911 ; GFX940-NEXT: s_waitcnt vmcnt(0)
9912 ; GFX940-NEXT: buffer_inv sc1
9913 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
9914 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
9915 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
9916 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
9917 ; GFX940-NEXT: s_cbranch_execnz .LBB41_1
9918 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
9919 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
9920 ; GFX940-NEXT: s_setpc_b64 s[30:31]
9921 ;
9922 ; GFX11-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
9923 ; GFX11: ; %bb.0:
9924 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9925 ; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
9926 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
9927 ; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v2
9928 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
9929 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v4
9930 ; GFX11-NEXT: v_and_b32_e32 v4, 3, v4
9931 ; GFX11-NEXT: s_mov_b32 s0, 0
9932 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
9933 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v4
9934 ; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
9935 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
9936 ; GFX11-NEXT: v_not_b32_e32 v5, v5
9937 ; GFX11-NEXT: .p2align 6
9938 ; GFX11-NEXT: .LBB41_1: ; %atomicrmw.start
9939 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
9940 ; GFX11-NEXT: s_waitcnt vmcnt(0)
9941 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
9942 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9943 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
9944 ; GFX11-NEXT: v_max_f32_e32 v2, v2, v6
9945 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
9946 ; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
9947 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v2
9948 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
9949 ; GFX11-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
9950 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9951 ; GFX11-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
9952 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v2
9953 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
9954 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, v4, v2
9955 ; GFX11-NEXT: v_and_or_b32 v2, v3, v5, v2
9956 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
9957 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
9958 ; GFX11-NEXT: s_waitcnt vmcnt(0)
9959 ; GFX11-NEXT: buffer_gl1_inv
9960 ; GFX11-NEXT: buffer_gl0_inv
9961 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
9962 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
9963 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
9964 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
9965 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
9966 ; GFX11-NEXT: s_cbranch_execnz .LBB41_1
9967 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
9968 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
9969 ; GFX11-NEXT: s_setpc_b64 s[30:31]
9970 ;
9971 ; GFX10-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
9972 ; GFX10: ; %bb.0:
9973 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9974 ; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
9975 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
9976 ; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v2
9977 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
9978 ; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
9979 ; GFX10-NEXT: s_mov_b32 s4, 0
9980 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
9981 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
9982 ; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
9983 ; GFX10-NEXT: v_not_b32_e32 v5, v5
9984 ; GFX10-NEXT: .LBB41_1: ; %atomicrmw.start
9985 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
9986 ; GFX10-NEXT: s_waitcnt vmcnt(0)
9987 ; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
9988 ; GFX10-NEXT: v_max_f32_e32 v2, v2, v6
9989 ; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
9990 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v2
9991 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
9992 ; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
9993 ; GFX10-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
9994 ; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
9995 ; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
9996 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
9997 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
9998 ; GFX10-NEXT: s_waitcnt vmcnt(0)
9999 ; GFX10-NEXT: buffer_gl1_inv
10000 ; GFX10-NEXT: buffer_gl0_inv
10001 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
10002 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
10003 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
10004 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
10005 ; GFX10-NEXT: s_cbranch_execnz .LBB41_1
10006 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
10007 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
10008 ; GFX10-NEXT: s_setpc_b64 s[30:31]
10009 ;
10010 ; GFX90A-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
10011 ; GFX90A: ; %bb.0:
10012 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10013 ; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
10014 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
10015 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
10016 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
10017 ; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
10018 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10019 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
10020 ; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
10021 ; GFX90A-NEXT: v_not_b32_e32 v5, v5
10022 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
10023 ; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10024 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
10025 ; GFX90A-NEXT: .LBB41_1: ; %atomicrmw.start
10026 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
10027 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
10028 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10029 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v6
10030 ; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
10031 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
10032 ; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s6
10033 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
10034 ; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
10035 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10036 ; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
10037 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
10038 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
10039 ; GFX90A-NEXT: buffer_wbinvl1
10040 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
10041 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
10042 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
10043 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
10044 ; GFX90A-NEXT: s_cbranch_execnz .LBB41_1
10045 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
10046 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
10047 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
10048 ;
10049 ; GFX908-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
10050 ; GFX908: ; %bb.0:
10051 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10052 ; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
10053 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
10054 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
10055 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
10056 ; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
10057 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10058 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
10059 ; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
10060 ; GFX908-NEXT: v_not_b32_e32 v5, v5
10061 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
10062 ; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10063 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
10064 ; GFX908-NEXT: .LBB41_1: ; %atomicrmw.start
10065 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
10066 ; GFX908-NEXT: s_waitcnt vmcnt(0)
10067 ; GFX908-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10068 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v6
10069 ; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
10070 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
10071 ; GFX908-NEXT: v_add3_u32 v7, v7, v2, s6
10072 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
10073 ; GFX908-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
10074 ; GFX908-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10075 ; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
10076 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
10077 ; GFX908-NEXT: s_waitcnt vmcnt(0)
10078 ; GFX908-NEXT: buffer_wbinvl1
10079 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
10080 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
10081 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
10082 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
10083 ; GFX908-NEXT: s_cbranch_execnz .LBB41_1
10084 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
10085 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
10086 ; GFX908-NEXT: s_setpc_b64 s[30:31]
10087 ;
10088 ; GFX8-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
10089 ; GFX8: ; %bb.0:
10090 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10091 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
10092 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
10093 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
10094 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
10095 ; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
10096 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10097 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
10098 ; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
10099 ; GFX8-NEXT: v_not_b32_e32 v5, v5
10100 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
10101 ; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v2
10102 ; GFX8-NEXT: .LBB41_1: ; %atomicrmw.start
10103 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
10104 ; GFX8-NEXT: s_waitcnt vmcnt(0)
10105 ; GFX8-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10106 ; GFX8-NEXT: v_max_f32_e32 v2, v2, v6
10107 ; GFX8-NEXT: v_bfe_u32 v8, v2, 16, 1
10108 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v2
10109 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
10110 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v2
10111 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
10112 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v8, v9, vcc
10113 ; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
10114 ; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10115 ; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
10116 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10117 ; GFX8-NEXT: s_waitcnt vmcnt(0)
10118 ; GFX8-NEXT: buffer_wbinvl1
10119 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
10120 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
10121 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
10122 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
10123 ; GFX8-NEXT: s_cbranch_execnz .LBB41_1
10124 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
10125 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
10126 ; GFX8-NEXT: s_setpc_b64 s[30:31]
10127 ;
10128 ; GFX7-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
10129 ; GFX7: ; %bb.0:
10130 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10131 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
10132 ; GFX7-NEXT: s_mov_b32 s6, 0
10133 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
10134 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
10135 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
10136 ; GFX7-NEXT: s_mov_b32 s4, s6
10137 ; GFX7-NEXT: s_mov_b32 s5, s6
10138 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
10139 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
10140 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10141 ; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
10142 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
10143 ; GFX7-NEXT: v_not_b32_e32 v5, v5
10144 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
10145 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
10146 ; GFX7-NEXT: .LBB41_1: ; %atomicrmw.start
10147 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
10148 ; GFX7-NEXT: s_waitcnt vmcnt(0)
10149 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
10150 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10151 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
10152 ; GFX7-NEXT: v_max_f32_e32 v2, v2, v6
10153 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10154 ; GFX7-NEXT: v_and_b32_e32 v7, v3, v5
10155 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v4, v2
10156 ; GFX7-NEXT: v_or_b32_e32 v2, v7, v2
10157 ; GFX7-NEXT: v_mov_b32_e32 v8, v3
10158 ; GFX7-NEXT: v_mov_b32_e32 v7, v2
10159 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
10160 ; GFX7-NEXT: s_waitcnt vmcnt(0)
10161 ; GFX7-NEXT: buffer_wbinvl1
10162 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
10163 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
10164 ; GFX7-NEXT: v_mov_b32_e32 v3, v7
10165 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
10166 ; GFX7-NEXT: s_cbranch_execnz .LBB41_1
10167 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
10168 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
10169 ; GFX7-NEXT: s_setpc_b64 s[30:31]
10170 ;
10171 ; GFX6-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
10172 ; GFX6: ; %bb.0:
10173 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10174 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
10175 ; GFX6-NEXT: s_mov_b32 s6, 0
10176 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
10177 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
10178 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
10179 ; GFX6-NEXT: s_mov_b32 s4, s6
10180 ; GFX6-NEXT: s_mov_b32 s5, s6
10181 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
10182 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
10183 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
10184 ; GFX6-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
10185 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
10186 ; GFX6-NEXT: v_not_b32_e32 v5, v5
10187 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
10188 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
10189 ; GFX6-NEXT: .LBB41_1: ; %atomicrmw.start
10190 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
10191 ; GFX6-NEXT: s_waitcnt vmcnt(0)
10192 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
10193 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10194 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
10195 ; GFX6-NEXT: v_max_f32_e32 v2, v2, v6
10196 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10197 ; GFX6-NEXT: s_waitcnt expcnt(0)
10198 ; GFX6-NEXT: v_and_b32_e32 v7, v3, v5
10199 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v4, v2
10200 ; GFX6-NEXT: v_or_b32_e32 v2, v7, v2
10201 ; GFX6-NEXT: v_mov_b32_e32 v8, v3
10202 ; GFX6-NEXT: v_mov_b32_e32 v7, v2
10203 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
10204 ; GFX6-NEXT: s_waitcnt vmcnt(0)
10205 ; GFX6-NEXT: buffer_wbinvl1
10206 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
10207 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
10208 ; GFX6-NEXT: v_mov_b32_e32 v3, v7
10209 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
10210 ; GFX6-NEXT: s_cbranch_execnz .LBB41_1
10211 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
10212 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
10213 ; GFX6-NEXT: s_waitcnt expcnt(0)
10214 ; GFX6-NEXT: s_setpc_b64 s[30:31]
10215 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 -1024
10216 %unused = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
10217 ret void
10218 }
10220 define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
10221 ; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
10222 ; GFX12: ; %bb.0:
10223 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
10224 ; GFX12-NEXT: s_wait_expcnt 0x0
10225 ; GFX12-NEXT: s_wait_samplecnt 0x0
10226 ; GFX12-NEXT: s_wait_bvhcnt 0x0
10227 ; GFX12-NEXT: s_wait_kmcnt 0x0
10228 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2046
10229 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10230 ; GFX12-NEXT: s_mov_b32 s0, 0
10231 ; GFX12-NEXT: .LBB42_1: ; %atomicrmw.start
10232 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
10233 ; GFX12-NEXT: s_wait_loadcnt 0x0
10234 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
10235 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10236 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v4
10237 ; GFX12-NEXT: v_max_num_f32_e32 v3, v3, v2
10238 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
10239 ; GFX12-NEXT: v_bfe_u32 v5, v3, 16, 1
10240 ; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v3
10241 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
10242 ; GFX12-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
10243 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10244 ; GFX12-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
10245 ; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
10246 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
10247 ; GFX12-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
10248 ; GFX12-NEXT: s_wait_storecnt 0x0
10249 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
10250 ; GFX12-NEXT: s_wait_loadcnt 0x0
10251 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
10252 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
10253 ; GFX12-NEXT: s_wait_alu 0xfffe
10254 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
10255 ; GFX12-NEXT: s_wait_alu 0xfffe
10256 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
10257 ; GFX12-NEXT: s_cbranch_execnz .LBB42_1
10258 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
10259 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
10260 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
10261 ; GFX12-NEXT: s_wait_alu 0xfffe
10262 ; GFX12-NEXT: s_setpc_b64 s[30:31]
10263 ;
10264 ; GFX940-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
10265 ; GFX940: ; %bb.0:
10266 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10267 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2046
10268 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
10269 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10270 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
10271 ; GFX940-NEXT: s_mov_b32 s3, 0xffff0000
10272 ; GFX940-NEXT: .LBB42_1: ; %atomicrmw.start
10273 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
10274 ; GFX940-NEXT: s_waitcnt vmcnt(0)
10275 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
10276 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 16, v5
10277 ; GFX940-NEXT: v_max_f32_e32 v3, v3, v2
10278 ; GFX940-NEXT: v_bfe_u32 v4, v3, 16, 1
10279 ; GFX940-NEXT: v_or_b32_e32 v6, 0x400000, v3
10280 ; GFX940-NEXT: v_add3_u32 v4, v4, v3, s2
10281 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
10282 ; GFX940-NEXT: s_nop 1
10283 ; GFX940-NEXT: v_cndmask_b32_e32 v3, v4, v6, vcc
10284 ; GFX940-NEXT: v_lshrrev_b32_e32 v3, 16, v3
10285 ; GFX940-NEXT: v_and_or_b32 v4, v5, s3, v3
10286 ; GFX940-NEXT: buffer_wbl2 sc1
10287 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 sc0
10288 ; GFX940-NEXT: s_waitcnt vmcnt(0)
10289 ; GFX940-NEXT: buffer_inv sc1
10290 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
10291 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
10292 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
10293 ; GFX940-NEXT: s_cbranch_execnz .LBB42_1
10294 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
10295 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
10296 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
10297 ; GFX940-NEXT: s_setpc_b64 s[30:31]
10298 ;
10299 ; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
10300 ; GFX11: ; %bb.0:
10301 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10302 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2046
10303 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10304 ; GFX11-NEXT: s_mov_b32 s0, 0
10305 ; GFX11-NEXT: .p2align 6
10306 ; GFX11-NEXT: .LBB42_1: ; %atomicrmw.start
10307 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
10308 ; GFX11-NEXT: s_waitcnt vmcnt(0)
10309 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
10310 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10311 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v4
10312 ; GFX11-NEXT: v_max_f32_e32 v3, v3, v2
10313 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
10314 ; GFX11-NEXT: v_bfe_u32 v5, v3, 16, 1
10315 ; GFX11-NEXT: v_or_b32_e32 v6, 0x400000, v3
10316 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
10317 ; GFX11-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
10318 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10319 ; GFX11-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
10320 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
10321 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
10322 ; GFX11-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
10323 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
10324 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
10325 ; GFX11-NEXT: s_waitcnt vmcnt(0)
10326 ; GFX11-NEXT: buffer_gl1_inv
10327 ; GFX11-NEXT: buffer_gl0_inv
10328 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
10329 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
10330 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
10331 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
10332 ; GFX11-NEXT: s_cbranch_execnz .LBB42_1
10333 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
10334 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
10335 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
10336 ; GFX11-NEXT: s_setpc_b64 s[30:31]
10337 ;
10338 ; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
10339 ; GFX10: ; %bb.0:
10340 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10341 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2046
10342 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10343 ; GFX10-NEXT: s_mov_b32 s4, 0
10344 ; GFX10-NEXT: .LBB42_1: ; %atomicrmw.start
10345 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
10346 ; GFX10-NEXT: s_waitcnt vmcnt(0)
10347 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
10348 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v4
10349 ; GFX10-NEXT: v_max_f32_e32 v3, v3, v2
10350 ; GFX10-NEXT: v_bfe_u32 v5, v3, 16, 1
10351 ; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v3
10352 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
10353 ; GFX10-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
10354 ; GFX10-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
10355 ; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v3
10356 ; GFX10-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
10357 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
10358 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2046 glc
10359 ; GFX10-NEXT: s_waitcnt vmcnt(0)
10360 ; GFX10-NEXT: buffer_gl1_inv
10361 ; GFX10-NEXT: buffer_gl0_inv
10362 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
10363 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
10364 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
10365 ; GFX10-NEXT: s_cbranch_execnz .LBB42_1
10366 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
10367 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
10368 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
10369 ; GFX10-NEXT: s_setpc_b64 s[30:31]
10370 ;
10371 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
10372 ; GFX90A: ; %bb.0:
10373 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10374 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2046
10375 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
10376 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10377 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
10378 ; GFX90A-NEXT: s_mov_b32 s7, 0xffff0000
10379 ; GFX90A-NEXT: .LBB42_1: ; %atomicrmw.start
10380 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
10381 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
10382 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
10383 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v5
10384 ; GFX90A-NEXT: v_max_f32_e32 v3, v3, v2
10385 ; GFX90A-NEXT: v_bfe_u32 v4, v3, 16, 1
10386 ; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v3
10387 ; GFX90A-NEXT: v_add3_u32 v4, v4, v3, s6
10388 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
10389 ; GFX90A-NEXT: v_cndmask_b32_e32 v3, v4, v6, vcc
10390 ; GFX90A-NEXT: v_lshrrev_b32_e32 v3, 16, v3
10391 ; GFX90A-NEXT: v_and_or_b32 v4, v5, s7, v3
10392 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 glc
10393 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
10394 ; GFX90A-NEXT: buffer_wbinvl1
10395 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
10396 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
10397 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
10398 ; GFX90A-NEXT: s_cbranch_execnz .LBB42_1
10399 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
10400 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
10401 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
10402 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
10403 ;
10404 ; GFX908-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
10405 ; GFX908: ; %bb.0:
10406 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10407 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2046
10408 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
10409 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10410 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
10411 ; GFX908-NEXT: s_mov_b32 s7, 0xffff0000
10412 ; GFX908-NEXT: .LBB42_1: ; %atomicrmw.start
10413 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
10414 ; GFX908-NEXT: s_waitcnt vmcnt(0)
10415 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
10416 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v4
10417 ; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
10418 ; GFX908-NEXT: v_bfe_u32 v5, v3, 16, 1
10419 ; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v3
10420 ; GFX908-NEXT: v_add3_u32 v5, v5, v3, s6
10421 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
10422 ; GFX908-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
10423 ; GFX908-NEXT: v_lshrrev_b32_e32 v3, 16, v3
10424 ; GFX908-NEXT: v_and_or_b32 v3, v4, s7, v3
10425 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2046 glc
10426 ; GFX908-NEXT: s_waitcnt vmcnt(0)
10427 ; GFX908-NEXT: buffer_wbinvl1
10428 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
10429 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
10430 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
10431 ; GFX908-NEXT: s_cbranch_execnz .LBB42_1
10432 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
10433 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
10434 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
10435 ; GFX908-NEXT: s_setpc_b64 s[30:31]
10436 ;
10437 ; GFX8-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
10438 ; GFX8: ; %bb.0:
10439 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10440 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
10441 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
10442 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
10443 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
10444 ; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
10445 ; GFX8-NEXT: .LBB42_1: ; %atomicrmw.start
10446 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
10447 ; GFX8-NEXT: s_waitcnt vmcnt(0)
10448 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
10449 ; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
10450 ; GFX8-NEXT: v_max_f32_e32 v0, v0, v1
10451 ; GFX8-NEXT: v_bfe_u32 v5, v0, 16, 1
10452 ; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v0
10453 ; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
10454 ; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v0
10455 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
10456 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
10457 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v5, v7, vcc
10458 ; GFX8-NEXT: v_or_b32_sdwa v5, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10459 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
10460 ; GFX8-NEXT: s_waitcnt vmcnt(0)
10461 ; GFX8-NEXT: buffer_wbinvl1
10462 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
10463 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
10464 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
10465 ; GFX8-NEXT: s_cbranch_execnz .LBB42_1
10466 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
10467 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
10468 ; GFX8-NEXT: s_setpc_b64 s[30:31]
10469 ;
10470 ; GFX7-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
10471 ; GFX7: ; %bb.0:
10472 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10473 ; GFX7-NEXT: s_mov_b32 s6, 0
10474 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
10475 ; GFX7-NEXT: s_mov_b32 s4, s6
10476 ; GFX7-NEXT: s_mov_b32 s5, s6
10477 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
10478 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
10479 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
10480 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
10481 ; GFX7-NEXT: .LBB42_1: ; %atomicrmw.start
10482 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
10483 ; GFX7-NEXT: s_waitcnt vmcnt(0)
10484 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
10485 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
10486 ; GFX7-NEXT: v_max_f32_e32 v2, v2, v6
10487 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
10488 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10489 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
10490 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
10491 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
10492 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 offset:2046 glc
10493 ; GFX7-NEXT: s_waitcnt vmcnt(0)
10494 ; GFX7-NEXT: buffer_wbinvl1
10495 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
10496 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
10497 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
10498 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
10499 ; GFX7-NEXT: s_cbranch_execnz .LBB42_1
10500 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
10501 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
10502 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v4
10503 ; GFX7-NEXT: s_setpc_b64 s[30:31]
10504 ;
10505 ; GFX6-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
10506 ; GFX6: ; %bb.0:
10507 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10508 ; GFX6-NEXT: s_mov_b32 s6, 0
10509 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
10510 ; GFX6-NEXT: s_mov_b32 s4, s6
10511 ; GFX6-NEXT: s_mov_b32 s5, s6
10512 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
10513 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
10514 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
10515 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
10516 ; GFX6-NEXT: .LBB42_1: ; %atomicrmw.start
10517 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
10518 ; GFX6-NEXT: s_waitcnt vmcnt(0)
10519 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
10520 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
10521 ; GFX6-NEXT: v_max_f32_e32 v2, v2, v6
10522 ; GFX6-NEXT: s_waitcnt expcnt(0)
10523 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
10524 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10525 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
10526 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
10527 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
10528 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 offset:2046 glc
10529 ; GFX6-NEXT: s_waitcnt vmcnt(0)
10530 ; GFX6-NEXT: buffer_wbinvl1
10531 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
10532 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
10533 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
10534 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
10535 ; GFX6-NEXT: s_cbranch_execnz .LBB42_1
10536 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
10537 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
10538 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v4
10539 ; GFX6-NEXT: s_waitcnt expcnt(0)
10540 ; GFX6-NEXT: s_setpc_b64 s[30:31]
10541 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
10542 %result = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
10543 ret bfloat %result
10544 }
10546 define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
10547 ; GFX12-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
10548 ; GFX12: ; %bb.0:
10549 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
10550 ; GFX12-NEXT: s_wait_expcnt 0x0
10551 ; GFX12-NEXT: s_wait_samplecnt 0x0
10552 ; GFX12-NEXT: s_wait_bvhcnt 0x0
10553 ; GFX12-NEXT: s_wait_kmcnt 0x0
10554 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2046
10555 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
10556 ; GFX12-NEXT: s_mov_b32 s0, 0
10557 ; GFX12-NEXT: .LBB43_1: ; %atomicrmw.start
10558 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
10559 ; GFX12-NEXT: s_wait_loadcnt 0x0
10560 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
10561 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10562 ; GFX12-NEXT: v_max_num_f32_e32 v2, v2, v4
10563 ; GFX12-NEXT: v_bfe_u32 v5, v2, 16, 1
10564 ; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v2
10565 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
10566 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
10567 ; GFX12-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
10568 ; GFX12-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
10569 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10570 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10571 ; GFX12-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
10572 ; GFX12-NEXT: s_wait_storecnt 0x0
10573 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
10574 ; GFX12-NEXT: s_wait_loadcnt 0x0
10575 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
10576 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
10577 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
10578 ; GFX12-NEXT: s_wait_alu 0xfffe
10579 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
10580 ; GFX12-NEXT: s_wait_alu 0xfffe
10581 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
10582 ; GFX12-NEXT: s_cbranch_execnz .LBB43_1
10583 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
10584 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
10585 ; GFX12-NEXT: s_wait_alu 0xfffe
10586 ; GFX12-NEXT: s_setpc_b64 s[30:31]
10587 ;
10588 ; GFX940-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
10589 ; GFX940: ; %bb.0:
10590 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10591 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2046
10592 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
10593 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
10594 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
10595 ; GFX940-NEXT: s_mov_b32 s3, 0xffff0000
10596 ; GFX940-NEXT: .LBB43_1: ; %atomicrmw.start
10597 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
10598 ; GFX940-NEXT: s_waitcnt vmcnt(0)
10599 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
10600 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
10601 ; GFX940-NEXT: v_bfe_u32 v5, v2, 16, 1
10602 ; GFX940-NEXT: v_or_b32_e32 v6, 0x400000, v2
10603 ; GFX940-NEXT: v_add3_u32 v5, v5, v2, s2
10604 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
10605 ; GFX940-NEXT: s_nop 1
10606 ; GFX940-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
10607 ; GFX940-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10608 ; GFX940-NEXT: v_and_or_b32 v2, v3, s3, v2
10609 ; GFX940-NEXT: buffer_wbl2 sc1
10610 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 sc0
10611 ; GFX940-NEXT: s_waitcnt vmcnt(0)
10612 ; GFX940-NEXT: buffer_inv sc1
10613 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
10614 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
10615 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
10616 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
10617 ; GFX940-NEXT: s_cbranch_execnz .LBB43_1
10618 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
10619 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
10620 ; GFX940-NEXT: s_setpc_b64 s[30:31]
10621 ;
10622 ; GFX11-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
10623 ; GFX11: ; %bb.0:
10624 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10625 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2046
10626 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
10627 ; GFX11-NEXT: s_mov_b32 s0, 0
10628 ; GFX11-NEXT: .p2align 6
10629 ; GFX11-NEXT: .LBB43_1: ; %atomicrmw.start
10630 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
10631 ; GFX11-NEXT: s_waitcnt vmcnt(0)
10632 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
10633 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10634 ; GFX11-NEXT: v_max_f32_e32 v2, v2, v4
10635 ; GFX11-NEXT: v_bfe_u32 v5, v2, 16, 1
10636 ; GFX11-NEXT: v_or_b32_e32 v6, 0x400000, v2
10637 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
10638 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
10639 ; GFX11-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
10640 ; GFX11-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
10641 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10642 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10643 ; GFX11-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
10644 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
10645 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 glc
10646 ; GFX11-NEXT: s_waitcnt vmcnt(0)
10647 ; GFX11-NEXT: buffer_gl1_inv
10648 ; GFX11-NEXT: buffer_gl0_inv
10649 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
10650 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
10651 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
10652 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
10653 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
10654 ; GFX11-NEXT: s_cbranch_execnz .LBB43_1
10655 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
10656 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
10657 ; GFX11-NEXT: s_setpc_b64 s[30:31]
10658 ;
10659 ; GFX10-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
10660 ; GFX10: ; %bb.0:
10661 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10662 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2046
10663 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
10664 ; GFX10-NEXT: s_mov_b32 s4, 0
10665 ; GFX10-NEXT: .LBB43_1: ; %atomicrmw.start
10666 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
10667 ; GFX10-NEXT: s_waitcnt vmcnt(0)
10668 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
10669 ; GFX10-NEXT: v_max_f32_e32 v2, v2, v4
10670 ; GFX10-NEXT: v_bfe_u32 v5, v2, 16, 1
10671 ; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v2
10672 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
10673 ; GFX10-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
10674 ; GFX10-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
10675 ; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10676 ; GFX10-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
10677 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
10678 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 glc
10679 ; GFX10-NEXT: s_waitcnt vmcnt(0)
10680 ; GFX10-NEXT: buffer_gl1_inv
10681 ; GFX10-NEXT: buffer_gl0_inv
10682 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
10683 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
10684 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
10685 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
10686 ; GFX10-NEXT: s_cbranch_execnz .LBB43_1
10687 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
10688 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
10689 ; GFX10-NEXT: s_setpc_b64 s[30:31]
10690 ;
10691 ; GFX90A-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
10692 ; GFX90A: ; %bb.0:
10693 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10694 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2046
10695 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
10696 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
10697 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
10698 ; GFX90A-NEXT: s_mov_b32 s7, 0xffff0000
10699 ; GFX90A-NEXT: .LBB43_1: ; %atomicrmw.start
10700 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
10701 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
10702 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
10703 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
10704 ; GFX90A-NEXT: v_bfe_u32 v5, v2, 16, 1
10705 ; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v2
10706 ; GFX90A-NEXT: v_add3_u32 v5, v5, v2, s6
10707 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
10708 ; GFX90A-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
10709 ; GFX90A-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10710 ; GFX90A-NEXT: v_and_or_b32 v2, v3, s7, v2
10711 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 glc
10712 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
10713 ; GFX90A-NEXT: buffer_wbinvl1
10714 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
10715 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
10716 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
10717 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
10718 ; GFX90A-NEXT: s_cbranch_execnz .LBB43_1
10719 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
10720 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
10721 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
10722 ;
10723 ; GFX908-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
10724 ; GFX908: ; %bb.0:
10725 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10726 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2046
10727 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
10728 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
10729 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
10730 ; GFX908-NEXT: s_mov_b32 s7, 0xffff0000
10731 ; GFX908-NEXT: .LBB43_1: ; %atomicrmw.start
10732 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
10733 ; GFX908-NEXT: s_waitcnt vmcnt(0)
10734 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
10735 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
10736 ; GFX908-NEXT: v_bfe_u32 v5, v2, 16, 1
10737 ; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v2
10738 ; GFX908-NEXT: v_add3_u32 v5, v5, v2, s6
10739 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
10740 ; GFX908-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
10741 ; GFX908-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10742 ; GFX908-NEXT: v_and_or_b32 v2, v3, s7, v2
10743 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 glc
10744 ; GFX908-NEXT: s_waitcnt vmcnt(0)
10745 ; GFX908-NEXT: buffer_wbinvl1
10746 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
10747 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
10748 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
10749 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
10750 ; GFX908-NEXT: s_cbranch_execnz .LBB43_1
10751 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
10752 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
10753 ; GFX908-NEXT: s_setpc_b64 s[30:31]
10754 ;
10755 ; GFX8-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
10756 ; GFX8: ; %bb.0:
10757 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10758 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fe, v0
10759 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
10760 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
10761 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
10762 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
10763 ; GFX8-NEXT: .LBB43_1: ; %atomicrmw.start
10764 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
10765 ; GFX8-NEXT: s_waitcnt vmcnt(0)
10766 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
10767 ; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
10768 ; GFX8-NEXT: v_bfe_u32 v6, v2, 16, 1
10769 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v2
10770 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
10771 ; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v2
10772 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
10773 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
10774 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
10775 ; GFX8-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10776 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10777 ; GFX8-NEXT: s_waitcnt vmcnt(0)
10778 ; GFX8-NEXT: buffer_wbinvl1
10779 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
10780 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
10781 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
10782 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
10783 ; GFX8-NEXT: s_cbranch_execnz .LBB43_1
10784 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
10785 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
10786 ; GFX8-NEXT: s_setpc_b64 s[30:31]
10787 ;
10788 ; GFX7-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
10789 ; GFX7: ; %bb.0:
10790 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10791 ; GFX7-NEXT: s_mov_b32 s6, 0
10792 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
10793 ; GFX7-NEXT: s_mov_b32 s4, s6
10794 ; GFX7-NEXT: s_mov_b32 s5, s6
10795 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
10796 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
10797 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
10798 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
10799 ; GFX7-NEXT: .LBB43_1: ; %atomicrmw.start
10800 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
10801 ; GFX7-NEXT: s_waitcnt vmcnt(0)
10802 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
10803 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
10804 ; GFX7-NEXT: v_max_f32_e32 v2, v2, v4
10805 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
10806 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10807 ; GFX7-NEXT: v_or_b32_e32 v2, v5, v2
10808 ; GFX7-NEXT: v_mov_b32_e32 v6, v3
10809 ; GFX7-NEXT: v_mov_b32_e32 v5, v2
10810 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2046 glc
10811 ; GFX7-NEXT: s_waitcnt vmcnt(0)
10812 ; GFX7-NEXT: buffer_wbinvl1
10813 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
10814 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
10815 ; GFX7-NEXT: v_mov_b32_e32 v3, v5
10816 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
10817 ; GFX7-NEXT: s_cbranch_execnz .LBB43_1
10818 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
10819 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
10820 ; GFX7-NEXT: s_setpc_b64 s[30:31]
10821 ;
10822 ; GFX6-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
10823 ; GFX6: ; %bb.0:
10824 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10825 ; GFX6-NEXT: s_mov_b32 s6, 0
10826 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
10827 ; GFX6-NEXT: s_mov_b32 s4, s6
10828 ; GFX6-NEXT: s_mov_b32 s5, s6
10829 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:2046
10830 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
10831 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
10832 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
10833 ; GFX6-NEXT: .LBB43_1: ; %atomicrmw.start
10834 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
10835 ; GFX6-NEXT: s_waitcnt vmcnt(0)
10836 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
10837 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
10838 ; GFX6-NEXT: v_max_f32_e32 v2, v2, v4
10839 ; GFX6-NEXT: s_waitcnt expcnt(0)
10840 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
10841 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
10842 ; GFX6-NEXT: v_or_b32_e32 v2, v5, v2
10843 ; GFX6-NEXT: v_mov_b32_e32 v6, v3
10844 ; GFX6-NEXT: v_mov_b32_e32 v5, v2
10845 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2046 glc
10846 ; GFX6-NEXT: s_waitcnt vmcnt(0)
10847 ; GFX6-NEXT: buffer_wbinvl1
10848 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
10849 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
10850 ; GFX6-NEXT: v_mov_b32_e32 v3, v5
10851 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
10852 ; GFX6-NEXT: s_cbranch_execnz .LBB43_1
10853 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
10854 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
10855 ; GFX6-NEXT: s_waitcnt expcnt(0)
10856 ; GFX6-NEXT: s_setpc_b64 s[30:31]
10857 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
10858 %unused = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
10859 ret void
10860 }
10862 define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
10863 ; GFX12-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
10864 ; GFX12: ; %bb.0:
10865 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
10866 ; GFX12-NEXT: s_wait_expcnt 0x0
10867 ; GFX12-NEXT: s_wait_samplecnt 0x0
10868 ; GFX12-NEXT: s_wait_bvhcnt 0x0
10869 ; GFX12-NEXT: s_wait_kmcnt 0x0
10870 ; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
10871 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
10872 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10873 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
10874 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
10875 ; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
10876 ; GFX12-NEXT: s_mov_b32 s0, 0
10877 ; GFX12-NEXT: global_load_b32 v5, v[0:1], off
10878 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
10879 ; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
10880 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
10881 ; GFX12-NEXT: v_not_b32_e32 v4, v4
10882 ; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
10883 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
10884 ; GFX12-NEXT: s_wait_loadcnt 0x0
10885 ; GFX12-NEXT: v_mov_b32_e32 v6, v5
10886 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10887 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
10888 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
10889 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10890 ; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
10891 ; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
10892 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
10893 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
10894 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
10895 ; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
10896 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
10897 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10898 ; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
10899 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
10900 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
10901 ; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
10902 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
10903 ; GFX12-NEXT: s_wait_storecnt 0x0
10904 ; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
10905 ; GFX12-NEXT: s_wait_loadcnt 0x0
10906 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
10907 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
10908 ; GFX12-NEXT: s_wait_alu 0xfffe
10909 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
10910 ; GFX12-NEXT: s_wait_alu 0xfffe
10911 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
10912 ; GFX12-NEXT: s_cbranch_execnz .LBB44_1
10913 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
10914 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
10915 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
10916 ; GFX12-NEXT: s_wait_alu 0xfffe
10917 ; GFX12-NEXT: s_setpc_b64 s[30:31]
10918 ;
10919 ; GFX940-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
10920 ; GFX940: ; %bb.0:
10921 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10922 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
10923 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
10924 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
10925 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
10926 ; GFX940-NEXT: global_load_dword v5, v[0:1], off
10927 ; GFX940-NEXT: v_and_b32_e32 v3, 3, v4
10928 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 3, v3
10929 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
10930 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v3, s0
10931 ; GFX940-NEXT: v_not_b32_e32 v4, v4
10932 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
10933 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10934 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
10935 ; GFX940-NEXT: .LBB44_1: ; %atomicrmw.start
10936 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
10937 ; GFX940-NEXT: s_waitcnt vmcnt(0)
10938 ; GFX940-NEXT: v_mov_b32_e32 v7, v5
10939 ; GFX940-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
10940 ; GFX940-NEXT: s_nop 0
10941 ; GFX940-NEXT: v_max_f32_e32 v5, v5, v2
10942 ; GFX940-NEXT: v_bfe_u32 v6, v5, 16, 1
10943 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v5
10944 ; GFX940-NEXT: v_add3_u32 v6, v6, v5, s2
10945 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
10946 ; GFX940-NEXT: s_nop 1
10947 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
10948 ; GFX940-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
10949 ; GFX940-NEXT: v_and_or_b32 v6, v7, v4, v5
10950 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
10951 ; GFX940-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off sc0 sc1
10952 ; GFX940-NEXT: s_waitcnt vmcnt(0)
10953 ; GFX940-NEXT: buffer_inv sc0 sc1
10954 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
10955 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
10956 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
10957 ; GFX940-NEXT: s_cbranch_execnz .LBB44_1
10958 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
10959 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
10960 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v3, v5
10961 ; GFX940-NEXT: s_setpc_b64 s[30:31]
10962 ;
10963 ; GFX11-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
10964 ; GFX11: ; %bb.0:
10965 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10966 ; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
10967 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
10968 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
10969 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
10970 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
10971 ; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
10972 ; GFX11-NEXT: s_mov_b32 s0, 0
10973 ; GFX11-NEXT: global_load_b32 v5, v[0:1], off
10974 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
10975 ; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
10976 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
10977 ; GFX11-NEXT: v_not_b32_e32 v4, v4
10978 ; GFX11-NEXT: .p2align 6
10979 ; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
10980 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
10981 ; GFX11-NEXT: s_waitcnt vmcnt(0)
10982 ; GFX11-NEXT: v_mov_b32_e32 v6, v5
10983 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10984 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
10985 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
10986 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10987 ; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
10988 ; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
10989 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
10990 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
10991 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
10992 ; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
10993 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
10994 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10995 ; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
10996 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
10997 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
10998 ; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
10999 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
11000 ; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
11001 ; GFX11-NEXT: s_waitcnt vmcnt(0)
11002 ; GFX11-NEXT: buffer_gl1_inv
11003 ; GFX11-NEXT: buffer_gl0_inv
11004 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
11005 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
11006 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
11007 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
11008 ; GFX11-NEXT: s_cbranch_execnz .LBB44_1
11009 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
11010 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
11011 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
11012 ; GFX11-NEXT: s_setpc_b64 s[30:31]
11013 ;
11014 ; GFX10-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
11015 ; GFX10: ; %bb.0:
11016 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11017 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
11018 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
11019 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11020 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
11021 ; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
11022 ; GFX10-NEXT: s_mov_b32 s4, 0
11023 ; GFX10-NEXT: global_load_dword v5, v[0:1], off
11024 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
11025 ; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
11026 ; GFX10-NEXT: v_not_b32_e32 v4, v4
11027 ; GFX10-NEXT: .LBB44_1: ; %atomicrmw.start
11028 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
11029 ; GFX10-NEXT: s_waitcnt vmcnt(0)
11030 ; GFX10-NEXT: v_mov_b32_e32 v6, v5
11031 ; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
11032 ; GFX10-NEXT: v_max_f32_e32 v5, v5, v2
11033 ; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
11034 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v5
11035 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
11036 ; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
11037 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
11038 ; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11039 ; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
11040 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
11041 ; GFX10-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
11042 ; GFX10-NEXT: s_waitcnt vmcnt(0)
11043 ; GFX10-NEXT: buffer_gl1_inv
11044 ; GFX10-NEXT: buffer_gl0_inv
11045 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
11046 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
11047 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
11048 ; GFX10-NEXT: s_cbranch_execnz .LBB44_1
11049 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
11050 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
11051 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v3, v5
11052 ; GFX10-NEXT: s_setpc_b64 s[30:31]
11053 ;
11054 ; GFX90A-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
11055 ; GFX90A: ; %bb.0:
11056 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11057 ; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
11058 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
11059 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
11060 ; GFX90A-NEXT: global_load_dword v5, v[0:1], off
11061 ; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
11062 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
11063 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
11064 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
11065 ; GFX90A-NEXT: v_not_b32_e32 v4, v4
11066 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
11067 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11068 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
11069 ; GFX90A-NEXT: .LBB44_1: ; %atomicrmw.start
11070 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
11071 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
11072 ; GFX90A-NEXT: v_mov_b32_e32 v7, v5
11073 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v5, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
11074 ; GFX90A-NEXT: v_max_f32_e32 v5, v5, v2
11075 ; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
11076 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v5
11077 ; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
11078 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
11079 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
11080 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11081 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v4, v5
11082 ; GFX90A-NEXT: buffer_wbl2
11083 ; GFX90A-NEXT: global_atomic_cmpswap v5, v[0:1], v[6:7], off glc
11084 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
11085 ; GFX90A-NEXT: buffer_invl2
11086 ; GFX90A-NEXT: buffer_wbinvl1
11087 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
11088 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11089 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
11090 ; GFX90A-NEXT: s_cbranch_execnz .LBB44_1
11091 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
11092 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
11093 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v5
11094 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
11095 ;
11096 ; GFX908-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
11097 ; GFX908: ; %bb.0:
11098 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11099 ; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
11100 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
11101 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
11102 ; GFX908-NEXT: global_load_dword v5, v[0:1], off
11103 ; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
11104 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v3
11105 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
11106 ; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
11107 ; GFX908-NEXT: v_not_b32_e32 v4, v4
11108 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
11109 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11110 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
11111 ; GFX908-NEXT: .LBB44_1: ; %atomicrmw.start
11112 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
11113 ; GFX908-NEXT: s_waitcnt vmcnt(0)
11114 ; GFX908-NEXT: v_mov_b32_e32 v6, v5
11115 ; GFX908-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
11116 ; GFX908-NEXT: v_max_f32_e32 v5, v5, v2
11117 ; GFX908-NEXT: v_bfe_u32 v7, v5, 16, 1
11118 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v5
11119 ; GFX908-NEXT: v_add3_u32 v7, v7, v5, s6
11120 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
11121 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc
11122 ; GFX908-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11123 ; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
11124 ; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
11125 ; GFX908-NEXT: s_waitcnt vmcnt(0)
11126 ; GFX908-NEXT: buffer_wbinvl1
11127 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
11128 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11129 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
11130 ; GFX908-NEXT: s_cbranch_execnz .LBB44_1
11131 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
11132 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
11133 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v3, v5
11134 ; GFX908-NEXT: s_setpc_b64 s[30:31]
11135 ;
11136 ; GFX8-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
11137 ; GFX8: ; %bb.0:
11138 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11139 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
11140 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
11141 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
11142 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
11143 ; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
11144 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
11145 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
11146 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
11147 ; GFX8-NEXT: v_not_b32_e32 v4, v4
11148 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
11149 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11150 ; GFX8-NEXT: .LBB44_1: ; %atomicrmw.start
11151 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
11152 ; GFX8-NEXT: s_waitcnt vmcnt(0)
11153 ; GFX8-NEXT: v_mov_b32_e32 v6, v5
11154 ; GFX8-NEXT: v_lshrrev_b32_sdwa v5, v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
11155 ; GFX8-NEXT: v_max_f32_e32 v5, v5, v2
11156 ; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
11157 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
11158 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
11159 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
11160 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
11161 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
11162 ; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
11163 ; GFX8-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11164 ; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
11165 ; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
11166 ; GFX8-NEXT: s_waitcnt vmcnt(0)
11167 ; GFX8-NEXT: buffer_wbinvl1
11168 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v6
11169 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11170 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
11171 ; GFX8-NEXT: s_cbranch_execnz .LBB44_1
11172 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
11173 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
11174 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v3, v5
11175 ; GFX8-NEXT: s_setpc_b64 s[30:31]
11176 ;
11177 ; GFX7-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
11178 ; GFX7: ; %bb.0:
11179 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11180 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
11181 ; GFX7-NEXT: s_mov_b32 s6, 0
11182 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
11183 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
11184 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
11185 ; GFX7-NEXT: s_mov_b32 s4, s6
11186 ; GFX7-NEXT: s_mov_b32 s5, s6
11187 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
11188 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
11189 ; GFX7-NEXT: v_lshlrev_b32_e32 v6, 3, v4
11190 ; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
11191 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
11192 ; GFX7-NEXT: v_not_b32_e32 v7, v4
11193 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
11194 ; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
11195 ; GFX7-NEXT: .LBB44_1: ; %atomicrmw.start
11196 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
11197 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11198 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v6, v3
11199 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11200 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
11201 ; GFX7-NEXT: v_max_f32_e32 v2, v2, v8
11202 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11203 ; GFX7-NEXT: v_and_b32_e32 v4, v3, v7
11204 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v6, v2
11205 ; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
11206 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
11207 ; GFX7-NEXT: v_mov_b32_e32 v4, v2
11208 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
11209 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11210 ; GFX7-NEXT: buffer_wbinvl1
11211 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
11212 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
11213 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
11214 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
11215 ; GFX7-NEXT: s_cbranch_execnz .LBB44_1
11216 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
11217 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
11218 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v6, v4
11219 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
11220 ; GFX7-NEXT: s_setpc_b64 s[30:31]
11221 ;
11222 ; GFX6-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
11223 ; GFX6: ; %bb.0:
11224 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11225 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
11226 ; GFX6-NEXT: s_mov_b32 s6, 0
11227 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
11228 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
11229 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
11230 ; GFX6-NEXT: s_mov_b32 s4, s6
11231 ; GFX6-NEXT: s_mov_b32 s5, s6
11232 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
11233 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
11234 ; GFX6-NEXT: v_lshlrev_b32_e32 v6, 3, v4
11235 ; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v6
11236 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
11237 ; GFX6-NEXT: v_not_b32_e32 v7, v4
11238 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
11239 ; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
11240 ; GFX6-NEXT: .LBB44_1: ; %atomicrmw.start
11241 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
11242 ; GFX6-NEXT: s_waitcnt vmcnt(0)
11243 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v3
11244 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11245 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
11246 ; GFX6-NEXT: v_max_f32_e32 v2, v2, v8
11247 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11248 ; GFX6-NEXT: s_waitcnt expcnt(0)
11249 ; GFX6-NEXT: v_and_b32_e32 v4, v3, v7
11250 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v6, v2
11251 ; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
11252 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
11253 ; GFX6-NEXT: v_mov_b32_e32 v4, v2
11254 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v[0:1], s[4:7], 0 addr64 glc
11255 ; GFX6-NEXT: s_waitcnt vmcnt(0)
11256 ; GFX6-NEXT: buffer_wbinvl1
11257 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
11258 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
11259 ; GFX6-NEXT: v_mov_b32_e32 v3, v4
11260 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
11261 ; GFX6-NEXT: s_cbranch_execnz .LBB44_1
11262 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
11263 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
11264 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v6, v4
11265 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
11266 ; GFX6-NEXT: s_waitcnt expcnt(0)
11267 ; GFX6-NEXT: s_setpc_b64 s[30:31]
11268 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
11269 %result = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val seq_cst, !amdgpu.no.fine.grained.memory !0
11270 ret bfloat %result
11271 }
11273 define void @global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
11274 ; GFX12-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
11275 ; GFX12: ; %bb.0:
11276 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
11277 ; GFX12-NEXT: s_wait_expcnt 0x0
11278 ; GFX12-NEXT: s_wait_samplecnt 0x0
11279 ; GFX12-NEXT: s_wait_bvhcnt 0x0
11280 ; GFX12-NEXT: s_wait_kmcnt 0x0
11281 ; GFX12-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
11282 ; GFX12-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
11283 ; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v2
11284 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
11285 ; GFX12-NEXT: v_and_b32_e32 v0, -4, v4
11286 ; GFX12-NEXT: v_and_b32_e32 v4, 3, v4
11287 ; GFX12-NEXT: s_mov_b32 s0, 0
11288 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
11289 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 3, v4
11290 ; GFX12-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
11291 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
11292 ; GFX12-NEXT: v_not_b32_e32 v5, v5
11293 ; GFX12-NEXT: .LBB45_1: ; %atomicrmw.start
11294 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
11295 ; GFX12-NEXT: s_wait_loadcnt 0x0
11296 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, v4, v3
11297 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11298 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11299 ; GFX12-NEXT: v_max_num_f32_e32 v2, v2, v6
11300 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
11301 ; GFX12-NEXT: v_bfe_u32 v7, v2, 16, 1
11302 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v2
11303 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
11304 ; GFX12-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
11305 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11306 ; GFX12-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
11307 ; GFX12-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11308 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11309 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, v4, v2
11310 ; GFX12-NEXT: v_and_or_b32 v2, v3, v5, v2
11311 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
11312 ; GFX12-NEXT: s_wait_storecnt 0x0
11313 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
11314 ; GFX12-NEXT: s_wait_loadcnt 0x0
11315 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
11316 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
11317 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
11318 ; GFX12-NEXT: s_wait_alu 0xfffe
11319 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
11320 ; GFX12-NEXT: s_wait_alu 0xfffe
11321 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
11322 ; GFX12-NEXT: s_cbranch_execnz .LBB45_1
11323 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
11324 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
11325 ; GFX12-NEXT: s_wait_alu 0xfffe
11326 ; GFX12-NEXT: s_setpc_b64 s[30:31]
11327 ;
11328 ; GFX940-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
11329 ; GFX940: ; %bb.0:
11330 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11331 ; GFX940-NEXT: s_mov_b64 s[0:1], 0x7fe
11332 ; GFX940-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
11333 ; GFX940-NEXT: v_and_b32_e32 v0, -4, v4
11334 ; GFX940-NEXT: v_mov_b32_e32 v1, v5
11335 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
11336 ; GFX940-NEXT: v_and_b32_e32 v4, 3, v4
11337 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 3, v4
11338 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
11339 ; GFX940-NEXT: v_lshlrev_b32_e64 v5, v4, s0
11340 ; GFX940-NEXT: v_not_b32_e32 v5, v5
11341 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
11342 ; GFX940-NEXT: v_lshlrev_b32_e32 v6, 16, v2
11343 ; GFX940-NEXT: s_movk_i32 s2, 0x7fff
11344 ; GFX940-NEXT: .LBB45_1: ; %atomicrmw.start
11345 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
11346 ; GFX940-NEXT: s_waitcnt vmcnt(0)
11347 ; GFX940-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
11348 ; GFX940-NEXT: s_nop 0
11349 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v6
11350 ; GFX940-NEXT: v_bfe_u32 v7, v2, 16, 1
11351 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v2
11352 ; GFX940-NEXT: v_add3_u32 v7, v7, v2, s2
11353 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
11354 ; GFX940-NEXT: s_nop 1
11355 ; GFX940-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
11356 ; GFX940-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11357 ; GFX940-NEXT: v_and_or_b32 v2, v3, v5, v2
11358 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
11359 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0 sc1
11360 ; GFX940-NEXT: s_waitcnt vmcnt(0)
11361 ; GFX940-NEXT: buffer_inv sc0 sc1
11362 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
11363 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
11364 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
11365 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
11366 ; GFX940-NEXT: s_cbranch_execnz .LBB45_1
11367 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
11368 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
11369 ; GFX940-NEXT: s_setpc_b64 s[30:31]
11370 ;
11371 ; GFX11-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
11372 ; GFX11: ; %bb.0:
11373 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11374 ; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
11375 ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
11376 ; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v2
11377 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
11378 ; GFX11-NEXT: v_and_b32_e32 v0, -4, v4
11379 ; GFX11-NEXT: v_and_b32_e32 v4, 3, v4
11380 ; GFX11-NEXT: s_mov_b32 s0, 0
11381 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
11382 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v4
11383 ; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
11384 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
11385 ; GFX11-NEXT: v_not_b32_e32 v5, v5
11386 ; GFX11-NEXT: .p2align 6
11387 ; GFX11-NEXT: .LBB45_1: ; %atomicrmw.start
11388 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
11389 ; GFX11-NEXT: s_waitcnt vmcnt(0)
11390 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v3
11391 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11392 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11393 ; GFX11-NEXT: v_max_f32_e32 v2, v2, v6
11394 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
11395 ; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
11396 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v2
11397 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
11398 ; GFX11-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
11399 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11400 ; GFX11-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
11401 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11402 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11403 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, v4, v2
11404 ; GFX11-NEXT: v_and_or_b32 v2, v3, v5, v2
11405 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
11406 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
11407 ; GFX11-NEXT: s_waitcnt vmcnt(0)
11408 ; GFX11-NEXT: buffer_gl1_inv
11409 ; GFX11-NEXT: buffer_gl0_inv
11410 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
11411 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
11412 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
11413 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
11414 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
11415 ; GFX11-NEXT: s_cbranch_execnz .LBB45_1
11416 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
11417 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
11418 ; GFX11-NEXT: s_setpc_b64 s[30:31]
11419 ;
11420 ; GFX10-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
11421 ; GFX10: ; %bb.0:
11422 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11423 ; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
11424 ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
11425 ; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v2
11426 ; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
11427 ; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
11428 ; GFX10-NEXT: s_mov_b32 s4, 0
11429 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
11430 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
11431 ; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
11432 ; GFX10-NEXT: v_not_b32_e32 v5, v5
11433 ; GFX10-NEXT: .LBB45_1: ; %atomicrmw.start
11434 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
11435 ; GFX10-NEXT: s_waitcnt vmcnt(0)
11436 ; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
11437 ; GFX10-NEXT: v_max_f32_e32 v2, v2, v6
11438 ; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
11439 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v2
11440 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
11441 ; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
11442 ; GFX10-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
11443 ; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11444 ; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
11445 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
11446 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
11447 ; GFX10-NEXT: s_waitcnt vmcnt(0)
11448 ; GFX10-NEXT: buffer_gl1_inv
11449 ; GFX10-NEXT: buffer_gl0_inv
11450 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
11451 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
11452 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
11453 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
11454 ; GFX10-NEXT: s_cbranch_execnz .LBB45_1
11455 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
11456 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
11457 ; GFX10-NEXT: s_setpc_b64 s[30:31]
11458 ;
11459 ; GFX90A-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
11460 ; GFX90A: ; %bb.0:
11461 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11462 ; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
11463 ; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
11464 ; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
11465 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
11466 ; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
11467 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
11468 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
11469 ; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
11470 ; GFX90A-NEXT: v_not_b32_e32 v5, v5
11471 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
11472 ; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v2
11473 ; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
11474 ; GFX90A-NEXT: .LBB45_1: ; %atomicrmw.start
11475 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
11476 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
11477 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
11478 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v6
11479 ; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
11480 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
11481 ; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s6
11482 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
11483 ; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
11484 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11485 ; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
11486 ; GFX90A-NEXT: buffer_wbl2
11487 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
11488 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
11489 ; GFX90A-NEXT: buffer_invl2
11490 ; GFX90A-NEXT: buffer_wbinvl1
11491 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
11492 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11493 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
11494 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
11495 ; GFX90A-NEXT: s_cbranch_execnz .LBB45_1
11496 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
11497 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
11498 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
11499 ;
11500 ; GFX908-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
11501 ; GFX908: ; %bb.0:
11502 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11503 ; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
11504 ; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
11505 ; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
11506 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
11507 ; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
11508 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
11509 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
11510 ; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
11511 ; GFX908-NEXT: v_not_b32_e32 v5, v5
11512 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
11513 ; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v2
11514 ; GFX908-NEXT: s_movk_i32 s6, 0x7fff
11515 ; GFX908-NEXT: .LBB45_1: ; %atomicrmw.start
11516 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
11517 ; GFX908-NEXT: s_waitcnt vmcnt(0)
11518 ; GFX908-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
11519 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v6
11520 ; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
11521 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
11522 ; GFX908-NEXT: v_add3_u32 v7, v7, v2, s6
11523 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
11524 ; GFX908-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
11525 ; GFX908-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11526 ; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
11527 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
11528 ; GFX908-NEXT: s_waitcnt vmcnt(0)
11529 ; GFX908-NEXT: buffer_wbinvl1
11530 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
11531 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11532 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
11533 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
11534 ; GFX908-NEXT: s_cbranch_execnz .LBB45_1
11535 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
11536 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
11537 ; GFX908-NEXT: s_setpc_b64 s[30:31]
11538 ;
11539 ; GFX8-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
11540 ; GFX8: ; %bb.0:
11541 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11542 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
11543 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
11544 ; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
11545 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
11546 ; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
11547 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
11548 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
11549 ; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
11550 ; GFX8-NEXT: v_not_b32_e32 v5, v5
11551 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
11552 ; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v2
11553 ; GFX8-NEXT: .LBB45_1: ; %atomicrmw.start
11554 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
11555 ; GFX8-NEXT: s_waitcnt vmcnt(0)
11556 ; GFX8-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
11557 ; GFX8-NEXT: v_max_f32_e32 v2, v2, v6
11558 ; GFX8-NEXT: v_bfe_u32 v8, v2, 16, 1
11559 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v2
11560 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
11561 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v2
11562 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
11563 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v8, v9, vcc
11564 ; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
11565 ; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
11566 ; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
11567 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11568 ; GFX8-NEXT: s_waitcnt vmcnt(0)
11569 ; GFX8-NEXT: buffer_wbinvl1
11570 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
11571 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11572 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
11573 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
11574 ; GFX8-NEXT: s_cbranch_execnz .LBB45_1
11575 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
11576 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
11577 ; GFX8-NEXT: s_setpc_b64 s[30:31]
11578 ;
11579 ; GFX7-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
11580 ; GFX7: ; %bb.0:
11581 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11582 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
11583 ; GFX7-NEXT: s_mov_b32 s6, 0
11584 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
11585 ; GFX7-NEXT: v_and_b32_e32 v0, -4, v4
11586 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
11587 ; GFX7-NEXT: s_mov_b32 s4, s6
11588 ; GFX7-NEXT: s_mov_b32 s5, s6
11589 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
11590 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
11591 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v4
11592 ; GFX7-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
11593 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
11594 ; GFX7-NEXT: v_not_b32_e32 v5, v5
11595 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
11596 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
11597 ; GFX7-NEXT: .LBB45_1: ; %atomicrmw.start
11598 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
11599 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11600 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, v4, v3
11601 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11602 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
11603 ; GFX7-NEXT: v_max_f32_e32 v2, v2, v6
11604 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11605 ; GFX7-NEXT: v_and_b32_e32 v7, v3, v5
11606 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, v4, v2
11607 ; GFX7-NEXT: v_or_b32_e32 v2, v7, v2
11608 ; GFX7-NEXT: v_mov_b32_e32 v8, v3
11609 ; GFX7-NEXT: v_mov_b32_e32 v7, v2
11610 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
11611 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11612 ; GFX7-NEXT: buffer_wbinvl1
11613 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
11614 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
11615 ; GFX7-NEXT: v_mov_b32_e32 v3, v7
11616 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
11617 ; GFX7-NEXT: s_cbranch_execnz .LBB45_1
11618 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
11619 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
11620 ; GFX7-NEXT: s_setpc_b64 s[30:31]
11621 ;
11622 ; GFX6-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
11623 ; GFX6: ; %bb.0:
11624 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11625 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x7fe, v0
11626 ; GFX6-NEXT: s_mov_b32 s6, 0
11627 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
11628 ; GFX6-NEXT: v_and_b32_e32 v0, -4, v4
11629 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
11630 ; GFX6-NEXT: s_mov_b32 s4, s6
11631 ; GFX6-NEXT: s_mov_b32 s5, s6
11632 ; GFX6-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
11633 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
11634 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v4
11635 ; GFX6-NEXT: v_lshl_b32_e32 v5, 0xffff, v4
11636 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
11637 ; GFX6-NEXT: v_not_b32_e32 v5, v5
11638 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
11639 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
11640 ; GFX6-NEXT: .LBB45_1: ; %atomicrmw.start
11641 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
11642 ; GFX6-NEXT: s_waitcnt vmcnt(0)
11643 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v3
11644 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
11645 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
11646 ; GFX6-NEXT: v_max_f32_e32 v2, v2, v6
11647 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
11648 ; GFX6-NEXT: s_waitcnt expcnt(0)
11649 ; GFX6-NEXT: v_and_b32_e32 v7, v3, v5
11650 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, v4, v2
11651 ; GFX6-NEXT: v_or_b32_e32 v2, v7, v2
11652 ; GFX6-NEXT: v_mov_b32_e32 v8, v3
11653 ; GFX6-NEXT: v_mov_b32_e32 v7, v2
11654 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
11655 ; GFX6-NEXT: s_waitcnt vmcnt(0)
11656 ; GFX6-NEXT: buffer_wbinvl1
11657 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v3
11658 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
11659 ; GFX6-NEXT: v_mov_b32_e32 v3, v7
11660 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
11661 ; GFX6-NEXT: s_cbranch_execnz .LBB45_1
11662 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
11663 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
11664 ; GFX6-NEXT: s_waitcnt expcnt(0)
11665 ; GFX6-NEXT: s_setpc_b64 s[30:31]
11666 %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
11667 %unused = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val seq_cst, !amdgpu.no.fine.grained.memory !0
11668 ret void
11669 }
11671 ; --------------------------------------------------------------------
11672 ; <2 x half>
11673 ; --------------------------------------------------------------------
11675 define <2 x half> @global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
11676 ; GFX12-LABEL: global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
11677 ; GFX12: ; %bb.0:
11678 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
11679 ; GFX12-NEXT: s_wait_expcnt 0x0
11680 ; GFX12-NEXT: s_wait_samplecnt 0x0
11681 ; GFX12-NEXT: s_wait_bvhcnt 0x0
11682 ; GFX12-NEXT: s_wait_kmcnt 0x0
11683 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
11684 ; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
11685 ; GFX12-NEXT: s_mov_b32 s0, 0
11686 ; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start
11687 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
11688 ; GFX12-NEXT: s_wait_loadcnt 0x0
11689 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
11690 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11691 ; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
11692 ; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v2
11693 ; GFX12-NEXT: s_wait_storecnt 0x0
11694 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
11695 ; GFX12-NEXT: s_wait_loadcnt 0x0
11696 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
11697 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
11698 ; GFX12-NEXT: s_wait_alu 0xfffe
11699 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
11700 ; GFX12-NEXT: s_wait_alu 0xfffe
11701 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
11702 ; GFX12-NEXT: s_cbranch_execnz .LBB46_1
11703 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
11704 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
11705 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
11706 ; GFX12-NEXT: s_wait_alu 0xfffe
11707 ; GFX12-NEXT: s_setpc_b64 s[30:31]
11708 ;
11709 ; GFX940-LABEL: global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
11710 ; GFX940: ; %bb.0:
11711 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11712 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
11713 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
11714 ; GFX940-NEXT: v_pk_max_f16 v2, v2, v2
11715 ; GFX940-NEXT: .LBB46_1: ; %atomicrmw.start
11716 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
11717 ; GFX940-NEXT: s_waitcnt vmcnt(0)
11718 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
11719 ; GFX940-NEXT: v_pk_max_f16 v3, v5, v5
11720 ; GFX940-NEXT: s_nop 0
11721 ; GFX940-NEXT: v_pk_max_f16 v4, v3, v2
11722 ; GFX940-NEXT: buffer_wbl2 sc1
11723 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
11724 ; GFX940-NEXT: s_waitcnt vmcnt(0)
11725 ; GFX940-NEXT: buffer_inv sc1
11726 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
11727 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
11728 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
11729 ; GFX940-NEXT: s_cbranch_execnz .LBB46_1
11730 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
11731 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
11732 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
11733 ; GFX940-NEXT: s_setpc_b64 s[30:31]
11734 ;
11735 ; GFX11-LABEL: global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
11736 ; GFX11: ; %bb.0:
11737 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11738 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
11739 ; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
11740 ; GFX11-NEXT: s_mov_b32 s0, 0
11741 ; GFX11-NEXT: .LBB46_1: ; %atomicrmw.start
11742 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
11743 ; GFX11-NEXT: s_waitcnt vmcnt(0)
11744 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
11745 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11746 ; GFX11-NEXT: v_pk_max_f16 v3, v4, v4
11747 ; GFX11-NEXT: v_pk_max_f16 v3, v3, v2
11748 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
11749 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
11750 ; GFX11-NEXT: s_waitcnt vmcnt(0)
11751 ; GFX11-NEXT: buffer_gl1_inv
11752 ; GFX11-NEXT: buffer_gl0_inv
11753 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
11754 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
11755 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
11756 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
11757 ; GFX11-NEXT: s_cbranch_execnz .LBB46_1
11758 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
11759 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
11760 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
11761 ; GFX11-NEXT: s_setpc_b64 s[30:31]
11762 ;
11763 ; GFX10-LABEL: global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
11764 ; GFX10: ; %bb.0:
11765 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11766 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
11767 ; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
11768 ; GFX10-NEXT: s_mov_b32 s4, 0
11769 ; GFX10-NEXT: .LBB46_1: ; %atomicrmw.start
11770 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
11771 ; GFX10-NEXT: s_waitcnt vmcnt(0)
11772 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
11773 ; GFX10-NEXT: v_pk_max_f16 v3, v4, v4
11774 ; GFX10-NEXT: v_pk_max_f16 v3, v3, v2
11775 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
11776 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
11777 ; GFX10-NEXT: s_waitcnt vmcnt(0)
11778 ; GFX10-NEXT: buffer_gl1_inv
11779 ; GFX10-NEXT: buffer_gl0_inv
11780 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
11781 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
11782 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
11783 ; GFX10-NEXT: s_cbranch_execnz .LBB46_1
11784 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
11785 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
11786 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
11787 ; GFX10-NEXT: s_setpc_b64 s[30:31]
11788 ;
11789 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
11790 ; GFX90A: ; %bb.0:
11791 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11792 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
11793 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
11794 ; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
11795 ; GFX90A-NEXT: .LBB46_1: ; %atomicrmw.start
11796 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
11797 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
11798 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
11799 ; GFX90A-NEXT: v_pk_max_f16 v3, v5, v5
11800 ; GFX90A-NEXT: v_pk_max_f16 v4, v3, v2
11801 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
11802 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
11803 ; GFX90A-NEXT: buffer_wbinvl1
11804 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
11805 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11806 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
11807 ; GFX90A-NEXT: s_cbranch_execnz .LBB46_1
11808 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
11809 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
11810 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
11811 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
11812 ;
11813 ; GFX908-LABEL: global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
11814 ; GFX908: ; %bb.0:
11815 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11816 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
11817 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
11818 ; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
11819 ; GFX908-NEXT: .LBB46_1: ; %atomicrmw.start
11820 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
11821 ; GFX908-NEXT: s_waitcnt vmcnt(0)
11822 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
11823 ; GFX908-NEXT: v_pk_max_f16 v3, v4, v4
11824 ; GFX908-NEXT: v_pk_max_f16 v3, v3, v2
11825 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
11826 ; GFX908-NEXT: s_waitcnt vmcnt(0)
11827 ; GFX908-NEXT: buffer_wbinvl1
11828 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
11829 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11830 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
11831 ; GFX908-NEXT: s_cbranch_execnz .LBB46_1
11832 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
11833 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
11834 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
11835 ; GFX908-NEXT: s_setpc_b64 s[30:31]
11836 ;
11837 ; GFX8-LABEL: global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
11838 ; GFX8: ; %bb.0:
11839 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11840 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
11841 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
11842 ; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
11843 ; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
11844 ; GFX8-NEXT: .LBB46_1: ; %atomicrmw.start
11845 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
11846 ; GFX8-NEXT: s_waitcnt vmcnt(0)
11847 ; GFX8-NEXT: v_mov_b32_e32 v6, v3
11848 ; GFX8-NEXT: v_max_f16_sdwa v3, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
11849 ; GFX8-NEXT: v_max_f16_e32 v5, v6, v6
11850 ; GFX8-NEXT: v_max_f16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
11851 ; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
11852 ; GFX8-NEXT: v_or_b32_e32 v5, v5, v3
11853 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
11854 ; GFX8-NEXT: s_waitcnt vmcnt(0)
11855 ; GFX8-NEXT: buffer_wbinvl1
11856 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
11857 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
11858 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
11859 ; GFX8-NEXT: s_cbranch_execnz .LBB46_1
11860 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
11861 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
11862 ; GFX8-NEXT: v_mov_b32_e32 v0, v3
11863 ; GFX8-NEXT: s_setpc_b64 s[30:31]
11864 ;
11865 ; GFX7-LABEL: global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
11866 ; GFX7: ; %bb.0:
11867 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11868 ; GFX7-NEXT: s_mov_b32 s6, 0
11869 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
11870 ; GFX7-NEXT: s_mov_b32 s4, s6
11871 ; GFX7-NEXT: s_mov_b32 s5, s6
11872 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
11873 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
11874 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v2
11875 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
11876 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v3
11877 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11878 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
11879 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v5
11880 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
11881 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v6
11882 ; GFX7-NEXT: .LBB46_1: ; %atomicrmw.start
11883 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
11884 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
11885 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
11886 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v3
11887 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
11888 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
11889 ; GFX7-NEXT: v_max_f32_e32 v6, v6, v4
11890 ; GFX7-NEXT: v_max_f32_e32 v7, v7, v5
11891 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6
11892 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v7
11893 ; GFX7-NEXT: v_or_b32_e32 v7, v2, v3
11894 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
11895 ; GFX7-NEXT: v_or_b32_e32 v6, v8, v2
11896 ; GFX7-NEXT: v_mov_b32_e32 v9, v7
11897 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
11898 ; GFX7-NEXT: buffer_atomic_cmpswap v[8:9], v[0:1], s[4:7], 0 addr64 glc
11899 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11900 ; GFX7-NEXT: buffer_wbinvl1
11901 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
11902 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v8
11903 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
11904 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
11905 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
11906 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
11907 ; GFX7-NEXT: s_cbranch_execnz .LBB46_1
11908 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
11909 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
11910 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
11911 ; GFX7-NEXT: v_mov_b32_e32 v1, v3
11912 ; GFX7-NEXT: s_setpc_b64 s[30:31]
11913 ;
11914 ; GFX6-LABEL: global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
11915 ; GFX6: ; %bb.0:
11916 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11917 ; GFX6-NEXT: s_mov_b32 s6, 0
11918 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
11919 ; GFX6-NEXT: s_mov_b32 s4, s6
11920 ; GFX6-NEXT: s_mov_b32 s5, s6
11921 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
11922 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
11923 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v2
11924 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
11925 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v3
11926 ; GFX6-NEXT: s_waitcnt vmcnt(0)
11927 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
11928 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v5
11929 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
11930 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v6
11931 ; GFX6-NEXT: .LBB46_1: ; %atomicrmw.start
11932 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
11933 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
11934 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
11935 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v3
11936 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
11937 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
11938 ; GFX6-NEXT: v_max_f32_e32 v6, v6, v4
11939 ; GFX6-NEXT: v_max_f32_e32 v7, v7, v5
11940 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
11941 ; GFX6-NEXT: s_waitcnt expcnt(0)
11942 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v7
11943 ; GFX6-NEXT: v_or_b32_e32 v7, v2, v3
11944 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v6
11945 ; GFX6-NEXT: v_or_b32_e32 v6, v8, v2
11946 ; GFX6-NEXT: v_mov_b32_e32 v9, v7
11947 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
11948 ; GFX6-NEXT: buffer_atomic_cmpswap v[8:9], v[0:1], s[4:7], 0 addr64 glc
11949 ; GFX6-NEXT: s_waitcnt vmcnt(0)
11950 ; GFX6-NEXT: buffer_wbinvl1
11951 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
11952 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v8
11953 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
11954 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
11955 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
11956 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
11957 ; GFX6-NEXT: s_cbranch_execnz .LBB46_1
11958 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
11959 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
11960 ; GFX6-NEXT: v_mov_b32_e32 v0, v2
11961 ; GFX6-NEXT: v_mov_b32_e32 v1, v3
11962 ; GFX6-NEXT: s_waitcnt expcnt(0)
11963 ; GFX6-NEXT: s_setpc_b64 s[30:31]
11964 %result = atomicrmw fmax ptr addrspace(1) %ptr, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
11965 ret <2 x half> %result
11966 }
11968 define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
11969 ; GFX12-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
11970 ; GFX12: ; %bb.0:
11971 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
11972 ; GFX12-NEXT: s_wait_expcnt 0x0
11973 ; GFX12-NEXT: s_wait_samplecnt 0x0
11974 ; GFX12-NEXT: s_wait_bvhcnt 0x0
11975 ; GFX12-NEXT: s_wait_kmcnt 0x0
11976 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
11977 ; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
11978 ; GFX12-NEXT: s_mov_b32 s0, 0
11979 ; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start
11980 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
11981 ; GFX12-NEXT: s_wait_loadcnt 0x0
11982 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
11983 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
11984 ; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
11985 ; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v2
11986 ; GFX12-NEXT: s_wait_storecnt 0x0
11987 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
11988 ; GFX12-NEXT: s_wait_loadcnt 0x0
11989 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
11990 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
11991 ; GFX12-NEXT: s_wait_alu 0xfffe
11992 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
11993 ; GFX12-NEXT: s_wait_alu 0xfffe
11994 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
11995 ; GFX12-NEXT: s_cbranch_execnz .LBB47_1
11996 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
11997 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
11998 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
11999 ; GFX12-NEXT: s_wait_alu 0xfffe
12000 ; GFX12-NEXT: s_setpc_b64 s[30:31]
12001 ;
12002 ; GFX940-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
12003 ; GFX940: ; %bb.0:
12004 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12005 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
12006 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
12007 ; GFX940-NEXT: v_pk_max_f16 v2, v2, v2
12008 ; GFX940-NEXT: .LBB47_1: ; %atomicrmw.start
12009 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
12010 ; GFX940-NEXT: s_waitcnt vmcnt(0)
12011 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
12012 ; GFX940-NEXT: v_pk_max_f16 v3, v5, v5
12013 ; GFX940-NEXT: s_nop 0
12014 ; GFX940-NEXT: v_pk_max_f16 v4, v3, v2
12015 ; GFX940-NEXT: buffer_wbl2 sc1
12016 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
12017 ; GFX940-NEXT: s_waitcnt vmcnt(0)
12018 ; GFX940-NEXT: buffer_inv sc1
12019 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
12020 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
12021 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
12022 ; GFX940-NEXT: s_cbranch_execnz .LBB47_1
12023 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
12024 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
12025 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
12026 ; GFX940-NEXT: s_setpc_b64 s[30:31]
12027 ;
12028 ; GFX11-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
12029 ; GFX11: ; %bb.0:
12030 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12031 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
12032 ; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
12033 ; GFX11-NEXT: s_mov_b32 s0, 0
12034 ; GFX11-NEXT: .LBB47_1: ; %atomicrmw.start
12035 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
12036 ; GFX11-NEXT: s_waitcnt vmcnt(0)
12037 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
12038 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
12039 ; GFX11-NEXT: v_pk_max_f16 v3, v4, v4
12040 ; GFX11-NEXT: v_pk_max_f16 v3, v3, v2
12041 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
12042 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
12043 ; GFX11-NEXT: s_waitcnt vmcnt(0)
12044 ; GFX11-NEXT: buffer_gl1_inv
12045 ; GFX11-NEXT: buffer_gl0_inv
12046 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
12047 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
12048 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
12049 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
12050 ; GFX11-NEXT: s_cbranch_execnz .LBB47_1
12051 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
12052 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
12053 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
12054 ; GFX11-NEXT: s_setpc_b64 s[30:31]
12055 ;
12056 ; GFX10-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
12057 ; GFX10: ; %bb.0:
12058 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12059 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
12060 ; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
12061 ; GFX10-NEXT: s_mov_b32 s4, 0
12062 ; GFX10-NEXT: .LBB47_1: ; %atomicrmw.start
12063 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
12064 ; GFX10-NEXT: s_waitcnt vmcnt(0)
12065 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
12066 ; GFX10-NEXT: v_pk_max_f16 v3, v4, v4
12067 ; GFX10-NEXT: v_pk_max_f16 v3, v3, v2
12068 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
12069 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
12070 ; GFX10-NEXT: s_waitcnt vmcnt(0)
12071 ; GFX10-NEXT: buffer_gl1_inv
12072 ; GFX10-NEXT: buffer_gl0_inv
12073 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
12074 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
12075 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
12076 ; GFX10-NEXT: s_cbranch_execnz .LBB47_1
12077 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
12078 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
12079 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
12080 ; GFX10-NEXT: s_setpc_b64 s[30:31]
12081 ;
12082 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
12083 ; GFX90A: ; %bb.0:
12084 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12085 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
12086 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
12087 ; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
12088 ; GFX90A-NEXT: .LBB47_1: ; %atomicrmw.start
12089 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
12090 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12091 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
12092 ; GFX90A-NEXT: v_pk_max_f16 v3, v5, v5
12093 ; GFX90A-NEXT: v_pk_max_f16 v4, v3, v2
12094 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
12095 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12096 ; GFX90A-NEXT: buffer_wbinvl1
12097 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
12098 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12099 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
12100 ; GFX90A-NEXT: s_cbranch_execnz .LBB47_1
12101 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
12102 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
12103 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
12104 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
12105 ;
12106 ; GFX908-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
12107 ; GFX908: ; %bb.0:
12108 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12109 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
12110 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
12111 ; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
12112 ; GFX908-NEXT: .LBB47_1: ; %atomicrmw.start
12113 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
12114 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12115 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
12116 ; GFX908-NEXT: v_pk_max_f16 v3, v4, v4
12117 ; GFX908-NEXT: v_pk_max_f16 v3, v3, v2
12118 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
12119 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12120 ; GFX908-NEXT: buffer_wbinvl1
12121 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
12122 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12123 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
12124 ; GFX908-NEXT: s_cbranch_execnz .LBB47_1
12125 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
12126 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
12127 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
12128 ; GFX908-NEXT: s_setpc_b64 s[30:31]
12129 ;
12130 ; GFX8-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
12131 ; GFX8: ; %bb.0:
12132 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12133 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
12134 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
12135 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
12136 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
12137 ; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
12138 ; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
12139 ; GFX8-NEXT: .LBB47_1: ; %atomicrmw.start
12140 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
12141 ; GFX8-NEXT: s_waitcnt vmcnt(0)
12142 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
12143 ; GFX8-NEXT: v_max_f16_sdwa v0, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
12144 ; GFX8-NEXT: v_max_f16_e32 v5, v6, v6
12145 ; GFX8-NEXT: v_max_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
12146 ; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
12147 ; GFX8-NEXT: v_or_b32_e32 v5, v5, v0
12148 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
12149 ; GFX8-NEXT: s_waitcnt vmcnt(0)
12150 ; GFX8-NEXT: buffer_wbinvl1
12151 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
12152 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12153 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
12154 ; GFX8-NEXT: s_cbranch_execnz .LBB47_1
12155 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
12156 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
12157 ; GFX8-NEXT: s_setpc_b64 s[30:31]
12158 ;
12159 ; GFX7-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
12160 ; GFX7: ; %bb.0:
12161 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12162 ; GFX7-NEXT: s_mov_b32 s6, 0
12163 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
12164 ; GFX7-NEXT: s_mov_b32 s4, s6
12165 ; GFX7-NEXT: s_mov_b32 s5, s6
12166 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
12167 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
12168 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v2
12169 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
12170 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v3
12171 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12172 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
12173 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v5
12174 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
12175 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v6
12176 ; GFX7-NEXT: .LBB47_1: ; %atomicrmw.start
12177 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
12178 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
12179 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
12180 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v3
12181 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
12182 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
12183 ; GFX7-NEXT: v_max_f32_e32 v6, v6, v4
12184 ; GFX7-NEXT: v_max_f32_e32 v7, v7, v5
12185 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6
12186 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v7
12187 ; GFX7-NEXT: v_or_b32_e32 v7, v2, v3
12188 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
12189 ; GFX7-NEXT: v_or_b32_e32 v6, v8, v2
12190 ; GFX7-NEXT: v_mov_b32_e32 v9, v7
12191 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
12192 ; GFX7-NEXT: buffer_atomic_cmpswap v[8:9], v[0:1], s[4:7], 0 addr64 offset:2044 glc
12193 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12194 ; GFX7-NEXT: buffer_wbinvl1
12195 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
12196 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v8
12197 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
12198 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
12199 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
12200 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
12201 ; GFX7-NEXT: s_cbranch_execnz .LBB47_1
12202 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
12203 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
12204 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
12205 ; GFX7-NEXT: v_mov_b32_e32 v1, v3
12206 ; GFX7-NEXT: s_setpc_b64 s[30:31]
12207 ;
12208 ; GFX6-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
12209 ; GFX6: ; %bb.0:
12210 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12211 ; GFX6-NEXT: s_mov_b32 s6, 0
12212 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
12213 ; GFX6-NEXT: s_mov_b32 s4, s6
12214 ; GFX6-NEXT: s_mov_b32 s5, s6
12215 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
12216 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
12217 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v2
12218 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
12219 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v3
12220 ; GFX6-NEXT: s_waitcnt vmcnt(0)
12221 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
12222 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v5
12223 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
12224 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v6
12225 ; GFX6-NEXT: .LBB47_1: ; %atomicrmw.start
12226 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
12227 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
12228 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
12229 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v3
12230 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
12231 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
12232 ; GFX6-NEXT: v_max_f32_e32 v6, v6, v4
12233 ; GFX6-NEXT: v_max_f32_e32 v7, v7, v5
12234 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
12235 ; GFX6-NEXT: s_waitcnt expcnt(0)
12236 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v7
12237 ; GFX6-NEXT: v_or_b32_e32 v7, v2, v3
12238 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v6
12239 ; GFX6-NEXT: v_or_b32_e32 v6, v8, v2
12240 ; GFX6-NEXT: v_mov_b32_e32 v9, v7
12241 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
12242 ; GFX6-NEXT: buffer_atomic_cmpswap v[8:9], v[0:1], s[4:7], 0 addr64 offset:2044 glc
12243 ; GFX6-NEXT: s_waitcnt vmcnt(0)
12244 ; GFX6-NEXT: buffer_wbinvl1
12245 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
12246 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v8
12247 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
12248 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
12249 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
12250 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
12251 ; GFX6-NEXT: s_cbranch_execnz .LBB47_1
12252 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
12253 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
12254 ; GFX6-NEXT: v_mov_b32_e32 v0, v2
12255 ; GFX6-NEXT: v_mov_b32_e32 v1, v3
12256 ; GFX6-NEXT: s_waitcnt expcnt(0)
12257 ; GFX6-NEXT: s_setpc_b64 s[30:31]
12258 %gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 511
12259 %result = atomicrmw fmax ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
12260 ret <2 x half> %result
12261 }
12263 define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
12264 ; GFX12-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
12265 ; GFX12: ; %bb.0:
12266 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
12267 ; GFX12-NEXT: s_wait_expcnt 0x0
12268 ; GFX12-NEXT: s_wait_samplecnt 0x0
12269 ; GFX12-NEXT: s_wait_bvhcnt 0x0
12270 ; GFX12-NEXT: s_wait_kmcnt 0x0
12271 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
12272 ; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
12273 ; GFX12-NEXT: s_mov_b32 s0, 0
12274 ; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start
12275 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
12276 ; GFX12-NEXT: s_wait_loadcnt 0x0
12277 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
12278 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
12279 ; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
12280 ; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v2
12281 ; GFX12-NEXT: s_wait_storecnt 0x0
12282 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
12283 ; GFX12-NEXT: s_wait_loadcnt 0x0
12284 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
12285 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
12286 ; GFX12-NEXT: s_wait_alu 0xfffe
12287 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
12288 ; GFX12-NEXT: s_wait_alu 0xfffe
12289 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
12290 ; GFX12-NEXT: s_cbranch_execnz .LBB48_1
12291 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
12292 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
12293 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
12294 ; GFX12-NEXT: s_wait_alu 0xfffe
12295 ; GFX12-NEXT: s_setpc_b64 s[30:31]
12296 ;
12297 ; GFX940-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
12298 ; GFX940: ; %bb.0:
12299 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12300 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
12301 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
12302 ; GFX940-NEXT: v_pk_max_f16 v2, v2, v2
12303 ; GFX940-NEXT: .LBB48_1: ; %atomicrmw.start
12304 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
12305 ; GFX940-NEXT: s_waitcnt vmcnt(0)
12306 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
12307 ; GFX940-NEXT: v_pk_max_f16 v3, v5, v5
12308 ; GFX940-NEXT: s_nop 0
12309 ; GFX940-NEXT: v_pk_max_f16 v4, v3, v2
12310 ; GFX940-NEXT: buffer_wbl2 sc1
12311 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
12312 ; GFX940-NEXT: s_waitcnt vmcnt(0)
12313 ; GFX940-NEXT: buffer_inv sc1
12314 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
12315 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
12316 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
12317 ; GFX940-NEXT: s_cbranch_execnz .LBB48_1
12318 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
12319 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
12320 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
12321 ; GFX940-NEXT: s_setpc_b64 s[30:31]
12322 ;
12323 ; GFX11-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
12324 ; GFX11: ; %bb.0:
12325 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12326 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
12327 ; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
12328 ; GFX11-NEXT: s_mov_b32 s0, 0
12329 ; GFX11-NEXT: .LBB48_1: ; %atomicrmw.start
12330 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
12331 ; GFX11-NEXT: s_waitcnt vmcnt(0)
12332 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
12333 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
12334 ; GFX11-NEXT: v_pk_max_f16 v3, v4, v4
12335 ; GFX11-NEXT: v_pk_max_f16 v3, v3, v2
12336 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
12337 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
12338 ; GFX11-NEXT: s_waitcnt vmcnt(0)
12339 ; GFX11-NEXT: buffer_gl1_inv
12340 ; GFX11-NEXT: buffer_gl0_inv
12341 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
12342 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
12343 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
12344 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
12345 ; GFX11-NEXT: s_cbranch_execnz .LBB48_1
12346 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
12347 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
12348 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
12349 ; GFX11-NEXT: s_setpc_b64 s[30:31]
12350 ;
12351 ; GFX10-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
12352 ; GFX10: ; %bb.0:
12353 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12354 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
12355 ; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
12356 ; GFX10-NEXT: s_mov_b32 s4, 0
12357 ; GFX10-NEXT: .LBB48_1: ; %atomicrmw.start
12358 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
12359 ; GFX10-NEXT: s_waitcnt vmcnt(0)
12360 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
12361 ; GFX10-NEXT: v_pk_max_f16 v3, v4, v4
12362 ; GFX10-NEXT: v_pk_max_f16 v3, v3, v2
12363 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
12364 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
12365 ; GFX10-NEXT: s_waitcnt vmcnt(0)
12366 ; GFX10-NEXT: buffer_gl1_inv
12367 ; GFX10-NEXT: buffer_gl0_inv
12368 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
12369 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
12370 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
12371 ; GFX10-NEXT: s_cbranch_execnz .LBB48_1
12372 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
12373 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
12374 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
12375 ; GFX10-NEXT: s_setpc_b64 s[30:31]
12376 ;
12377 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
12378 ; GFX90A: ; %bb.0:
12379 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12380 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
12381 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
12382 ; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
12383 ; GFX90A-NEXT: .LBB48_1: ; %atomicrmw.start
12384 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
12385 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12386 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
12387 ; GFX90A-NEXT: v_pk_max_f16 v3, v5, v5
12388 ; GFX90A-NEXT: v_pk_max_f16 v4, v3, v2
12389 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
12390 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12391 ; GFX90A-NEXT: buffer_wbinvl1
12392 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
12393 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12394 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
12395 ; GFX90A-NEXT: s_cbranch_execnz .LBB48_1
12396 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
12397 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
12398 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
12399 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
12400 ;
12401 ; GFX908-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
12402 ; GFX908: ; %bb.0:
12403 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12404 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
12405 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
12406 ; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
12407 ; GFX908-NEXT: .LBB48_1: ; %atomicrmw.start
12408 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
12409 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12410 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
12411 ; GFX908-NEXT: v_pk_max_f16 v3, v4, v4
12412 ; GFX908-NEXT: v_pk_max_f16 v3, v3, v2
12413 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
12414 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12415 ; GFX908-NEXT: buffer_wbinvl1
12416 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
12417 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12418 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
12419 ; GFX908-NEXT: s_cbranch_execnz .LBB48_1
12420 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
12421 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
12422 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
12423 ; GFX908-NEXT: s_setpc_b64 s[30:31]
12424 ;
12425 ; GFX8-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
12426 ; GFX8: ; %bb.0:
12427 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12428 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
12429 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
12430 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
12431 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
12432 ; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
12433 ; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
12434 ; GFX8-NEXT: .LBB48_1: ; %atomicrmw.start
12435 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
12436 ; GFX8-NEXT: s_waitcnt vmcnt(0)
12437 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
12438 ; GFX8-NEXT: v_max_f16_sdwa v0, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
12439 ; GFX8-NEXT: v_max_f16_e32 v5, v6, v6
12440 ; GFX8-NEXT: v_max_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
12441 ; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
12442 ; GFX8-NEXT: v_or_b32_e32 v5, v5, v0
12443 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
12444 ; GFX8-NEXT: s_waitcnt vmcnt(0)
12445 ; GFX8-NEXT: buffer_wbinvl1
12446 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
12447 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12448 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
12449 ; GFX8-NEXT: s_cbranch_execnz .LBB48_1
12450 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
12451 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
12452 ; GFX8-NEXT: s_setpc_b64 s[30:31]
12453 ;
12454 ; GFX7-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
12455 ; GFX7: ; %bb.0:
12456 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12457 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
12458 ; GFX7-NEXT: s_mov_b32 s5, -1
12459 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
12460 ; GFX7-NEXT: s_mov_b32 s6, 0
12461 ; GFX7-NEXT: buffer_load_dword v6, v[0:1], s[4:7], 0 addr64
12462 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
12463 ; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v3
12464 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v2
12465 ; GFX7-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
12466 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v0
12467 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
12468 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
12469 ; GFX7-NEXT: s_mov_b32 s4, s6
12470 ; GFX7-NEXT: s_mov_b32 s5, s6
12471 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12472 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v6
12473 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v6
12474 ; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
12475 ; GFX7-NEXT: .LBB48_1: ; %atomicrmw.start
12476 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
12477 ; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
12478 ; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
12479 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v1
12480 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v0
12481 ; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
12482 ; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
12483 ; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
12484 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6
12485 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v7
12486 ; GFX7-NEXT: v_or_b32_e32 v7, v0, v1
12487 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v6
12488 ; GFX7-NEXT: v_or_b32_e32 v6, v8, v0
12489 ; GFX7-NEXT: v_mov_b32_e32 v9, v7
12490 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
12491 ; GFX7-NEXT: buffer_atomic_cmpswap v[8:9], v[4:5], s[4:7], 0 addr64 glc
12492 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12493 ; GFX7-NEXT: buffer_wbinvl1
12494 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v8
12495 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v8
12496 ; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
12497 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
12498 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
12499 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
12500 ; GFX7-NEXT: s_cbranch_execnz .LBB48_1
12501 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
12502 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
12503 ; GFX7-NEXT: s_setpc_b64 s[30:31]
12504 ;
12505 ; GFX6-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
12506 ; GFX6: ; %bb.0:
12507 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12508 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
12509 ; GFX6-NEXT: s_mov_b32 s5, -1
12510 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
12511 ; GFX6-NEXT: s_mov_b32 s6, 0
12512 ; GFX6-NEXT: buffer_load_dword v6, v[0:1], s[4:7], 0 addr64
12513 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
12514 ; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v3
12515 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v2
12516 ; GFX6-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
12517 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v0
12518 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
12519 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
12520 ; GFX6-NEXT: s_mov_b32 s4, s6
12521 ; GFX6-NEXT: s_mov_b32 s5, s6
12522 ; GFX6-NEXT: s_waitcnt vmcnt(0)
12523 ; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v6
12524 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v6
12525 ; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
12526 ; GFX6-NEXT: .LBB48_1: ; %atomicrmw.start
12527 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
12528 ; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
12529 ; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
12530 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v1
12531 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v0
12532 ; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
12533 ; GFX6-NEXT: v_max_f32_e32 v6, v6, v2
12534 ; GFX6-NEXT: v_max_f32_e32 v7, v7, v3
12535 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
12536 ; GFX6-NEXT: s_waitcnt expcnt(0)
12537 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v7
12538 ; GFX6-NEXT: v_or_b32_e32 v7, v0, v1
12539 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v6
12540 ; GFX6-NEXT: v_or_b32_e32 v6, v8, v0
12541 ; GFX6-NEXT: v_mov_b32_e32 v9, v7
12542 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
12543 ; GFX6-NEXT: buffer_atomic_cmpswap v[8:9], v[4:5], s[4:7], 0 addr64 glc
12544 ; GFX6-NEXT: s_waitcnt vmcnt(0)
12545 ; GFX6-NEXT: buffer_wbinvl1
12546 ; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v8
12547 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v8
12548 ; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
12549 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
12550 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
12551 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
12552 ; GFX6-NEXT: s_cbranch_execnz .LBB48_1
12553 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
12554 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
12555 ; GFX6-NEXT: s_waitcnt expcnt(0)
12556 ; GFX6-NEXT: s_setpc_b64 s[30:31]
12557 %gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 -512
12558 %result = atomicrmw fmax ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
12559 ret <2 x half> %result
12560 }
12562 define void @global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
12563 ; GFX12-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
12564 ; GFX12: ; %bb.0:
12565 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
12566 ; GFX12-NEXT: s_wait_expcnt 0x0
12567 ; GFX12-NEXT: s_wait_samplecnt 0x0
12568 ; GFX12-NEXT: s_wait_bvhcnt 0x0
12569 ; GFX12-NEXT: s_wait_kmcnt 0x0
12570 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
12571 ; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
12572 ; GFX12-NEXT: s_mov_b32 s0, 0
12573 ; GFX12-NEXT: .LBB49_1: ; %atomicrmw.start
12574 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
12575 ; GFX12-NEXT: s_wait_loadcnt 0x0
12576 ; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
12577 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
12578 ; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v4
12579 ; GFX12-NEXT: s_wait_storecnt 0x0
12580 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
12581 ; GFX12-NEXT: s_wait_loadcnt 0x0
12582 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
12583 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
12584 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
12585 ; GFX12-NEXT: s_wait_alu 0xfffe
12586 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
12587 ; GFX12-NEXT: s_wait_alu 0xfffe
12588 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
12589 ; GFX12-NEXT: s_cbranch_execnz .LBB49_1
12590 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
12591 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
12592 ; GFX12-NEXT: s_wait_alu 0xfffe
12593 ; GFX12-NEXT: s_setpc_b64 s[30:31]
12594 ;
12595 ; GFX940-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
12596 ; GFX940: ; %bb.0:
12597 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12598 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
12599 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
12600 ; GFX940-NEXT: v_pk_max_f16 v4, v2, v2
12601 ; GFX940-NEXT: .LBB49_1: ; %atomicrmw.start
12602 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
12603 ; GFX940-NEXT: s_waitcnt vmcnt(0)
12604 ; GFX940-NEXT: v_pk_max_f16 v2, v3, v3
12605 ; GFX940-NEXT: s_nop 0
12606 ; GFX940-NEXT: v_pk_max_f16 v2, v2, v4
12607 ; GFX940-NEXT: buffer_wbl2 sc1
12608 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
12609 ; GFX940-NEXT: s_waitcnt vmcnt(0)
12610 ; GFX940-NEXT: buffer_inv sc1
12611 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
12612 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
12613 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
12614 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
12615 ; GFX940-NEXT: s_cbranch_execnz .LBB49_1
12616 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
12617 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
12618 ; GFX940-NEXT: s_setpc_b64 s[30:31]
12619 ;
12620 ; GFX11-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
12621 ; GFX11: ; %bb.0:
12622 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12623 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
12624 ; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
12625 ; GFX11-NEXT: s_mov_b32 s0, 0
12626 ; GFX11-NEXT: .LBB49_1: ; %atomicrmw.start
12627 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
12628 ; GFX11-NEXT: s_waitcnt vmcnt(0)
12629 ; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
12630 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
12631 ; GFX11-NEXT: v_pk_max_f16 v2, v2, v4
12632 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
12633 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
12634 ; GFX11-NEXT: s_waitcnt vmcnt(0)
12635 ; GFX11-NEXT: buffer_gl1_inv
12636 ; GFX11-NEXT: buffer_gl0_inv
12637 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
12638 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
12639 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
12640 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
12641 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
12642 ; GFX11-NEXT: s_cbranch_execnz .LBB49_1
12643 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
12644 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
12645 ; GFX11-NEXT: s_setpc_b64 s[30:31]
12646 ;
12647 ; GFX10-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
12648 ; GFX10: ; %bb.0:
12649 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12650 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
12651 ; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
12652 ; GFX10-NEXT: s_mov_b32 s4, 0
12653 ; GFX10-NEXT: .LBB49_1: ; %atomicrmw.start
12654 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
12655 ; GFX10-NEXT: s_waitcnt vmcnt(0)
12656 ; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
12657 ; GFX10-NEXT: v_pk_max_f16 v2, v2, v4
12658 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
12659 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
12660 ; GFX10-NEXT: s_waitcnt vmcnt(0)
12661 ; GFX10-NEXT: buffer_gl1_inv
12662 ; GFX10-NEXT: buffer_gl0_inv
12663 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
12664 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
12665 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
12666 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
12667 ; GFX10-NEXT: s_cbranch_execnz .LBB49_1
12668 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
12669 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
12670 ; GFX10-NEXT: s_setpc_b64 s[30:31]
12671 ;
12672 ; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
12673 ; GFX90A: ; %bb.0:
12674 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12675 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
12676 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
12677 ; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
12678 ; GFX90A-NEXT: .LBB49_1: ; %atomicrmw.start
12679 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
12680 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12681 ; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
12682 ; GFX90A-NEXT: v_pk_max_f16 v2, v2, v4
12683 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
12684 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12685 ; GFX90A-NEXT: buffer_wbinvl1
12686 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
12687 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12688 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
12689 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
12690 ; GFX90A-NEXT: s_cbranch_execnz .LBB49_1
12691 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
12692 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
12693 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
12694 ;
12695 ; GFX908-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
12696 ; GFX908: ; %bb.0:
12697 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12698 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
12699 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
12700 ; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
12701 ; GFX908-NEXT: .LBB49_1: ; %atomicrmw.start
12702 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
12703 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12704 ; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
12705 ; GFX908-NEXT: v_pk_max_f16 v2, v2, v4
12706 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
12707 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12708 ; GFX908-NEXT: buffer_wbinvl1
12709 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
12710 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12711 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
12712 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
12713 ; GFX908-NEXT: s_cbranch_execnz .LBB49_1
12714 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
12715 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
12716 ; GFX908-NEXT: s_setpc_b64 s[30:31]
12717 ;
12718 ; GFX8-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
12719 ; GFX8: ; %bb.0:
12720 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12721 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
12722 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
12723 ; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
12724 ; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
12725 ; GFX8-NEXT: .LBB49_1: ; %atomicrmw.start
12726 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
12727 ; GFX8-NEXT: s_waitcnt vmcnt(0)
12728 ; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
12729 ; GFX8-NEXT: v_max_f16_e32 v6, v3, v3
12730 ; GFX8-NEXT: v_max_f16_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
12731 ; GFX8-NEXT: v_max_f16_e32 v6, v6, v5
12732 ; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
12733 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12734 ; GFX8-NEXT: s_waitcnt vmcnt(0)
12735 ; GFX8-NEXT: buffer_wbinvl1
12736 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
12737 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12738 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
12739 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
12740 ; GFX8-NEXT: s_cbranch_execnz .LBB49_1
12741 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
12742 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
12743 ; GFX8-NEXT: s_setpc_b64 s[30:31]
12744 ;
12745 ; GFX7-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
12746 ; GFX7: ; %bb.0:
12747 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12748 ; GFX7-NEXT: s_mov_b32 s6, 0
12749 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
12750 ; GFX7-NEXT: s_mov_b32 s4, s6
12751 ; GFX7-NEXT: s_mov_b32 s5, s6
12752 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
12753 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
12754 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v2
12755 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
12756 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
12757 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12758 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
12759 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
12760 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
12761 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
12762 ; GFX7-NEXT: .LBB49_1: ; %atomicrmw.start
12763 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
12764 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
12765 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
12766 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v5
12767 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v4
12768 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
12769 ; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
12770 ; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
12771 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v6
12772 ; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v7
12773 ; GFX7-NEXT: v_or_b32_e32 v6, v4, v5
12774 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v8
12775 ; GFX7-NEXT: v_or_b32_e32 v5, v7, v4
12776 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
12777 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
12778 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
12779 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12780 ; GFX7-NEXT: buffer_wbinvl1
12781 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v7
12782 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v7
12783 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
12784 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
12785 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
12786 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
12787 ; GFX7-NEXT: s_cbranch_execnz .LBB49_1
12788 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
12789 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
12790 ; GFX7-NEXT: s_setpc_b64 s[30:31]
12791 ;
12792 ; GFX6-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
12793 ; GFX6: ; %bb.0:
12794 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12795 ; GFX6-NEXT: s_mov_b32 s6, 0
12796 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
12797 ; GFX6-NEXT: s_mov_b32 s4, s6
12798 ; GFX6-NEXT: s_mov_b32 s5, s6
12799 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
12800 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
12801 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v2
12802 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
12803 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v3
12804 ; GFX6-NEXT: s_waitcnt vmcnt(0)
12805 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
12806 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v5
12807 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v3
12808 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v6
12809 ; GFX6-NEXT: .LBB49_1: ; %atomicrmw.start
12810 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
12811 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
12812 ; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
12813 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v5
12814 ; GFX6-NEXT: s_waitcnt expcnt(0)
12815 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v4
12816 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
12817 ; GFX6-NEXT: v_max_f32_e32 v6, v6, v2
12818 ; GFX6-NEXT: v_max_f32_e32 v7, v7, v3
12819 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v6
12820 ; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v7
12821 ; GFX6-NEXT: v_or_b32_e32 v6, v4, v5
12822 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v8
12823 ; GFX6-NEXT: v_or_b32_e32 v5, v7, v4
12824 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
12825 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
12826 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
12827 ; GFX6-NEXT: s_waitcnt vmcnt(0)
12828 ; GFX6-NEXT: buffer_wbinvl1
12829 ; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v7
12830 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v7
12831 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
12832 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
12833 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
12834 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
12835 ; GFX6-NEXT: s_cbranch_execnz .LBB49_1
12836 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
12837 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
12838 ; GFX6-NEXT: s_waitcnt expcnt(0)
12839 ; GFX6-NEXT: s_setpc_b64 s[30:31]
12840 %unused = atomicrmw fmax ptr addrspace(1) %ptr, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
12841 ret void
12842 }
12844 define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
12845 ; GFX12-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
12846 ; GFX12: ; %bb.0:
12847 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
12848 ; GFX12-NEXT: s_wait_expcnt 0x0
12849 ; GFX12-NEXT: s_wait_samplecnt 0x0
12850 ; GFX12-NEXT: s_wait_bvhcnt 0x0
12851 ; GFX12-NEXT: s_wait_kmcnt 0x0
12852 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
12853 ; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
12854 ; GFX12-NEXT: s_mov_b32 s0, 0
12855 ; GFX12-NEXT: .LBB50_1: ; %atomicrmw.start
12856 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
12857 ; GFX12-NEXT: s_wait_loadcnt 0x0
12858 ; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
12859 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
12860 ; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v4
12861 ; GFX12-NEXT: s_wait_storecnt 0x0
12862 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
12863 ; GFX12-NEXT: s_wait_loadcnt 0x0
12864 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
12865 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
12866 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
12867 ; GFX12-NEXT: s_wait_alu 0xfffe
12868 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
12869 ; GFX12-NEXT: s_wait_alu 0xfffe
12870 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
12871 ; GFX12-NEXT: s_cbranch_execnz .LBB50_1
12872 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
12873 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
12874 ; GFX12-NEXT: s_wait_alu 0xfffe
12875 ; GFX12-NEXT: s_setpc_b64 s[30:31]
12876 ;
12877 ; GFX940-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
12878 ; GFX940: ; %bb.0:
12879 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12880 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
12881 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
12882 ; GFX940-NEXT: v_pk_max_f16 v4, v2, v2
12883 ; GFX940-NEXT: .LBB50_1: ; %atomicrmw.start
12884 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
12885 ; GFX940-NEXT: s_waitcnt vmcnt(0)
12886 ; GFX940-NEXT: v_pk_max_f16 v2, v3, v3
12887 ; GFX940-NEXT: s_nop 0
12888 ; GFX940-NEXT: v_pk_max_f16 v2, v2, v4
12889 ; GFX940-NEXT: buffer_wbl2 sc1
12890 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
12891 ; GFX940-NEXT: s_waitcnt vmcnt(0)
12892 ; GFX940-NEXT: buffer_inv sc1
12893 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
12894 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
12895 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
12896 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
12897 ; GFX940-NEXT: s_cbranch_execnz .LBB50_1
12898 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
12899 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
12900 ; GFX940-NEXT: s_setpc_b64 s[30:31]
12901 ;
12902 ; GFX11-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
12903 ; GFX11: ; %bb.0:
12904 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12905 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
12906 ; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
12907 ; GFX11-NEXT: s_mov_b32 s0, 0
12908 ; GFX11-NEXT: .LBB50_1: ; %atomicrmw.start
12909 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
12910 ; GFX11-NEXT: s_waitcnt vmcnt(0)
12911 ; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
12912 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
12913 ; GFX11-NEXT: v_pk_max_f16 v2, v2, v4
12914 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
12915 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
12916 ; GFX11-NEXT: s_waitcnt vmcnt(0)
12917 ; GFX11-NEXT: buffer_gl1_inv
12918 ; GFX11-NEXT: buffer_gl0_inv
12919 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
12920 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
12921 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
12922 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
12923 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
12924 ; GFX11-NEXT: s_cbranch_execnz .LBB50_1
12925 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
12926 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
12927 ; GFX11-NEXT: s_setpc_b64 s[30:31]
12928 ;
12929 ; GFX10-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
12930 ; GFX10: ; %bb.0:
12931 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12932 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
12933 ; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
12934 ; GFX10-NEXT: s_mov_b32 s4, 0
12935 ; GFX10-NEXT: .LBB50_1: ; %atomicrmw.start
12936 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
12937 ; GFX10-NEXT: s_waitcnt vmcnt(0)
12938 ; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
12939 ; GFX10-NEXT: v_pk_max_f16 v2, v2, v4
12940 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
12941 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
12942 ; GFX10-NEXT: s_waitcnt vmcnt(0)
12943 ; GFX10-NEXT: buffer_gl1_inv
12944 ; GFX10-NEXT: buffer_gl0_inv
12945 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
12946 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
12947 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
12948 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
12949 ; GFX10-NEXT: s_cbranch_execnz .LBB50_1
12950 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
12951 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
12952 ; GFX10-NEXT: s_setpc_b64 s[30:31]
12953 ;
12954 ; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
12955 ; GFX90A: ; %bb.0:
12956 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12957 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
12958 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
12959 ; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
12960 ; GFX90A-NEXT: .LBB50_1: ; %atomicrmw.start
12961 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
12962 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12963 ; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
12964 ; GFX90A-NEXT: v_pk_max_f16 v2, v2, v4
12965 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
12966 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
12967 ; GFX90A-NEXT: buffer_wbinvl1
12968 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
12969 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12970 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
12971 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
12972 ; GFX90A-NEXT: s_cbranch_execnz .LBB50_1
12973 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
12974 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
12975 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
12976 ;
12977 ; GFX908-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
12978 ; GFX908: ; %bb.0:
12979 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12980 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
12981 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
12982 ; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
12983 ; GFX908-NEXT: .LBB50_1: ; %atomicrmw.start
12984 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
12985 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12986 ; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
12987 ; GFX908-NEXT: v_pk_max_f16 v2, v2, v4
12988 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
12989 ; GFX908-NEXT: s_waitcnt vmcnt(0)
12990 ; GFX908-NEXT: buffer_wbinvl1
12991 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
12992 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
12993 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
12994 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
12995 ; GFX908-NEXT: s_cbranch_execnz .LBB50_1
12996 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
12997 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
12998 ; GFX908-NEXT: s_setpc_b64 s[30:31]
12999 ;
13000 ; GFX8-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
13001 ; GFX8: ; %bb.0:
13002 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13003 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
13004 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
13005 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
13006 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
13007 ; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
13008 ; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
13009 ; GFX8-NEXT: .LBB50_1: ; %atomicrmw.start
13010 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
13011 ; GFX8-NEXT: s_waitcnt vmcnt(0)
13012 ; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
13013 ; GFX8-NEXT: v_max_f16_e32 v6, v3, v3
13014 ; GFX8-NEXT: v_max_f16_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
13015 ; GFX8-NEXT: v_max_f16_e32 v6, v6, v5
13016 ; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
13017 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13018 ; GFX8-NEXT: s_waitcnt vmcnt(0)
13019 ; GFX8-NEXT: buffer_wbinvl1
13020 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
13021 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13022 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
13023 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
13024 ; GFX8-NEXT: s_cbranch_execnz .LBB50_1
13025 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
13026 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
13027 ; GFX8-NEXT: s_setpc_b64 s[30:31]
13028 ;
13029 ; GFX7-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
13030 ; GFX7: ; %bb.0:
13031 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13032 ; GFX7-NEXT: s_mov_b32 s6, 0
13033 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
13034 ; GFX7-NEXT: s_mov_b32 s4, s6
13035 ; GFX7-NEXT: s_mov_b32 s5, s6
13036 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
13037 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
13038 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v2
13039 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
13040 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
13041 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13042 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
13043 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
13044 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
13045 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
13046 ; GFX7-NEXT: .LBB50_1: ; %atomicrmw.start
13047 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
13048 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
13049 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
13050 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v5
13051 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v4
13052 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
13053 ; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
13054 ; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
13055 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v6
13056 ; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v7
13057 ; GFX7-NEXT: v_or_b32_e32 v6, v4, v5
13058 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v8
13059 ; GFX7-NEXT: v_or_b32_e32 v5, v7, v4
13060 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
13061 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
13062 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 offset:2044 glc
13063 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13064 ; GFX7-NEXT: buffer_wbinvl1
13065 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v7
13066 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v7
13067 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
13068 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
13069 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
13070 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
13071 ; GFX7-NEXT: s_cbranch_execnz .LBB50_1
13072 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
13073 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
13074 ; GFX7-NEXT: s_setpc_b64 s[30:31]
13075 ;
13076 ; GFX6-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
13077 ; GFX6: ; %bb.0:
13078 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13079 ; GFX6-NEXT: s_mov_b32 s6, 0
13080 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
13081 ; GFX6-NEXT: s_mov_b32 s4, s6
13082 ; GFX6-NEXT: s_mov_b32 s5, s6
13083 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
13084 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
13085 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v2
13086 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
13087 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v3
13088 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13089 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
13090 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v5
13091 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v3
13092 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v6
13093 ; GFX6-NEXT: .LBB50_1: ; %atomicrmw.start
13094 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
13095 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
13096 ; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
13097 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v5
13098 ; GFX6-NEXT: s_waitcnt expcnt(0)
13099 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v4
13100 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
13101 ; GFX6-NEXT: v_max_f32_e32 v6, v6, v2
13102 ; GFX6-NEXT: v_max_f32_e32 v7, v7, v3
13103 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v6
13104 ; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v7
13105 ; GFX6-NEXT: v_or_b32_e32 v6, v4, v5
13106 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v8
13107 ; GFX6-NEXT: v_or_b32_e32 v5, v7, v4
13108 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
13109 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
13110 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 offset:2044 glc
13111 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13112 ; GFX6-NEXT: buffer_wbinvl1
13113 ; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v7
13114 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v7
13115 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
13116 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
13117 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
13118 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
13119 ; GFX6-NEXT: s_cbranch_execnz .LBB50_1
13120 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
13121 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
13122 ; GFX6-NEXT: s_waitcnt expcnt(0)
13123 ; GFX6-NEXT: s_setpc_b64 s[30:31]
13124 %gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 511
13125 %unused = atomicrmw fmax ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
13126 ret void
13127 }
13129 define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
13130 ; GFX12-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
13131 ; GFX12: ; %bb.0:
13132 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
13133 ; GFX12-NEXT: s_wait_expcnt 0x0
13134 ; GFX12-NEXT: s_wait_samplecnt 0x0
13135 ; GFX12-NEXT: s_wait_bvhcnt 0x0
13136 ; GFX12-NEXT: s_wait_kmcnt 0x0
13137 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
13138 ; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
13139 ; GFX12-NEXT: s_mov_b32 s0, 0
13140 ; GFX12-NEXT: .LBB51_1: ; %atomicrmw.start
13141 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
13142 ; GFX12-NEXT: s_wait_loadcnt 0x0
13143 ; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
13144 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
13145 ; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v4
13146 ; GFX12-NEXT: s_wait_storecnt 0x0
13147 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
13148 ; GFX12-NEXT: s_wait_loadcnt 0x0
13149 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
13150 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
13151 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
13152 ; GFX12-NEXT: s_wait_alu 0xfffe
13153 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
13154 ; GFX12-NEXT: s_wait_alu 0xfffe
13155 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
13156 ; GFX12-NEXT: s_cbranch_execnz .LBB51_1
13157 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
13158 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
13159 ; GFX12-NEXT: s_wait_alu 0xfffe
13160 ; GFX12-NEXT: s_setpc_b64 s[30:31]
13161 ;
13162 ; GFX940-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
13163 ; GFX940: ; %bb.0:
13164 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13165 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
13166 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
13167 ; GFX940-NEXT: v_pk_max_f16 v4, v2, v2
13168 ; GFX940-NEXT: .LBB51_1: ; %atomicrmw.start
13169 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
13170 ; GFX940-NEXT: s_waitcnt vmcnt(0)
13171 ; GFX940-NEXT: v_pk_max_f16 v2, v3, v3
13172 ; GFX940-NEXT: s_nop 0
13173 ; GFX940-NEXT: v_pk_max_f16 v2, v2, v4
13174 ; GFX940-NEXT: buffer_wbl2 sc1
13175 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
13176 ; GFX940-NEXT: s_waitcnt vmcnt(0)
13177 ; GFX940-NEXT: buffer_inv sc1
13178 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
13179 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
13180 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
13181 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
13182 ; GFX940-NEXT: s_cbranch_execnz .LBB51_1
13183 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
13184 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
13185 ; GFX940-NEXT: s_setpc_b64 s[30:31]
13186 ;
13187 ; GFX11-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
13188 ; GFX11: ; %bb.0:
13189 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13190 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
13191 ; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
13192 ; GFX11-NEXT: s_mov_b32 s0, 0
13193 ; GFX11-NEXT: .LBB51_1: ; %atomicrmw.start
13194 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
13195 ; GFX11-NEXT: s_waitcnt vmcnt(0)
13196 ; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
13197 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
13198 ; GFX11-NEXT: v_pk_max_f16 v2, v2, v4
13199 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
13200 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 glc
13201 ; GFX11-NEXT: s_waitcnt vmcnt(0)
13202 ; GFX11-NEXT: buffer_gl1_inv
13203 ; GFX11-NEXT: buffer_gl0_inv
13204 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
13205 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
13206 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
13207 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
13208 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
13209 ; GFX11-NEXT: s_cbranch_execnz .LBB51_1
13210 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
13211 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
13212 ; GFX11-NEXT: s_setpc_b64 s[30:31]
13213 ;
13214 ; GFX10-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
13215 ; GFX10: ; %bb.0:
13216 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13217 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
13218 ; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
13219 ; GFX10-NEXT: s_mov_b32 s4, 0
13220 ; GFX10-NEXT: .LBB51_1: ; %atomicrmw.start
13221 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
13222 ; GFX10-NEXT: s_waitcnt vmcnt(0)
13223 ; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
13224 ; GFX10-NEXT: v_pk_max_f16 v2, v2, v4
13225 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
13226 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
13227 ; GFX10-NEXT: s_waitcnt vmcnt(0)
13228 ; GFX10-NEXT: buffer_gl1_inv
13229 ; GFX10-NEXT: buffer_gl0_inv
13230 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
13231 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
13232 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
13233 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
13234 ; GFX10-NEXT: s_cbranch_execnz .LBB51_1
13235 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
13236 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
13237 ; GFX10-NEXT: s_setpc_b64 s[30:31]
13238 ;
13239 ; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
13240 ; GFX90A: ; %bb.0:
13241 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13242 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
13243 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
13244 ; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
13245 ; GFX90A-NEXT: .LBB51_1: ; %atomicrmw.start
13246 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
13247 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
13248 ; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
13249 ; GFX90A-NEXT: v_pk_max_f16 v2, v2, v4
13250 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
13251 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
13252 ; GFX90A-NEXT: buffer_wbinvl1
13253 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
13254 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13255 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
13256 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
13257 ; GFX90A-NEXT: s_cbranch_execnz .LBB51_1
13258 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
13259 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
13260 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
13261 ;
13262 ; GFX908-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
13263 ; GFX908: ; %bb.0:
13264 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13265 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
13266 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
13267 ; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
13268 ; GFX908-NEXT: .LBB51_1: ; %atomicrmw.start
13269 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
13270 ; GFX908-NEXT: s_waitcnt vmcnt(0)
13271 ; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
13272 ; GFX908-NEXT: v_pk_max_f16 v2, v2, v4
13273 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
13274 ; GFX908-NEXT: s_waitcnt vmcnt(0)
13275 ; GFX908-NEXT: buffer_wbinvl1
13276 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
13277 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13278 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
13279 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
13280 ; GFX908-NEXT: s_cbranch_execnz .LBB51_1
13281 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
13282 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
13283 ; GFX908-NEXT: s_setpc_b64 s[30:31]
13284 ;
13285 ; GFX8-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
13286 ; GFX8: ; %bb.0:
13287 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13288 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
13289 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
13290 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
13291 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
13292 ; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
13293 ; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
13294 ; GFX8-NEXT: .LBB51_1: ; %atomicrmw.start
13295 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
13296 ; GFX8-NEXT: s_waitcnt vmcnt(0)
13297 ; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
13298 ; GFX8-NEXT: v_max_f16_e32 v6, v3, v3
13299 ; GFX8-NEXT: v_max_f16_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
13300 ; GFX8-NEXT: v_max_f16_e32 v6, v6, v5
13301 ; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
13302 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13303 ; GFX8-NEXT: s_waitcnt vmcnt(0)
13304 ; GFX8-NEXT: buffer_wbinvl1
13305 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
13306 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13307 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
13308 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
13309 ; GFX8-NEXT: s_cbranch_execnz .LBB51_1
13310 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
13311 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
13312 ; GFX8-NEXT: s_setpc_b64 s[30:31]
13313 ;
13314 ; GFX7-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
13315 ; GFX7: ; %bb.0:
13316 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13317 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
13318 ; GFX7-NEXT: s_mov_b32 s5, -1
13319 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
13320 ; GFX7-NEXT: s_mov_b32 s6, 0
13321 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
13322 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
13323 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v2
13324 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
13325 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
13326 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
13327 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
13328 ; GFX7-NEXT: s_mov_b32 s4, s6
13329 ; GFX7-NEXT: s_mov_b32 s5, s6
13330 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13331 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
13332 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
13333 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
13334 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
13335 ; GFX7-NEXT: .LBB51_1: ; %atomicrmw.start
13336 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
13337 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
13338 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
13339 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v5
13340 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v4
13341 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
13342 ; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
13343 ; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
13344 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v6
13345 ; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v7
13346 ; GFX7-NEXT: v_or_b32_e32 v6, v4, v5
13347 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v8
13348 ; GFX7-NEXT: v_or_b32_e32 v5, v7, v4
13349 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
13350 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
13351 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
13352 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13353 ; GFX7-NEXT: buffer_wbinvl1
13354 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v7
13355 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v7
13356 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
13357 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
13358 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
13359 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
13360 ; GFX7-NEXT: s_cbranch_execnz .LBB51_1
13361 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
13362 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
13363 ; GFX7-NEXT: s_setpc_b64 s[30:31]
13364 ;
13365 ; GFX6-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
13366 ; GFX6: ; %bb.0:
13367 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13368 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
13369 ; GFX6-NEXT: s_mov_b32 s5, -1
13370 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
13371 ; GFX6-NEXT: s_mov_b32 s6, 0
13372 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
13373 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
13374 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v2
13375 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
13376 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v3
13377 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
13378 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
13379 ; GFX6-NEXT: s_mov_b32 s4, s6
13380 ; GFX6-NEXT: s_mov_b32 s5, s6
13381 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13382 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
13383 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v5
13384 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v3
13385 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v6
13386 ; GFX6-NEXT: .LBB51_1: ; %atomicrmw.start
13387 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
13388 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
13389 ; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
13390 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v5
13391 ; GFX6-NEXT: s_waitcnt expcnt(0)
13392 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v4
13393 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
13394 ; GFX6-NEXT: v_max_f32_e32 v6, v6, v2
13395 ; GFX6-NEXT: v_max_f32_e32 v7, v7, v3
13396 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v6
13397 ; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v7
13398 ; GFX6-NEXT: v_or_b32_e32 v6, v4, v5
13399 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v8
13400 ; GFX6-NEXT: v_or_b32_e32 v5, v7, v4
13401 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
13402 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
13403 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 glc
13404 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13405 ; GFX6-NEXT: buffer_wbinvl1
13406 ; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v7
13407 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v7
13408 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
13409 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
13410 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
13411 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
13412 ; GFX6-NEXT: s_cbranch_execnz .LBB51_1
13413 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
13414 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
13415 ; GFX6-NEXT: s_waitcnt expcnt(0)
13416 ; GFX6-NEXT: s_setpc_b64 s[30:31]
13417 %gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 -512
13418 %unused = atomicrmw fmax ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
13419 ret void
13420 }
13422 define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
13423 ; GFX12-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
13424 ; GFX12: ; %bb.0:
13425 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
13426 ; GFX12-NEXT: s_wait_expcnt 0x0
13427 ; GFX12-NEXT: s_wait_samplecnt 0x0
13428 ; GFX12-NEXT: s_wait_bvhcnt 0x0
13429 ; GFX12-NEXT: s_wait_kmcnt 0x0
13430 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
13431 ; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
13432 ; GFX12-NEXT: s_mov_b32 s0, 0
13433 ; GFX12-NEXT: .LBB52_1: ; %atomicrmw.start
13434 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
13435 ; GFX12-NEXT: s_wait_loadcnt 0x0
13436 ; GFX12-NEXT: v_mov_b32_e32 v4, v3
13437 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
13438 ; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
13439 ; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v2
13440 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
13441 ; GFX12-NEXT: s_wait_storecnt 0x0
13442 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
13443 ; GFX12-NEXT: s_wait_loadcnt 0x0
13444 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
13445 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13446 ; GFX12-NEXT: s_wait_alu 0xfffe
13447 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
13448 ; GFX12-NEXT: s_wait_alu 0xfffe
13449 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
13450 ; GFX12-NEXT: s_cbranch_execnz .LBB52_1
13451 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
13452 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
13453 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
13454 ; GFX12-NEXT: s_wait_alu 0xfffe
13455 ; GFX12-NEXT: s_setpc_b64 s[30:31]
13456 ;
13457 ; GFX940-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
13458 ; GFX940: ; %bb.0:
13459 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13460 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
13461 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
13462 ; GFX940-NEXT: v_pk_max_f16 v2, v2, v2
13463 ; GFX940-NEXT: .LBB52_1: ; %atomicrmw.start
13464 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
13465 ; GFX940-NEXT: s_waitcnt vmcnt(0)
13466 ; GFX940-NEXT: v_mov_b32_e32 v5, v3
13467 ; GFX940-NEXT: v_pk_max_f16 v3, v5, v5
13468 ; GFX940-NEXT: s_nop 0
13469 ; GFX940-NEXT: v_pk_max_f16 v4, v3, v2
13470 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
13471 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
13472 ; GFX940-NEXT: s_waitcnt vmcnt(0)
13473 ; GFX940-NEXT: buffer_inv sc0 sc1
13474 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
13475 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
13476 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
13477 ; GFX940-NEXT: s_cbranch_execnz .LBB52_1
13478 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
13479 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
13480 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
13481 ; GFX940-NEXT: s_setpc_b64 s[30:31]
13482 ;
13483 ; GFX11-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
13484 ; GFX11: ; %bb.0:
13485 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13486 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
13487 ; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
13488 ; GFX11-NEXT: s_mov_b32 s0, 0
13489 ; GFX11-NEXT: .LBB52_1: ; %atomicrmw.start
13490 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
13491 ; GFX11-NEXT: s_waitcnt vmcnt(0)
13492 ; GFX11-NEXT: v_mov_b32_e32 v4, v3
13493 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
13494 ; GFX11-NEXT: v_pk_max_f16 v3, v4, v4
13495 ; GFX11-NEXT: v_pk_max_f16 v3, v3, v2
13496 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
13497 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
13498 ; GFX11-NEXT: s_waitcnt vmcnt(0)
13499 ; GFX11-NEXT: buffer_gl1_inv
13500 ; GFX11-NEXT: buffer_gl0_inv
13501 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13502 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
13503 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
13504 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
13505 ; GFX11-NEXT: s_cbranch_execnz .LBB52_1
13506 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
13507 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
13508 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
13509 ; GFX11-NEXT: s_setpc_b64 s[30:31]
13510 ;
13511 ; GFX10-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
13512 ; GFX10: ; %bb.0:
13513 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13514 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
13515 ; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
13516 ; GFX10-NEXT: s_mov_b32 s4, 0
13517 ; GFX10-NEXT: .LBB52_1: ; %atomicrmw.start
13518 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
13519 ; GFX10-NEXT: s_waitcnt vmcnt(0)
13520 ; GFX10-NEXT: v_mov_b32_e32 v4, v3
13521 ; GFX10-NEXT: v_pk_max_f16 v3, v4, v4
13522 ; GFX10-NEXT: v_pk_max_f16 v3, v3, v2
13523 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
13524 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
13525 ; GFX10-NEXT: s_waitcnt vmcnt(0)
13526 ; GFX10-NEXT: buffer_gl1_inv
13527 ; GFX10-NEXT: buffer_gl0_inv
13528 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
13529 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
13530 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
13531 ; GFX10-NEXT: s_cbranch_execnz .LBB52_1
13532 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
13533 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
13534 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
13535 ; GFX10-NEXT: s_setpc_b64 s[30:31]
13536 ;
13537 ; GFX90A-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
13538 ; GFX90A: ; %bb.0:
13539 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13540 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
13541 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
13542 ; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
13543 ; GFX90A-NEXT: .LBB52_1: ; %atomicrmw.start
13544 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
13545 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
13546 ; GFX90A-NEXT: v_mov_b32_e32 v5, v3
13547 ; GFX90A-NEXT: v_pk_max_f16 v3, v5, v5
13548 ; GFX90A-NEXT: v_pk_max_f16 v4, v3, v2
13549 ; GFX90A-NEXT: buffer_wbl2
13550 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
13551 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
13552 ; GFX90A-NEXT: buffer_invl2
13553 ; GFX90A-NEXT: buffer_wbinvl1
13554 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
13555 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13556 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
13557 ; GFX90A-NEXT: s_cbranch_execnz .LBB52_1
13558 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
13559 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
13560 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
13561 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
13562 ;
13563 ; GFX908-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
13564 ; GFX908: ; %bb.0:
13565 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13566 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
13567 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
13568 ; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
13569 ; GFX908-NEXT: .LBB52_1: ; %atomicrmw.start
13570 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
13571 ; GFX908-NEXT: s_waitcnt vmcnt(0)
13572 ; GFX908-NEXT: v_mov_b32_e32 v4, v3
13573 ; GFX908-NEXT: v_pk_max_f16 v3, v4, v4
13574 ; GFX908-NEXT: v_pk_max_f16 v3, v3, v2
13575 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
13576 ; GFX908-NEXT: s_waitcnt vmcnt(0)
13577 ; GFX908-NEXT: buffer_wbinvl1
13578 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
13579 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13580 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
13581 ; GFX908-NEXT: s_cbranch_execnz .LBB52_1
13582 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
13583 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
13584 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
13585 ; GFX908-NEXT: s_setpc_b64 s[30:31]
13586 ;
13587 ; GFX8-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
13588 ; GFX8: ; %bb.0:
13589 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13590 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
13591 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
13592 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
13593 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
13594 ; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
13595 ; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
13596 ; GFX8-NEXT: .LBB52_1: ; %atomicrmw.start
13597 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
13598 ; GFX8-NEXT: s_waitcnt vmcnt(0)
13599 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
13600 ; GFX8-NEXT: v_max_f16_sdwa v0, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
13601 ; GFX8-NEXT: v_max_f16_e32 v5, v6, v6
13602 ; GFX8-NEXT: v_max_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
13603 ; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
13604 ; GFX8-NEXT: v_or_b32_e32 v5, v5, v0
13605 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
13606 ; GFX8-NEXT: s_waitcnt vmcnt(0)
13607 ; GFX8-NEXT: buffer_wbinvl1
13608 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
13609 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13610 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
13611 ; GFX8-NEXT: s_cbranch_execnz .LBB52_1
13612 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
13613 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
13614 ; GFX8-NEXT: s_setpc_b64 s[30:31]
13615 ;
13616 ; GFX7-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
13617 ; GFX7: ; %bb.0:
13618 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13619 ; GFX7-NEXT: s_mov_b32 s6, 0
13620 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
13621 ; GFX7-NEXT: s_mov_b32 s4, s6
13622 ; GFX7-NEXT: s_mov_b32 s5, s6
13623 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
13624 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
13625 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v2
13626 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
13627 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v3
13628 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13629 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
13630 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v5
13631 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
13632 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v6
13633 ; GFX7-NEXT: .LBB52_1: ; %atomicrmw.start
13634 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
13635 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
13636 ; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
13637 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v3
13638 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v2
13639 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
13640 ; GFX7-NEXT: v_max_f32_e32 v6, v6, v4
13641 ; GFX7-NEXT: v_max_f32_e32 v7, v7, v5
13642 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6
13643 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v7
13644 ; GFX7-NEXT: v_or_b32_e32 v7, v2, v3
13645 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
13646 ; GFX7-NEXT: v_or_b32_e32 v6, v8, v2
13647 ; GFX7-NEXT: v_mov_b32_e32 v9, v7
13648 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
13649 ; GFX7-NEXT: buffer_atomic_cmpswap v[8:9], v[0:1], s[4:7], 0 addr64 offset:2044 glc
13650 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13651 ; GFX7-NEXT: buffer_wbinvl1
13652 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v8
13653 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v8
13654 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
13655 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
13656 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
13657 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
13658 ; GFX7-NEXT: s_cbranch_execnz .LBB52_1
13659 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
13660 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
13661 ; GFX7-NEXT: v_mov_b32_e32 v0, v2
13662 ; GFX7-NEXT: v_mov_b32_e32 v1, v3
13663 ; GFX7-NEXT: s_setpc_b64 s[30:31]
13664 ;
13665 ; GFX6-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
13666 ; GFX6: ; %bb.0:
13667 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13668 ; GFX6-NEXT: s_mov_b32 s6, 0
13669 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
13670 ; GFX6-NEXT: s_mov_b32 s4, s6
13671 ; GFX6-NEXT: s_mov_b32 s5, s6
13672 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
13673 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
13674 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v2
13675 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
13676 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v3
13677 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13678 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
13679 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v5
13680 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
13681 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v6
13682 ; GFX6-NEXT: .LBB52_1: ; %atomicrmw.start
13683 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
13684 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
13685 ; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
13686 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v3
13687 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
13688 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
13689 ; GFX6-NEXT: v_max_f32_e32 v6, v6, v4
13690 ; GFX6-NEXT: v_max_f32_e32 v7, v7, v5
13691 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
13692 ; GFX6-NEXT: s_waitcnt expcnt(0)
13693 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v7
13694 ; GFX6-NEXT: v_or_b32_e32 v7, v2, v3
13695 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v6
13696 ; GFX6-NEXT: v_or_b32_e32 v6, v8, v2
13697 ; GFX6-NEXT: v_mov_b32_e32 v9, v7
13698 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
13699 ; GFX6-NEXT: buffer_atomic_cmpswap v[8:9], v[0:1], s[4:7], 0 addr64 offset:2044 glc
13700 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13701 ; GFX6-NEXT: buffer_wbinvl1
13702 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v8
13703 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v8
13704 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
13705 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
13706 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
13707 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
13708 ; GFX6-NEXT: s_cbranch_execnz .LBB52_1
13709 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
13710 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
13711 ; GFX6-NEXT: v_mov_b32_e32 v0, v2
13712 ; GFX6-NEXT: v_mov_b32_e32 v1, v3
13713 ; GFX6-NEXT: s_waitcnt expcnt(0)
13714 ; GFX6-NEXT: s_setpc_b64 s[30:31]
13715 %gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 511
13716 %result = atomicrmw fmax ptr addrspace(1) %gep, <2 x half> %val seq_cst, !amdgpu.no.fine.grained.memory !0
13717 ret <2 x half> %result
13718 }
13720 define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x half> %val) #0 {
13721 ; GFX12-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
13722 ; GFX12: ; %bb.0:
13723 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
13724 ; GFX12-NEXT: s_wait_expcnt 0x0
13725 ; GFX12-NEXT: s_wait_samplecnt 0x0
13726 ; GFX12-NEXT: s_wait_bvhcnt 0x0
13727 ; GFX12-NEXT: s_wait_kmcnt 0x0
13728 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
13729 ; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
13730 ; GFX12-NEXT: s_mov_b32 s0, 0
13731 ; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start
13732 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
13733 ; GFX12-NEXT: s_wait_loadcnt 0x0
13734 ; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
13735 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
13736 ; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v4
13737 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
13738 ; GFX12-NEXT: s_wait_storecnt 0x0
13739 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
13740 ; GFX12-NEXT: s_wait_loadcnt 0x0
13741 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
13742 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
13743 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
13744 ; GFX12-NEXT: s_wait_alu 0xfffe
13745 ; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
13746 ; GFX12-NEXT: s_wait_alu 0xfffe
13747 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
13748 ; GFX12-NEXT: s_cbranch_execnz .LBB53_1
13749 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
13750 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
13751 ; GFX12-NEXT: s_wait_alu 0xfffe
13752 ; GFX12-NEXT: s_setpc_b64 s[30:31]
13753 ;
13754 ; GFX940-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
13755 ; GFX940: ; %bb.0:
13756 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13757 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
13758 ; GFX940-NEXT: s_mov_b64 s[0:1], 0
13759 ; GFX940-NEXT: v_pk_max_f16 v4, v2, v2
13760 ; GFX940-NEXT: .LBB53_1: ; %atomicrmw.start
13761 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
13762 ; GFX940-NEXT: s_waitcnt vmcnt(0)
13763 ; GFX940-NEXT: v_pk_max_f16 v2, v3, v3
13764 ; GFX940-NEXT: s_nop 0
13765 ; GFX940-NEXT: v_pk_max_f16 v2, v2, v4
13766 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
13767 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
13768 ; GFX940-NEXT: s_waitcnt vmcnt(0)
13769 ; GFX940-NEXT: buffer_inv sc0 sc1
13770 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
13771 ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
13772 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
13773 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
13774 ; GFX940-NEXT: s_cbranch_execnz .LBB53_1
13775 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
13776 ; GFX940-NEXT: s_or_b64 exec, exec, s[0:1]
13777 ; GFX940-NEXT: s_setpc_b64 s[30:31]
13778 ;
13779 ; GFX11-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
13780 ; GFX11: ; %bb.0:
13781 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13782 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
13783 ; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
13784 ; GFX11-NEXT: s_mov_b32 s0, 0
13785 ; GFX11-NEXT: .LBB53_1: ; %atomicrmw.start
13786 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
13787 ; GFX11-NEXT: s_waitcnt vmcnt(0)
13788 ; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
13789 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
13790 ; GFX11-NEXT: v_pk_max_f16 v2, v2, v4
13791 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
13792 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
13793 ; GFX11-NEXT: s_waitcnt vmcnt(0)
13794 ; GFX11-NEXT: buffer_gl1_inv
13795 ; GFX11-NEXT: buffer_gl0_inv
13796 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
13797 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
13798 ; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
13799 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
13800 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
13801 ; GFX11-NEXT: s_cbranch_execnz .LBB53_1
13802 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
13803 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
13804 ; GFX11-NEXT: s_setpc_b64 s[30:31]
13805 ;
13806 ; GFX10-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
13807 ; GFX10: ; %bb.0:
13808 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13809 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
13810 ; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
13811 ; GFX10-NEXT: s_mov_b32 s4, 0
13812 ; GFX10-NEXT: .LBB53_1: ; %atomicrmw.start
13813 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
13814 ; GFX10-NEXT: s_waitcnt vmcnt(0)
13815 ; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
13816 ; GFX10-NEXT: v_pk_max_f16 v2, v2, v4
13817 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
13818 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
13819 ; GFX10-NEXT: s_waitcnt vmcnt(0)
13820 ; GFX10-NEXT: buffer_gl1_inv
13821 ; GFX10-NEXT: buffer_gl0_inv
13822 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
13823 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
13824 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
13825 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
13826 ; GFX10-NEXT: s_cbranch_execnz .LBB53_1
13827 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
13828 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
13829 ; GFX10-NEXT: s_setpc_b64 s[30:31]
13830 ;
13831 ; GFX90A-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
13832 ; GFX90A: ; %bb.0:
13833 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13834 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
13835 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
13836 ; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
13837 ; GFX90A-NEXT: .LBB53_1: ; %atomicrmw.start
13838 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
13839 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
13840 ; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
13841 ; GFX90A-NEXT: v_pk_max_f16 v2, v2, v4
13842 ; GFX90A-NEXT: buffer_wbl2
13843 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
13844 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
13845 ; GFX90A-NEXT: buffer_invl2
13846 ; GFX90A-NEXT: buffer_wbinvl1
13847 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
13848 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13849 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
13850 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
13851 ; GFX90A-NEXT: s_cbranch_execnz .LBB53_1
13852 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
13853 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
13854 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
13855 ;
13856 ; GFX908-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
13857 ; GFX908: ; %bb.0:
13858 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13859 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
13860 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
13861 ; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
13862 ; GFX908-NEXT: .LBB53_1: ; %atomicrmw.start
13863 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
13864 ; GFX908-NEXT: s_waitcnt vmcnt(0)
13865 ; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
13866 ; GFX908-NEXT: v_pk_max_f16 v2, v2, v4
13867 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
13868 ; GFX908-NEXT: s_waitcnt vmcnt(0)
13869 ; GFX908-NEXT: buffer_wbinvl1
13870 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
13871 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13872 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
13873 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
13874 ; GFX908-NEXT: s_cbranch_execnz .LBB53_1
13875 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
13876 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
13877 ; GFX908-NEXT: s_setpc_b64 s[30:31]
13878 ;
13879 ; GFX8-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
13880 ; GFX8: ; %bb.0:
13881 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13882 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
13883 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
13884 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
13885 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
13886 ; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
13887 ; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
13888 ; GFX8-NEXT: .LBB53_1: ; %atomicrmw.start
13889 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
13890 ; GFX8-NEXT: s_waitcnt vmcnt(0)
13891 ; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
13892 ; GFX8-NEXT: v_max_f16_e32 v6, v3, v3
13893 ; GFX8-NEXT: v_max_f16_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
13894 ; GFX8-NEXT: v_max_f16_e32 v6, v6, v5
13895 ; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
13896 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13897 ; GFX8-NEXT: s_waitcnt vmcnt(0)
13898 ; GFX8-NEXT: buffer_wbinvl1
13899 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
13900 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
13901 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
13902 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
13903 ; GFX8-NEXT: s_cbranch_execnz .LBB53_1
13904 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
13905 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
13906 ; GFX8-NEXT: s_setpc_b64 s[30:31]
13907 ;
13908 ; GFX7-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
13909 ; GFX7: ; %bb.0:
13910 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13911 ; GFX7-NEXT: s_mov_b32 s6, 0
13912 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
13913 ; GFX7-NEXT: s_mov_b32 s4, s6
13914 ; GFX7-NEXT: s_mov_b32 s5, s6
13915 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
13916 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
13917 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v2
13918 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
13919 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v3
13920 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13921 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
13922 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v5
13923 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
13924 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
13925 ; GFX7-NEXT: .LBB53_1: ; %atomicrmw.start
13926 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
13927 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
13928 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
13929 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v5
13930 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v4
13931 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
13932 ; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
13933 ; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
13934 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v6
13935 ; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v7
13936 ; GFX7-NEXT: v_or_b32_e32 v6, v4, v5
13937 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v8
13938 ; GFX7-NEXT: v_or_b32_e32 v5, v7, v4
13939 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
13940 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
13941 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 offset:2044 glc
13942 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13943 ; GFX7-NEXT: buffer_wbinvl1
13944 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v7
13945 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v7
13946 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
13947 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
13948 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
13949 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
13950 ; GFX7-NEXT: s_cbranch_execnz .LBB53_1
13951 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
13952 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
13953 ; GFX7-NEXT: s_setpc_b64 s[30:31]
13954 ;
13955 ; GFX6-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
13956 ; GFX6: ; %bb.0:
13957 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13958 ; GFX6-NEXT: s_mov_b32 s6, 0
13959 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
13960 ; GFX6-NEXT: s_mov_b32 s4, s6
13961 ; GFX6-NEXT: s_mov_b32 s5, s6
13962 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
13963 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
13964 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v2
13965 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
13966 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v3
13967 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13968 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
13969 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v5
13970 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v3
13971 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v6
13972 ; GFX6-NEXT: .LBB53_1: ; %atomicrmw.start
13973 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
13974 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
13975 ; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
13976 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v5
13977 ; GFX6-NEXT: s_waitcnt expcnt(0)
13978 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v4
13979 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
13980 ; GFX6-NEXT: v_max_f32_e32 v6, v6, v2
13981 ; GFX6-NEXT: v_max_f32_e32 v7, v7, v3
13982 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v6
13983 ; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v7
13984 ; GFX6-NEXT: v_or_b32_e32 v6, v4, v5
13985 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v8
13986 ; GFX6-NEXT: v_or_b32_e32 v5, v7, v4
13987 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
13988 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
13989 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v[0:1], s[4:7], 0 addr64 offset:2044 glc
13990 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13991 ; GFX6-NEXT: buffer_wbinvl1
13992 ; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v7
13993 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v7
13994 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
13995 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
13996 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
13997 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
13998 ; GFX6-NEXT: s_cbranch_execnz .LBB53_1
13999 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
14000 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
14001 ; GFX6-NEXT: s_waitcnt expcnt(0)
14002 ; GFX6-NEXT: s_setpc_b64 s[30:31]
14003 %gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i64 511
14004 %unused = atomicrmw fmax ptr addrspace(1) %gep, <2 x half> %val seq_cst, !amdgpu.no.fine.grained.memory !0
14005 ret void
14006 }
14008 ; --------------------------------------------------------------------
14009 ; <2 x bfloat>
14010 ; --------------------------------------------------------------------
14012 define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
14013 ; GFX12-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
14014 ; GFX12: ; %bb.0:
14015 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
14016 ; GFX12-NEXT: s_wait_expcnt 0x0
14017 ; GFX12-NEXT: s_wait_samplecnt 0x0
14018 ; GFX12-NEXT: s_wait_bvhcnt 0x0
14019 ; GFX12-NEXT: s_wait_kmcnt 0x0
14020 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
14021 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14022 ; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14023 ; GFX12-NEXT: s_mov_b32 s1, 0
14024 ; GFX12-NEXT: .LBB54_1: ; %atomicrmw.start
14025 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
14026 ; GFX12-NEXT: s_wait_loadcnt 0x0
14027 ; GFX12-NEXT: v_mov_b32_e32 v6, v3
14028 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14029 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14030 ; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
14031 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14032 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
14033 ; GFX12-NEXT: v_bfe_u32 v8, v5, 16, 1
14034 ; GFX12-NEXT: v_max_num_f32_e32 v3, v3, v4
14035 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v5
14036 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
14037 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14038 ; GFX12-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
14039 ; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
14040 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v3
14041 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v3, v3
14042 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14043 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
14044 ; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
14045 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14046 ; GFX12-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
14047 ; GFX12-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
14048 ; GFX12-NEXT: s_wait_storecnt 0x0
14049 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
14050 ; GFX12-NEXT: s_wait_loadcnt 0x0
14051 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
14052 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
14053 ; GFX12-NEXT: s_wait_alu 0xfffe
14054 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
14055 ; GFX12-NEXT: s_wait_alu 0xfffe
14056 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
14057 ; GFX12-NEXT: s_cbranch_execnz .LBB54_1
14058 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
14059 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
14060 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
14061 ; GFX12-NEXT: s_wait_alu 0xfffe
14062 ; GFX12-NEXT: s_setpc_b64 s[30:31]
14063 ;
14064 ; GFX940-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
14065 ; GFX940: ; %bb.0:
14066 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14067 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
14068 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
14069 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14070 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
14071 ; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14072 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
14073 ; GFX940-NEXT: .LBB54_1: ; %atomicrmw.start
14074 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
14075 ; GFX940-NEXT: s_waitcnt vmcnt(0)
14076 ; GFX940-NEXT: v_mov_b32_e32 v7, v3
14077 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 16, v7
14078 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
14079 ; GFX940-NEXT: v_max_f32_e32 v3, v3, v4
14080 ; GFX940-NEXT: v_max_f32_e32 v5, v5, v2
14081 ; GFX940-NEXT: v_bfe_u32 v6, v3, 16, 1
14082 ; GFX940-NEXT: v_bfe_u32 v9, v5, 16, 1
14083 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v3
14084 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v5
14085 ; GFX940-NEXT: v_add3_u32 v6, v6, v3, s4
14086 ; GFX940-NEXT: v_add3_u32 v9, v9, v5, s4
14087 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
14088 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
14089 ; GFX940-NEXT: s_nop 0
14090 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
14091 ; GFX940-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[0:1]
14092 ; GFX940-NEXT: v_perm_b32 v6, v5, v3, s5
14093 ; GFX940-NEXT: buffer_wbl2 sc1
14094 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off sc0
14095 ; GFX940-NEXT: s_waitcnt vmcnt(0)
14096 ; GFX940-NEXT: buffer_inv sc1
14097 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
14098 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
14099 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
14100 ; GFX940-NEXT: s_cbranch_execnz .LBB54_1
14101 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
14102 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
14103 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
14104 ; GFX940-NEXT: s_setpc_b64 s[30:31]
14105 ;
14106 ; GFX11-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
14107 ; GFX11: ; %bb.0:
14108 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14109 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
14110 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14111 ; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14112 ; GFX11-NEXT: s_mov_b32 s1, 0
14113 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
14114 ; GFX11-NEXT: .p2align 6
14115 ; GFX11-NEXT: .LBB54_1: ; %atomicrmw.start
14116 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
14117 ; GFX11-NEXT: s_waitcnt vmcnt(0)
14118 ; GFX11-NEXT: v_mov_b32_e32 v6, v3
14119 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14120 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14121 ; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
14122 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14123 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
14124 ; GFX11-NEXT: v_bfe_u32 v8, v5, 16, 1
14125 ; GFX11-NEXT: v_max_f32_e32 v3, v3, v4
14126 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v5
14127 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
14128 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14129 ; GFX11-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
14130 ; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
14131 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v3
14132 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v3, v3
14133 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14134 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
14135 ; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
14136 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14137 ; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
14138 ; GFX11-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
14139 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
14140 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
14141 ; GFX11-NEXT: s_waitcnt vmcnt(0)
14142 ; GFX11-NEXT: buffer_gl1_inv
14143 ; GFX11-NEXT: buffer_gl0_inv
14144 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
14145 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
14146 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
14147 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
14148 ; GFX11-NEXT: s_cbranch_execnz .LBB54_1
14149 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
14150 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
14151 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
14152 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
14153 ; GFX11-NEXT: s_setpc_b64 s[30:31]
14154 ;
14155 ; GFX10-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
14156 ; GFX10: ; %bb.0:
14157 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14158 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
14159 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14160 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14161 ; GFX10-NEXT: s_mov_b32 s5, 0
14162 ; GFX10-NEXT: .LBB54_1: ; %atomicrmw.start
14163 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
14164 ; GFX10-NEXT: s_waitcnt vmcnt(0)
14165 ; GFX10-NEXT: v_mov_b32_e32 v6, v3
14166 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14167 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14168 ; GFX10-NEXT: v_max_f32_e32 v3, v3, v4
14169 ; GFX10-NEXT: v_max_f32_e32 v5, v5, v2
14170 ; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
14171 ; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
14172 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
14173 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
14174 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
14175 ; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
14176 ; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
14177 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
14178 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
14179 ; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
14180 ; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
14181 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
14182 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
14183 ; GFX10-NEXT: s_waitcnt vmcnt(0)
14184 ; GFX10-NEXT: buffer_gl1_inv
14185 ; GFX10-NEXT: buffer_gl0_inv
14186 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
14187 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
14188 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
14189 ; GFX10-NEXT: s_cbranch_execnz .LBB54_1
14190 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
14191 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
14192 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
14193 ; GFX10-NEXT: s_setpc_b64 s[30:31]
14194 ;
14195 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
14196 ; GFX90A: ; %bb.0:
14197 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14198 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
14199 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
14200 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14201 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
14202 ; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14203 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
14204 ; GFX90A-NEXT: .LBB54_1: ; %atomicrmw.start
14205 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
14206 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
14207 ; GFX90A-NEXT: v_mov_b32_e32 v7, v3
14208 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v7
14209 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
14210 ; GFX90A-NEXT: v_max_f32_e32 v3, v3, v4
14211 ; GFX90A-NEXT: v_max_f32_e32 v5, v5, v2
14212 ; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
14213 ; GFX90A-NEXT: v_bfe_u32 v9, v5, 16, 1
14214 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v3
14215 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v5
14216 ; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
14217 ; GFX90A-NEXT: v_add3_u32 v9, v9, v5, s8
14218 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
14219 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
14220 ; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[4:5]
14221 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
14222 ; GFX90A-NEXT: v_perm_b32 v6, v5, v3, s9
14223 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off glc
14224 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
14225 ; GFX90A-NEXT: buffer_wbinvl1
14226 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
14227 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
14228 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
14229 ; GFX90A-NEXT: s_cbranch_execnz .LBB54_1
14230 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
14231 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
14232 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
14233 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
14234 ;
14235 ; GFX908-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
14236 ; GFX908: ; %bb.0:
14237 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14238 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
14239 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
14240 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14241 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
14242 ; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14243 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
14244 ; GFX908-NEXT: .LBB54_1: ; %atomicrmw.start
14245 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
14246 ; GFX908-NEXT: s_waitcnt vmcnt(0)
14247 ; GFX908-NEXT: v_mov_b32_e32 v6, v3
14248 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14249 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14250 ; GFX908-NEXT: v_max_f32_e32 v3, v3, v4
14251 ; GFX908-NEXT: v_max_f32_e32 v5, v5, v2
14252 ; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
14253 ; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
14254 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
14255 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
14256 ; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
14257 ; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
14258 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
14259 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
14260 ; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
14261 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
14262 ; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
14263 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
14264 ; GFX908-NEXT: s_waitcnt vmcnt(0)
14265 ; GFX908-NEXT: buffer_wbinvl1
14266 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
14267 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
14268 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
14269 ; GFX908-NEXT: s_cbranch_execnz .LBB54_1
14270 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
14271 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
14272 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
14273 ; GFX908-NEXT: s_setpc_b64 s[30:31]
14274 ;
14275 ; GFX8-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
14276 ; GFX8: ; %bb.0:
14277 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14278 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
14279 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
14280 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14281 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14282 ; GFX8-NEXT: .LBB54_1: ; %atomicrmw.start
14283 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
14284 ; GFX8-NEXT: s_waitcnt vmcnt(0)
14285 ; GFX8-NEXT: v_mov_b32_e32 v6, v3
14286 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14287 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14288 ; GFX8-NEXT: v_max_f32_e32 v3, v3, v4
14289 ; GFX8-NEXT: v_max_f32_e32 v5, v5, v2
14290 ; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
14291 ; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
14292 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
14293 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
14294 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
14295 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
14296 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
14297 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
14298 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v3
14299 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
14300 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
14301 ; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
14302 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
14303 ; GFX8-NEXT: v_alignbit_b32 v5, v5, v3, 16
14304 ; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
14305 ; GFX8-NEXT: s_waitcnt vmcnt(0)
14306 ; GFX8-NEXT: buffer_wbinvl1
14307 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
14308 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
14309 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
14310 ; GFX8-NEXT: s_cbranch_execnz .LBB54_1
14311 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
14312 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
14313 ; GFX8-NEXT: v_mov_b32_e32 v0, v3
14314 ; GFX8-NEXT: s_setpc_b64 s[30:31]
14315 ;
14316 ; GFX7-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
14317 ; GFX7: ; %bb.0:
14318 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14319 ; GFX7-NEXT: s_mov_b32 s6, 0
14320 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
14321 ; GFX7-NEXT: s_mov_b32 s4, s6
14322 ; GFX7-NEXT: s_mov_b32 s5, s6
14323 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
14324 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
14325 ; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
14326 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
14327 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
14328 ; GFX7-NEXT: s_waitcnt vmcnt(0)
14329 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
14330 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
14331 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14332 ; GFX7-NEXT: .LBB54_1: ; %atomicrmw.start
14333 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
14334 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
14335 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
14336 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
14337 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
14338 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
14339 ; GFX7-NEXT: v_max_f32_e32 v6, v6, v4
14340 ; GFX7-NEXT: v_max_f32_e32 v7, v7, v5
14341 ; GFX7-NEXT: v_alignbit_b32 v3, v2, v3, 16
14342 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v6
14343 ; GFX7-NEXT: v_alignbit_b32 v2, v2, v7, 16
14344 ; GFX7-NEXT: v_mov_b32_e32 v7, v3
14345 ; GFX7-NEXT: v_mov_b32_e32 v6, v2
14346 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
14347 ; GFX7-NEXT: s_waitcnt vmcnt(0)
14348 ; GFX7-NEXT: buffer_wbinvl1
14349 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v3
14350 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
14351 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
14352 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14353 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
14354 ; GFX7-NEXT: s_cbranch_execnz .LBB54_1
14355 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
14356 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
14357 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
14358 ; GFX7-NEXT: v_mov_b32_e32 v1, v2
14359 ; GFX7-NEXT: s_setpc_b64 s[30:31]
14360 ;
14361 ; GFX6-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
14362 ; GFX6: ; %bb.0:
14363 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14364 ; GFX6-NEXT: s_mov_b32 s6, 0
14365 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
14366 ; GFX6-NEXT: s_mov_b32 s4, s6
14367 ; GFX6-NEXT: s_mov_b32 s5, s6
14368 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64
14369 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
14370 ; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
14371 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
14372 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
14373 ; GFX6-NEXT: s_waitcnt vmcnt(0)
14374 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
14375 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
14376 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14377 ; GFX6-NEXT: .LBB54_1: ; %atomicrmw.start
14378 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
14379 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
14380 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
14381 ; GFX6-NEXT: s_waitcnt expcnt(0)
14382 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
14383 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
14384 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
14385 ; GFX6-NEXT: v_max_f32_e32 v6, v6, v4
14386 ; GFX6-NEXT: v_max_f32_e32 v7, v7, v5
14387 ; GFX6-NEXT: v_alignbit_b32 v3, v2, v3, 16
14388 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v6
14389 ; GFX6-NEXT: v_alignbit_b32 v2, v2, v7, 16
14390 ; GFX6-NEXT: v_mov_b32_e32 v7, v3
14391 ; GFX6-NEXT: v_mov_b32_e32 v6, v2
14392 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
14393 ; GFX6-NEXT: s_waitcnt vmcnt(0)
14394 ; GFX6-NEXT: buffer_wbinvl1
14395 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v3
14396 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
14397 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
14398 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14399 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
14400 ; GFX6-NEXT: s_cbranch_execnz .LBB54_1
14401 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
14402 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
14403 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
14404 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
14405 ; GFX6-NEXT: s_waitcnt expcnt(0)
14406 ; GFX6-NEXT: s_setpc_b64 s[30:31]
14407 %result = atomicrmw fmax ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
14408 ret <2 x bfloat> %result
14409 }
14411 define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
14412 ; GFX12-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
14413 ; GFX12: ; %bb.0:
14414 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
14415 ; GFX12-NEXT: s_wait_expcnt 0x0
14416 ; GFX12-NEXT: s_wait_samplecnt 0x0
14417 ; GFX12-NEXT: s_wait_bvhcnt 0x0
14418 ; GFX12-NEXT: s_wait_kmcnt 0x0
14419 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
14420 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14421 ; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14422 ; GFX12-NEXT: s_mov_b32 s1, 0
14423 ; GFX12-NEXT: .LBB55_1: ; %atomicrmw.start
14424 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
14425 ; GFX12-NEXT: s_wait_loadcnt 0x0
14426 ; GFX12-NEXT: v_mov_b32_e32 v6, v3
14427 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14428 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14429 ; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
14430 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14431 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
14432 ; GFX12-NEXT: v_bfe_u32 v8, v5, 16, 1
14433 ; GFX12-NEXT: v_max_num_f32_e32 v3, v3, v4
14434 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v5
14435 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
14436 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14437 ; GFX12-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
14438 ; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
14439 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v3
14440 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v3, v3
14441 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14442 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
14443 ; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
14444 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14445 ; GFX12-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
14446 ; GFX12-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
14447 ; GFX12-NEXT: s_wait_storecnt 0x0
14448 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
14449 ; GFX12-NEXT: s_wait_loadcnt 0x0
14450 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
14451 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
14452 ; GFX12-NEXT: s_wait_alu 0xfffe
14453 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
14454 ; GFX12-NEXT: s_wait_alu 0xfffe
14455 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
14456 ; GFX12-NEXT: s_cbranch_execnz .LBB55_1
14457 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
14458 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
14459 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
14460 ; GFX12-NEXT: s_wait_alu 0xfffe
14461 ; GFX12-NEXT: s_setpc_b64 s[30:31]
14462 ;
14463 ; GFX940-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
14464 ; GFX940: ; %bb.0:
14465 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14466 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
14467 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
14468 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14469 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
14470 ; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14471 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
14472 ; GFX940-NEXT: .LBB55_1: ; %atomicrmw.start
14473 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
14474 ; GFX940-NEXT: s_waitcnt vmcnt(0)
14475 ; GFX940-NEXT: v_mov_b32_e32 v7, v3
14476 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 16, v7
14477 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
14478 ; GFX940-NEXT: v_max_f32_e32 v3, v3, v4
14479 ; GFX940-NEXT: v_max_f32_e32 v5, v5, v2
14480 ; GFX940-NEXT: v_bfe_u32 v6, v3, 16, 1
14481 ; GFX940-NEXT: v_bfe_u32 v9, v5, 16, 1
14482 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v3
14483 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v5
14484 ; GFX940-NEXT: v_add3_u32 v6, v6, v3, s4
14485 ; GFX940-NEXT: v_add3_u32 v9, v9, v5, s4
14486 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
14487 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
14488 ; GFX940-NEXT: s_nop 0
14489 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
14490 ; GFX940-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[0:1]
14491 ; GFX940-NEXT: v_perm_b32 v6, v5, v3, s5
14492 ; GFX940-NEXT: buffer_wbl2 sc1
14493 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off offset:2044 sc0
14494 ; GFX940-NEXT: s_waitcnt vmcnt(0)
14495 ; GFX940-NEXT: buffer_inv sc1
14496 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
14497 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
14498 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
14499 ; GFX940-NEXT: s_cbranch_execnz .LBB55_1
14500 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
14501 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
14502 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
14503 ; GFX940-NEXT: s_setpc_b64 s[30:31]
14504 ;
14505 ; GFX11-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
14506 ; GFX11: ; %bb.0:
14507 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14508 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
14509 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14510 ; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14511 ; GFX11-NEXT: s_mov_b32 s1, 0
14512 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
14513 ; GFX11-NEXT: .p2align 6
14514 ; GFX11-NEXT: .LBB55_1: ; %atomicrmw.start
14515 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
14516 ; GFX11-NEXT: s_waitcnt vmcnt(0)
14517 ; GFX11-NEXT: v_mov_b32_e32 v6, v3
14518 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14519 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14520 ; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
14521 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14522 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
14523 ; GFX11-NEXT: v_bfe_u32 v8, v5, 16, 1
14524 ; GFX11-NEXT: v_max_f32_e32 v3, v3, v4
14525 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v5
14526 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
14527 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14528 ; GFX11-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
14529 ; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
14530 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v3
14531 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v3, v3
14532 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14533 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
14534 ; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
14535 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14536 ; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
14537 ; GFX11-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
14538 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
14539 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
14540 ; GFX11-NEXT: s_waitcnt vmcnt(0)
14541 ; GFX11-NEXT: buffer_gl1_inv
14542 ; GFX11-NEXT: buffer_gl0_inv
14543 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
14544 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
14545 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
14546 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
14547 ; GFX11-NEXT: s_cbranch_execnz .LBB55_1
14548 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
14549 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
14550 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
14551 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
14552 ; GFX11-NEXT: s_setpc_b64 s[30:31]
14553 ;
14554 ; GFX10-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
14555 ; GFX10: ; %bb.0:
14556 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14557 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
14558 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14559 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14560 ; GFX10-NEXT: s_mov_b32 s5, 0
14561 ; GFX10-NEXT: .LBB55_1: ; %atomicrmw.start
14562 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
14563 ; GFX10-NEXT: s_waitcnt vmcnt(0)
14564 ; GFX10-NEXT: v_mov_b32_e32 v6, v3
14565 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14566 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14567 ; GFX10-NEXT: v_max_f32_e32 v3, v3, v4
14568 ; GFX10-NEXT: v_max_f32_e32 v5, v5, v2
14569 ; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
14570 ; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
14571 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
14572 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
14573 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
14574 ; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
14575 ; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
14576 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
14577 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
14578 ; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
14579 ; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
14580 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
14581 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
14582 ; GFX10-NEXT: s_waitcnt vmcnt(0)
14583 ; GFX10-NEXT: buffer_gl1_inv
14584 ; GFX10-NEXT: buffer_gl0_inv
14585 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
14586 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
14587 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
14588 ; GFX10-NEXT: s_cbranch_execnz .LBB55_1
14589 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
14590 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
14591 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
14592 ; GFX10-NEXT: s_setpc_b64 s[30:31]
14593 ;
14594 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
14595 ; GFX90A: ; %bb.0:
14596 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14597 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
14598 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
14599 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14600 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
14601 ; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14602 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
14603 ; GFX90A-NEXT: .LBB55_1: ; %atomicrmw.start
14604 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
14605 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
14606 ; GFX90A-NEXT: v_mov_b32_e32 v7, v3
14607 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v7
14608 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
14609 ; GFX90A-NEXT: v_max_f32_e32 v3, v3, v4
14610 ; GFX90A-NEXT: v_max_f32_e32 v5, v5, v2
14611 ; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
14612 ; GFX90A-NEXT: v_bfe_u32 v9, v5, 16, 1
14613 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v3
14614 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v5
14615 ; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
14616 ; GFX90A-NEXT: v_add3_u32 v9, v9, v5, s8
14617 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
14618 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
14619 ; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[4:5]
14620 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
14621 ; GFX90A-NEXT: v_perm_b32 v6, v5, v3, s9
14622 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off offset:2044 glc
14623 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
14624 ; GFX90A-NEXT: buffer_wbinvl1
14625 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
14626 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
14627 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
14628 ; GFX90A-NEXT: s_cbranch_execnz .LBB55_1
14629 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
14630 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
14631 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
14632 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
14633 ;
14634 ; GFX908-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
14635 ; GFX908: ; %bb.0:
14636 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14637 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
14638 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
14639 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14640 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
14641 ; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14642 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
14643 ; GFX908-NEXT: .LBB55_1: ; %atomicrmw.start
14644 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
14645 ; GFX908-NEXT: s_waitcnt vmcnt(0)
14646 ; GFX908-NEXT: v_mov_b32_e32 v6, v3
14647 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14648 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14649 ; GFX908-NEXT: v_max_f32_e32 v3, v3, v4
14650 ; GFX908-NEXT: v_max_f32_e32 v5, v5, v2
14651 ; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
14652 ; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
14653 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
14654 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
14655 ; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
14656 ; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
14657 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
14658 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
14659 ; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
14660 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
14661 ; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
14662 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
14663 ; GFX908-NEXT: s_waitcnt vmcnt(0)
14664 ; GFX908-NEXT: buffer_wbinvl1
14665 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
14666 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
14667 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
14668 ; GFX908-NEXT: s_cbranch_execnz .LBB55_1
14669 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
14670 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
14671 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
14672 ; GFX908-NEXT: s_setpc_b64 s[30:31]
14673 ;
14674 ; GFX8-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
14675 ; GFX8: ; %bb.0:
14676 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14677 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
14678 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
14679 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
14680 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
14681 ; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
14682 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14683 ; GFX8-NEXT: .LBB55_1: ; %atomicrmw.start
14684 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
14685 ; GFX8-NEXT: s_waitcnt vmcnt(0)
14686 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
14687 ; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
14688 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14689 ; GFX8-NEXT: v_max_f32_e32 v0, v0, v1
14690 ; GFX8-NEXT: v_max_f32_e32 v5, v5, v2
14691 ; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
14692 ; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
14693 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
14694 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
14695 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
14696 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
14697 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
14698 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
14699 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
14700 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
14701 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
14702 ; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
14703 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
14704 ; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
14705 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
14706 ; GFX8-NEXT: s_waitcnt vmcnt(0)
14707 ; GFX8-NEXT: buffer_wbinvl1
14708 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
14709 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
14710 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
14711 ; GFX8-NEXT: s_cbranch_execnz .LBB55_1
14712 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
14713 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
14714 ; GFX8-NEXT: s_setpc_b64 s[30:31]
14715 ;
14716 ; GFX7-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
14717 ; GFX7: ; %bb.0:
14718 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14719 ; GFX7-NEXT: s_mov_b32 s6, 0
14720 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
14721 ; GFX7-NEXT: s_mov_b32 s4, s6
14722 ; GFX7-NEXT: s_mov_b32 s5, s6
14723 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
14724 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
14725 ; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
14726 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
14727 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
14728 ; GFX7-NEXT: s_waitcnt vmcnt(0)
14729 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
14730 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
14731 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14732 ; GFX7-NEXT: .LBB55_1: ; %atomicrmw.start
14733 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
14734 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
14735 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
14736 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
14737 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
14738 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
14739 ; GFX7-NEXT: v_max_f32_e32 v6, v6, v4
14740 ; GFX7-NEXT: v_max_f32_e32 v7, v7, v5
14741 ; GFX7-NEXT: v_alignbit_b32 v3, v2, v3, 16
14742 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v6
14743 ; GFX7-NEXT: v_alignbit_b32 v2, v2, v7, 16
14744 ; GFX7-NEXT: v_mov_b32_e32 v7, v3
14745 ; GFX7-NEXT: v_mov_b32_e32 v6, v2
14746 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
14747 ; GFX7-NEXT: s_waitcnt vmcnt(0)
14748 ; GFX7-NEXT: buffer_wbinvl1
14749 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v3
14750 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
14751 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
14752 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14753 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
14754 ; GFX7-NEXT: s_cbranch_execnz .LBB55_1
14755 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
14756 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
14757 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
14758 ; GFX7-NEXT: v_mov_b32_e32 v1, v2
14759 ; GFX7-NEXT: s_setpc_b64 s[30:31]
14760 ;
14761 ; GFX6-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
14762 ; GFX6: ; %bb.0:
14763 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14764 ; GFX6-NEXT: s_mov_b32 s6, 0
14765 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
14766 ; GFX6-NEXT: s_mov_b32 s4, s6
14767 ; GFX6-NEXT: s_mov_b32 s5, s6
14768 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
14769 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
14770 ; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
14771 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
14772 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
14773 ; GFX6-NEXT: s_waitcnt vmcnt(0)
14774 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
14775 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
14776 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14777 ; GFX6-NEXT: .LBB55_1: ; %atomicrmw.start
14778 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
14779 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
14780 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
14781 ; GFX6-NEXT: s_waitcnt expcnt(0)
14782 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
14783 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
14784 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
14785 ; GFX6-NEXT: v_max_f32_e32 v6, v6, v4
14786 ; GFX6-NEXT: v_max_f32_e32 v7, v7, v5
14787 ; GFX6-NEXT: v_alignbit_b32 v3, v2, v3, 16
14788 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v6
14789 ; GFX6-NEXT: v_alignbit_b32 v2, v2, v7, 16
14790 ; GFX6-NEXT: v_mov_b32_e32 v7, v3
14791 ; GFX6-NEXT: v_mov_b32_e32 v6, v2
14792 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
14793 ; GFX6-NEXT: s_waitcnt vmcnt(0)
14794 ; GFX6-NEXT: buffer_wbinvl1
14795 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v3
14796 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
14797 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
14798 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14799 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
14800 ; GFX6-NEXT: s_cbranch_execnz .LBB55_1
14801 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
14802 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
14803 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
14804 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
14805 ; GFX6-NEXT: s_waitcnt expcnt(0)
14806 ; GFX6-NEXT: s_setpc_b64 s[30:31]
14807 %gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
14808 %result = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
14809 ret <2 x bfloat> %result
14810 }
14812 define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
14813 ; GFX12-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
14814 ; GFX12: ; %bb.0:
14815 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
14816 ; GFX12-NEXT: s_wait_expcnt 0x0
14817 ; GFX12-NEXT: s_wait_samplecnt 0x0
14818 ; GFX12-NEXT: s_wait_bvhcnt 0x0
14819 ; GFX12-NEXT: s_wait_kmcnt 0x0
14820 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
14821 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14822 ; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14823 ; GFX12-NEXT: s_mov_b32 s1, 0
14824 ; GFX12-NEXT: .LBB56_1: ; %atomicrmw.start
14825 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
14826 ; GFX12-NEXT: s_wait_loadcnt 0x0
14827 ; GFX12-NEXT: v_mov_b32_e32 v6, v3
14828 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14829 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14830 ; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
14831 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14832 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
14833 ; GFX12-NEXT: v_bfe_u32 v8, v5, 16, 1
14834 ; GFX12-NEXT: v_max_num_f32_e32 v3, v3, v4
14835 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v5
14836 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
14837 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14838 ; GFX12-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
14839 ; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
14840 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v3
14841 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v3, v3
14842 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14843 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
14844 ; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
14845 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14846 ; GFX12-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
14847 ; GFX12-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
14848 ; GFX12-NEXT: s_wait_storecnt 0x0
14849 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
14850 ; GFX12-NEXT: s_wait_loadcnt 0x0
14851 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
14852 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
14853 ; GFX12-NEXT: s_wait_alu 0xfffe
14854 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
14855 ; GFX12-NEXT: s_wait_alu 0xfffe
14856 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
14857 ; GFX12-NEXT: s_cbranch_execnz .LBB56_1
14858 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
14859 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
14860 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
14861 ; GFX12-NEXT: s_wait_alu 0xfffe
14862 ; GFX12-NEXT: s_setpc_b64 s[30:31]
14863 ;
14864 ; GFX940-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
14865 ; GFX940: ; %bb.0:
14866 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14867 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
14868 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
14869 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14870 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
14871 ; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14872 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
14873 ; GFX940-NEXT: .LBB56_1: ; %atomicrmw.start
14874 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
14875 ; GFX940-NEXT: s_waitcnt vmcnt(0)
14876 ; GFX940-NEXT: v_mov_b32_e32 v7, v3
14877 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 16, v7
14878 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
14879 ; GFX940-NEXT: v_max_f32_e32 v3, v3, v4
14880 ; GFX940-NEXT: v_max_f32_e32 v5, v5, v2
14881 ; GFX940-NEXT: v_bfe_u32 v6, v3, 16, 1
14882 ; GFX940-NEXT: v_bfe_u32 v9, v5, 16, 1
14883 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v3
14884 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v5
14885 ; GFX940-NEXT: v_add3_u32 v6, v6, v3, s4
14886 ; GFX940-NEXT: v_add3_u32 v9, v9, v5, s4
14887 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
14888 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
14889 ; GFX940-NEXT: s_nop 0
14890 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
14891 ; GFX940-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[0:1]
14892 ; GFX940-NEXT: v_perm_b32 v6, v5, v3, s5
14893 ; GFX940-NEXT: buffer_wbl2 sc1
14894 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off offset:-2048 sc0
14895 ; GFX940-NEXT: s_waitcnt vmcnt(0)
14896 ; GFX940-NEXT: buffer_inv sc1
14897 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
14898 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
14899 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
14900 ; GFX940-NEXT: s_cbranch_execnz .LBB56_1
14901 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
14902 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
14903 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
14904 ; GFX940-NEXT: s_setpc_b64 s[30:31]
14905 ;
14906 ; GFX11-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
14907 ; GFX11: ; %bb.0:
14908 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14909 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
14910 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14911 ; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14912 ; GFX11-NEXT: s_mov_b32 s1, 0
14913 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
14914 ; GFX11-NEXT: .p2align 6
14915 ; GFX11-NEXT: .LBB56_1: ; %atomicrmw.start
14916 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
14917 ; GFX11-NEXT: s_waitcnt vmcnt(0)
14918 ; GFX11-NEXT: v_mov_b32_e32 v6, v3
14919 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14920 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14921 ; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
14922 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14923 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
14924 ; GFX11-NEXT: v_bfe_u32 v8, v5, 16, 1
14925 ; GFX11-NEXT: v_max_f32_e32 v3, v3, v4
14926 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v5
14927 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
14928 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14929 ; GFX11-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
14930 ; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
14931 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v3
14932 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v3, v3
14933 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
14934 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
14935 ; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
14936 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
14937 ; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
14938 ; GFX11-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
14939 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
14940 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 glc
14941 ; GFX11-NEXT: s_waitcnt vmcnt(0)
14942 ; GFX11-NEXT: buffer_gl1_inv
14943 ; GFX11-NEXT: buffer_gl0_inv
14944 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
14945 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
14946 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
14947 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
14948 ; GFX11-NEXT: s_cbranch_execnz .LBB56_1
14949 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
14950 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
14951 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
14952 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
14953 ; GFX11-NEXT: s_setpc_b64 s[30:31]
14954 ;
14955 ; GFX10-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
14956 ; GFX10: ; %bb.0:
14957 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14958 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
14959 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
14960 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
14961 ; GFX10-NEXT: s_mov_b32 s5, 0
14962 ; GFX10-NEXT: .LBB56_1: ; %atomicrmw.start
14963 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
14964 ; GFX10-NEXT: s_waitcnt vmcnt(0)
14965 ; GFX10-NEXT: v_mov_b32_e32 v6, v3
14966 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
14967 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
14968 ; GFX10-NEXT: v_max_f32_e32 v3, v3, v4
14969 ; GFX10-NEXT: v_max_f32_e32 v5, v5, v2
14970 ; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
14971 ; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
14972 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
14973 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
14974 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
14975 ; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
14976 ; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
14977 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
14978 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
14979 ; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
14980 ; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
14981 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
14982 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:-2048 glc
14983 ; GFX10-NEXT: s_waitcnt vmcnt(0)
14984 ; GFX10-NEXT: buffer_gl1_inv
14985 ; GFX10-NEXT: buffer_gl0_inv
14986 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
14987 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
14988 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
14989 ; GFX10-NEXT: s_cbranch_execnz .LBB56_1
14990 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
14991 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
14992 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
14993 ; GFX10-NEXT: s_setpc_b64 s[30:31]
14994 ;
14995 ; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
14996 ; GFX90A: ; %bb.0:
14997 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14998 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
14999 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
15000 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15001 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
15002 ; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15003 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
15004 ; GFX90A-NEXT: .LBB56_1: ; %atomicrmw.start
15005 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
15006 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
15007 ; GFX90A-NEXT: v_mov_b32_e32 v7, v3
15008 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v7
15009 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
15010 ; GFX90A-NEXT: v_max_f32_e32 v3, v3, v4
15011 ; GFX90A-NEXT: v_max_f32_e32 v5, v5, v2
15012 ; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
15013 ; GFX90A-NEXT: v_bfe_u32 v9, v5, 16, 1
15014 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v3
15015 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v5
15016 ; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
15017 ; GFX90A-NEXT: v_add3_u32 v9, v9, v5, s8
15018 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
15019 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
15020 ; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[4:5]
15021 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
15022 ; GFX90A-NEXT: v_perm_b32 v6, v5, v3, s9
15023 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off offset:-2048 glc
15024 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
15025 ; GFX90A-NEXT: buffer_wbinvl1
15026 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
15027 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15028 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
15029 ; GFX90A-NEXT: s_cbranch_execnz .LBB56_1
15030 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
15031 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
15032 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
15033 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
15034 ;
15035 ; GFX908-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
15036 ; GFX908: ; %bb.0:
15037 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15038 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
15039 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
15040 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15041 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
15042 ; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15043 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
15044 ; GFX908-NEXT: .LBB56_1: ; %atomicrmw.start
15045 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
15046 ; GFX908-NEXT: s_waitcnt vmcnt(0)
15047 ; GFX908-NEXT: v_mov_b32_e32 v6, v3
15048 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
15049 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15050 ; GFX908-NEXT: v_max_f32_e32 v3, v3, v4
15051 ; GFX908-NEXT: v_max_f32_e32 v5, v5, v2
15052 ; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
15053 ; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
15054 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
15055 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
15056 ; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
15057 ; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
15058 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
15059 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
15060 ; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
15061 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
15062 ; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
15063 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:-2048 glc
15064 ; GFX908-NEXT: s_waitcnt vmcnt(0)
15065 ; GFX908-NEXT: buffer_wbinvl1
15066 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
15067 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15068 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
15069 ; GFX908-NEXT: s_cbranch_execnz .LBB56_1
15070 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
15071 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
15072 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
15073 ; GFX908-NEXT: s_setpc_b64 s[30:31]
15074 ;
15075 ; GFX8-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
15076 ; GFX8: ; %bb.0:
15077 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15078 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
15079 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
15080 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
15081 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
15082 ; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
15083 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
15084 ; GFX8-NEXT: .LBB56_1: ; %atomicrmw.start
15085 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
15086 ; GFX8-NEXT: s_waitcnt vmcnt(0)
15087 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
15088 ; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
15089 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15090 ; GFX8-NEXT: v_max_f32_e32 v0, v0, v1
15091 ; GFX8-NEXT: v_max_f32_e32 v5, v5, v2
15092 ; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
15093 ; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
15094 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
15095 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
15096 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
15097 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
15098 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
15099 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
15100 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
15101 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
15102 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
15103 ; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
15104 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
15105 ; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
15106 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
15107 ; GFX8-NEXT: s_waitcnt vmcnt(0)
15108 ; GFX8-NEXT: buffer_wbinvl1
15109 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
15110 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15111 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
15112 ; GFX8-NEXT: s_cbranch_execnz .LBB56_1
15113 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
15114 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
15115 ; GFX8-NEXT: s_setpc_b64 s[30:31]
15116 ;
15117 ; GFX7-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
15118 ; GFX7: ; %bb.0:
15119 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15120 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
15121 ; GFX7-NEXT: s_mov_b32 s5, -1
15122 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
15123 ; GFX7-NEXT: s_mov_b32 s6, 0
15124 ; GFX7-NEXT: buffer_load_dword v6, v[0:1], s[4:7], 0 addr64
15125 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
15126 ; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v3
15127 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v2
15128 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
15129 ; GFX7-NEXT: s_mov_b32 s4, s6
15130 ; GFX7-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
15131 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
15132 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
15133 ; GFX7-NEXT: s_mov_b32 s5, s6
15134 ; GFX7-NEXT: s_waitcnt vmcnt(0)
15135 ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
15136 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v6
15137 ; GFX7-NEXT: .LBB56_1: ; %atomicrmw.start
15138 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
15139 ; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
15140 ; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
15141 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
15142 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v0
15143 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
15144 ; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
15145 ; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
15146 ; GFX7-NEXT: v_alignbit_b32 v1, v1, v0, 16
15147 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v6
15148 ; GFX7-NEXT: v_alignbit_b32 v0, v0, v7, 16
15149 ; GFX7-NEXT: v_mov_b32_e32 v7, v1
15150 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
15151 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[4:5], s[4:7], 0 addr64 glc
15152 ; GFX7-NEXT: s_waitcnt vmcnt(0)
15153 ; GFX7-NEXT: buffer_wbinvl1
15154 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v1
15155 ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
15156 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
15157 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v6
15158 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
15159 ; GFX7-NEXT: s_cbranch_execnz .LBB56_1
15160 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
15161 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
15162 ; GFX7-NEXT: s_setpc_b64 s[30:31]
15163 ;
15164 ; GFX6-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
15165 ; GFX6: ; %bb.0:
15166 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15167 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
15168 ; GFX6-NEXT: s_mov_b32 s5, -1
15169 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
15170 ; GFX6-NEXT: s_mov_b32 s6, 0
15171 ; GFX6-NEXT: buffer_load_dword v6, v[0:1], s[4:7], 0 addr64
15172 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffff800, v0
15173 ; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v3
15174 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v2
15175 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
15176 ; GFX6-NEXT: s_mov_b32 s4, s6
15177 ; GFX6-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
15178 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
15179 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
15180 ; GFX6-NEXT: s_mov_b32 s5, s6
15181 ; GFX6-NEXT: s_waitcnt vmcnt(0)
15182 ; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
15183 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v6
15184 ; GFX6-NEXT: .LBB56_1: ; %atomicrmw.start
15185 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
15186 ; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
15187 ; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
15188 ; GFX6-NEXT: s_waitcnt expcnt(0)
15189 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
15190 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v0
15191 ; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
15192 ; GFX6-NEXT: v_max_f32_e32 v6, v6, v2
15193 ; GFX6-NEXT: v_max_f32_e32 v7, v7, v3
15194 ; GFX6-NEXT: v_alignbit_b32 v1, v1, v0, 16
15195 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v6
15196 ; GFX6-NEXT: v_alignbit_b32 v0, v0, v7, 16
15197 ; GFX6-NEXT: v_mov_b32_e32 v7, v1
15198 ; GFX6-NEXT: v_mov_b32_e32 v6, v0
15199 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[4:5], s[4:7], 0 addr64 glc
15200 ; GFX6-NEXT: s_waitcnt vmcnt(0)
15201 ; GFX6-NEXT: buffer_wbinvl1
15202 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v1
15203 ; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
15204 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
15205 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v6
15206 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
15207 ; GFX6-NEXT: s_cbranch_execnz .LBB56_1
15208 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
15209 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
15210 ; GFX6-NEXT: s_waitcnt expcnt(0)
15211 ; GFX6-NEXT: s_setpc_b64 s[30:31]
15212 %gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 -512
15213 %result = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
15214 ret <2 x bfloat> %result
15215 }
15217 define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
15218 ; GFX12-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
15219 ; GFX12: ; %bb.0:
15220 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
15221 ; GFX12-NEXT: s_wait_expcnt 0x0
15222 ; GFX12-NEXT: s_wait_samplecnt 0x0
15223 ; GFX12-NEXT: s_wait_bvhcnt 0x0
15224 ; GFX12-NEXT: s_wait_kmcnt 0x0
15225 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off
15226 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15227 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15228 ; GFX12-NEXT: s_mov_b32 s1, 0
15229 ; GFX12-NEXT: .LBB57_1: ; %atomicrmw.start
15230 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
15231 ; GFX12-NEXT: s_wait_loadcnt 0x0
15232 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15233 ; GFX12-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15234 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15235 ; GFX12-NEXT: v_max_num_f32_e32 v2, v2, v4
15236 ; GFX12-NEXT: v_max_num_f32_e32 v6, v6, v5
15237 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15238 ; GFX12-NEXT: v_bfe_u32 v7, v2, 16, 1
15239 ; GFX12-NEXT: v_bfe_u32 v8, v6, 16, 1
15240 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v2
15241 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v6
15242 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
15243 ; GFX12-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
15244 ; GFX12-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
15245 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v2, v2
15246 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15247 ; GFX12-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
15248 ; GFX12-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
15249 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
15250 ; GFX12-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
15251 ; GFX12-NEXT: s_wait_storecnt 0x0
15252 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
15253 ; GFX12-NEXT: s_wait_loadcnt 0x0
15254 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
15255 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
15256 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
15257 ; GFX12-NEXT: s_wait_alu 0xfffe
15258 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
15259 ; GFX12-NEXT: s_wait_alu 0xfffe
15260 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
15261 ; GFX12-NEXT: s_cbranch_execnz .LBB57_1
15262 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
15263 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
15264 ; GFX12-NEXT: s_wait_alu 0xfffe
15265 ; GFX12-NEXT: s_setpc_b64 s[30:31]
15266 ;
15267 ; GFX940-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
15268 ; GFX940: ; %bb.0:
15269 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15270 ; GFX940-NEXT: global_load_dword v3, v[0:1], off
15271 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
15272 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15273 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
15274 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15275 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
15276 ; GFX940-NEXT: .LBB57_1: ; %atomicrmw.start
15277 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
15278 ; GFX940-NEXT: s_waitcnt vmcnt(0)
15279 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15280 ; GFX940-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15281 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
15282 ; GFX940-NEXT: v_max_f32_e32 v6, v6, v5
15283 ; GFX940-NEXT: v_bfe_u32 v7, v2, 16, 1
15284 ; GFX940-NEXT: v_bfe_u32 v9, v6, 16, 1
15285 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v2
15286 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v6
15287 ; GFX940-NEXT: v_add3_u32 v7, v7, v2, s4
15288 ; GFX940-NEXT: v_add3_u32 v9, v9, v6, s4
15289 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
15290 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
15291 ; GFX940-NEXT: s_nop 0
15292 ; GFX940-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
15293 ; GFX940-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
15294 ; GFX940-NEXT: v_perm_b32 v2, v6, v2, s5
15295 ; GFX940-NEXT: buffer_wbl2 sc1
15296 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
15297 ; GFX940-NEXT: s_waitcnt vmcnt(0)
15298 ; GFX940-NEXT: buffer_inv sc1
15299 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
15300 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
15301 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
15302 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
15303 ; GFX940-NEXT: s_cbranch_execnz .LBB57_1
15304 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
15305 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
15306 ; GFX940-NEXT: s_setpc_b64 s[30:31]
15307 ;
15308 ; GFX11-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
15309 ; GFX11: ; %bb.0:
15310 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15311 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off
15312 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15313 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15314 ; GFX11-NEXT: s_mov_b32 s1, 0
15315 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
15316 ; GFX11-NEXT: .p2align 6
15317 ; GFX11-NEXT: .LBB57_1: ; %atomicrmw.start
15318 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
15319 ; GFX11-NEXT: s_waitcnt vmcnt(0)
15320 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15321 ; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15322 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15323 ; GFX11-NEXT: v_max_f32_e32 v2, v2, v4
15324 ; GFX11-NEXT: v_max_f32_e32 v6, v6, v5
15325 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15326 ; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
15327 ; GFX11-NEXT: v_bfe_u32 v8, v6, 16, 1
15328 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v2
15329 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v6
15330 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
15331 ; GFX11-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
15332 ; GFX11-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
15333 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v2, v2
15334 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15335 ; GFX11-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
15336 ; GFX11-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
15337 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
15338 ; GFX11-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
15339 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
15340 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
15341 ; GFX11-NEXT: s_waitcnt vmcnt(0)
15342 ; GFX11-NEXT: buffer_gl1_inv
15343 ; GFX11-NEXT: buffer_gl0_inv
15344 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
15345 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
15346 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
15347 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
15348 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
15349 ; GFX11-NEXT: s_cbranch_execnz .LBB57_1
15350 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
15351 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
15352 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
15353 ; GFX11-NEXT: s_setpc_b64 s[30:31]
15354 ;
15355 ; GFX10-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
15356 ; GFX10: ; %bb.0:
15357 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15358 ; GFX10-NEXT: global_load_dword v3, v[0:1], off
15359 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15360 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15361 ; GFX10-NEXT: s_mov_b32 s5, 0
15362 ; GFX10-NEXT: .LBB57_1: ; %atomicrmw.start
15363 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
15364 ; GFX10-NEXT: s_waitcnt vmcnt(0)
15365 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15366 ; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15367 ; GFX10-NEXT: v_max_f32_e32 v2, v2, v4
15368 ; GFX10-NEXT: v_max_f32_e32 v6, v6, v5
15369 ; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
15370 ; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
15371 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
15372 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
15373 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
15374 ; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
15375 ; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
15376 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
15377 ; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
15378 ; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
15379 ; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
15380 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
15381 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
15382 ; GFX10-NEXT: s_waitcnt vmcnt(0)
15383 ; GFX10-NEXT: buffer_gl1_inv
15384 ; GFX10-NEXT: buffer_gl0_inv
15385 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
15386 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
15387 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
15388 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
15389 ; GFX10-NEXT: s_cbranch_execnz .LBB57_1
15390 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
15391 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
15392 ; GFX10-NEXT: s_setpc_b64 s[30:31]
15393 ;
15394 ; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
15395 ; GFX90A: ; %bb.0:
15396 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15397 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off
15398 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
15399 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15400 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
15401 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15402 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
15403 ; GFX90A-NEXT: .LBB57_1: ; %atomicrmw.start
15404 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
15405 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
15406 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15407 ; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15408 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
15409 ; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
15410 ; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
15411 ; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
15412 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
15413 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
15414 ; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
15415 ; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
15416 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
15417 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
15418 ; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
15419 ; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
15420 ; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
15421 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
15422 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
15423 ; GFX90A-NEXT: buffer_wbinvl1
15424 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
15425 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15426 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
15427 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
15428 ; GFX90A-NEXT: s_cbranch_execnz .LBB57_1
15429 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
15430 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
15431 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
15432 ;
15433 ; GFX908-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
15434 ; GFX908: ; %bb.0:
15435 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15436 ; GFX908-NEXT: global_load_dword v3, v[0:1], off
15437 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
15438 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15439 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
15440 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15441 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
15442 ; GFX908-NEXT: .LBB57_1: ; %atomicrmw.start
15443 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
15444 ; GFX908-NEXT: s_waitcnt vmcnt(0)
15445 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15446 ; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15447 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
15448 ; GFX908-NEXT: v_max_f32_e32 v6, v6, v5
15449 ; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
15450 ; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
15451 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
15452 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
15453 ; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
15454 ; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
15455 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
15456 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
15457 ; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
15458 ; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
15459 ; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
15460 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
15461 ; GFX908-NEXT: s_waitcnt vmcnt(0)
15462 ; GFX908-NEXT: buffer_wbinvl1
15463 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
15464 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15465 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
15466 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
15467 ; GFX908-NEXT: s_cbranch_execnz .LBB57_1
15468 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
15469 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
15470 ; GFX908-NEXT: s_setpc_b64 s[30:31]
15471 ;
15472 ; GFX8-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
15473 ; GFX8: ; %bb.0:
15474 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15475 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
15476 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
15477 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15478 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15479 ; GFX8-NEXT: .LBB57_1: ; %atomicrmw.start
15480 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
15481 ; GFX8-NEXT: s_waitcnt vmcnt(0)
15482 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15483 ; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15484 ; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
15485 ; GFX8-NEXT: v_max_f32_e32 v6, v6, v5
15486 ; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
15487 ; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
15488 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
15489 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
15490 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
15491 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
15492 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
15493 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
15494 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
15495 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
15496 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
15497 ; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
15498 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
15499 ; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
15500 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
15501 ; GFX8-NEXT: s_waitcnt vmcnt(0)
15502 ; GFX8-NEXT: buffer_wbinvl1
15503 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
15504 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15505 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
15506 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
15507 ; GFX8-NEXT: s_cbranch_execnz .LBB57_1
15508 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
15509 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
15510 ; GFX8-NEXT: s_setpc_b64 s[30:31]
15511 ;
15512 ; GFX7-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
15513 ; GFX7: ; %bb.0:
15514 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15515 ; GFX7-NEXT: s_mov_b32 s6, 0
15516 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
15517 ; GFX7-NEXT: s_mov_b32 s4, s6
15518 ; GFX7-NEXT: s_mov_b32 s5, s6
15519 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
15520 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
15521 ; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
15522 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
15523 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
15524 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
15525 ; GFX7-NEXT: s_waitcnt vmcnt(0)
15526 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
15527 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
15528 ; GFX7-NEXT: .LBB57_1: ; %atomicrmw.start
15529 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
15530 ; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
15531 ; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
15532 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
15533 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
15534 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
15535 ; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
15536 ; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
15537 ; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
15538 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
15539 ; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
15540 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
15541 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
15542 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
15543 ; GFX7-NEXT: s_waitcnt vmcnt(0)
15544 ; GFX7-NEXT: buffer_wbinvl1
15545 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
15546 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15547 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
15548 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
15549 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
15550 ; GFX7-NEXT: s_cbranch_execnz .LBB57_1
15551 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
15552 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
15553 ; GFX7-NEXT: s_setpc_b64 s[30:31]
15554 ;
15555 ; GFX6-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
15556 ; GFX6: ; %bb.0:
15557 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15558 ; GFX6-NEXT: s_mov_b32 s6, 0
15559 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
15560 ; GFX6-NEXT: s_mov_b32 s4, s6
15561 ; GFX6-NEXT: s_mov_b32 s5, s6
15562 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
15563 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
15564 ; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
15565 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
15566 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
15567 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
15568 ; GFX6-NEXT: s_waitcnt vmcnt(0)
15569 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
15570 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
15571 ; GFX6-NEXT: .LBB57_1: ; %atomicrmw.start
15572 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
15573 ; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
15574 ; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
15575 ; GFX6-NEXT: s_waitcnt expcnt(0)
15576 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
15577 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
15578 ; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
15579 ; GFX6-NEXT: v_max_f32_e32 v6, v6, v2
15580 ; GFX6-NEXT: v_max_f32_e32 v7, v7, v3
15581 ; GFX6-NEXT: v_alignbit_b32 v5, v5, v4, 16
15582 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
15583 ; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
15584 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
15585 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
15586 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
15587 ; GFX6-NEXT: s_waitcnt vmcnt(0)
15588 ; GFX6-NEXT: buffer_wbinvl1
15589 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
15590 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15591 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
15592 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
15593 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
15594 ; GFX6-NEXT: s_cbranch_execnz .LBB57_1
15595 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
15596 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
15597 ; GFX6-NEXT: s_waitcnt expcnt(0)
15598 ; GFX6-NEXT: s_setpc_b64 s[30:31]
15599 %unused = atomicrmw fmax ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
15600 ret void
15601 }
15603 define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
15604 ; GFX12-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
15605 ; GFX12: ; %bb.0:
15606 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
15607 ; GFX12-NEXT: s_wait_expcnt 0x0
15608 ; GFX12-NEXT: s_wait_samplecnt 0x0
15609 ; GFX12-NEXT: s_wait_bvhcnt 0x0
15610 ; GFX12-NEXT: s_wait_kmcnt 0x0
15611 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
15612 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15613 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15614 ; GFX12-NEXT: s_mov_b32 s1, 0
15615 ; GFX12-NEXT: .LBB58_1: ; %atomicrmw.start
15616 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
15617 ; GFX12-NEXT: s_wait_loadcnt 0x0
15618 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15619 ; GFX12-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15620 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15621 ; GFX12-NEXT: v_max_num_f32_e32 v2, v2, v4
15622 ; GFX12-NEXT: v_max_num_f32_e32 v6, v6, v5
15623 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15624 ; GFX12-NEXT: v_bfe_u32 v7, v2, 16, 1
15625 ; GFX12-NEXT: v_bfe_u32 v8, v6, 16, 1
15626 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v2
15627 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v6
15628 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
15629 ; GFX12-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
15630 ; GFX12-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
15631 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v2, v2
15632 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15633 ; GFX12-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
15634 ; GFX12-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
15635 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
15636 ; GFX12-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
15637 ; GFX12-NEXT: s_wait_storecnt 0x0
15638 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
15639 ; GFX12-NEXT: s_wait_loadcnt 0x0
15640 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
15641 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
15642 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
15643 ; GFX12-NEXT: s_wait_alu 0xfffe
15644 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
15645 ; GFX12-NEXT: s_wait_alu 0xfffe
15646 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
15647 ; GFX12-NEXT: s_cbranch_execnz .LBB58_1
15648 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
15649 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
15650 ; GFX12-NEXT: s_wait_alu 0xfffe
15651 ; GFX12-NEXT: s_setpc_b64 s[30:31]
15652 ;
15653 ; GFX940-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
15654 ; GFX940: ; %bb.0:
15655 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15656 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
15657 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
15658 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15659 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
15660 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15661 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
15662 ; GFX940-NEXT: .LBB58_1: ; %atomicrmw.start
15663 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
15664 ; GFX940-NEXT: s_waitcnt vmcnt(0)
15665 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15666 ; GFX940-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15667 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
15668 ; GFX940-NEXT: v_max_f32_e32 v6, v6, v5
15669 ; GFX940-NEXT: v_bfe_u32 v7, v2, 16, 1
15670 ; GFX940-NEXT: v_bfe_u32 v9, v6, 16, 1
15671 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v2
15672 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v6
15673 ; GFX940-NEXT: v_add3_u32 v7, v7, v2, s4
15674 ; GFX940-NEXT: v_add3_u32 v9, v9, v6, s4
15675 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
15676 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
15677 ; GFX940-NEXT: s_nop 0
15678 ; GFX940-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
15679 ; GFX940-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
15680 ; GFX940-NEXT: v_perm_b32 v2, v6, v2, s5
15681 ; GFX940-NEXT: buffer_wbl2 sc1
15682 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
15683 ; GFX940-NEXT: s_waitcnt vmcnt(0)
15684 ; GFX940-NEXT: buffer_inv sc1
15685 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
15686 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
15687 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
15688 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
15689 ; GFX940-NEXT: s_cbranch_execnz .LBB58_1
15690 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
15691 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
15692 ; GFX940-NEXT: s_setpc_b64 s[30:31]
15693 ;
15694 ; GFX11-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
15695 ; GFX11: ; %bb.0:
15696 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15697 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
15698 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15699 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15700 ; GFX11-NEXT: s_mov_b32 s1, 0
15701 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
15702 ; GFX11-NEXT: .p2align 6
15703 ; GFX11-NEXT: .LBB58_1: ; %atomicrmw.start
15704 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
15705 ; GFX11-NEXT: s_waitcnt vmcnt(0)
15706 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15707 ; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15708 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15709 ; GFX11-NEXT: v_max_f32_e32 v2, v2, v4
15710 ; GFX11-NEXT: v_max_f32_e32 v6, v6, v5
15711 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15712 ; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
15713 ; GFX11-NEXT: v_bfe_u32 v8, v6, 16, 1
15714 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v2
15715 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v6
15716 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
15717 ; GFX11-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
15718 ; GFX11-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
15719 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v2, v2
15720 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
15721 ; GFX11-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
15722 ; GFX11-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
15723 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
15724 ; GFX11-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
15725 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
15726 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
15727 ; GFX11-NEXT: s_waitcnt vmcnt(0)
15728 ; GFX11-NEXT: buffer_gl1_inv
15729 ; GFX11-NEXT: buffer_gl0_inv
15730 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
15731 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
15732 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
15733 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
15734 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
15735 ; GFX11-NEXT: s_cbranch_execnz .LBB58_1
15736 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
15737 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
15738 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
15739 ; GFX11-NEXT: s_setpc_b64 s[30:31]
15740 ;
15741 ; GFX10-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
15742 ; GFX10: ; %bb.0:
15743 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15744 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
15745 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15746 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15747 ; GFX10-NEXT: s_mov_b32 s5, 0
15748 ; GFX10-NEXT: .LBB58_1: ; %atomicrmw.start
15749 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
15750 ; GFX10-NEXT: s_waitcnt vmcnt(0)
15751 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15752 ; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15753 ; GFX10-NEXT: v_max_f32_e32 v2, v2, v4
15754 ; GFX10-NEXT: v_max_f32_e32 v6, v6, v5
15755 ; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
15756 ; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
15757 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
15758 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
15759 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
15760 ; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
15761 ; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
15762 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
15763 ; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
15764 ; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
15765 ; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
15766 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
15767 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
15768 ; GFX10-NEXT: s_waitcnt vmcnt(0)
15769 ; GFX10-NEXT: buffer_gl1_inv
15770 ; GFX10-NEXT: buffer_gl0_inv
15771 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
15772 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
15773 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
15774 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
15775 ; GFX10-NEXT: s_cbranch_execnz .LBB58_1
15776 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
15777 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
15778 ; GFX10-NEXT: s_setpc_b64 s[30:31]
15779 ;
15780 ; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
15781 ; GFX90A: ; %bb.0:
15782 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15783 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
15784 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
15785 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15786 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
15787 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15788 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
15789 ; GFX90A-NEXT: .LBB58_1: ; %atomicrmw.start
15790 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
15791 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
15792 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15793 ; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15794 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
15795 ; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
15796 ; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
15797 ; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
15798 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
15799 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
15800 ; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
15801 ; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
15802 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
15803 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
15804 ; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
15805 ; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
15806 ; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
15807 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
15808 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
15809 ; GFX90A-NEXT: buffer_wbinvl1
15810 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
15811 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15812 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
15813 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
15814 ; GFX90A-NEXT: s_cbranch_execnz .LBB58_1
15815 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
15816 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
15817 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
15818 ;
15819 ; GFX908-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
15820 ; GFX908: ; %bb.0:
15821 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15822 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
15823 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
15824 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15825 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
15826 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15827 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
15828 ; GFX908-NEXT: .LBB58_1: ; %atomicrmw.start
15829 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
15830 ; GFX908-NEXT: s_waitcnt vmcnt(0)
15831 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15832 ; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15833 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
15834 ; GFX908-NEXT: v_max_f32_e32 v6, v6, v5
15835 ; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
15836 ; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
15837 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
15838 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
15839 ; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
15840 ; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
15841 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
15842 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
15843 ; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
15844 ; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
15845 ; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
15846 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
15847 ; GFX908-NEXT: s_waitcnt vmcnt(0)
15848 ; GFX908-NEXT: buffer_wbinvl1
15849 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
15850 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15851 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
15852 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
15853 ; GFX908-NEXT: s_cbranch_execnz .LBB58_1
15854 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
15855 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
15856 ; GFX908-NEXT: s_setpc_b64 s[30:31]
15857 ;
15858 ; GFX8-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
15859 ; GFX8: ; %bb.0:
15860 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15861 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
15862 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
15863 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
15864 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
15865 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
15866 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
15867 ; GFX8-NEXT: .LBB58_1: ; %atomicrmw.start
15868 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
15869 ; GFX8-NEXT: s_waitcnt vmcnt(0)
15870 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
15871 ; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
15872 ; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
15873 ; GFX8-NEXT: v_max_f32_e32 v6, v6, v5
15874 ; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
15875 ; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
15876 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
15877 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
15878 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
15879 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
15880 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
15881 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
15882 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
15883 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
15884 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
15885 ; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
15886 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
15887 ; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
15888 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
15889 ; GFX8-NEXT: s_waitcnt vmcnt(0)
15890 ; GFX8-NEXT: buffer_wbinvl1
15891 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
15892 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
15893 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
15894 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
15895 ; GFX8-NEXT: s_cbranch_execnz .LBB58_1
15896 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
15897 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
15898 ; GFX8-NEXT: s_setpc_b64 s[30:31]
15899 ;
15900 ; GFX7-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
15901 ; GFX7: ; %bb.0:
15902 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15903 ; GFX7-NEXT: s_mov_b32 s6, 0
15904 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
15905 ; GFX7-NEXT: s_mov_b32 s4, s6
15906 ; GFX7-NEXT: s_mov_b32 s5, s6
15907 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:2044
15908 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
15909 ; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
15910 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
15911 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
15912 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
15913 ; GFX7-NEXT: s_waitcnt vmcnt(0)
15914 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
15915 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
15916 ; GFX7-NEXT: .LBB58_1: ; %atomicrmw.start
15917 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
15918 ; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
15919 ; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
15920 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
15921 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
15922 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
15923 ; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
15924 ; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
15925 ; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
15926 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
15927 ; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
15928 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
15929 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
15930 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
15931 ; GFX7-NEXT: s_waitcnt vmcnt(0)
15932 ; GFX7-NEXT: buffer_wbinvl1
15933 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
15934 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15935 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
15936 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
15937 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
15938 ; GFX7-NEXT: s_cbranch_execnz .LBB58_1
15939 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
15940 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
15941 ; GFX7-NEXT: s_setpc_b64 s[30:31]
15942 ;
15943 ; GFX6-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
15944 ; GFX6: ; %bb.0:
15945 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
15946 ; GFX6-NEXT: s_mov_b32 s6, 0
15947 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
15948 ; GFX6-NEXT: s_mov_b32 s4, s6
15949 ; GFX6-NEXT: s_mov_b32 s5, s6
15950 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:2044
15951 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
15952 ; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
15953 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
15954 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
15955 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
15956 ; GFX6-NEXT: s_waitcnt vmcnt(0)
15957 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
15958 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
15959 ; GFX6-NEXT: .LBB58_1: ; %atomicrmw.start
15960 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
15961 ; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
15962 ; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
15963 ; GFX6-NEXT: s_waitcnt expcnt(0)
15964 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
15965 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
15966 ; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
15967 ; GFX6-NEXT: v_max_f32_e32 v6, v6, v2
15968 ; GFX6-NEXT: v_max_f32_e32 v7, v7, v3
15969 ; GFX6-NEXT: v_alignbit_b32 v5, v5, v4, 16
15970 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
15971 ; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
15972 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
15973 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
15974 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
15975 ; GFX6-NEXT: s_waitcnt vmcnt(0)
15976 ; GFX6-NEXT: buffer_wbinvl1
15977 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
15978 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
15979 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
15980 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
15981 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
15982 ; GFX6-NEXT: s_cbranch_execnz .LBB58_1
15983 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
15984 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
15985 ; GFX6-NEXT: s_waitcnt expcnt(0)
15986 ; GFX6-NEXT: s_setpc_b64 s[30:31]
15987 %gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
15988 %unused = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
15989 ret void
15990 }
15992 define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
15993 ; GFX12-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
15994 ; GFX12: ; %bb.0:
15995 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
15996 ; GFX12-NEXT: s_wait_expcnt 0x0
15997 ; GFX12-NEXT: s_wait_samplecnt 0x0
15998 ; GFX12-NEXT: s_wait_bvhcnt 0x0
15999 ; GFX12-NEXT: s_wait_kmcnt 0x0
16000 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
16001 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16002 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16003 ; GFX12-NEXT: s_mov_b32 s1, 0
16004 ; GFX12-NEXT: .LBB59_1: ; %atomicrmw.start
16005 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
16006 ; GFX12-NEXT: s_wait_loadcnt 0x0
16007 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16008 ; GFX12-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16009 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16010 ; GFX12-NEXT: v_max_num_f32_e32 v2, v2, v4
16011 ; GFX12-NEXT: v_max_num_f32_e32 v6, v6, v5
16012 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16013 ; GFX12-NEXT: v_bfe_u32 v7, v2, 16, 1
16014 ; GFX12-NEXT: v_bfe_u32 v8, v6, 16, 1
16015 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v2
16016 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v6
16017 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
16018 ; GFX12-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
16019 ; GFX12-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
16020 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v2, v2
16021 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16022 ; GFX12-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
16023 ; GFX12-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
16024 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
16025 ; GFX12-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
16026 ; GFX12-NEXT: s_wait_storecnt 0x0
16027 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
16028 ; GFX12-NEXT: s_wait_loadcnt 0x0
16029 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
16030 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
16031 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
16032 ; GFX12-NEXT: s_wait_alu 0xfffe
16033 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
16034 ; GFX12-NEXT: s_wait_alu 0xfffe
16035 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
16036 ; GFX12-NEXT: s_cbranch_execnz .LBB59_1
16037 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
16038 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
16039 ; GFX12-NEXT: s_wait_alu 0xfffe
16040 ; GFX12-NEXT: s_setpc_b64 s[30:31]
16041 ;
16042 ; GFX940-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
16043 ; GFX940: ; %bb.0:
16044 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16045 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:-2048
16046 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
16047 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16048 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
16049 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16050 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
16051 ; GFX940-NEXT: .LBB59_1: ; %atomicrmw.start
16052 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
16053 ; GFX940-NEXT: s_waitcnt vmcnt(0)
16054 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16055 ; GFX940-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16056 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
16057 ; GFX940-NEXT: v_max_f32_e32 v6, v6, v5
16058 ; GFX940-NEXT: v_bfe_u32 v7, v2, 16, 1
16059 ; GFX940-NEXT: v_bfe_u32 v9, v6, 16, 1
16060 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v2
16061 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v6
16062 ; GFX940-NEXT: v_add3_u32 v7, v7, v2, s4
16063 ; GFX940-NEXT: v_add3_u32 v9, v9, v6, s4
16064 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
16065 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
16066 ; GFX940-NEXT: s_nop 0
16067 ; GFX940-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
16068 ; GFX940-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
16069 ; GFX940-NEXT: v_perm_b32 v2, v6, v2, s5
16070 ; GFX940-NEXT: buffer_wbl2 sc1
16071 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
16072 ; GFX940-NEXT: s_waitcnt vmcnt(0)
16073 ; GFX940-NEXT: buffer_inv sc1
16074 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
16075 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
16076 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
16077 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
16078 ; GFX940-NEXT: s_cbranch_execnz .LBB59_1
16079 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
16080 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
16081 ; GFX940-NEXT: s_setpc_b64 s[30:31]
16082 ;
16083 ; GFX11-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
16084 ; GFX11: ; %bb.0:
16085 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16086 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
16087 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16088 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16089 ; GFX11-NEXT: s_mov_b32 s1, 0
16090 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
16091 ; GFX11-NEXT: .p2align 6
16092 ; GFX11-NEXT: .LBB59_1: ; %atomicrmw.start
16093 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
16094 ; GFX11-NEXT: s_waitcnt vmcnt(0)
16095 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16096 ; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16097 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16098 ; GFX11-NEXT: v_max_f32_e32 v2, v2, v4
16099 ; GFX11-NEXT: v_max_f32_e32 v6, v6, v5
16100 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16101 ; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
16102 ; GFX11-NEXT: v_bfe_u32 v8, v6, 16, 1
16103 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v2
16104 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v6
16105 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
16106 ; GFX11-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
16107 ; GFX11-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
16108 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v2, v2
16109 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16110 ; GFX11-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
16111 ; GFX11-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
16112 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
16113 ; GFX11-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
16114 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
16115 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 glc
16116 ; GFX11-NEXT: s_waitcnt vmcnt(0)
16117 ; GFX11-NEXT: buffer_gl1_inv
16118 ; GFX11-NEXT: buffer_gl0_inv
16119 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
16120 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
16121 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
16122 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
16123 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
16124 ; GFX11-NEXT: s_cbranch_execnz .LBB59_1
16125 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
16126 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
16127 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
16128 ; GFX11-NEXT: s_setpc_b64 s[30:31]
16129 ;
16130 ; GFX10-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
16131 ; GFX10: ; %bb.0:
16132 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16133 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
16134 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16135 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16136 ; GFX10-NEXT: s_mov_b32 s5, 0
16137 ; GFX10-NEXT: .LBB59_1: ; %atomicrmw.start
16138 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
16139 ; GFX10-NEXT: s_waitcnt vmcnt(0)
16140 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16141 ; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16142 ; GFX10-NEXT: v_max_f32_e32 v2, v2, v4
16143 ; GFX10-NEXT: v_max_f32_e32 v6, v6, v5
16144 ; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
16145 ; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
16146 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
16147 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
16148 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
16149 ; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
16150 ; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
16151 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
16152 ; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
16153 ; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
16154 ; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
16155 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
16156 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
16157 ; GFX10-NEXT: s_waitcnt vmcnt(0)
16158 ; GFX10-NEXT: buffer_gl1_inv
16159 ; GFX10-NEXT: buffer_gl0_inv
16160 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
16161 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
16162 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
16163 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
16164 ; GFX10-NEXT: s_cbranch_execnz .LBB59_1
16165 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
16166 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
16167 ; GFX10-NEXT: s_setpc_b64 s[30:31]
16168 ;
16169 ; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
16170 ; GFX90A: ; %bb.0:
16171 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16172 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
16173 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
16174 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16175 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
16176 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16177 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
16178 ; GFX90A-NEXT: .LBB59_1: ; %atomicrmw.start
16179 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
16180 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
16181 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16182 ; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16183 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
16184 ; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
16185 ; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
16186 ; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
16187 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
16188 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
16189 ; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
16190 ; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
16191 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
16192 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
16193 ; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
16194 ; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
16195 ; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
16196 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
16197 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
16198 ; GFX90A-NEXT: buffer_wbinvl1
16199 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
16200 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
16201 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
16202 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
16203 ; GFX90A-NEXT: s_cbranch_execnz .LBB59_1
16204 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
16205 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
16206 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
16207 ;
16208 ; GFX908-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
16209 ; GFX908: ; %bb.0:
16210 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16211 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
16212 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
16213 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16214 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
16215 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16216 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
16217 ; GFX908-NEXT: .LBB59_1: ; %atomicrmw.start
16218 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
16219 ; GFX908-NEXT: s_waitcnt vmcnt(0)
16220 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16221 ; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16222 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
16223 ; GFX908-NEXT: v_max_f32_e32 v6, v6, v5
16224 ; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
16225 ; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
16226 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
16227 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
16228 ; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
16229 ; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
16230 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
16231 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
16232 ; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
16233 ; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
16234 ; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
16235 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
16236 ; GFX908-NEXT: s_waitcnt vmcnt(0)
16237 ; GFX908-NEXT: buffer_wbinvl1
16238 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
16239 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
16240 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
16241 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
16242 ; GFX908-NEXT: s_cbranch_execnz .LBB59_1
16243 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
16244 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
16245 ; GFX908-NEXT: s_setpc_b64 s[30:31]
16246 ;
16247 ; GFX8-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
16248 ; GFX8: ; %bb.0:
16249 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16250 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
16251 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
16252 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
16253 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
16254 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16255 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16256 ; GFX8-NEXT: .LBB59_1: ; %atomicrmw.start
16257 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
16258 ; GFX8-NEXT: s_waitcnt vmcnt(0)
16259 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16260 ; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16261 ; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
16262 ; GFX8-NEXT: v_max_f32_e32 v6, v6, v5
16263 ; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
16264 ; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
16265 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
16266 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
16267 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
16268 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
16269 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
16270 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
16271 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
16272 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
16273 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
16274 ; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
16275 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
16276 ; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
16277 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
16278 ; GFX8-NEXT: s_waitcnt vmcnt(0)
16279 ; GFX8-NEXT: buffer_wbinvl1
16280 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
16281 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
16282 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
16283 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
16284 ; GFX8-NEXT: s_cbranch_execnz .LBB59_1
16285 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
16286 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
16287 ; GFX8-NEXT: s_setpc_b64 s[30:31]
16288 ;
16289 ; GFX7-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
16290 ; GFX7: ; %bb.0:
16291 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16292 ; GFX7-NEXT: s_movk_i32 s4, 0xf800
16293 ; GFX7-NEXT: s_mov_b32 s5, -1
16294 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
16295 ; GFX7-NEXT: s_mov_b32 s6, 0
16296 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
16297 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
16298 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
16299 ; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v2
16300 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
16301 ; GFX7-NEXT: s_mov_b32 s4, s6
16302 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
16303 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
16304 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
16305 ; GFX7-NEXT: s_mov_b32 s5, s6
16306 ; GFX7-NEXT: s_waitcnt vmcnt(0)
16307 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
16308 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
16309 ; GFX7-NEXT: .LBB59_1: ; %atomicrmw.start
16310 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
16311 ; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
16312 ; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
16313 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
16314 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
16315 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
16316 ; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
16317 ; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
16318 ; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
16319 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
16320 ; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
16321 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
16322 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
16323 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
16324 ; GFX7-NEXT: s_waitcnt vmcnt(0)
16325 ; GFX7-NEXT: buffer_wbinvl1
16326 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
16327 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
16328 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
16329 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
16330 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
16331 ; GFX7-NEXT: s_cbranch_execnz .LBB59_1
16332 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
16333 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
16334 ; GFX7-NEXT: s_setpc_b64 s[30:31]
16335 ;
16336 ; GFX6-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
16337 ; GFX6: ; %bb.0:
16338 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16339 ; GFX6-NEXT: s_movk_i32 s4, 0xf800
16340 ; GFX6-NEXT: s_mov_b32 s5, -1
16341 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
16342 ; GFX6-NEXT: s_mov_b32 s6, 0
16343 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
16344 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffff800, v0
16345 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
16346 ; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v2
16347 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
16348 ; GFX6-NEXT: s_mov_b32 s4, s6
16349 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
16350 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
16351 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
16352 ; GFX6-NEXT: s_mov_b32 s5, s6
16353 ; GFX6-NEXT: s_waitcnt vmcnt(0)
16354 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
16355 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
16356 ; GFX6-NEXT: .LBB59_1: ; %atomicrmw.start
16357 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
16358 ; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
16359 ; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
16360 ; GFX6-NEXT: s_waitcnt expcnt(0)
16361 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
16362 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
16363 ; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
16364 ; GFX6-NEXT: v_max_f32_e32 v6, v6, v2
16365 ; GFX6-NEXT: v_max_f32_e32 v7, v7, v3
16366 ; GFX6-NEXT: v_alignbit_b32 v5, v5, v4, 16
16367 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
16368 ; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
16369 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
16370 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
16371 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 glc
16372 ; GFX6-NEXT: s_waitcnt vmcnt(0)
16373 ; GFX6-NEXT: buffer_wbinvl1
16374 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
16375 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
16376 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
16377 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
16378 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
16379 ; GFX6-NEXT: s_cbranch_execnz .LBB59_1
16380 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
16381 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
16382 ; GFX6-NEXT: s_waitcnt expcnt(0)
16383 ; GFX6-NEXT: s_setpc_b64 s[30:31]
16384 %gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 -512
16385 %unused = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
16386 ret void
16387 }
16389 define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
16390 ; GFX12-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
16391 ; GFX12: ; %bb.0:
16392 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
16393 ; GFX12-NEXT: s_wait_expcnt 0x0
16394 ; GFX12-NEXT: s_wait_samplecnt 0x0
16395 ; GFX12-NEXT: s_wait_bvhcnt 0x0
16396 ; GFX12-NEXT: s_wait_kmcnt 0x0
16397 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
16398 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16399 ; GFX12-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16400 ; GFX12-NEXT: s_mov_b32 s1, 0
16401 ; GFX12-NEXT: .LBB60_1: ; %atomicrmw.start
16402 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
16403 ; GFX12-NEXT: s_wait_loadcnt 0x0
16404 ; GFX12-NEXT: v_mov_b32_e32 v6, v3
16405 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
16406 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
16407 ; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
16408 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v6
16409 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16410 ; GFX12-NEXT: v_bfe_u32 v8, v5, 16, 1
16411 ; GFX12-NEXT: v_max_num_f32_e32 v3, v3, v4
16412 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v5
16413 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
16414 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
16415 ; GFX12-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
16416 ; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
16417 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v3
16418 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v3, v3
16419 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
16420 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
16421 ; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
16422 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
16423 ; GFX12-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
16424 ; GFX12-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
16425 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
16426 ; GFX12-NEXT: s_wait_storecnt 0x0
16427 ; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
16428 ; GFX12-NEXT: s_wait_loadcnt 0x0
16429 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
16430 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
16431 ; GFX12-NEXT: s_wait_alu 0xfffe
16432 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
16433 ; GFX12-NEXT: s_wait_alu 0xfffe
16434 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
16435 ; GFX12-NEXT: s_cbranch_execnz .LBB60_1
16436 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
16437 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
16438 ; GFX12-NEXT: v_mov_b32_e32 v0, v3
16439 ; GFX12-NEXT: s_wait_alu 0xfffe
16440 ; GFX12-NEXT: s_setpc_b64 s[30:31]
16441 ;
16442 ; GFX940-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
16443 ; GFX940: ; %bb.0:
16444 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16445 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
16446 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
16447 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16448 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
16449 ; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16450 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
16451 ; GFX940-NEXT: .LBB60_1: ; %atomicrmw.start
16452 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
16453 ; GFX940-NEXT: s_waitcnt vmcnt(0)
16454 ; GFX940-NEXT: v_mov_b32_e32 v7, v3
16455 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 16, v7
16456 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
16457 ; GFX940-NEXT: v_max_f32_e32 v3, v3, v4
16458 ; GFX940-NEXT: v_max_f32_e32 v5, v5, v2
16459 ; GFX940-NEXT: v_bfe_u32 v6, v3, 16, 1
16460 ; GFX940-NEXT: v_bfe_u32 v9, v5, 16, 1
16461 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v3
16462 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v5
16463 ; GFX940-NEXT: v_add3_u32 v6, v6, v3, s4
16464 ; GFX940-NEXT: v_add3_u32 v9, v9, v5, s4
16465 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
16466 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
16467 ; GFX940-NEXT: s_nop 0
16468 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
16469 ; GFX940-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[0:1]
16470 ; GFX940-NEXT: v_perm_b32 v6, v5, v3, s5
16471 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
16472 ; GFX940-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off offset:2044 sc0 sc1
16473 ; GFX940-NEXT: s_waitcnt vmcnt(0)
16474 ; GFX940-NEXT: buffer_inv sc0 sc1
16475 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
16476 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
16477 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
16478 ; GFX940-NEXT: s_cbranch_execnz .LBB60_1
16479 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
16480 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
16481 ; GFX940-NEXT: v_mov_b32_e32 v0, v3
16482 ; GFX940-NEXT: s_setpc_b64 s[30:31]
16483 ;
16484 ; GFX11-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
16485 ; GFX11: ; %bb.0:
16486 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16487 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
16488 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16489 ; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16490 ; GFX11-NEXT: s_mov_b32 s1, 0
16491 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
16492 ; GFX11-NEXT: .p2align 6
16493 ; GFX11-NEXT: .LBB60_1: ; %atomicrmw.start
16494 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
16495 ; GFX11-NEXT: s_waitcnt vmcnt(0)
16496 ; GFX11-NEXT: v_mov_b32_e32 v6, v3
16497 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
16498 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
16499 ; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
16500 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v6
16501 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16502 ; GFX11-NEXT: v_bfe_u32 v8, v5, 16, 1
16503 ; GFX11-NEXT: v_max_f32_e32 v3, v3, v4
16504 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v5
16505 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
16506 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
16507 ; GFX11-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
16508 ; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
16509 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v3
16510 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v3, v3
16511 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
16512 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
16513 ; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
16514 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
16515 ; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
16516 ; GFX11-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
16517 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
16518 ; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
16519 ; GFX11-NEXT: s_waitcnt vmcnt(0)
16520 ; GFX11-NEXT: buffer_gl1_inv
16521 ; GFX11-NEXT: buffer_gl0_inv
16522 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
16523 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
16524 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
16525 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
16526 ; GFX11-NEXT: s_cbranch_execnz .LBB60_1
16527 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
16528 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
16529 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
16530 ; GFX11-NEXT: v_mov_b32_e32 v0, v3
16531 ; GFX11-NEXT: s_setpc_b64 s[30:31]
16532 ;
16533 ; GFX10-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
16534 ; GFX10: ; %bb.0:
16535 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16536 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
16537 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16538 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16539 ; GFX10-NEXT: s_mov_b32 s5, 0
16540 ; GFX10-NEXT: .LBB60_1: ; %atomicrmw.start
16541 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
16542 ; GFX10-NEXT: s_waitcnt vmcnt(0)
16543 ; GFX10-NEXT: v_mov_b32_e32 v6, v3
16544 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
16545 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
16546 ; GFX10-NEXT: v_max_f32_e32 v3, v3, v4
16547 ; GFX10-NEXT: v_max_f32_e32 v5, v5, v2
16548 ; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
16549 ; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
16550 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
16551 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
16552 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
16553 ; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
16554 ; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
16555 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
16556 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
16557 ; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
16558 ; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
16559 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
16560 ; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
16561 ; GFX10-NEXT: s_waitcnt vmcnt(0)
16562 ; GFX10-NEXT: buffer_gl1_inv
16563 ; GFX10-NEXT: buffer_gl0_inv
16564 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
16565 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
16566 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
16567 ; GFX10-NEXT: s_cbranch_execnz .LBB60_1
16568 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
16569 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
16570 ; GFX10-NEXT: v_mov_b32_e32 v0, v3
16571 ; GFX10-NEXT: s_setpc_b64 s[30:31]
16572 ;
16573 ; GFX90A-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
16574 ; GFX90A: ; %bb.0:
16575 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16576 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
16577 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
16578 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16579 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
16580 ; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16581 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
16582 ; GFX90A-NEXT: .LBB60_1: ; %atomicrmw.start
16583 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
16584 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
16585 ; GFX90A-NEXT: v_mov_b32_e32 v7, v3
16586 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v7
16587 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
16588 ; GFX90A-NEXT: v_max_f32_e32 v3, v3, v4
16589 ; GFX90A-NEXT: v_max_f32_e32 v5, v5, v2
16590 ; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
16591 ; GFX90A-NEXT: v_bfe_u32 v9, v5, 16, 1
16592 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v3
16593 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v5
16594 ; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
16595 ; GFX90A-NEXT: v_add3_u32 v9, v9, v5, s8
16596 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
16597 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
16598 ; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[4:5]
16599 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
16600 ; GFX90A-NEXT: v_perm_b32 v6, v5, v3, s9
16601 ; GFX90A-NEXT: buffer_wbl2
16602 ; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off offset:2044 glc
16603 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
16604 ; GFX90A-NEXT: buffer_invl2
16605 ; GFX90A-NEXT: buffer_wbinvl1
16606 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
16607 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
16608 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
16609 ; GFX90A-NEXT: s_cbranch_execnz .LBB60_1
16610 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
16611 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
16612 ; GFX90A-NEXT: v_mov_b32_e32 v0, v3
16613 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
16614 ;
16615 ; GFX908-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
16616 ; GFX908: ; %bb.0:
16617 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16618 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
16619 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
16620 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16621 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
16622 ; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16623 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
16624 ; GFX908-NEXT: .LBB60_1: ; %atomicrmw.start
16625 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
16626 ; GFX908-NEXT: s_waitcnt vmcnt(0)
16627 ; GFX908-NEXT: v_mov_b32_e32 v6, v3
16628 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
16629 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
16630 ; GFX908-NEXT: v_max_f32_e32 v3, v3, v4
16631 ; GFX908-NEXT: v_max_f32_e32 v5, v5, v2
16632 ; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
16633 ; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
16634 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
16635 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
16636 ; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
16637 ; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
16638 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
16639 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
16640 ; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
16641 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
16642 ; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
16643 ; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
16644 ; GFX908-NEXT: s_waitcnt vmcnt(0)
16645 ; GFX908-NEXT: buffer_wbinvl1
16646 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
16647 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
16648 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
16649 ; GFX908-NEXT: s_cbranch_execnz .LBB60_1
16650 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
16651 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
16652 ; GFX908-NEXT: v_mov_b32_e32 v0, v3
16653 ; GFX908-NEXT: s_setpc_b64 s[30:31]
16654 ;
16655 ; GFX8-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
16656 ; GFX8: ; %bb.0:
16657 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16658 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fc, v0
16659 ; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
16660 ; GFX8-NEXT: flat_load_dword v0, v[3:4]
16661 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
16662 ; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
16663 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
16664 ; GFX8-NEXT: .LBB60_1: ; %atomicrmw.start
16665 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
16666 ; GFX8-NEXT: s_waitcnt vmcnt(0)
16667 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
16668 ; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
16669 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
16670 ; GFX8-NEXT: v_max_f32_e32 v0, v0, v1
16671 ; GFX8-NEXT: v_max_f32_e32 v5, v5, v2
16672 ; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
16673 ; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
16674 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
16675 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
16676 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
16677 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
16678 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
16679 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
16680 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
16681 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
16682 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
16683 ; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
16684 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
16685 ; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
16686 ; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
16687 ; GFX8-NEXT: s_waitcnt vmcnt(0)
16688 ; GFX8-NEXT: buffer_wbinvl1
16689 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
16690 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
16691 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
16692 ; GFX8-NEXT: s_cbranch_execnz .LBB60_1
16693 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
16694 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
16695 ; GFX8-NEXT: s_setpc_b64 s[30:31]
16696 ;
16697 ; GFX7-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
16698 ; GFX7: ; %bb.0:
16699 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16700 ; GFX7-NEXT: s_mov_b32 s6, 0
16701 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
16702 ; GFX7-NEXT: s_mov_b32 s4, s6
16703 ; GFX7-NEXT: s_mov_b32 s5, s6
16704 ; GFX7-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
16705 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
16706 ; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
16707 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
16708 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
16709 ; GFX7-NEXT: s_waitcnt vmcnt(0)
16710 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
16711 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
16712 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
16713 ; GFX7-NEXT: .LBB60_1: ; %atomicrmw.start
16714 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
16715 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
16716 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
16717 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
16718 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
16719 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
16720 ; GFX7-NEXT: v_max_f32_e32 v6, v6, v4
16721 ; GFX7-NEXT: v_max_f32_e32 v7, v7, v5
16722 ; GFX7-NEXT: v_alignbit_b32 v3, v2, v3, 16
16723 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v6
16724 ; GFX7-NEXT: v_alignbit_b32 v2, v2, v7, 16
16725 ; GFX7-NEXT: v_mov_b32_e32 v7, v3
16726 ; GFX7-NEXT: v_mov_b32_e32 v6, v2
16727 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
16728 ; GFX7-NEXT: s_waitcnt vmcnt(0)
16729 ; GFX7-NEXT: buffer_wbinvl1
16730 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v3
16731 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
16732 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
16733 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v6
16734 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
16735 ; GFX7-NEXT: s_cbranch_execnz .LBB60_1
16736 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
16737 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
16738 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
16739 ; GFX7-NEXT: v_mov_b32_e32 v1, v2
16740 ; GFX7-NEXT: s_setpc_b64 s[30:31]
16741 ;
16742 ; GFX6-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
16743 ; GFX6: ; %bb.0:
16744 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16745 ; GFX6-NEXT: s_mov_b32 s6, 0
16746 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
16747 ; GFX6-NEXT: s_mov_b32 s4, s6
16748 ; GFX6-NEXT: s_mov_b32 s5, s6
16749 ; GFX6-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:2044
16750 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
16751 ; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
16752 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
16753 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
16754 ; GFX6-NEXT: s_waitcnt vmcnt(0)
16755 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
16756 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
16757 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
16758 ; GFX6-NEXT: .LBB60_1: ; %atomicrmw.start
16759 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
16760 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
16761 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
16762 ; GFX6-NEXT: s_waitcnt expcnt(0)
16763 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
16764 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
16765 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
16766 ; GFX6-NEXT: v_max_f32_e32 v6, v6, v4
16767 ; GFX6-NEXT: v_max_f32_e32 v7, v7, v5
16768 ; GFX6-NEXT: v_alignbit_b32 v3, v2, v3, 16
16769 ; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v6
16770 ; GFX6-NEXT: v_alignbit_b32 v2, v2, v7, 16
16771 ; GFX6-NEXT: v_mov_b32_e32 v7, v3
16772 ; GFX6-NEXT: v_mov_b32_e32 v6, v2
16773 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
16774 ; GFX6-NEXT: s_waitcnt vmcnt(0)
16775 ; GFX6-NEXT: buffer_wbinvl1
16776 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v3
16777 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
16778 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
16779 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v6
16780 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
16781 ; GFX6-NEXT: s_cbranch_execnz .LBB60_1
16782 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
16783 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
16784 ; GFX6-NEXT: v_mov_b32_e32 v0, v3
16785 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
16786 ; GFX6-NEXT: s_waitcnt expcnt(0)
16787 ; GFX6-NEXT: s_setpc_b64 s[30:31]
16788 %gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
16789 %result = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val seq_cst, !amdgpu.no.fine.grained.memory !0
16790 ret <2 x bfloat> %result
16791 }
16793 define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, <2 x bfloat> %val) #0 {
16794 ; GFX12-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
16795 ; GFX12: ; %bb.0:
16796 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
16797 ; GFX12-NEXT: s_wait_expcnt 0x0
16798 ; GFX12-NEXT: s_wait_samplecnt 0x0
16799 ; GFX12-NEXT: s_wait_bvhcnt 0x0
16800 ; GFX12-NEXT: s_wait_kmcnt 0x0
16801 ; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
16802 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16803 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16804 ; GFX12-NEXT: s_mov_b32 s1, 0
16805 ; GFX12-NEXT: .LBB61_1: ; %atomicrmw.start
16806 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
16807 ; GFX12-NEXT: s_wait_loadcnt 0x0
16808 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16809 ; GFX12-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16810 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16811 ; GFX12-NEXT: v_max_num_f32_e32 v2, v2, v4
16812 ; GFX12-NEXT: v_max_num_f32_e32 v6, v6, v5
16813 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16814 ; GFX12-NEXT: v_bfe_u32 v7, v2, 16, 1
16815 ; GFX12-NEXT: v_bfe_u32 v8, v6, 16, 1
16816 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v2
16817 ; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v6
16818 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
16819 ; GFX12-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
16820 ; GFX12-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
16821 ; GFX12-NEXT: v_cmp_u_f32_e64 s0, v2, v2
16822 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16823 ; GFX12-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
16824 ; GFX12-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
16825 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
16826 ; GFX12-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
16827 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
16828 ; GFX12-NEXT: s_wait_storecnt 0x0
16829 ; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
16830 ; GFX12-NEXT: s_wait_loadcnt 0x0
16831 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
16832 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
16833 ; GFX12-NEXT: v_mov_b32_e32 v3, v2
16834 ; GFX12-NEXT: s_wait_alu 0xfffe
16835 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
16836 ; GFX12-NEXT: s_wait_alu 0xfffe
16837 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
16838 ; GFX12-NEXT: s_cbranch_execnz .LBB61_1
16839 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
16840 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
16841 ; GFX12-NEXT: s_wait_alu 0xfffe
16842 ; GFX12-NEXT: s_setpc_b64 s[30:31]
16843 ;
16844 ; GFX940-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
16845 ; GFX940: ; %bb.0:
16846 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16847 ; GFX940-NEXT: global_load_dword v3, v[0:1], off offset:2044
16848 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
16849 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16850 ; GFX940-NEXT: s_movk_i32 s4, 0x7fff
16851 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16852 ; GFX940-NEXT: s_mov_b32 s5, 0x7060302
16853 ; GFX940-NEXT: .LBB61_1: ; %atomicrmw.start
16854 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
16855 ; GFX940-NEXT: s_waitcnt vmcnt(0)
16856 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16857 ; GFX940-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16858 ; GFX940-NEXT: v_max_f32_e32 v2, v2, v4
16859 ; GFX940-NEXT: v_max_f32_e32 v6, v6, v5
16860 ; GFX940-NEXT: v_bfe_u32 v7, v2, 16, 1
16861 ; GFX940-NEXT: v_bfe_u32 v9, v6, 16, 1
16862 ; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v2
16863 ; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v6
16864 ; GFX940-NEXT: v_add3_u32 v7, v7, v2, s4
16865 ; GFX940-NEXT: v_add3_u32 v9, v9, v6, s4
16866 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
16867 ; GFX940-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
16868 ; GFX940-NEXT: s_nop 0
16869 ; GFX940-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
16870 ; GFX940-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
16871 ; GFX940-NEXT: v_perm_b32 v2, v6, v2, s5
16872 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
16873 ; GFX940-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
16874 ; GFX940-NEXT: s_waitcnt vmcnt(0)
16875 ; GFX940-NEXT: buffer_inv sc0 sc1
16876 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
16877 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
16878 ; GFX940-NEXT: v_mov_b32_e32 v3, v2
16879 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
16880 ; GFX940-NEXT: s_cbranch_execnz .LBB61_1
16881 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
16882 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
16883 ; GFX940-NEXT: s_setpc_b64 s[30:31]
16884 ;
16885 ; GFX11-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
16886 ; GFX11: ; %bb.0:
16887 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16888 ; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
16889 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16890 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16891 ; GFX11-NEXT: s_mov_b32 s1, 0
16892 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
16893 ; GFX11-NEXT: .p2align 6
16894 ; GFX11-NEXT: .LBB61_1: ; %atomicrmw.start
16895 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
16896 ; GFX11-NEXT: s_waitcnt vmcnt(0)
16897 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16898 ; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16899 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16900 ; GFX11-NEXT: v_max_f32_e32 v2, v2, v4
16901 ; GFX11-NEXT: v_max_f32_e32 v6, v6, v5
16902 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16903 ; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
16904 ; GFX11-NEXT: v_bfe_u32 v8, v6, 16, 1
16905 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v2
16906 ; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v6
16907 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
16908 ; GFX11-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
16909 ; GFX11-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
16910 ; GFX11-NEXT: v_cmp_u_f32_e64 s0, v2, v2
16911 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
16912 ; GFX11-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
16913 ; GFX11-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
16914 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
16915 ; GFX11-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
16916 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
16917 ; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
16918 ; GFX11-NEXT: s_waitcnt vmcnt(0)
16919 ; GFX11-NEXT: buffer_gl1_inv
16920 ; GFX11-NEXT: buffer_gl0_inv
16921 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
16922 ; GFX11-NEXT: v_mov_b32_e32 v3, v2
16923 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
16924 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
16925 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
16926 ; GFX11-NEXT: s_cbranch_execnz .LBB61_1
16927 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
16928 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
16929 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
16930 ; GFX11-NEXT: s_setpc_b64 s[30:31]
16931 ;
16932 ; GFX10-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
16933 ; GFX10: ; %bb.0:
16934 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16935 ; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
16936 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16937 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16938 ; GFX10-NEXT: s_mov_b32 s5, 0
16939 ; GFX10-NEXT: .LBB61_1: ; %atomicrmw.start
16940 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
16941 ; GFX10-NEXT: s_waitcnt vmcnt(0)
16942 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16943 ; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16944 ; GFX10-NEXT: v_max_f32_e32 v2, v2, v4
16945 ; GFX10-NEXT: v_max_f32_e32 v6, v6, v5
16946 ; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
16947 ; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
16948 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
16949 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
16950 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
16951 ; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
16952 ; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
16953 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
16954 ; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
16955 ; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
16956 ; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
16957 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
16958 ; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
16959 ; GFX10-NEXT: s_waitcnt vmcnt(0)
16960 ; GFX10-NEXT: buffer_gl1_inv
16961 ; GFX10-NEXT: buffer_gl0_inv
16962 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
16963 ; GFX10-NEXT: v_mov_b32_e32 v3, v2
16964 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
16965 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
16966 ; GFX10-NEXT: s_cbranch_execnz .LBB61_1
16967 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
16968 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
16969 ; GFX10-NEXT: s_setpc_b64 s[30:31]
16970 ;
16971 ; GFX90A-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
16972 ; GFX90A: ; %bb.0:
16973 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16974 ; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
16975 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
16976 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
16977 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
16978 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
16979 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
16980 ; GFX90A-NEXT: .LBB61_1: ; %atomicrmw.start
16981 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
16982 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
16983 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
16984 ; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
16985 ; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
16986 ; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
16987 ; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
16988 ; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
16989 ; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
16990 ; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
16991 ; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
16992 ; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
16993 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
16994 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
16995 ; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
16996 ; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
16997 ; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
16998 ; GFX90A-NEXT: buffer_wbl2
16999 ; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
17000 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
17001 ; GFX90A-NEXT: buffer_invl2
17002 ; GFX90A-NEXT: buffer_wbinvl1
17003 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
17004 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
17005 ; GFX90A-NEXT: v_mov_b32_e32 v3, v2
17006 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
17007 ; GFX90A-NEXT: s_cbranch_execnz .LBB61_1
17008 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
17009 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
17010 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
17011 ;
17012 ; GFX908-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
17013 ; GFX908: ; %bb.0:
17014 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17015 ; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
17016 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
17017 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
17018 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
17019 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
17020 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
17021 ; GFX908-NEXT: .LBB61_1: ; %atomicrmw.start
17022 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
17023 ; GFX908-NEXT: s_waitcnt vmcnt(0)
17024 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
17025 ; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
17026 ; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
17027 ; GFX908-NEXT: v_max_f32_e32 v6, v6, v5
17028 ; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
17029 ; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
17030 ; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
17031 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
17032 ; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
17033 ; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
17034 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
17035 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
17036 ; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
17037 ; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
17038 ; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
17039 ; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
17040 ; GFX908-NEXT: s_waitcnt vmcnt(0)
17041 ; GFX908-NEXT: buffer_wbinvl1
17042 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
17043 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
17044 ; GFX908-NEXT: v_mov_b32_e32 v3, v2
17045 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
17046 ; GFX908-NEXT: s_cbranch_execnz .LBB61_1
17047 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
17048 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
17049 ; GFX908-NEXT: s_setpc_b64 s[30:31]
17050 ;
17051 ; GFX8-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
17052 ; GFX8: ; %bb.0:
17053 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17054 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
17055 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
17056 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
17057 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
17058 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
17059 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
17060 ; GFX8-NEXT: .LBB61_1: ; %atomicrmw.start
17061 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
17062 ; GFX8-NEXT: s_waitcnt vmcnt(0)
17063 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
17064 ; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
17065 ; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
17066 ; GFX8-NEXT: v_max_f32_e32 v6, v6, v5
17067 ; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
17068 ; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
17069 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
17070 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
17071 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
17072 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
17073 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
17074 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
17075 ; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
17076 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
17077 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
17078 ; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
17079 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
17080 ; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
17081 ; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
17082 ; GFX8-NEXT: s_waitcnt vmcnt(0)
17083 ; GFX8-NEXT: buffer_wbinvl1
17084 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
17085 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
17086 ; GFX8-NEXT: v_mov_b32_e32 v3, v2
17087 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
17088 ; GFX8-NEXT: s_cbranch_execnz .LBB61_1
17089 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
17090 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
17091 ; GFX8-NEXT: s_setpc_b64 s[30:31]
17092 ;
17093 ; GFX7-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
17094 ; GFX7: ; %bb.0:
17095 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17096 ; GFX7-NEXT: s_mov_b32 s6, 0
17097 ; GFX7-NEXT: s_mov_b32 s7, 0xf000
17098 ; GFX7-NEXT: s_mov_b32 s4, s6
17099 ; GFX7-NEXT: s_mov_b32 s5, s6
17100 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:2044
17101 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
17102 ; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v2
17103 ; GFX7-NEXT: s_mov_b64 s[8:9], 0
17104 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
17105 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
17106 ; GFX7-NEXT: s_waitcnt vmcnt(0)
17107 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
17108 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
17109 ; GFX7-NEXT: .LBB61_1: ; %atomicrmw.start
17110 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
17111 ; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
17112 ; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
17113 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
17114 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
17115 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
17116 ; GFX7-NEXT: v_max_f32_e32 v6, v6, v2
17117 ; GFX7-NEXT: v_max_f32_e32 v7, v7, v3
17118 ; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
17119 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
17120 ; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
17121 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
17122 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
17123 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
17124 ; GFX7-NEXT: s_waitcnt vmcnt(0)
17125 ; GFX7-NEXT: buffer_wbinvl1
17126 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
17127 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
17128 ; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
17129 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
17130 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
17131 ; GFX7-NEXT: s_cbranch_execnz .LBB61_1
17132 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
17133 ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
17134 ; GFX7-NEXT: s_setpc_b64 s[30:31]
17135 ;
17136 ; GFX6-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
17137 ; GFX6: ; %bb.0:
17138 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
17139 ; GFX6-NEXT: s_mov_b32 s6, 0
17140 ; GFX6-NEXT: s_mov_b32 s7, 0xf000
17141 ; GFX6-NEXT: s_mov_b32 s4, s6
17142 ; GFX6-NEXT: s_mov_b32 s5, s6
17143 ; GFX6-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:2044
17144 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
17145 ; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v2
17146 ; GFX6-NEXT: s_mov_b64 s[8:9], 0
17147 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
17148 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
17149 ; GFX6-NEXT: s_waitcnt vmcnt(0)
17150 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
17151 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
17152 ; GFX6-NEXT: .LBB61_1: ; %atomicrmw.start
17153 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
17154 ; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
17155 ; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
17156 ; GFX6-NEXT: s_waitcnt expcnt(0)
17157 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
17158 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
17159 ; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
17160 ; GFX6-NEXT: v_max_f32_e32 v6, v6, v2
17161 ; GFX6-NEXT: v_max_f32_e32 v7, v7, v3
17162 ; GFX6-NEXT: v_alignbit_b32 v5, v5, v4, 16
17163 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
17164 ; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
17165 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
17166 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
17167 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v[0:1], s[4:7], 0 addr64 offset:2044 glc
17168 ; GFX6-NEXT: s_waitcnt vmcnt(0)
17169 ; GFX6-NEXT: buffer_wbinvl1
17170 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
17171 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
17172 ; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
17173 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
17174 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
17175 ; GFX6-NEXT: s_cbranch_execnz .LBB61_1
17176 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
17177 ; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
17178 ; GFX6-NEXT: s_waitcnt expcnt(0)
17179 ; GFX6-NEXT: s_setpc_b64 s[30:31]
17180 %gep = getelementptr <2 x bfloat>, ptr addrspace(1) %ptr, i64 511
17181 %unused = atomicrmw fmax ptr addrspace(1) %gep, <2 x bfloat> %val seq_cst, !amdgpu.no.fine.grained.memory !0
17182 ret void
17183 }
17185 attributes #0 = { nounwind }
17186 attributes #1 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
17188 !0 = !{}