| blob | a6eb81fcbf5155af8daf2c2e551f91120d1a470b |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
2 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
3 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx940 < %s | FileCheck -check-prefix=GFX940 %s
4 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 < %s | FileCheck -check-prefix=GFX11 %s
5 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
6 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx90a < %s | FileCheck -check-prefix=GFX90A %s
7 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx908 < %s | FileCheck -check-prefix=GFX908 %s
8 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=tonga < %s | FileCheck -check-prefix=GFX8 %s
9 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=hawaii < %s | FileCheck -check-prefix=GFX7 %s
10 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=tahiti < %s | FileCheck -check-prefix=GFX6 %s
12 ; --------------------------------------------------------------------
13 ; float
14 ; --------------------------------------------------------------------
16 define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, float %val) #0 {
17 ; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
18 ; GFX12: ; %bb.0:
19 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
20 ; GFX12-NEXT: s_wait_expcnt 0x0
21 ; GFX12-NEXT: s_wait_samplecnt 0x0
22 ; GFX12-NEXT: s_wait_bvhcnt 0x0
23 ; GFX12-NEXT: s_wait_kmcnt 0x0
24 ; GFX12-NEXT: v_mov_b32_e32 v1, s16
25 ; GFX12-NEXT: s_wait_storecnt 0x0
26 ; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
27 ; GFX12-NEXT: s_wait_loadcnt 0x0
28 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
29 ; GFX12-NEXT: s_setpc_b64 s[30:31]
30 ;
31 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
32 ; GFX940: ; %bb.0:
33 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
34 ; GFX940-NEXT: v_mov_b32_e32 v1, v0
35 ; GFX940-NEXT: v_mov_b32_e32 v0, s16
36 ; GFX940-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
37 ; GFX940-NEXT: s_add_i32 s6, s16, 0x400
38 ; GFX940-NEXT: s_mov_b64 s[4:5], 0
39 ; GFX940-NEXT: v_max_f32_e32 v2, v1, v1
40 ; GFX940-NEXT: v_mov_b32_e32 v3, s6
41 ; GFX940-NEXT: .LBB0_1: ; %atomicrmw.start
42 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
43 ; GFX940-NEXT: s_waitcnt vmcnt(0)
44 ; GFX940-NEXT: v_mov_b32_e32 v5, v0
45 ; GFX940-NEXT: v_max_f32_e32 v0, v5, v5
46 ; GFX940-NEXT: v_min_f32_e32 v4, v0, v2
47 ; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
48 ; GFX940-NEXT: buffer_wbl2 sc1
49 ; GFX940-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen sc0
50 ; GFX940-NEXT: s_waitcnt vmcnt(0)
51 ; GFX940-NEXT: buffer_inv sc1
52 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
53 ; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
54 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
55 ; GFX940-NEXT: s_cbranch_execnz .LBB0_1
56 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
57 ; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
58 ; GFX940-NEXT: s_setpc_b64 s[30:31]
59 ;
60 ; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
61 ; GFX11: ; %bb.0:
62 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
63 ; GFX11-NEXT: v_mov_b32_e32 v1, s16
64 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
65 ; GFX11-NEXT: buffer_atomic_min_f32 v0, v1, s[0:3], 0 offen offset:1024 glc
66 ; GFX11-NEXT: s_waitcnt vmcnt(0)
67 ; GFX11-NEXT: buffer_gl1_inv
68 ; GFX11-NEXT: buffer_gl0_inv
69 ; GFX11-NEXT: s_setpc_b64 s[30:31]
70 ;
71 ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
72 ; GFX10: ; %bb.0:
73 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
74 ; GFX10-NEXT: v_mov_b32_e32 v1, s20
75 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
76 ; GFX10-NEXT: buffer_atomic_fmin v0, v1, s[16:19], 0 offen offset:1024 glc
77 ; GFX10-NEXT: s_waitcnt vmcnt(0)
78 ; GFX10-NEXT: buffer_gl1_inv
79 ; GFX10-NEXT: buffer_gl0_inv
80 ; GFX10-NEXT: s_setpc_b64 s[30:31]
81 ;
82 ; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
83 ; GFX90A: ; %bb.0:
84 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
85 ; GFX90A-NEXT: v_mov_b32_e32 v1, v0
86 ; GFX90A-NEXT: v_mov_b32_e32 v0, s20
87 ; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
88 ; GFX90A-NEXT: s_add_i32 s6, s20, 0x400
89 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
90 ; GFX90A-NEXT: v_max_f32_e32 v2, v1, v1
91 ; GFX90A-NEXT: v_mov_b32_e32 v3, s6
92 ; GFX90A-NEXT: .LBB0_1: ; %atomicrmw.start
93 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
94 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
95 ; GFX90A-NEXT: v_mov_b32_e32 v5, v0
96 ; GFX90A-NEXT: v_max_f32_e32 v0, v5, v5
97 ; GFX90A-NEXT: v_min_f32_e32 v4, v0, v2
98 ; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
99 ; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
100 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
101 ; GFX90A-NEXT: buffer_wbinvl1
102 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
103 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
104 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
105 ; GFX90A-NEXT: s_cbranch_execnz .LBB0_1
106 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
107 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
108 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
109 ;
110 ; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
111 ; GFX908: ; %bb.0:
112 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
113 ; GFX908-NEXT: v_mov_b32_e32 v1, v0
114 ; GFX908-NEXT: v_mov_b32_e32 v0, s20
115 ; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
116 ; GFX908-NEXT: s_add_i32 s6, s20, 0x400
117 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
118 ; GFX908-NEXT: v_max_f32_e32 v2, v1, v1
119 ; GFX908-NEXT: v_mov_b32_e32 v3, s6
120 ; GFX908-NEXT: .LBB0_1: ; %atomicrmw.start
121 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
122 ; GFX908-NEXT: s_waitcnt vmcnt(0)
123 ; GFX908-NEXT: v_mov_b32_e32 v5, v0
124 ; GFX908-NEXT: v_max_f32_e32 v0, v5, v5
125 ; GFX908-NEXT: v_min_f32_e32 v4, v0, v2
126 ; GFX908-NEXT: v_mov_b32_e32 v0, v4
127 ; GFX908-NEXT: v_mov_b32_e32 v1, v5
128 ; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
129 ; GFX908-NEXT: s_waitcnt vmcnt(0)
130 ; GFX908-NEXT: buffer_wbinvl1
131 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
132 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
133 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
134 ; GFX908-NEXT: s_cbranch_execnz .LBB0_1
135 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
136 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
137 ; GFX908-NEXT: s_setpc_b64 s[30:31]
138 ;
139 ; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
140 ; GFX8: ; %bb.0:
141 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
142 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
143 ; GFX8-NEXT: v_mov_b32_e32 v0, s20
144 ; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
145 ; GFX8-NEXT: s_add_i32 s6, s20, 0x400
146 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
147 ; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v1
148 ; GFX8-NEXT: v_mov_b32_e32 v3, s6
149 ; GFX8-NEXT: .LBB0_1: ; %atomicrmw.start
150 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
151 ; GFX8-NEXT: s_waitcnt vmcnt(0)
152 ; GFX8-NEXT: v_mov_b32_e32 v5, v0
153 ; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v5
154 ; GFX8-NEXT: v_min_f32_e32 v4, v0, v2
155 ; GFX8-NEXT: v_mov_b32_e32 v0, v4
156 ; GFX8-NEXT: v_mov_b32_e32 v1, v5
157 ; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
158 ; GFX8-NEXT: s_waitcnt vmcnt(0)
159 ; GFX8-NEXT: buffer_wbinvl1
160 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
161 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
162 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
163 ; GFX8-NEXT: s_cbranch_execnz .LBB0_1
164 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
165 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
166 ; GFX8-NEXT: s_setpc_b64 s[30:31]
167 ;
168 ; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
169 ; GFX7: ; %bb.0:
170 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
171 ; GFX7-NEXT: v_mov_b32_e32 v1, s20
172 ; GFX7-NEXT: buffer_atomic_fmin v0, v1, s[16:19], 0 offen offset:1024 glc
173 ; GFX7-NEXT: s_waitcnt vmcnt(0)
174 ; GFX7-NEXT: buffer_wbinvl1
175 ; GFX7-NEXT: s_setpc_b64 s[30:31]
176 ;
177 ; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
178 ; GFX6: ; %bb.0:
179 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
180 ; GFX6-NEXT: v_mov_b32_e32 v1, s20
181 ; GFX6-NEXT: buffer_atomic_fmin v0, v1, s[16:19], 0 offen offset:1024 glc
182 ; GFX6-NEXT: s_waitcnt vmcnt(0)
183 ; GFX6-NEXT: buffer_wbinvl1
184 ; GFX6-NEXT: s_waitcnt expcnt(0)
185 ; GFX6-NEXT: s_setpc_b64 s[30:31]
186 %gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
187 %result = atomicrmw fmin ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
188 ret float %result
189 }
191 define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, float %val) #0 {
192 ; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory:
193 ; GFX12: ; %bb.0:
194 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
195 ; GFX12-NEXT: s_wait_expcnt 0x0
196 ; GFX12-NEXT: s_wait_samplecnt 0x0
197 ; GFX12-NEXT: s_wait_bvhcnt 0x0
198 ; GFX12-NEXT: s_wait_kmcnt 0x0
199 ; GFX12-NEXT: v_mov_b32_e32 v1, s16
200 ; GFX12-NEXT: s_wait_storecnt 0x0
201 ; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen offset:1024
202 ; GFX12-NEXT: s_wait_storecnt 0x0
203 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
204 ; GFX12-NEXT: s_setpc_b64 s[30:31]
205 ;
206 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory:
207 ; GFX940: ; %bb.0:
208 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
209 ; GFX940-NEXT: v_mov_b32_e32 v1, s16
210 ; GFX940-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen offset:1024
211 ; GFX940-NEXT: s_add_i32 s6, s16, 0x400
212 ; GFX940-NEXT: s_mov_b64 s[4:5], 0
213 ; GFX940-NEXT: v_max_f32_e32 v2, v0, v0
214 ; GFX940-NEXT: v_mov_b32_e32 v3, s6
215 ; GFX940-NEXT: .LBB1_1: ; %atomicrmw.start
216 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
217 ; GFX940-NEXT: s_waitcnt vmcnt(0)
218 ; GFX940-NEXT: v_max_f32_e32 v0, v1, v1
219 ; GFX940-NEXT: v_min_f32_e32 v0, v0, v2
220 ; GFX940-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
221 ; GFX940-NEXT: buffer_wbl2 sc1
222 ; GFX940-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[0:3], 0 offen sc0
223 ; GFX940-NEXT: s_waitcnt vmcnt(0)
224 ; GFX940-NEXT: buffer_inv sc1
225 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
226 ; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
227 ; GFX940-NEXT: v_mov_b32_e32 v1, v4
228 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
229 ; GFX940-NEXT: s_cbranch_execnz .LBB1_1
230 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
231 ; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
232 ; GFX940-NEXT: s_setpc_b64 s[30:31]
233 ;
234 ; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory:
235 ; GFX11: ; %bb.0:
236 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
237 ; GFX11-NEXT: v_mov_b32_e32 v1, s16
238 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
239 ; GFX11-NEXT: buffer_atomic_min_f32 v0, v1, s[0:3], 0 offen offset:1024
240 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
241 ; GFX11-NEXT: buffer_gl1_inv
242 ; GFX11-NEXT: buffer_gl0_inv
243 ; GFX11-NEXT: s_setpc_b64 s[30:31]
244 ;
245 ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory:
246 ; GFX10: ; %bb.0:
247 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
248 ; GFX10-NEXT: v_mov_b32_e32 v1, s20
249 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
250 ; GFX10-NEXT: buffer_atomic_fmin v0, v1, s[16:19], 0 offen offset:1024
251 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
252 ; GFX10-NEXT: buffer_gl1_inv
253 ; GFX10-NEXT: buffer_gl0_inv
254 ; GFX10-NEXT: s_setpc_b64 s[30:31]
255 ;
256 ; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory:
257 ; GFX90A: ; %bb.0:
258 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
259 ; GFX90A-NEXT: v_mov_b32_e32 v1, s20
260 ; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
261 ; GFX90A-NEXT: s_add_i32 s6, s20, 0x400
262 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
263 ; GFX90A-NEXT: v_max_f32_e32 v2, v0, v0
264 ; GFX90A-NEXT: v_mov_b32_e32 v3, s6
265 ; GFX90A-NEXT: .LBB1_1: ; %atomicrmw.start
266 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
267 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
268 ; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
269 ; GFX90A-NEXT: v_min_f32_e32 v0, v0, v2
270 ; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
271 ; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen glc
272 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
273 ; GFX90A-NEXT: buffer_wbinvl1
274 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
275 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
276 ; GFX90A-NEXT: v_mov_b32_e32 v1, v4
277 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
278 ; GFX90A-NEXT: s_cbranch_execnz .LBB1_1
279 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
280 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
281 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
282 ;
283 ; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory:
284 ; GFX908: ; %bb.0:
285 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
286 ; GFX908-NEXT: v_mov_b32_e32 v1, s20
287 ; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
288 ; GFX908-NEXT: s_add_i32 s6, s20, 0x400
289 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
290 ; GFX908-NEXT: v_max_f32_e32 v2, v0, v0
291 ; GFX908-NEXT: v_mov_b32_e32 v3, s6
292 ; GFX908-NEXT: .LBB1_1: ; %atomicrmw.start
293 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
294 ; GFX908-NEXT: s_waitcnt vmcnt(0)
295 ; GFX908-NEXT: v_max_f32_e32 v0, v1, v1
296 ; GFX908-NEXT: v_min_f32_e32 v0, v0, v2
297 ; GFX908-NEXT: v_mov_b32_e32 v5, v1
298 ; GFX908-NEXT: v_mov_b32_e32 v4, v0
299 ; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen glc
300 ; GFX908-NEXT: s_waitcnt vmcnt(0)
301 ; GFX908-NEXT: buffer_wbinvl1
302 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
303 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
304 ; GFX908-NEXT: v_mov_b32_e32 v1, v4
305 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
306 ; GFX908-NEXT: s_cbranch_execnz .LBB1_1
307 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
308 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
309 ; GFX908-NEXT: s_setpc_b64 s[30:31]
310 ;
311 ; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory:
312 ; GFX8: ; %bb.0:
313 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
314 ; GFX8-NEXT: v_mov_b32_e32 v1, s20
315 ; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
316 ; GFX8-NEXT: s_add_i32 s6, s20, 0x400
317 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
318 ; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v0
319 ; GFX8-NEXT: v_mov_b32_e32 v3, s6
320 ; GFX8-NEXT: .LBB1_1: ; %atomicrmw.start
321 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
322 ; GFX8-NEXT: s_waitcnt vmcnt(0)
323 ; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v1
324 ; GFX8-NEXT: v_min_f32_e32 v0, v0, v2
325 ; GFX8-NEXT: v_mov_b32_e32 v5, v1
326 ; GFX8-NEXT: v_mov_b32_e32 v4, v0
327 ; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen glc
328 ; GFX8-NEXT: s_waitcnt vmcnt(0)
329 ; GFX8-NEXT: buffer_wbinvl1
330 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
331 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
332 ; GFX8-NEXT: v_mov_b32_e32 v1, v4
333 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
334 ; GFX8-NEXT: s_cbranch_execnz .LBB1_1
335 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
336 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
337 ; GFX8-NEXT: s_setpc_b64 s[30:31]
338 ;
339 ; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory:
340 ; GFX7: ; %bb.0:
341 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
342 ; GFX7-NEXT: v_mov_b32_e32 v1, s20
343 ; GFX7-NEXT: buffer_atomic_fmin v0, v1, s[16:19], 0 offen offset:1024
344 ; GFX7-NEXT: s_waitcnt vmcnt(0)
345 ; GFX7-NEXT: buffer_wbinvl1
346 ; GFX7-NEXT: s_setpc_b64 s[30:31]
347 ;
348 ; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_grained_memory:
349 ; GFX6: ; %bb.0:
350 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
351 ; GFX6-NEXT: v_mov_b32_e32 v1, s20
352 ; GFX6-NEXT: buffer_atomic_fmin v0, v1, s[16:19], 0 offen offset:1024
353 ; GFX6-NEXT: s_waitcnt vmcnt(0)
354 ; GFX6-NEXT: buffer_wbinvl1
355 ; GFX6-NEXT: s_waitcnt expcnt(0)
356 ; GFX6-NEXT: s_setpc_b64 s[30:31]
357 %gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
358 %unused = atomicrmw fmin ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
359 ret void
360 }
362 define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, float %val) #0 {
363 ; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
364 ; GFX12: ; %bb.0:
365 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
366 ; GFX12-NEXT: s_wait_expcnt 0x0
367 ; GFX12-NEXT: s_wait_samplecnt 0x0
368 ; GFX12-NEXT: s_wait_bvhcnt 0x0
369 ; GFX12-NEXT: s_wait_kmcnt 0x0
370 ; GFX12-NEXT: s_mov_b32 s1, exec_lo
371 ; GFX12-NEXT: s_wait_storecnt 0x0
372 ; GFX12-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
373 ; GFX12-NEXT: v_readfirstlane_b32 s4, v0
374 ; GFX12-NEXT: v_readfirstlane_b32 s5, v1
375 ; GFX12-NEXT: v_readfirstlane_b32 s6, v2
376 ; GFX12-NEXT: v_readfirstlane_b32 s7, v3
377 ; GFX12-NEXT: s_wait_alu 0xf1ff
378 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
379 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
380 ; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
381 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
382 ; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
383 ; GFX12-NEXT: s_wait_alu 0xfffe
384 ; GFX12-NEXT: s_and_saveexec_b32 s0, s0
385 ; GFX12-NEXT: s_wait_loadcnt 0x0
386 ; GFX12-NEXT: buffer_atomic_min_num_f32 v5, v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
387 ; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
388 ; GFX12-NEXT: ; implicit-def: $vgpr4
389 ; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
390 ; GFX12-NEXT: s_cbranch_execnz .LBB2_1
391 ; GFX12-NEXT: ; %bb.2:
392 ; GFX12-NEXT: s_mov_b32 exec_lo, s1
393 ; GFX12-NEXT: s_wait_loadcnt 0x0
394 ; GFX12-NEXT: v_mov_b32_e32 v0, v5
395 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
396 ; GFX12-NEXT: s_setpc_b64 s[30:31]
397 ;
398 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
399 ; GFX940: ; %bb.0:
400 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
401 ; GFX940-NEXT: v_add_u32_e32 v8, 0x400, v4
402 ; GFX940-NEXT: s_mov_b64 s[2:3], exec
403 ; GFX940-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
404 ; GFX940-NEXT: v_readfirstlane_b32 s4, v0
405 ; GFX940-NEXT: v_readfirstlane_b32 s5, v1
406 ; GFX940-NEXT: v_readfirstlane_b32 s6, v2
407 ; GFX940-NEXT: v_readfirstlane_b32 s7, v3
408 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
409 ; GFX940-NEXT: s_nop 0
410 ; GFX940-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
411 ; GFX940-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
412 ; GFX940-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
413 ; GFX940-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
414 ; GFX940-NEXT: ; implicit-def: $vgpr4
415 ; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
416 ; GFX940-NEXT: s_cbranch_execnz .LBB2_1
417 ; GFX940-NEXT: ; %bb.2:
418 ; GFX940-NEXT: s_mov_b64 exec, s[2:3]
419 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
420 ; GFX940-NEXT: v_max_f32_e32 v9, v5, v5
421 ; GFX940-NEXT: .LBB2_3: ; %atomicrmw.start
422 ; GFX940-NEXT: ; =>This Loop Header: Depth=1
423 ; GFX940-NEXT: ; Child Loop BB2_4 Depth 2
424 ; GFX940-NEXT: s_waitcnt vmcnt(0)
425 ; GFX940-NEXT: v_max_f32_e32 v4, v7, v7
426 ; GFX940-NEXT: v_min_f32_e32 v6, v4, v9
427 ; GFX940-NEXT: s_mov_b64 s[8:9], exec
428 ; GFX940-NEXT: v_mov_b64_e32 v[4:5], v[6:7]
429 ; GFX940-NEXT: buffer_wbl2 sc1
430 ; GFX940-NEXT: .LBB2_4: ; Parent Loop BB2_3 Depth=1
431 ; GFX940-NEXT: ; => This Inner Loop Header: Depth=2
432 ; GFX940-NEXT: v_readfirstlane_b32 s4, v0
433 ; GFX940-NEXT: v_readfirstlane_b32 s5, v1
434 ; GFX940-NEXT: v_readfirstlane_b32 s6, v2
435 ; GFX940-NEXT: v_readfirstlane_b32 s7, v3
436 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
437 ; GFX940-NEXT: s_nop 0
438 ; GFX940-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
439 ; GFX940-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
440 ; GFX940-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
441 ; GFX940-NEXT: s_waitcnt vmcnt(0)
442 ; GFX940-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen sc0
443 ; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
444 ; GFX940-NEXT: s_cbranch_execnz .LBB2_4
445 ; GFX940-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1
446 ; GFX940-NEXT: s_mov_b64 exec, s[8:9]
447 ; GFX940-NEXT: s_waitcnt vmcnt(0)
448 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
449 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
450 ; GFX940-NEXT: v_mov_b32_e32 v7, v4
451 ; GFX940-NEXT: buffer_inv sc1
452 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
453 ; GFX940-NEXT: s_cbranch_execnz .LBB2_3
454 ; GFX940-NEXT: ; %bb.6: ; %atomicrmw.end
455 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
456 ; GFX940-NEXT: v_mov_b32_e32 v0, v4
457 ; GFX940-NEXT: s_setpc_b64 s[30:31]
458 ;
459 ; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
460 ; GFX11: ; %bb.0:
461 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
462 ; GFX11-NEXT: s_mov_b32 s1, exec_lo
463 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
464 ; GFX11-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
465 ; GFX11-NEXT: v_readfirstlane_b32 s4, v0
466 ; GFX11-NEXT: v_readfirstlane_b32 s5, v1
467 ; GFX11-NEXT: v_readfirstlane_b32 s6, v2
468 ; GFX11-NEXT: v_readfirstlane_b32 s7, v3
469 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
470 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
471 ; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
472 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
473 ; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
474 ; GFX11-NEXT: s_and_saveexec_b32 s0, s0
475 ; GFX11-NEXT: s_waitcnt vmcnt(0)
476 ; GFX11-NEXT: buffer_atomic_min_f32 v5, v4, s[4:7], 0 offen offset:1024 glc
477 ; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
478 ; GFX11-NEXT: ; implicit-def: $vgpr4
479 ; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
480 ; GFX11-NEXT: s_cbranch_execnz .LBB2_1
481 ; GFX11-NEXT: ; %bb.2:
482 ; GFX11-NEXT: s_mov_b32 exec_lo, s1
483 ; GFX11-NEXT: s_waitcnt vmcnt(0)
484 ; GFX11-NEXT: v_mov_b32_e32 v0, v5
485 ; GFX11-NEXT: buffer_gl1_inv
486 ; GFX11-NEXT: buffer_gl0_inv
487 ; GFX11-NEXT: s_setpc_b64 s[30:31]
488 ;
489 ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
490 ; GFX10: ; %bb.0:
491 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
492 ; GFX10-NEXT: s_mov_b32 s5, exec_lo
493 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
494 ; GFX10-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
495 ; GFX10-NEXT: v_readfirstlane_b32 s8, v0
496 ; GFX10-NEXT: v_readfirstlane_b32 s9, v1
497 ; GFX10-NEXT: v_readfirstlane_b32 s10, v2
498 ; GFX10-NEXT: v_readfirstlane_b32 s11, v3
499 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
500 ; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
501 ; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
502 ; GFX10-NEXT: s_and_saveexec_b32 s4, s4
503 ; GFX10-NEXT: s_waitcnt vmcnt(0)
504 ; GFX10-NEXT: buffer_atomic_fmin v5, v4, s[8:11], 0 offen offset:1024 glc
505 ; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
506 ; GFX10-NEXT: ; implicit-def: $vgpr4
507 ; GFX10-NEXT: s_waitcnt_depctr 0xffe3
508 ; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
509 ; GFX10-NEXT: s_cbranch_execnz .LBB2_1
510 ; GFX10-NEXT: ; %bb.2:
511 ; GFX10-NEXT: s_mov_b32 exec_lo, s5
512 ; GFX10-NEXT: s_waitcnt vmcnt(0)
513 ; GFX10-NEXT: v_mov_b32_e32 v0, v5
514 ; GFX10-NEXT: buffer_gl1_inv
515 ; GFX10-NEXT: buffer_gl0_inv
516 ; GFX10-NEXT: s_setpc_b64 s[30:31]
517 ;
518 ; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
519 ; GFX90A: ; %bb.0:
520 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
521 ; GFX90A-NEXT: v_add_u32_e32 v8, 0x400, v4
522 ; GFX90A-NEXT: s_mov_b64 s[6:7], exec
523 ; GFX90A-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
524 ; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
525 ; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
526 ; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
527 ; GFX90A-NEXT: v_readfirstlane_b32 s11, v3
528 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
529 ; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
530 ; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
531 ; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
532 ; GFX90A-NEXT: s_nop 0
533 ; GFX90A-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
534 ; GFX90A-NEXT: ; implicit-def: $vgpr4
535 ; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
536 ; GFX90A-NEXT: s_cbranch_execnz .LBB2_1
537 ; GFX90A-NEXT: ; %bb.2:
538 ; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
539 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
540 ; GFX90A-NEXT: v_max_f32_e32 v9, v5, v5
541 ; GFX90A-NEXT: .LBB2_3: ; %atomicrmw.start
542 ; GFX90A-NEXT: ; =>This Loop Header: Depth=1
543 ; GFX90A-NEXT: ; Child Loop BB2_4 Depth 2
544 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
545 ; GFX90A-NEXT: v_max_f32_e32 v4, v7, v7
546 ; GFX90A-NEXT: v_min_f32_e32 v6, v4, v9
547 ; GFX90A-NEXT: s_mov_b64 s[12:13], exec
548 ; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[6:7] op_sel:[0,1]
549 ; GFX90A-NEXT: .LBB2_4: ; Parent Loop BB2_3 Depth=1
550 ; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
551 ; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
552 ; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
553 ; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
554 ; GFX90A-NEXT: v_readfirstlane_b32 s11, v3
555 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
556 ; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
557 ; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
558 ; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
559 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
560 ; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
561 ; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
562 ; GFX90A-NEXT: s_cbranch_execnz .LBB2_4
563 ; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1
564 ; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
565 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
566 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
567 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
568 ; GFX90A-NEXT: v_mov_b32_e32 v7, v4
569 ; GFX90A-NEXT: buffer_wbinvl1
570 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
571 ; GFX90A-NEXT: s_cbranch_execnz .LBB2_3
572 ; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.end
573 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
574 ; GFX90A-NEXT: v_mov_b32_e32 v0, v4
575 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
576 ;
577 ; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
578 ; GFX908: ; %bb.0:
579 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
580 ; GFX908-NEXT: v_add_u32_e32 v7, 0x400, v4
581 ; GFX908-NEXT: s_mov_b64 s[6:7], exec
582 ; GFX908-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
583 ; GFX908-NEXT: v_readfirstlane_b32 s8, v0
584 ; GFX908-NEXT: v_readfirstlane_b32 s9, v1
585 ; GFX908-NEXT: v_readfirstlane_b32 s10, v2
586 ; GFX908-NEXT: v_readfirstlane_b32 s11, v3
587 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
588 ; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
589 ; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
590 ; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
591 ; GFX908-NEXT: s_nop 0
592 ; GFX908-NEXT: buffer_load_dword v6, v4, s[8:11], 0 offen offset:1024
593 ; GFX908-NEXT: ; implicit-def: $vgpr4
594 ; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
595 ; GFX908-NEXT: s_cbranch_execnz .LBB2_1
596 ; GFX908-NEXT: ; %bb.2:
597 ; GFX908-NEXT: s_mov_b64 exec, s[6:7]
598 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
599 ; GFX908-NEXT: v_max_f32_e32 v8, v5, v5
600 ; GFX908-NEXT: .LBB2_3: ; %atomicrmw.start
601 ; GFX908-NEXT: ; =>This Loop Header: Depth=1
602 ; GFX908-NEXT: ; Child Loop BB2_4 Depth 2
603 ; GFX908-NEXT: s_waitcnt vmcnt(0)
604 ; GFX908-NEXT: v_max_f32_e32 v4, v6, v6
605 ; GFX908-NEXT: v_min_f32_e32 v5, v4, v8
606 ; GFX908-NEXT: v_mov_b32_e32 v4, v5
607 ; GFX908-NEXT: s_mov_b64 s[12:13], exec
608 ; GFX908-NEXT: v_mov_b32_e32 v5, v6
609 ; GFX908-NEXT: .LBB2_4: ; Parent Loop BB2_3 Depth=1
610 ; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
611 ; GFX908-NEXT: v_readfirstlane_b32 s8, v0
612 ; GFX908-NEXT: v_readfirstlane_b32 s9, v1
613 ; GFX908-NEXT: v_readfirstlane_b32 s10, v2
614 ; GFX908-NEXT: v_readfirstlane_b32 s11, v3
615 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
616 ; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
617 ; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
618 ; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
619 ; GFX908-NEXT: s_waitcnt vmcnt(0)
620 ; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v7, s[8:11], 0 offen glc
621 ; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
622 ; GFX908-NEXT: s_cbranch_execnz .LBB2_4
623 ; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1
624 ; GFX908-NEXT: s_mov_b64 exec, s[12:13]
625 ; GFX908-NEXT: s_waitcnt vmcnt(0)
626 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
627 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
628 ; GFX908-NEXT: v_mov_b32_e32 v6, v4
629 ; GFX908-NEXT: buffer_wbinvl1
630 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
631 ; GFX908-NEXT: s_cbranch_execnz .LBB2_3
632 ; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
633 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
634 ; GFX908-NEXT: v_mov_b32_e32 v0, v4
635 ; GFX908-NEXT: s_setpc_b64 s[30:31]
636 ;
637 ; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
638 ; GFX8: ; %bb.0:
639 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
640 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x400, v4
641 ; GFX8-NEXT: s_mov_b64 s[6:7], exec
642 ; GFX8-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
643 ; GFX8-NEXT: v_readfirstlane_b32 s8, v0
644 ; GFX8-NEXT: v_readfirstlane_b32 s9, v1
645 ; GFX8-NEXT: v_readfirstlane_b32 s10, v2
646 ; GFX8-NEXT: v_readfirstlane_b32 s11, v3
647 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
648 ; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
649 ; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
650 ; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
651 ; GFX8-NEXT: s_nop 0
652 ; GFX8-NEXT: buffer_load_dword v6, v4, s[8:11], 0 offen offset:1024
653 ; GFX8-NEXT: ; implicit-def: $vgpr4
654 ; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
655 ; GFX8-NEXT: s_cbranch_execnz .LBB2_1
656 ; GFX8-NEXT: ; %bb.2:
657 ; GFX8-NEXT: s_mov_b64 exec, s[6:7]
658 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
659 ; GFX8-NEXT: v_mul_f32_e32 v8, 1.0, v5
660 ; GFX8-NEXT: .LBB2_3: ; %atomicrmw.start
661 ; GFX8-NEXT: ; =>This Loop Header: Depth=1
662 ; GFX8-NEXT: ; Child Loop BB2_4 Depth 2
663 ; GFX8-NEXT: s_waitcnt vmcnt(0)
664 ; GFX8-NEXT: v_mul_f32_e32 v4, 1.0, v6
665 ; GFX8-NEXT: v_min_f32_e32 v5, v4, v8
666 ; GFX8-NEXT: v_mov_b32_e32 v4, v5
667 ; GFX8-NEXT: s_mov_b64 s[12:13], exec
668 ; GFX8-NEXT: v_mov_b32_e32 v5, v6
669 ; GFX8-NEXT: .LBB2_4: ; Parent Loop BB2_3 Depth=1
670 ; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
671 ; GFX8-NEXT: v_readfirstlane_b32 s8, v0
672 ; GFX8-NEXT: v_readfirstlane_b32 s9, v1
673 ; GFX8-NEXT: v_readfirstlane_b32 s10, v2
674 ; GFX8-NEXT: v_readfirstlane_b32 s11, v3
675 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
676 ; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
677 ; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
678 ; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
679 ; GFX8-NEXT: s_waitcnt vmcnt(0)
680 ; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v7, s[8:11], 0 offen glc
681 ; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
682 ; GFX8-NEXT: s_cbranch_execnz .LBB2_4
683 ; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1
684 ; GFX8-NEXT: s_mov_b64 exec, s[12:13]
685 ; GFX8-NEXT: s_waitcnt vmcnt(0)
686 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
687 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
688 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
689 ; GFX8-NEXT: buffer_wbinvl1
690 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
691 ; GFX8-NEXT: s_cbranch_execnz .LBB2_3
692 ; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
693 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
694 ; GFX8-NEXT: v_mov_b32_e32 v0, v4
695 ; GFX8-NEXT: s_setpc_b64 s[30:31]
696 ;
697 ; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
698 ; GFX7: ; %bb.0:
699 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
700 ; GFX7-NEXT: s_mov_b64 s[6:7], exec
701 ; GFX7-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
702 ; GFX7-NEXT: v_readfirstlane_b32 s8, v0
703 ; GFX7-NEXT: v_readfirstlane_b32 s9, v1
704 ; GFX7-NEXT: v_readfirstlane_b32 s10, v2
705 ; GFX7-NEXT: v_readfirstlane_b32 s11, v3
706 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
707 ; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
708 ; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
709 ; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
710 ; GFX7-NEXT: s_waitcnt vmcnt(0)
711 ; GFX7-NEXT: buffer_atomic_fmin v5, v4, s[8:11], 0 offen offset:1024 glc
712 ; GFX7-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
713 ; GFX7-NEXT: ; implicit-def: $vgpr4
714 ; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
715 ; GFX7-NEXT: s_cbranch_execnz .LBB2_1
716 ; GFX7-NEXT: ; %bb.2:
717 ; GFX7-NEXT: s_mov_b64 exec, s[6:7]
718 ; GFX7-NEXT: s_waitcnt vmcnt(0)
719 ; GFX7-NEXT: v_mov_b32_e32 v0, v5
720 ; GFX7-NEXT: buffer_wbinvl1
721 ; GFX7-NEXT: s_setpc_b64 s[30:31]
722 ;
723 ; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
724 ; GFX6: ; %bb.0:
725 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
726 ; GFX6-NEXT: s_mov_b64 s[6:7], exec
727 ; GFX6-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
728 ; GFX6-NEXT: v_readfirstlane_b32 s8, v0
729 ; GFX6-NEXT: v_readfirstlane_b32 s9, v1
730 ; GFX6-NEXT: v_readfirstlane_b32 s10, v2
731 ; GFX6-NEXT: v_readfirstlane_b32 s11, v3
732 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
733 ; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
734 ; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
735 ; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
736 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
737 ; GFX6-NEXT: buffer_atomic_fmin v5, v4, s[8:11], 0 offen offset:1024 glc
738 ; GFX6-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
739 ; GFX6-NEXT: ; implicit-def: $vgpr4
740 ; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
741 ; GFX6-NEXT: s_cbranch_execnz .LBB2_1
742 ; GFX6-NEXT: ; %bb.2:
743 ; GFX6-NEXT: s_mov_b64 exec, s[6:7]
744 ; GFX6-NEXT: s_waitcnt vmcnt(0)
745 ; GFX6-NEXT: v_mov_b32_e32 v0, v5
746 ; GFX6-NEXT: buffer_wbinvl1
747 ; GFX6-NEXT: s_waitcnt expcnt(0)
748 ; GFX6-NEXT: s_setpc_b64 s[30:31]
749 %gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
750 %result = atomicrmw fmin ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
751 ret float %result
752 }
754 define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory(ptr addrspace(7) inreg %ptr, float %val) #0 {
755 ; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory:
756 ; GFX12: ; %bb.0:
757 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
758 ; GFX12-NEXT: s_wait_expcnt 0x0
759 ; GFX12-NEXT: s_wait_samplecnt 0x0
760 ; GFX12-NEXT: s_wait_bvhcnt 0x0
761 ; GFX12-NEXT: s_wait_kmcnt 0x0
762 ; GFX12-NEXT: v_mov_b32_e32 v1, s16
763 ; GFX12-NEXT: s_wait_storecnt 0x0
764 ; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
765 ; GFX12-NEXT: s_wait_loadcnt 0x0
766 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
767 ; GFX12-NEXT: s_setpc_b64 s[30:31]
768 ;
769 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory:
770 ; GFX940: ; %bb.0:
771 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
772 ; GFX940-NEXT: v_mov_b32_e32 v1, v0
773 ; GFX940-NEXT: v_mov_b32_e32 v0, s16
774 ; GFX940-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
775 ; GFX940-NEXT: s_add_i32 s6, s16, 0x400
776 ; GFX940-NEXT: s_mov_b64 s[4:5], 0
777 ; GFX940-NEXT: v_max_f32_e32 v2, v1, v1
778 ; GFX940-NEXT: v_mov_b32_e32 v3, s6
779 ; GFX940-NEXT: .LBB3_1: ; %atomicrmw.start
780 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
781 ; GFX940-NEXT: s_waitcnt vmcnt(0)
782 ; GFX940-NEXT: v_mov_b32_e32 v5, v0
783 ; GFX940-NEXT: v_max_f32_e32 v0, v5, v5
784 ; GFX940-NEXT: v_min_f32_e32 v4, v0, v2
785 ; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
786 ; GFX940-NEXT: buffer_wbl2 sc1
787 ; GFX940-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen sc0
788 ; GFX940-NEXT: s_waitcnt vmcnt(0)
789 ; GFX940-NEXT: buffer_inv sc1
790 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
791 ; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
792 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
793 ; GFX940-NEXT: s_cbranch_execnz .LBB3_1
794 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
795 ; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
796 ; GFX940-NEXT: s_setpc_b64 s[30:31]
797 ;
798 ; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory:
799 ; GFX11: ; %bb.0:
800 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
801 ; GFX11-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
802 ; GFX11-NEXT: s_add_i32 s4, s16, 0x400
803 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
804 ; GFX11-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_max_f32 v2, v1, v1
805 ; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
806 ; GFX11-NEXT: s_mov_b32 s4, 0
807 ; GFX11-NEXT: .LBB3_1: ; %atomicrmw.start
808 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
809 ; GFX11-NEXT: s_waitcnt vmcnt(0)
810 ; GFX11-NEXT: v_mov_b32_e32 v5, v0
811 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
812 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
813 ; GFX11-NEXT: v_max_f32_e32 v0, v5, v5
814 ; GFX11-NEXT: v_min_f32_e32 v4, v0, v2
815 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
816 ; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
817 ; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen glc
818 ; GFX11-NEXT: s_waitcnt vmcnt(0)
819 ; GFX11-NEXT: buffer_gl1_inv
820 ; GFX11-NEXT: buffer_gl0_inv
821 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
822 ; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
823 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
824 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
825 ; GFX11-NEXT: s_cbranch_execnz .LBB3_1
826 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
827 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
828 ; GFX11-NEXT: s_setpc_b64 s[30:31]
829 ;
830 ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory:
831 ; GFX10: ; %bb.0:
832 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
833 ; GFX10-NEXT: v_mov_b32_e32 v1, v0
834 ; GFX10-NEXT: v_mov_b32_e32 v0, s20
835 ; GFX10-NEXT: s_add_i32 s4, s20, 0x400
836 ; GFX10-NEXT: v_mov_b32_e32 v3, s4
837 ; GFX10-NEXT: v_max_f32_e32 v2, v1, v1
838 ; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
839 ; GFX10-NEXT: s_mov_b32 s4, 0
840 ; GFX10-NEXT: .LBB3_1: ; %atomicrmw.start
841 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
842 ; GFX10-NEXT: s_waitcnt vmcnt(0)
843 ; GFX10-NEXT: v_mov_b32_e32 v5, v0
844 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
845 ; GFX10-NEXT: v_max_f32_e32 v0, v5, v5
846 ; GFX10-NEXT: v_min_f32_e32 v4, v0, v2
847 ; GFX10-NEXT: v_mov_b32_e32 v0, v4
848 ; GFX10-NEXT: v_mov_b32_e32 v1, v5
849 ; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
850 ; GFX10-NEXT: s_waitcnt vmcnt(0)
851 ; GFX10-NEXT: buffer_gl1_inv
852 ; GFX10-NEXT: buffer_gl0_inv
853 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
854 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
855 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
856 ; GFX10-NEXT: s_cbranch_execnz .LBB3_1
857 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
858 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
859 ; GFX10-NEXT: s_setpc_b64 s[30:31]
860 ;
861 ; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory:
862 ; GFX90A: ; %bb.0:
863 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
864 ; GFX90A-NEXT: v_mov_b32_e32 v1, v0
865 ; GFX90A-NEXT: v_mov_b32_e32 v0, s20
866 ; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
867 ; GFX90A-NEXT: s_add_i32 s6, s20, 0x400
868 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
869 ; GFX90A-NEXT: v_max_f32_e32 v2, v1, v1
870 ; GFX90A-NEXT: v_mov_b32_e32 v3, s6
871 ; GFX90A-NEXT: .LBB3_1: ; %atomicrmw.start
872 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
873 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
874 ; GFX90A-NEXT: v_mov_b32_e32 v5, v0
875 ; GFX90A-NEXT: v_max_f32_e32 v0, v5, v5
876 ; GFX90A-NEXT: v_min_f32_e32 v4, v0, v2
877 ; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
878 ; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
879 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
880 ; GFX90A-NEXT: buffer_wbinvl1
881 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
882 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
883 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
884 ; GFX90A-NEXT: s_cbranch_execnz .LBB3_1
885 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
886 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
887 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
888 ;
889 ; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory:
890 ; GFX908: ; %bb.0:
891 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
892 ; GFX908-NEXT: v_mov_b32_e32 v1, v0
893 ; GFX908-NEXT: v_mov_b32_e32 v0, s20
894 ; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
895 ; GFX908-NEXT: s_add_i32 s6, s20, 0x400
896 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
897 ; GFX908-NEXT: v_max_f32_e32 v2, v1, v1
898 ; GFX908-NEXT: v_mov_b32_e32 v3, s6
899 ; GFX908-NEXT: .LBB3_1: ; %atomicrmw.start
900 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
901 ; GFX908-NEXT: s_waitcnt vmcnt(0)
902 ; GFX908-NEXT: v_mov_b32_e32 v5, v0
903 ; GFX908-NEXT: v_max_f32_e32 v0, v5, v5
904 ; GFX908-NEXT: v_min_f32_e32 v4, v0, v2
905 ; GFX908-NEXT: v_mov_b32_e32 v0, v4
906 ; GFX908-NEXT: v_mov_b32_e32 v1, v5
907 ; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
908 ; GFX908-NEXT: s_waitcnt vmcnt(0)
909 ; GFX908-NEXT: buffer_wbinvl1
910 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
911 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
912 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
913 ; GFX908-NEXT: s_cbranch_execnz .LBB3_1
914 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
915 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
916 ; GFX908-NEXT: s_setpc_b64 s[30:31]
917 ;
918 ; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory:
919 ; GFX8: ; %bb.0:
920 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
921 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
922 ; GFX8-NEXT: v_mov_b32_e32 v0, s20
923 ; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
924 ; GFX8-NEXT: s_add_i32 s6, s20, 0x400
925 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
926 ; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v1
927 ; GFX8-NEXT: v_mov_b32_e32 v3, s6
928 ; GFX8-NEXT: .LBB3_1: ; %atomicrmw.start
929 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
930 ; GFX8-NEXT: s_waitcnt vmcnt(0)
931 ; GFX8-NEXT: v_mov_b32_e32 v5, v0
932 ; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v5
933 ; GFX8-NEXT: v_min_f32_e32 v4, v0, v2
934 ; GFX8-NEXT: v_mov_b32_e32 v0, v4
935 ; GFX8-NEXT: v_mov_b32_e32 v1, v5
936 ; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
937 ; GFX8-NEXT: s_waitcnt vmcnt(0)
938 ; GFX8-NEXT: buffer_wbinvl1
939 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
940 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
941 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
942 ; GFX8-NEXT: s_cbranch_execnz .LBB3_1
943 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
944 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
945 ; GFX8-NEXT: s_setpc_b64 s[30:31]
946 ;
947 ; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory:
948 ; GFX7: ; %bb.0:
949 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
950 ; GFX7-NEXT: v_mov_b32_e32 v1, v0
951 ; GFX7-NEXT: v_mov_b32_e32 v0, s20
952 ; GFX7-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
953 ; GFX7-NEXT: s_add_i32 s6, s20, 0x400
954 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
955 ; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v1
956 ; GFX7-NEXT: v_mov_b32_e32 v3, s6
957 ; GFX7-NEXT: .LBB3_1: ; %atomicrmw.start
958 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
959 ; GFX7-NEXT: s_waitcnt vmcnt(0)
960 ; GFX7-NEXT: v_mov_b32_e32 v5, v0
961 ; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v5
962 ; GFX7-NEXT: v_min_f32_e32 v4, v0, v2
963 ; GFX7-NEXT: v_mov_b32_e32 v0, v4
964 ; GFX7-NEXT: v_mov_b32_e32 v1, v5
965 ; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
966 ; GFX7-NEXT: s_waitcnt vmcnt(0)
967 ; GFX7-NEXT: buffer_wbinvl1
968 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
969 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
970 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
971 ; GFX7-NEXT: s_cbranch_execnz .LBB3_1
972 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
973 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
974 ; GFX7-NEXT: s_setpc_b64 s[30:31]
975 ;
976 ; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory:
977 ; GFX6: ; %bb.0:
978 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
979 ; GFX6-NEXT: v_mov_b32_e32 v1, v0
980 ; GFX6-NEXT: v_mov_b32_e32 v0, s20
981 ; GFX6-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
982 ; GFX6-NEXT: s_add_i32 s6, s20, 0x400
983 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
984 ; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v1
985 ; GFX6-NEXT: v_mov_b32_e32 v3, s6
986 ; GFX6-NEXT: .LBB3_1: ; %atomicrmw.start
987 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
988 ; GFX6-NEXT: s_waitcnt vmcnt(0)
989 ; GFX6-NEXT: v_mov_b32_e32 v5, v0
990 ; GFX6-NEXT: s_waitcnt expcnt(0)
991 ; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v5
992 ; GFX6-NEXT: v_min_f32_e32 v4, v0, v2
993 ; GFX6-NEXT: v_mov_b32_e32 v0, v4
994 ; GFX6-NEXT: v_mov_b32_e32 v1, v5
995 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
996 ; GFX6-NEXT: s_waitcnt vmcnt(0)
997 ; GFX6-NEXT: buffer_wbinvl1
998 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
999 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1000 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
1001 ; GFX6-NEXT: s_cbranch_execnz .LBB3_1
1002 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
1003 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
1004 ; GFX6-NEXT: s_waitcnt expcnt(0)
1005 ; GFX6-NEXT: s_setpc_b64 s[30:31]
1006 %gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
1007 %result = atomicrmw fmin ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
1008 ret float %result
1009 }
1011 define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(7) inreg %ptr, float %val) #0 {
1012 ; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
1013 ; GFX12: ; %bb.0:
1014 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
1015 ; GFX12-NEXT: s_wait_expcnt 0x0
1016 ; GFX12-NEXT: s_wait_samplecnt 0x0
1017 ; GFX12-NEXT: s_wait_bvhcnt 0x0
1018 ; GFX12-NEXT: s_wait_kmcnt 0x0
1019 ; GFX12-NEXT: v_mov_b32_e32 v1, s16
1020 ; GFX12-NEXT: s_wait_storecnt 0x0
1021 ; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
1022 ; GFX12-NEXT: s_wait_loadcnt 0x0
1023 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
1024 ; GFX12-NEXT: s_setpc_b64 s[30:31]
1025 ;
1026 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
1027 ; GFX940: ; %bb.0:
1028 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1029 ; GFX940-NEXT: v_mov_b32_e32 v1, v0
1030 ; GFX940-NEXT: v_mov_b32_e32 v0, s16
1031 ; GFX940-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
1032 ; GFX940-NEXT: s_add_i32 s6, s16, 0x400
1033 ; GFX940-NEXT: s_mov_b64 s[4:5], 0
1034 ; GFX940-NEXT: v_max_f32_e32 v2, v1, v1
1035 ; GFX940-NEXT: v_mov_b32_e32 v3, s6
1036 ; GFX940-NEXT: .LBB4_1: ; %atomicrmw.start
1037 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
1038 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1039 ; GFX940-NEXT: v_mov_b32_e32 v5, v0
1040 ; GFX940-NEXT: v_max_f32_e32 v0, v5, v5
1041 ; GFX940-NEXT: v_min_f32_e32 v4, v0, v2
1042 ; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
1043 ; GFX940-NEXT: buffer_wbl2 sc1
1044 ; GFX940-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen sc0
1045 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1046 ; GFX940-NEXT: buffer_inv sc1
1047 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
1048 ; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1049 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
1050 ; GFX940-NEXT: s_cbranch_execnz .LBB4_1
1051 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
1052 ; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
1053 ; GFX940-NEXT: s_setpc_b64 s[30:31]
1054 ;
1055 ; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
1056 ; GFX11: ; %bb.0:
1057 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1058 ; GFX11-NEXT: v_mov_b32_e32 v1, s16
1059 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1060 ; GFX11-NEXT: buffer_atomic_min_f32 v0, v1, s[0:3], 0 offen offset:1024 glc
1061 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1062 ; GFX11-NEXT: buffer_gl1_inv
1063 ; GFX11-NEXT: buffer_gl0_inv
1064 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1065 ;
1066 ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
1067 ; GFX10: ; %bb.0:
1068 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1069 ; GFX10-NEXT: v_mov_b32_e32 v1, s20
1070 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1071 ; GFX10-NEXT: buffer_atomic_fmin v0, v1, s[16:19], 0 offen offset:1024 glc
1072 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1073 ; GFX10-NEXT: buffer_gl1_inv
1074 ; GFX10-NEXT: buffer_gl0_inv
1075 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1076 ;
1077 ; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
1078 ; GFX90A: ; %bb.0:
1079 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1080 ; GFX90A-NEXT: v_mov_b32_e32 v1, v0
1081 ; GFX90A-NEXT: v_mov_b32_e32 v0, s20
1082 ; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
1083 ; GFX90A-NEXT: s_add_i32 s6, s20, 0x400
1084 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
1085 ; GFX90A-NEXT: v_max_f32_e32 v2, v1, v1
1086 ; GFX90A-NEXT: v_mov_b32_e32 v3, s6
1087 ; GFX90A-NEXT: .LBB4_1: ; %atomicrmw.start
1088 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
1089 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1090 ; GFX90A-NEXT: v_mov_b32_e32 v5, v0
1091 ; GFX90A-NEXT: v_max_f32_e32 v0, v5, v5
1092 ; GFX90A-NEXT: v_min_f32_e32 v4, v0, v2
1093 ; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
1094 ; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
1095 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1096 ; GFX90A-NEXT: buffer_wbinvl1
1097 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
1098 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1099 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
1100 ; GFX90A-NEXT: s_cbranch_execnz .LBB4_1
1101 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
1102 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
1103 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1104 ;
1105 ; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
1106 ; GFX908: ; %bb.0:
1107 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1108 ; GFX908-NEXT: v_mov_b32_e32 v1, v0
1109 ; GFX908-NEXT: v_mov_b32_e32 v0, s20
1110 ; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
1111 ; GFX908-NEXT: s_add_i32 s6, s20, 0x400
1112 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
1113 ; GFX908-NEXT: v_max_f32_e32 v2, v1, v1
1114 ; GFX908-NEXT: v_mov_b32_e32 v3, s6
1115 ; GFX908-NEXT: .LBB4_1: ; %atomicrmw.start
1116 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
1117 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1118 ; GFX908-NEXT: v_mov_b32_e32 v5, v0
1119 ; GFX908-NEXT: v_max_f32_e32 v0, v5, v5
1120 ; GFX908-NEXT: v_min_f32_e32 v4, v0, v2
1121 ; GFX908-NEXT: v_mov_b32_e32 v0, v4
1122 ; GFX908-NEXT: v_mov_b32_e32 v1, v5
1123 ; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
1124 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1125 ; GFX908-NEXT: buffer_wbinvl1
1126 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
1127 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1128 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
1129 ; GFX908-NEXT: s_cbranch_execnz .LBB4_1
1130 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
1131 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
1132 ; GFX908-NEXT: s_setpc_b64 s[30:31]
1133 ;
1134 ; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
1135 ; GFX8: ; %bb.0:
1136 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1137 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
1138 ; GFX8-NEXT: v_mov_b32_e32 v0, s20
1139 ; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
1140 ; GFX8-NEXT: s_add_i32 s6, s20, 0x400
1141 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
1142 ; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v1
1143 ; GFX8-NEXT: v_mov_b32_e32 v3, s6
1144 ; GFX8-NEXT: .LBB4_1: ; %atomicrmw.start
1145 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
1146 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1147 ; GFX8-NEXT: v_mov_b32_e32 v5, v0
1148 ; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v5
1149 ; GFX8-NEXT: v_min_f32_e32 v4, v0, v2
1150 ; GFX8-NEXT: v_mov_b32_e32 v0, v4
1151 ; GFX8-NEXT: v_mov_b32_e32 v1, v5
1152 ; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
1153 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1154 ; GFX8-NEXT: buffer_wbinvl1
1155 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
1156 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1157 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1158 ; GFX8-NEXT: s_cbranch_execnz .LBB4_1
1159 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1160 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1161 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1162 ;
1163 ; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
1164 ; GFX7: ; %bb.0:
1165 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1166 ; GFX7-NEXT: v_mov_b32_e32 v1, s20
1167 ; GFX7-NEXT: buffer_atomic_fmin v0, v1, s[16:19], 0 offen offset:1024 glc
1168 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1169 ; GFX7-NEXT: buffer_wbinvl1
1170 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1171 ;
1172 ; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
1173 ; GFX6: ; %bb.0:
1174 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1175 ; GFX6-NEXT: v_mov_b32_e32 v1, s20
1176 ; GFX6-NEXT: buffer_atomic_fmin v0, v1, s[16:19], 0 offen offset:1024 glc
1177 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1178 ; GFX6-NEXT: buffer_wbinvl1
1179 ; GFX6-NEXT: s_waitcnt expcnt(0)
1180 ; GFX6-NEXT: s_setpc_b64 s[30:31]
1181 %gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
1182 %result = atomicrmw fmin ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
1183 ret float %result
1184 }
1186 ; --------------------------------------------------------------------
1187 ; double
1188 ; --------------------------------------------------------------------
1190 define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, double %val) #0 {
1191 ; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
1192 ; GFX12: ; %bb.0:
1193 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
1194 ; GFX12-NEXT: s_wait_expcnt 0x0
1195 ; GFX12-NEXT: s_wait_samplecnt 0x0
1196 ; GFX12-NEXT: s_wait_bvhcnt 0x0
1197 ; GFX12-NEXT: s_wait_kmcnt 0x0
1198 ; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
1199 ; GFX12-NEXT: v_mov_b32_e32 v0, s16
1200 ; GFX12-NEXT: s_add_co_i32 s4, s16, 0x800
1201 ; GFX12-NEXT: s_wait_alu 0xfffe
1202 ; GFX12-NEXT: v_mov_b32_e32 v6, s4
1203 ; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
1204 ; GFX12-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen offset:2048
1205 ; GFX12-NEXT: s_mov_b32 s4, 0
1206 ; GFX12-NEXT: .LBB5_1: ; %atomicrmw.start
1207 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
1208 ; GFX12-NEXT: s_wait_loadcnt 0x0
1209 ; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
1210 ; GFX12-NEXT: s_wait_storecnt 0x0
1211 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
1212 ; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[9:10], v[9:10]
1213 ; GFX12-NEXT: v_min_num_f64_e32 v[7:8], v[0:1], v[4:5]
1214 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
1215 ; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
1216 ; GFX12-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
1217 ; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], null offen th:TH_ATOMIC_RETURN
1218 ; GFX12-NEXT: s_wait_loadcnt 0x0
1219 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
1220 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
1221 ; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
1222 ; GFX12-NEXT: s_wait_alu 0xfffe
1223 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
1224 ; GFX12-NEXT: s_cbranch_execnz .LBB5_1
1225 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
1226 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
1227 ; GFX12-NEXT: s_setpc_b64 s[30:31]
1228 ;
1229 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
1230 ; GFX940: ; %bb.0:
1231 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1232 ; GFX940-NEXT: v_mov_b32_e32 v2, s16
1233 ; GFX940-NEXT: buffer_wbl2 sc1
1234 ; GFX940-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen offset:2048 sc0
1235 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1236 ; GFX940-NEXT: buffer_inv sc1
1237 ; GFX940-NEXT: s_setpc_b64 s[30:31]
1238 ;
1239 ; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
1240 ; GFX11: ; %bb.0:
1241 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1242 ; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
1243 ; GFX11-NEXT: v_mov_b32_e32 v0, s16
1244 ; GFX11-NEXT: s_add_i32 s4, s16, 0x800
1245 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
1246 ; GFX11-NEXT: v_mov_b32_e32 v6, s4
1247 ; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
1248 ; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:2048
1249 ; GFX11-NEXT: s_mov_b32 s4, 0
1250 ; GFX11-NEXT: .LBB5_1: ; %atomicrmw.start
1251 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
1252 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1253 ; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
1254 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1255 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
1256 ; GFX11-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
1257 ; GFX11-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
1258 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
1259 ; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
1260 ; GFX11-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
1261 ; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], 0 offen glc
1262 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1263 ; GFX11-NEXT: buffer_gl1_inv
1264 ; GFX11-NEXT: buffer_gl0_inv
1265 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
1266 ; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
1267 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1268 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
1269 ; GFX11-NEXT: s_cbranch_execnz .LBB5_1
1270 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
1271 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
1272 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1273 ;
1274 ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
1275 ; GFX10: ; %bb.0:
1276 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1277 ; GFX10-NEXT: v_mov_b32_e32 v2, s20
1278 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1279 ; GFX10-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[16:19], 0 offen offset:2048 glc
1280 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1281 ; GFX10-NEXT: buffer_gl1_inv
1282 ; GFX10-NEXT: buffer_gl0_inv
1283 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1284 ;
1285 ; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
1286 ; GFX90A: ; %bb.0:
1287 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1288 ; GFX90A-NEXT: v_mov_b32_e32 v2, s20
1289 ; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[16:19], 0 offen offset:2048 glc
1290 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1291 ; GFX90A-NEXT: buffer_wbinvl1
1292 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1293 ;
1294 ; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
1295 ; GFX908: ; %bb.0:
1296 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1297 ; GFX908-NEXT: v_mov_b32_e32 v2, v0
1298 ; GFX908-NEXT: v_mov_b32_e32 v0, s20
1299 ; GFX908-NEXT: v_mov_b32_e32 v3, v1
1300 ; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
1301 ; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
1302 ; GFX908-NEXT: s_add_i32 s6, s20, 0x800
1303 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
1304 ; GFX908-NEXT: v_mov_b32_e32 v6, s6
1305 ; GFX908-NEXT: .LBB5_1: ; %atomicrmw.start
1306 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
1307 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1308 ; GFX908-NEXT: v_mov_b32_e32 v10, v1
1309 ; GFX908-NEXT: v_mov_b32_e32 v9, v0
1310 ; GFX908-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
1311 ; GFX908-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
1312 ; GFX908-NEXT: v_mov_b32_e32 v0, v7
1313 ; GFX908-NEXT: v_mov_b32_e32 v1, v8
1314 ; GFX908-NEXT: v_mov_b32_e32 v2, v9
1315 ; GFX908-NEXT: v_mov_b32_e32 v3, v10
1316 ; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen glc
1317 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1318 ; GFX908-NEXT: buffer_wbinvl1
1319 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
1320 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1321 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
1322 ; GFX908-NEXT: s_cbranch_execnz .LBB5_1
1323 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
1324 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
1325 ; GFX908-NEXT: s_setpc_b64 s[30:31]
1326 ;
1327 ; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
1328 ; GFX8: ; %bb.0:
1329 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1330 ; GFX8-NEXT: v_mov_b32_e32 v2, v0
1331 ; GFX8-NEXT: v_mov_b32_e32 v0, s20
1332 ; GFX8-NEXT: v_mov_b32_e32 v3, v1
1333 ; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
1334 ; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
1335 ; GFX8-NEXT: s_add_i32 s6, s20, 0x800
1336 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
1337 ; GFX8-NEXT: v_mov_b32_e32 v6, s6
1338 ; GFX8-NEXT: .LBB5_1: ; %atomicrmw.start
1339 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
1340 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1341 ; GFX8-NEXT: v_mov_b32_e32 v10, v1
1342 ; GFX8-NEXT: v_mov_b32_e32 v9, v0
1343 ; GFX8-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
1344 ; GFX8-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
1345 ; GFX8-NEXT: v_mov_b32_e32 v0, v7
1346 ; GFX8-NEXT: v_mov_b32_e32 v1, v8
1347 ; GFX8-NEXT: v_mov_b32_e32 v2, v9
1348 ; GFX8-NEXT: v_mov_b32_e32 v3, v10
1349 ; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen glc
1350 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1351 ; GFX8-NEXT: buffer_wbinvl1
1352 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
1353 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1354 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1355 ; GFX8-NEXT: s_cbranch_execnz .LBB5_1
1356 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1357 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1358 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1359 ;
1360 ; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
1361 ; GFX7: ; %bb.0:
1362 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1363 ; GFX7-NEXT: v_mov_b32_e32 v2, s20
1364 ; GFX7-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[16:19], 0 offen offset:2048 glc
1365 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1366 ; GFX7-NEXT: buffer_wbinvl1
1367 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1368 ;
1369 ; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
1370 ; GFX6: ; %bb.0:
1371 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1372 ; GFX6-NEXT: v_mov_b32_e32 v2, s20
1373 ; GFX6-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[16:19], 0 offen offset:2048 glc
1374 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1375 ; GFX6-NEXT: buffer_wbinvl1
1376 ; GFX6-NEXT: s_waitcnt expcnt(0)
1377 ; GFX6-NEXT: s_setpc_b64 s[30:31]
1378 %gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
1379 %result = atomicrmw fmin ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
1380 ret double %result
1381 }
1383 define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, double %val) #0 {
1384 ; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory:
1385 ; GFX12: ; %bb.0:
1386 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
1387 ; GFX12-NEXT: s_wait_expcnt 0x0
1388 ; GFX12-NEXT: s_wait_samplecnt 0x0
1389 ; GFX12-NEXT: s_wait_bvhcnt 0x0
1390 ; GFX12-NEXT: s_wait_kmcnt 0x0
1391 ; GFX12-NEXT: v_mov_b32_e32 v2, s16
1392 ; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[0:1], v[0:1]
1393 ; GFX12-NEXT: s_add_co_i32 s4, s16, 0x800
1394 ; GFX12-NEXT: s_wait_alu 0xfffe
1395 ; GFX12-NEXT: v_mov_b32_e32 v6, s4
1396 ; GFX12-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], null offen offset:2048
1397 ; GFX12-NEXT: s_mov_b32 s4, 0
1398 ; GFX12-NEXT: .LBB6_1: ; %atomicrmw.start
1399 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
1400 ; GFX12-NEXT: s_wait_loadcnt 0x0
1401 ; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
1402 ; GFX12-NEXT: s_wait_storecnt 0x0
1403 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
1404 ; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[4:5]
1405 ; GFX12-NEXT: v_dual_mov_b32 v10, v3 :: v_dual_mov_b32 v9, v2
1406 ; GFX12-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
1407 ; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], null offen th:TH_ATOMIC_RETURN
1408 ; GFX12-NEXT: s_wait_loadcnt 0x0
1409 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
1410 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
1411 ; GFX12-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
1412 ; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
1413 ; GFX12-NEXT: s_wait_alu 0xfffe
1414 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
1415 ; GFX12-NEXT: s_cbranch_execnz .LBB6_1
1416 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
1417 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
1418 ; GFX12-NEXT: s_setpc_b64 s[30:31]
1419 ;
1420 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory:
1421 ; GFX940: ; %bb.0:
1422 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1423 ; GFX940-NEXT: v_mov_b32_e32 v2, s16
1424 ; GFX940-NEXT: buffer_wbl2 sc1
1425 ; GFX940-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen offset:2048
1426 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1427 ; GFX940-NEXT: buffer_inv sc1
1428 ; GFX940-NEXT: s_setpc_b64 s[30:31]
1429 ;
1430 ; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory:
1431 ; GFX11: ; %bb.0:
1432 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1433 ; GFX11-NEXT: v_mov_b32_e32 v2, s16
1434 ; GFX11-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
1435 ; GFX11-NEXT: s_add_i32 s4, s16, 0x800
1436 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1437 ; GFX11-NEXT: v_mov_b32_e32 v6, s4
1438 ; GFX11-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], 0 offen offset:2048
1439 ; GFX11-NEXT: s_mov_b32 s4, 0
1440 ; GFX11-NEXT: .LBB6_1: ; %atomicrmw.start
1441 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
1442 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1443 ; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
1444 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1445 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
1446 ; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
1447 ; GFX11-NEXT: v_dual_mov_b32 v10, v3 :: v_dual_mov_b32 v9, v2
1448 ; GFX11-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
1449 ; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], 0 offen glc
1450 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1451 ; GFX11-NEXT: buffer_gl1_inv
1452 ; GFX11-NEXT: buffer_gl0_inv
1453 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
1454 ; GFX11-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
1455 ; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
1456 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1457 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
1458 ; GFX11-NEXT: s_cbranch_execnz .LBB6_1
1459 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
1460 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
1461 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1462 ;
1463 ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory:
1464 ; GFX10: ; %bb.0:
1465 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1466 ; GFX10-NEXT: v_mov_b32_e32 v2, s20
1467 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1468 ; GFX10-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[16:19], 0 offen offset:2048
1469 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1470 ; GFX10-NEXT: buffer_gl1_inv
1471 ; GFX10-NEXT: buffer_gl0_inv
1472 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1473 ;
1474 ; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory:
1475 ; GFX90A: ; %bb.0:
1476 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1477 ; GFX90A-NEXT: v_mov_b32_e32 v2, s20
1478 ; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[16:19], 0 offen offset:2048
1479 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1480 ; GFX90A-NEXT: buffer_wbinvl1
1481 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1482 ;
1483 ; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory:
1484 ; GFX908: ; %bb.0:
1485 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1486 ; GFX908-NEXT: v_mov_b32_e32 v2, s20
1487 ; GFX908-NEXT: buffer_load_dwordx2 v[2:3], v2, s[16:19], 0 offen offset:2048
1488 ; GFX908-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
1489 ; GFX908-NEXT: s_add_i32 s6, s20, 0x800
1490 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
1491 ; GFX908-NEXT: v_mov_b32_e32 v6, s6
1492 ; GFX908-NEXT: .LBB6_1: ; %atomicrmw.start
1493 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
1494 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1495 ; GFX908-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
1496 ; GFX908-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
1497 ; GFX908-NEXT: v_mov_b32_e32 v10, v3
1498 ; GFX908-NEXT: v_mov_b32_e32 v9, v2
1499 ; GFX908-NEXT: v_mov_b32_e32 v8, v1
1500 ; GFX908-NEXT: v_mov_b32_e32 v7, v0
1501 ; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[16:19], 0 offen glc
1502 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1503 ; GFX908-NEXT: buffer_wbinvl1
1504 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[2:3]
1505 ; GFX908-NEXT: v_mov_b32_e32 v2, v7
1506 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1507 ; GFX908-NEXT: v_mov_b32_e32 v3, v8
1508 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
1509 ; GFX908-NEXT: s_cbranch_execnz .LBB6_1
1510 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
1511 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
1512 ; GFX908-NEXT: s_setpc_b64 s[30:31]
1513 ;
1514 ; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory:
1515 ; GFX8: ; %bb.0:
1516 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1517 ; GFX8-NEXT: v_mov_b32_e32 v2, s20
1518 ; GFX8-NEXT: buffer_load_dwordx2 v[2:3], v2, s[16:19], 0 offen offset:2048
1519 ; GFX8-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
1520 ; GFX8-NEXT: s_add_i32 s6, s20, 0x800
1521 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
1522 ; GFX8-NEXT: v_mov_b32_e32 v6, s6
1523 ; GFX8-NEXT: .LBB6_1: ; %atomicrmw.start
1524 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
1525 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1526 ; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
1527 ; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
1528 ; GFX8-NEXT: v_mov_b32_e32 v10, v3
1529 ; GFX8-NEXT: v_mov_b32_e32 v9, v2
1530 ; GFX8-NEXT: v_mov_b32_e32 v8, v1
1531 ; GFX8-NEXT: v_mov_b32_e32 v7, v0
1532 ; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[16:19], 0 offen glc
1533 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1534 ; GFX8-NEXT: buffer_wbinvl1
1535 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[2:3]
1536 ; GFX8-NEXT: v_mov_b32_e32 v2, v7
1537 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1538 ; GFX8-NEXT: v_mov_b32_e32 v3, v8
1539 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1540 ; GFX8-NEXT: s_cbranch_execnz .LBB6_1
1541 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1542 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1543 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1544 ;
1545 ; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory:
1546 ; GFX7: ; %bb.0:
1547 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1548 ; GFX7-NEXT: v_mov_b32_e32 v2, s20
1549 ; GFX7-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[16:19], 0 offen offset:2048
1550 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1551 ; GFX7-NEXT: buffer_wbinvl1
1552 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1553 ;
1554 ; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_grained_memory:
1555 ; GFX6: ; %bb.0:
1556 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1557 ; GFX6-NEXT: v_mov_b32_e32 v2, s20
1558 ; GFX6-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[16:19], 0 offen offset:2048
1559 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1560 ; GFX6-NEXT: buffer_wbinvl1
1561 ; GFX6-NEXT: s_waitcnt expcnt(0)
1562 ; GFX6-NEXT: s_setpc_b64 s[30:31]
1563 %gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
1564 %unused = atomicrmw fmin ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
1565 ret void
1566 }
1568 define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, double %val) #0 {
1569 ; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
1570 ; GFX12: ; %bb.0:
1571 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
1572 ; GFX12-NEXT: s_wait_expcnt 0x0
1573 ; GFX12-NEXT: s_wait_samplecnt 0x0
1574 ; GFX12-NEXT: s_wait_bvhcnt 0x0
1575 ; GFX12-NEXT: s_wait_kmcnt 0x0
1576 ; GFX12-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
1577 ; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
1578 ; GFX12-NEXT: v_add_nc_u32_e32 v15, 0x800, v4
1579 ; GFX12-NEXT: s_mov_b32 s1, exec_lo
1580 ; GFX12-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
1581 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
1582 ; GFX12-NEXT: v_readfirstlane_b32 s4, v9
1583 ; GFX12-NEXT: v_readfirstlane_b32 s5, v10
1584 ; GFX12-NEXT: v_readfirstlane_b32 s6, v7
1585 ; GFX12-NEXT: v_readfirstlane_b32 s7, v8
1586 ; GFX12-NEXT: s_wait_alu 0xf1ff
1587 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
1588 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
1589 ; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
1590 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
1591 ; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
1592 ; GFX12-NEXT: s_wait_alu 0xfffe
1593 ; GFX12-NEXT: s_and_saveexec_b32 s0, s0
1594 ; GFX12-NEXT: s_wait_loadcnt 0x0
1595 ; GFX12-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], null offen offset:2048
1596 ; GFX12-NEXT: ; implicit-def: $vgpr4
1597 ; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
1598 ; GFX12-NEXT: s_cbranch_execnz .LBB7_1
1599 ; GFX12-NEXT: ; %bb.2:
1600 ; GFX12-NEXT: s_mov_b32 exec_lo, s1
1601 ; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[5:6], v[5:6]
1602 ; GFX12-NEXT: s_mov_b32 s1, 0
1603 ; GFX12-NEXT: .LBB7_3: ; %atomicrmw.start
1604 ; GFX12-NEXT: ; =>This Loop Header: Depth=1
1605 ; GFX12-NEXT: ; Child Loop BB7_4 Depth 2
1606 ; GFX12-NEXT: s_wait_loadcnt 0x0
1607 ; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[13:14], v[13:14]
1608 ; GFX12-NEXT: s_mov_b32 s2, exec_lo
1609 ; GFX12-NEXT: s_wait_storecnt 0x0
1610 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
1611 ; GFX12-NEXT: v_min_num_f64_e32 v[11:12], v[0:1], v[4:5]
1612 ; GFX12-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
1613 ; GFX12-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
1614 ; GFX12-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
1615 ; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
1616 ; GFX12-NEXT: v_readfirstlane_b32 s4, v9
1617 ; GFX12-NEXT: v_readfirstlane_b32 s5, v10
1618 ; GFX12-NEXT: v_readfirstlane_b32 s6, v7
1619 ; GFX12-NEXT: v_readfirstlane_b32 s7, v8
1620 ; GFX12-NEXT: s_wait_alu 0xf1ff
1621 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
1622 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
1623 ; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
1624 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
1625 ; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
1626 ; GFX12-NEXT: s_wait_alu 0xfffe
1627 ; GFX12-NEXT: s_and_saveexec_b32 s0, s0
1628 ; GFX12-NEXT: s_wait_loadcnt 0x0
1629 ; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v15, s[4:7], null offen th:TH_ATOMIC_RETURN
1630 ; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
1631 ; GFX12-NEXT: s_cbranch_execnz .LBB7_4
1632 ; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
1633 ; GFX12-NEXT: s_mov_b32 exec_lo, s2
1634 ; GFX12-NEXT: s_wait_loadcnt 0x0
1635 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
1636 ; GFX12-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
1637 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
1638 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
1639 ; GFX12-NEXT: s_wait_alu 0xfffe
1640 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
1641 ; GFX12-NEXT: s_cbranch_execnz .LBB7_3
1642 ; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
1643 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
1644 ; GFX12-NEXT: s_setpc_b64 s[30:31]
1645 ;
1646 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
1647 ; GFX940: ; %bb.0:
1648 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1649 ; GFX940-NEXT: v_mov_b32_e32 v7, v6
1650 ; GFX940-NEXT: v_mov_b32_e32 v6, v5
1651 ; GFX940-NEXT: s_mov_b64 s[2:3], exec
1652 ; GFX940-NEXT: buffer_wbl2 sc1
1653 ; GFX940-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
1654 ; GFX940-NEXT: v_readfirstlane_b32 s4, v0
1655 ; GFX940-NEXT: v_readfirstlane_b32 s5, v1
1656 ; GFX940-NEXT: v_readfirstlane_b32 s6, v2
1657 ; GFX940-NEXT: v_readfirstlane_b32 s7, v3
1658 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
1659 ; GFX940-NEXT: s_nop 0
1660 ; GFX940-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
1661 ; GFX940-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
1662 ; GFX940-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
1663 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1664 ; GFX940-NEXT: buffer_atomic_min_f64 v[6:7], v4, s[4:7], 0 offen offset:2048 sc0
1665 ; GFX940-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
1666 ; GFX940-NEXT: ; implicit-def: $vgpr4
1667 ; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
1668 ; GFX940-NEXT: s_cbranch_execnz .LBB7_1
1669 ; GFX940-NEXT: ; %bb.2:
1670 ; GFX940-NEXT: s_mov_b64 exec, s[2:3]
1671 ; GFX940-NEXT: s_waitcnt vmcnt(0)
1672 ; GFX940-NEXT: v_mov_b32_e32 v0, v6
1673 ; GFX940-NEXT: v_mov_b32_e32 v1, v7
1674 ; GFX940-NEXT: buffer_inv sc1
1675 ; GFX940-NEXT: s_setpc_b64 s[30:31]
1676 ;
1677 ; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
1678 ; GFX11: ; %bb.0:
1679 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1680 ; GFX11-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
1681 ; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
1682 ; GFX11-NEXT: v_add_nc_u32_e32 v15, 0x800, v4
1683 ; GFX11-NEXT: s_mov_b32 s1, 0
1684 ; GFX11-NEXT: s_mov_b32 s2, exec_lo
1685 ; GFX11-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
1686 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
1687 ; GFX11-NEXT: v_readfirstlane_b32 s4, v9
1688 ; GFX11-NEXT: v_readfirstlane_b32 s5, v10
1689 ; GFX11-NEXT: v_readfirstlane_b32 s6, v7
1690 ; GFX11-NEXT: v_readfirstlane_b32 s7, v8
1691 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
1692 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
1693 ; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
1694 ; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
1695 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1696 ; GFX11-NEXT: s_and_saveexec_b32 s0, s0
1697 ; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], 0 offen offset:2048
1698 ; GFX11-NEXT: ; implicit-def: $vgpr4
1699 ; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
1700 ; GFX11-NEXT: s_cbranch_execnz .LBB7_1
1701 ; GFX11-NEXT: ; %bb.2:
1702 ; GFX11-NEXT: s_mov_b32 exec_lo, s2
1703 ; GFX11-NEXT: v_max_f64 v[4:5], v[5:6], v[5:6]
1704 ; GFX11-NEXT: .p2align 6
1705 ; GFX11-NEXT: .LBB7_3: ; %atomicrmw.start
1706 ; GFX11-NEXT: ; =>This Loop Header: Depth=1
1707 ; GFX11-NEXT: ; Child Loop BB7_4 Depth 2
1708 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1709 ; GFX11-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
1710 ; GFX11-NEXT: s_mov_b32 s2, exec_lo
1711 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
1712 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
1713 ; GFX11-NEXT: v_min_f64 v[11:12], v[0:1], v[4:5]
1714 ; GFX11-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
1715 ; GFX11-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
1716 ; GFX11-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
1717 ; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
1718 ; GFX11-NEXT: v_readfirstlane_b32 s4, v9
1719 ; GFX11-NEXT: v_readfirstlane_b32 s5, v10
1720 ; GFX11-NEXT: v_readfirstlane_b32 s6, v7
1721 ; GFX11-NEXT: v_readfirstlane_b32 s7, v8
1722 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
1723 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
1724 ; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
1725 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
1726 ; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
1727 ; GFX11-NEXT: s_and_saveexec_b32 s0, s0
1728 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1729 ; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v15, s[4:7], 0 offen glc
1730 ; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
1731 ; GFX11-NEXT: s_cbranch_execnz .LBB7_4
1732 ; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
1733 ; GFX11-NEXT: s_mov_b32 exec_lo, s2
1734 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1735 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
1736 ; GFX11-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
1737 ; GFX11-NEXT: buffer_gl1_inv
1738 ; GFX11-NEXT: buffer_gl0_inv
1739 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
1740 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
1741 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
1742 ; GFX11-NEXT: s_cbranch_execnz .LBB7_3
1743 ; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
1744 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
1745 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1746 ;
1747 ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
1748 ; GFX10: ; %bb.0:
1749 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1750 ; GFX10-NEXT: s_mov_b32 s5, exec_lo
1751 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
1752 ; GFX10-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
1753 ; GFX10-NEXT: v_readfirstlane_b32 s8, v0
1754 ; GFX10-NEXT: v_readfirstlane_b32 s9, v1
1755 ; GFX10-NEXT: v_readfirstlane_b32 s10, v2
1756 ; GFX10-NEXT: v_readfirstlane_b32 s11, v3
1757 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
1758 ; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
1759 ; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
1760 ; GFX10-NEXT: s_and_saveexec_b32 s4, s4
1761 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1762 ; GFX10-NEXT: buffer_atomic_fmin_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
1763 ; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
1764 ; GFX10-NEXT: ; implicit-def: $vgpr4
1765 ; GFX10-NEXT: s_waitcnt_depctr 0xffe3
1766 ; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
1767 ; GFX10-NEXT: s_cbranch_execnz .LBB7_1
1768 ; GFX10-NEXT: ; %bb.2:
1769 ; GFX10-NEXT: s_mov_b32 exec_lo, s5
1770 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1771 ; GFX10-NEXT: v_mov_b32_e32 v0, v5
1772 ; GFX10-NEXT: v_mov_b32_e32 v1, v6
1773 ; GFX10-NEXT: buffer_gl1_inv
1774 ; GFX10-NEXT: buffer_gl0_inv
1775 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1776 ;
1777 ; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
1778 ; GFX90A: ; %bb.0:
1779 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1780 ; GFX90A-NEXT: v_mov_b32_e32 v7, v6
1781 ; GFX90A-NEXT: v_mov_b32_e32 v6, v5
1782 ; GFX90A-NEXT: s_mov_b64 s[6:7], exec
1783 ; GFX90A-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
1784 ; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
1785 ; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
1786 ; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
1787 ; GFX90A-NEXT: v_readfirstlane_b32 s11, v3
1788 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
1789 ; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
1790 ; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
1791 ; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
1792 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1793 ; GFX90A-NEXT: buffer_atomic_min_f64 v[6:7], v4, s[8:11], 0 offen offset:2048 glc
1794 ; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
1795 ; GFX90A-NEXT: ; implicit-def: $vgpr4
1796 ; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
1797 ; GFX90A-NEXT: s_cbranch_execnz .LBB7_1
1798 ; GFX90A-NEXT: ; %bb.2:
1799 ; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
1800 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
1801 ; GFX90A-NEXT: v_mov_b32_e32 v0, v6
1802 ; GFX90A-NEXT: v_mov_b32_e32 v1, v7
1803 ; GFX90A-NEXT: buffer_wbinvl1
1804 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
1805 ;
1806 ; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
1807 ; GFX908: ; %bb.0:
1808 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1809 ; GFX908-NEXT: v_mov_b32_e32 v8, v3
1810 ; GFX908-NEXT: v_mov_b32_e32 v7, v2
1811 ; GFX908-NEXT: v_mov_b32_e32 v10, v1
1812 ; GFX908-NEXT: v_mov_b32_e32 v9, v0
1813 ; GFX908-NEXT: v_add_u32_e32 v15, 0x800, v4
1814 ; GFX908-NEXT: s_mov_b64 s[6:7], exec
1815 ; GFX908-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
1816 ; GFX908-NEXT: v_readfirstlane_b32 s8, v9
1817 ; GFX908-NEXT: v_readfirstlane_b32 s9, v10
1818 ; GFX908-NEXT: v_readfirstlane_b32 s10, v7
1819 ; GFX908-NEXT: v_readfirstlane_b32 s11, v8
1820 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]
1821 ; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
1822 ; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
1823 ; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
1824 ; GFX908-NEXT: s_nop 0
1825 ; GFX908-NEXT: buffer_load_dwordx2 v[13:14], v4, s[8:11], 0 offen offset:2048
1826 ; GFX908-NEXT: ; implicit-def: $vgpr4
1827 ; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
1828 ; GFX908-NEXT: s_cbranch_execnz .LBB7_1
1829 ; GFX908-NEXT: ; %bb.2:
1830 ; GFX908-NEXT: s_mov_b64 exec, s[6:7]
1831 ; GFX908-NEXT: v_max_f64 v[4:5], v[5:6], v[5:6]
1832 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
1833 ; GFX908-NEXT: .LBB7_3: ; %atomicrmw.start
1834 ; GFX908-NEXT: ; =>This Loop Header: Depth=1
1835 ; GFX908-NEXT: ; Child Loop BB7_4 Depth 2
1836 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1837 ; GFX908-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
1838 ; GFX908-NEXT: s_mov_b64 s[12:13], exec
1839 ; GFX908-NEXT: v_min_f64 v[11:12], v[0:1], v[4:5]
1840 ; GFX908-NEXT: v_mov_b32_e32 v0, v11
1841 ; GFX908-NEXT: v_mov_b32_e32 v1, v12
1842 ; GFX908-NEXT: v_mov_b32_e32 v2, v13
1843 ; GFX908-NEXT: v_mov_b32_e32 v3, v14
1844 ; GFX908-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
1845 ; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
1846 ; GFX908-NEXT: v_readfirstlane_b32 s8, v9
1847 ; GFX908-NEXT: v_readfirstlane_b32 s9, v10
1848 ; GFX908-NEXT: v_readfirstlane_b32 s10, v7
1849 ; GFX908-NEXT: v_readfirstlane_b32 s11, v8
1850 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]
1851 ; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
1852 ; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
1853 ; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
1854 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1855 ; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v15, s[8:11], 0 offen glc
1856 ; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
1857 ; GFX908-NEXT: s_cbranch_execnz .LBB7_4
1858 ; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
1859 ; GFX908-NEXT: s_mov_b64 exec, s[12:13]
1860 ; GFX908-NEXT: s_waitcnt vmcnt(0)
1861 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[13:14]
1862 ; GFX908-NEXT: v_mov_b32_e32 v14, v1
1863 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
1864 ; GFX908-NEXT: v_mov_b32_e32 v13, v0
1865 ; GFX908-NEXT: buffer_wbinvl1
1866 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
1867 ; GFX908-NEXT: s_cbranch_execnz .LBB7_3
1868 ; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
1869 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
1870 ; GFX908-NEXT: s_setpc_b64 s[30:31]
1871 ;
1872 ; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
1873 ; GFX8: ; %bb.0:
1874 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1875 ; GFX8-NEXT: v_mov_b32_e32 v8, v3
1876 ; GFX8-NEXT: v_mov_b32_e32 v7, v2
1877 ; GFX8-NEXT: v_mov_b32_e32 v10, v1
1878 ; GFX8-NEXT: v_mov_b32_e32 v9, v0
1879 ; GFX8-NEXT: v_add_u32_e32 v15, vcc, 0x800, v4
1880 ; GFX8-NEXT: s_mov_b64 s[6:7], exec
1881 ; GFX8-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
1882 ; GFX8-NEXT: v_readfirstlane_b32 s8, v9
1883 ; GFX8-NEXT: v_readfirstlane_b32 s9, v10
1884 ; GFX8-NEXT: v_readfirstlane_b32 s10, v7
1885 ; GFX8-NEXT: v_readfirstlane_b32 s11, v8
1886 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]
1887 ; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
1888 ; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
1889 ; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
1890 ; GFX8-NEXT: s_nop 0
1891 ; GFX8-NEXT: buffer_load_dwordx2 v[13:14], v4, s[8:11], 0 offen offset:2048
1892 ; GFX8-NEXT: ; implicit-def: $vgpr4
1893 ; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
1894 ; GFX8-NEXT: s_cbranch_execnz .LBB7_1
1895 ; GFX8-NEXT: ; %bb.2:
1896 ; GFX8-NEXT: s_mov_b64 exec, s[6:7]
1897 ; GFX8-NEXT: v_max_f64 v[4:5], v[5:6], v[5:6]
1898 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
1899 ; GFX8-NEXT: .LBB7_3: ; %atomicrmw.start
1900 ; GFX8-NEXT: ; =>This Loop Header: Depth=1
1901 ; GFX8-NEXT: ; Child Loop BB7_4 Depth 2
1902 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1903 ; GFX8-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
1904 ; GFX8-NEXT: s_mov_b64 s[12:13], exec
1905 ; GFX8-NEXT: v_min_f64 v[11:12], v[0:1], v[4:5]
1906 ; GFX8-NEXT: v_mov_b32_e32 v0, v11
1907 ; GFX8-NEXT: v_mov_b32_e32 v1, v12
1908 ; GFX8-NEXT: v_mov_b32_e32 v2, v13
1909 ; GFX8-NEXT: v_mov_b32_e32 v3, v14
1910 ; GFX8-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
1911 ; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
1912 ; GFX8-NEXT: v_readfirstlane_b32 s8, v9
1913 ; GFX8-NEXT: v_readfirstlane_b32 s9, v10
1914 ; GFX8-NEXT: v_readfirstlane_b32 s10, v7
1915 ; GFX8-NEXT: v_readfirstlane_b32 s11, v8
1916 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]
1917 ; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]
1918 ; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
1919 ; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
1920 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1921 ; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v15, s[8:11], 0 offen glc
1922 ; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
1923 ; GFX8-NEXT: s_cbranch_execnz .LBB7_4
1924 ; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
1925 ; GFX8-NEXT: s_mov_b64 exec, s[12:13]
1926 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1927 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[13:14]
1928 ; GFX8-NEXT: v_mov_b32_e32 v14, v1
1929 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
1930 ; GFX8-NEXT: v_mov_b32_e32 v13, v0
1931 ; GFX8-NEXT: buffer_wbinvl1
1932 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
1933 ; GFX8-NEXT: s_cbranch_execnz .LBB7_3
1934 ; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
1935 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
1936 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1937 ;
1938 ; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
1939 ; GFX7: ; %bb.0:
1940 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1941 ; GFX7-NEXT: s_mov_b64 s[6:7], exec
1942 ; GFX7-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
1943 ; GFX7-NEXT: v_readfirstlane_b32 s8, v0
1944 ; GFX7-NEXT: v_readfirstlane_b32 s9, v1
1945 ; GFX7-NEXT: v_readfirstlane_b32 s10, v2
1946 ; GFX7-NEXT: v_readfirstlane_b32 s11, v3
1947 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
1948 ; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
1949 ; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
1950 ; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
1951 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1952 ; GFX7-NEXT: buffer_atomic_fmin_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
1953 ; GFX7-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
1954 ; GFX7-NEXT: ; implicit-def: $vgpr4
1955 ; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
1956 ; GFX7-NEXT: s_cbranch_execnz .LBB7_1
1957 ; GFX7-NEXT: ; %bb.2:
1958 ; GFX7-NEXT: s_mov_b64 exec, s[6:7]
1959 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1960 ; GFX7-NEXT: v_mov_b32_e32 v0, v5
1961 ; GFX7-NEXT: v_mov_b32_e32 v1, v6
1962 ; GFX7-NEXT: buffer_wbinvl1
1963 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1964 ;
1965 ; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
1966 ; GFX6: ; %bb.0:
1967 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1968 ; GFX6-NEXT: s_mov_b64 s[6:7], exec
1969 ; GFX6-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
1970 ; GFX6-NEXT: v_readfirstlane_b32 s8, v0
1971 ; GFX6-NEXT: v_readfirstlane_b32 s9, v1
1972 ; GFX6-NEXT: v_readfirstlane_b32 s10, v2
1973 ; GFX6-NEXT: v_readfirstlane_b32 s11, v3
1974 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
1975 ; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
1976 ; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
1977 ; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
1978 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
1979 ; GFX6-NEXT: buffer_atomic_fmin_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
1980 ; GFX6-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
1981 ; GFX6-NEXT: ; implicit-def: $vgpr4
1982 ; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
1983 ; GFX6-NEXT: s_cbranch_execnz .LBB7_1
1984 ; GFX6-NEXT: ; %bb.2:
1985 ; GFX6-NEXT: s_mov_b64 exec, s[6:7]
1986 ; GFX6-NEXT: s_waitcnt vmcnt(0)
1987 ; GFX6-NEXT: v_mov_b32_e32 v0, v5
1988 ; GFX6-NEXT: v_mov_b32_e32 v1, v6
1989 ; GFX6-NEXT: buffer_wbinvl1
1990 ; GFX6-NEXT: s_waitcnt expcnt(0)
1991 ; GFX6-NEXT: s_setpc_b64 s[30:31]
1992 %gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
1993 %result = atomicrmw fmin ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
1994 ret double %result
1995 }
1997 define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory(ptr addrspace(7) inreg %ptr, double %val) #0 {
1998 ; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
1999 ; GFX12: ; %bb.0:
2000 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
2001 ; GFX12-NEXT: s_wait_expcnt 0x0
2002 ; GFX12-NEXT: s_wait_samplecnt 0x0
2003 ; GFX12-NEXT: s_wait_bvhcnt 0x0
2004 ; GFX12-NEXT: s_wait_kmcnt 0x0
2005 ; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
2006 ; GFX12-NEXT: v_mov_b32_e32 v0, s16
2007 ; GFX12-NEXT: s_add_co_i32 s4, s16, 0x800
2008 ; GFX12-NEXT: s_wait_alu 0xfffe
2009 ; GFX12-NEXT: v_mov_b32_e32 v6, s4
2010 ; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
2011 ; GFX12-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen offset:2048
2012 ; GFX12-NEXT: s_mov_b32 s4, 0
2013 ; GFX12-NEXT: .LBB8_1: ; %atomicrmw.start
2014 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
2015 ; GFX12-NEXT: s_wait_loadcnt 0x0
2016 ; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
2017 ; GFX12-NEXT: s_wait_storecnt 0x0
2018 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2019 ; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[9:10], v[9:10]
2020 ; GFX12-NEXT: v_min_num_f64_e32 v[7:8], v[0:1], v[4:5]
2021 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
2022 ; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
2023 ; GFX12-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
2024 ; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], null offen th:TH_ATOMIC_RETURN
2025 ; GFX12-NEXT: s_wait_loadcnt 0x0
2026 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
2027 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
2028 ; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
2029 ; GFX12-NEXT: s_wait_alu 0xfffe
2030 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
2031 ; GFX12-NEXT: s_cbranch_execnz .LBB8_1
2032 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
2033 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
2034 ; GFX12-NEXT: s_setpc_b64 s[30:31]
2035 ;
2036 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
2037 ; GFX940: ; %bb.0:
2038 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2039 ; GFX940-NEXT: v_mov_b32_e32 v2, s16
2040 ; GFX940-NEXT: buffer_wbl2 sc1
2041 ; GFX940-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen offset:2048 sc0
2042 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2043 ; GFX940-NEXT: buffer_inv sc1
2044 ; GFX940-NEXT: s_setpc_b64 s[30:31]
2045 ;
2046 ; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
2047 ; GFX11: ; %bb.0:
2048 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2049 ; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
2050 ; GFX11-NEXT: v_mov_b32_e32 v0, s16
2051 ; GFX11-NEXT: s_add_i32 s4, s16, 0x800
2052 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
2053 ; GFX11-NEXT: v_mov_b32_e32 v6, s4
2054 ; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
2055 ; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:2048
2056 ; GFX11-NEXT: s_mov_b32 s4, 0
2057 ; GFX11-NEXT: .LBB8_1: ; %atomicrmw.start
2058 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
2059 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2060 ; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
2061 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2062 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2063 ; GFX11-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
2064 ; GFX11-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
2065 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
2066 ; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
2067 ; GFX11-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
2068 ; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], 0 offen glc
2069 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2070 ; GFX11-NEXT: buffer_gl1_inv
2071 ; GFX11-NEXT: buffer_gl0_inv
2072 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
2073 ; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
2074 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
2075 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
2076 ; GFX11-NEXT: s_cbranch_execnz .LBB8_1
2077 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
2078 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
2079 ; GFX11-NEXT: s_setpc_b64 s[30:31]
2080 ;
2081 ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
2082 ; GFX10: ; %bb.0:
2083 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2084 ; GFX10-NEXT: v_mov_b32_e32 v2, v0
2085 ; GFX10-NEXT: v_mov_b32_e32 v0, s20
2086 ; GFX10-NEXT: v_mov_b32_e32 v3, v1
2087 ; GFX10-NEXT: s_add_i32 s4, s20, 0x800
2088 ; GFX10-NEXT: v_mov_b32_e32 v6, s4
2089 ; GFX10-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
2090 ; GFX10-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
2091 ; GFX10-NEXT: s_mov_b32 s4, 0
2092 ; GFX10-NEXT: .LBB8_1: ; %atomicrmw.start
2093 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
2094 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2095 ; GFX10-NEXT: v_mov_b32_e32 v10, v1
2096 ; GFX10-NEXT: v_mov_b32_e32 v9, v0
2097 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2098 ; GFX10-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
2099 ; GFX10-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
2100 ; GFX10-NEXT: v_mov_b32_e32 v0, v7
2101 ; GFX10-NEXT: v_mov_b32_e32 v1, v8
2102 ; GFX10-NEXT: v_mov_b32_e32 v2, v9
2103 ; GFX10-NEXT: v_mov_b32_e32 v3, v10
2104 ; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen glc
2105 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2106 ; GFX10-NEXT: buffer_gl1_inv
2107 ; GFX10-NEXT: buffer_gl0_inv
2108 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
2109 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
2110 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
2111 ; GFX10-NEXT: s_cbranch_execnz .LBB8_1
2112 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
2113 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
2114 ; GFX10-NEXT: s_setpc_b64 s[30:31]
2115 ;
2116 ; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
2117 ; GFX90A: ; %bb.0:
2118 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2119 ; GFX90A-NEXT: v_mov_b32_e32 v2, v0
2120 ; GFX90A-NEXT: v_mov_b32_e32 v0, s20
2121 ; GFX90A-NEXT: v_mov_b32_e32 v3, v1
2122 ; GFX90A-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
2123 ; GFX90A-NEXT: s_add_i32 s6, s20, 0x800
2124 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
2125 ; GFX90A-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
2126 ; GFX90A-NEXT: v_mov_b32_e32 v6, s6
2127 ; GFX90A-NEXT: .LBB8_1: ; %atomicrmw.start
2128 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
2129 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2130 ; GFX90A-NEXT: v_pk_mov_b32 v[10:11], v[0:1], v[0:1] op_sel:[0,1]
2131 ; GFX90A-NEXT: v_max_f64 v[0:1], v[10:11], v[10:11]
2132 ; GFX90A-NEXT: v_min_f64 v[8:9], v[0:1], v[4:5]
2133 ; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[8:9], v[8:9] op_sel:[0,1]
2134 ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[10:11], v[10:11] op_sel:[0,1]
2135 ; GFX90A-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen glc
2136 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2137 ; GFX90A-NEXT: buffer_wbinvl1
2138 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
2139 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2140 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
2141 ; GFX90A-NEXT: s_cbranch_execnz .LBB8_1
2142 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
2143 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
2144 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
2145 ;
2146 ; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
2147 ; GFX908: ; %bb.0:
2148 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2149 ; GFX908-NEXT: v_mov_b32_e32 v2, v0
2150 ; GFX908-NEXT: v_mov_b32_e32 v0, s20
2151 ; GFX908-NEXT: v_mov_b32_e32 v3, v1
2152 ; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
2153 ; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
2154 ; GFX908-NEXT: s_add_i32 s6, s20, 0x800
2155 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
2156 ; GFX908-NEXT: v_mov_b32_e32 v6, s6
2157 ; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start
2158 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
2159 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2160 ; GFX908-NEXT: v_mov_b32_e32 v10, v1
2161 ; GFX908-NEXT: v_mov_b32_e32 v9, v0
2162 ; GFX908-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
2163 ; GFX908-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
2164 ; GFX908-NEXT: v_mov_b32_e32 v0, v7
2165 ; GFX908-NEXT: v_mov_b32_e32 v1, v8
2166 ; GFX908-NEXT: v_mov_b32_e32 v2, v9
2167 ; GFX908-NEXT: v_mov_b32_e32 v3, v10
2168 ; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen glc
2169 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2170 ; GFX908-NEXT: buffer_wbinvl1
2171 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
2172 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2173 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
2174 ; GFX908-NEXT: s_cbranch_execnz .LBB8_1
2175 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
2176 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
2177 ; GFX908-NEXT: s_setpc_b64 s[30:31]
2178 ;
2179 ; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
2180 ; GFX8: ; %bb.0:
2181 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2182 ; GFX8-NEXT: v_mov_b32_e32 v2, v0
2183 ; GFX8-NEXT: v_mov_b32_e32 v0, s20
2184 ; GFX8-NEXT: v_mov_b32_e32 v3, v1
2185 ; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
2186 ; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
2187 ; GFX8-NEXT: s_add_i32 s6, s20, 0x800
2188 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2189 ; GFX8-NEXT: v_mov_b32_e32 v6, s6
2190 ; GFX8-NEXT: .LBB8_1: ; %atomicrmw.start
2191 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2192 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2193 ; GFX8-NEXT: v_mov_b32_e32 v10, v1
2194 ; GFX8-NEXT: v_mov_b32_e32 v9, v0
2195 ; GFX8-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
2196 ; GFX8-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
2197 ; GFX8-NEXT: v_mov_b32_e32 v0, v7
2198 ; GFX8-NEXT: v_mov_b32_e32 v1, v8
2199 ; GFX8-NEXT: v_mov_b32_e32 v2, v9
2200 ; GFX8-NEXT: v_mov_b32_e32 v3, v10
2201 ; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen glc
2202 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2203 ; GFX8-NEXT: buffer_wbinvl1
2204 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
2205 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2206 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2207 ; GFX8-NEXT: s_cbranch_execnz .LBB8_1
2208 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2209 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2210 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2211 ;
2212 ; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
2213 ; GFX7: ; %bb.0:
2214 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2215 ; GFX7-NEXT: v_mov_b32_e32 v2, v0
2216 ; GFX7-NEXT: v_mov_b32_e32 v0, s20
2217 ; GFX7-NEXT: v_mov_b32_e32 v3, v1
2218 ; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
2219 ; GFX7-NEXT: s_add_i32 s6, s20, 0x800
2220 ; GFX7-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
2221 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
2222 ; GFX7-NEXT: v_mov_b32_e32 v6, s6
2223 ; GFX7-NEXT: .LBB8_1: ; %atomicrmw.start
2224 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
2225 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2226 ; GFX7-NEXT: v_mov_b32_e32 v10, v1
2227 ; GFX7-NEXT: v_mov_b32_e32 v9, v0
2228 ; GFX7-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
2229 ; GFX7-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
2230 ; GFX7-NEXT: v_mov_b32_e32 v0, v7
2231 ; GFX7-NEXT: v_mov_b32_e32 v1, v8
2232 ; GFX7-NEXT: v_mov_b32_e32 v2, v9
2233 ; GFX7-NEXT: v_mov_b32_e32 v3, v10
2234 ; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen glc
2235 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2236 ; GFX7-NEXT: buffer_wbinvl1
2237 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
2238 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2239 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
2240 ; GFX7-NEXT: s_cbranch_execnz .LBB8_1
2241 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
2242 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
2243 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2244 ;
2245 ; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
2246 ; GFX6: ; %bb.0:
2247 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2248 ; GFX6-NEXT: v_mov_b32_e32 v2, v0
2249 ; GFX6-NEXT: v_mov_b32_e32 v0, s20
2250 ; GFX6-NEXT: v_mov_b32_e32 v3, v1
2251 ; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
2252 ; GFX6-NEXT: s_add_i32 s6, s20, 0x800
2253 ; GFX6-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
2254 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
2255 ; GFX6-NEXT: v_mov_b32_e32 v6, s6
2256 ; GFX6-NEXT: .LBB8_1: ; %atomicrmw.start
2257 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
2258 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2259 ; GFX6-NEXT: v_mov_b32_e32 v10, v1
2260 ; GFX6-NEXT: v_mov_b32_e32 v9, v0
2261 ; GFX6-NEXT: s_waitcnt expcnt(0)
2262 ; GFX6-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
2263 ; GFX6-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
2264 ; GFX6-NEXT: v_mov_b32_e32 v0, v7
2265 ; GFX6-NEXT: v_mov_b32_e32 v1, v8
2266 ; GFX6-NEXT: v_mov_b32_e32 v2, v9
2267 ; GFX6-NEXT: v_mov_b32_e32 v3, v10
2268 ; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen glc
2269 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2270 ; GFX6-NEXT: buffer_wbinvl1
2271 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
2272 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2273 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
2274 ; GFX6-NEXT: s_cbranch_execnz .LBB8_1
2275 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
2276 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
2277 ; GFX6-NEXT: s_waitcnt expcnt(0)
2278 ; GFX6-NEXT: s_setpc_b64 s[30:31]
2279 %gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
2280 %result = atomicrmw fmin ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
2281 ret double %result
2282 }
2284 define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(7) inreg %ptr, double %val) #0 {
2285 ; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
2286 ; GFX12: ; %bb.0:
2287 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
2288 ; GFX12-NEXT: s_wait_expcnt 0x0
2289 ; GFX12-NEXT: s_wait_samplecnt 0x0
2290 ; GFX12-NEXT: s_wait_bvhcnt 0x0
2291 ; GFX12-NEXT: s_wait_kmcnt 0x0
2292 ; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
2293 ; GFX12-NEXT: v_mov_b32_e32 v0, s16
2294 ; GFX12-NEXT: s_add_co_i32 s4, s16, 0x800
2295 ; GFX12-NEXT: s_wait_alu 0xfffe
2296 ; GFX12-NEXT: v_mov_b32_e32 v6, s4
2297 ; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
2298 ; GFX12-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen offset:2048
2299 ; GFX12-NEXT: s_mov_b32 s4, 0
2300 ; GFX12-NEXT: .LBB9_1: ; %atomicrmw.start
2301 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
2302 ; GFX12-NEXT: s_wait_loadcnt 0x0
2303 ; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
2304 ; GFX12-NEXT: s_wait_storecnt 0x0
2305 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2306 ; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[9:10], v[9:10]
2307 ; GFX12-NEXT: v_min_num_f64_e32 v[7:8], v[0:1], v[4:5]
2308 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
2309 ; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
2310 ; GFX12-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
2311 ; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], null offen th:TH_ATOMIC_RETURN
2312 ; GFX12-NEXT: s_wait_loadcnt 0x0
2313 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
2314 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
2315 ; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
2316 ; GFX12-NEXT: s_wait_alu 0xfffe
2317 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
2318 ; GFX12-NEXT: s_cbranch_execnz .LBB9_1
2319 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
2320 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
2321 ; GFX12-NEXT: s_setpc_b64 s[30:31]
2322 ;
2323 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
2324 ; GFX940: ; %bb.0:
2325 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2326 ; GFX940-NEXT: v_mov_b32_e32 v2, s16
2327 ; GFX940-NEXT: buffer_wbl2 sc1
2328 ; GFX940-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen offset:2048 sc0
2329 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2330 ; GFX940-NEXT: buffer_inv sc1
2331 ; GFX940-NEXT: s_setpc_b64 s[30:31]
2332 ;
2333 ; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
2334 ; GFX11: ; %bb.0:
2335 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2336 ; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
2337 ; GFX11-NEXT: v_mov_b32_e32 v0, s16
2338 ; GFX11-NEXT: s_add_i32 s4, s16, 0x800
2339 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
2340 ; GFX11-NEXT: v_mov_b32_e32 v6, s4
2341 ; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
2342 ; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:2048
2343 ; GFX11-NEXT: s_mov_b32 s4, 0
2344 ; GFX11-NEXT: .LBB9_1: ; %atomicrmw.start
2345 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
2346 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2347 ; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
2348 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2349 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2350 ; GFX11-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
2351 ; GFX11-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
2352 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
2353 ; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
2354 ; GFX11-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
2355 ; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], 0 offen glc
2356 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2357 ; GFX11-NEXT: buffer_gl1_inv
2358 ; GFX11-NEXT: buffer_gl0_inv
2359 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
2360 ; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
2361 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
2362 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
2363 ; GFX11-NEXT: s_cbranch_execnz .LBB9_1
2364 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
2365 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
2366 ; GFX11-NEXT: s_setpc_b64 s[30:31]
2367 ;
2368 ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
2369 ; GFX10: ; %bb.0:
2370 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2371 ; GFX10-NEXT: v_mov_b32_e32 v2, s20
2372 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2373 ; GFX10-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[16:19], 0 offen offset:2048 glc
2374 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2375 ; GFX10-NEXT: buffer_gl1_inv
2376 ; GFX10-NEXT: buffer_gl0_inv
2377 ; GFX10-NEXT: s_setpc_b64 s[30:31]
2378 ;
2379 ; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
2380 ; GFX90A: ; %bb.0:
2381 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2382 ; GFX90A-NEXT: v_mov_b32_e32 v2, s20
2383 ; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[16:19], 0 offen offset:2048 glc
2384 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2385 ; GFX90A-NEXT: buffer_wbinvl1
2386 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
2387 ;
2388 ; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
2389 ; GFX908: ; %bb.0:
2390 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2391 ; GFX908-NEXT: v_mov_b32_e32 v2, v0
2392 ; GFX908-NEXT: v_mov_b32_e32 v0, s20
2393 ; GFX908-NEXT: v_mov_b32_e32 v3, v1
2394 ; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
2395 ; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
2396 ; GFX908-NEXT: s_add_i32 s6, s20, 0x800
2397 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
2398 ; GFX908-NEXT: v_mov_b32_e32 v6, s6
2399 ; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
2400 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
2401 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2402 ; GFX908-NEXT: v_mov_b32_e32 v10, v1
2403 ; GFX908-NEXT: v_mov_b32_e32 v9, v0
2404 ; GFX908-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
2405 ; GFX908-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
2406 ; GFX908-NEXT: v_mov_b32_e32 v0, v7
2407 ; GFX908-NEXT: v_mov_b32_e32 v1, v8
2408 ; GFX908-NEXT: v_mov_b32_e32 v2, v9
2409 ; GFX908-NEXT: v_mov_b32_e32 v3, v10
2410 ; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen glc
2411 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2412 ; GFX908-NEXT: buffer_wbinvl1
2413 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
2414 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2415 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
2416 ; GFX908-NEXT: s_cbranch_execnz .LBB9_1
2417 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
2418 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
2419 ; GFX908-NEXT: s_setpc_b64 s[30:31]
2420 ;
2421 ; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
2422 ; GFX8: ; %bb.0:
2423 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2424 ; GFX8-NEXT: v_mov_b32_e32 v2, v0
2425 ; GFX8-NEXT: v_mov_b32_e32 v0, s20
2426 ; GFX8-NEXT: v_mov_b32_e32 v3, v1
2427 ; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
2428 ; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
2429 ; GFX8-NEXT: s_add_i32 s6, s20, 0x800
2430 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2431 ; GFX8-NEXT: v_mov_b32_e32 v6, s6
2432 ; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start
2433 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2434 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2435 ; GFX8-NEXT: v_mov_b32_e32 v10, v1
2436 ; GFX8-NEXT: v_mov_b32_e32 v9, v0
2437 ; GFX8-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
2438 ; GFX8-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
2439 ; GFX8-NEXT: v_mov_b32_e32 v0, v7
2440 ; GFX8-NEXT: v_mov_b32_e32 v1, v8
2441 ; GFX8-NEXT: v_mov_b32_e32 v2, v9
2442 ; GFX8-NEXT: v_mov_b32_e32 v3, v10
2443 ; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen glc
2444 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2445 ; GFX8-NEXT: buffer_wbinvl1
2446 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
2447 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2448 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2449 ; GFX8-NEXT: s_cbranch_execnz .LBB9_1
2450 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2451 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2452 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2453 ;
2454 ; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
2455 ; GFX7: ; %bb.0:
2456 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2457 ; GFX7-NEXT: v_mov_b32_e32 v2, s20
2458 ; GFX7-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[16:19], 0 offen offset:2048 glc
2459 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2460 ; GFX7-NEXT: buffer_wbinvl1
2461 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2462 ;
2463 ; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
2464 ; GFX6: ; %bb.0:
2465 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2466 ; GFX6-NEXT: v_mov_b32_e32 v2, s20
2467 ; GFX6-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[16:19], 0 offen offset:2048 glc
2468 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2469 ; GFX6-NEXT: buffer_wbinvl1
2470 ; GFX6-NEXT: s_waitcnt expcnt(0)
2471 ; GFX6-NEXT: s_setpc_b64 s[30:31]
2472 %gep = getelementptr double, ptr addrspace(7) %ptr, i32 256
2473 %result = atomicrmw fmin ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
2474 ret double %result
2475 }
2477 ; --------------------------------------------------------------------
2478 ; half
2479 ; --------------------------------------------------------------------
2481 define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, half %val) #0 {
2482 ; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
2483 ; GFX12: ; %bb.0:
2484 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
2485 ; GFX12-NEXT: s_wait_expcnt 0x0
2486 ; GFX12-NEXT: s_wait_samplecnt 0x0
2487 ; GFX12-NEXT: s_wait_bvhcnt 0x0
2488 ; GFX12-NEXT: s_wait_kmcnt 0x0
2489 ; GFX12-NEXT: s_addk_co_i32 s16, 0x200
2490 ; GFX12-NEXT: v_max_num_f16_e32 v5, v0, v0
2491 ; GFX12-NEXT: s_wait_alu 0xfffe
2492 ; GFX12-NEXT: s_and_b32 s4, s16, -4
2493 ; GFX12-NEXT: s_wait_alu 0xfffe
2494 ; GFX12-NEXT: v_mov_b32_e32 v4, s4
2495 ; GFX12-NEXT: s_and_b32 s4, s16, 3
2496 ; GFX12-NEXT: s_wait_alu 0xfffe
2497 ; GFX12-NEXT: s_lshl_b32 s4, s4, 3
2498 ; GFX12-NEXT: s_wait_alu 0xfffe
2499 ; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
2500 ; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
2501 ; GFX12-NEXT: s_not_b32 s6, s5
2502 ; GFX12-NEXT: s_mov_b32 s5, 0
2503 ; GFX12-NEXT: .LBB10_1: ; %atomicrmw.start
2504 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
2505 ; GFX12-NEXT: s_wait_loadcnt 0x0
2506 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
2507 ; GFX12-NEXT: s_wait_storecnt 0x0
2508 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2509 ; GFX12-NEXT: v_max_num_f16_e32 v0, v0, v0
2510 ; GFX12-NEXT: v_min_num_f16_e32 v0, v0, v5
2511 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2512 ; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
2513 ; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
2514 ; GFX12-NEXT: s_wait_alu 0xfffe
2515 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2516 ; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
2517 ; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
2518 ; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
2519 ; GFX12-NEXT: s_wait_loadcnt 0x0
2520 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
2521 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
2522 ; GFX12-NEXT: v_mov_b32_e32 v1, v2
2523 ; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
2524 ; GFX12-NEXT: s_wait_alu 0xfffe
2525 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
2526 ; GFX12-NEXT: s_cbranch_execnz .LBB10_1
2527 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
2528 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
2529 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2
2530 ; GFX12-NEXT: s_setpc_b64 s[30:31]
2531 ;
2532 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
2533 ; GFX940: ; %bb.0:
2534 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2535 ; GFX940-NEXT: s_addk_i32 s16, 0x200
2536 ; GFX940-NEXT: s_and_b32 s4, s16, -4
2537 ; GFX940-NEXT: v_mov_b32_e32 v4, s4
2538 ; GFX940-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen
2539 ; GFX940-NEXT: s_and_b32 s4, s16, 3
2540 ; GFX940-NEXT: s_lshl_b32 s6, s4, 3
2541 ; GFX940-NEXT: s_lshl_b32 s4, 0xffff, s6
2542 ; GFX940-NEXT: s_not_b32 s7, s4
2543 ; GFX940-NEXT: s_mov_b64 s[4:5], 0
2544 ; GFX940-NEXT: v_max_f16_e32 v5, v0, v0
2545 ; GFX940-NEXT: .LBB10_1: ; %atomicrmw.start
2546 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
2547 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2548 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, s6, v1
2549 ; GFX940-NEXT: v_max_f16_e32 v0, v0, v0
2550 ; GFX940-NEXT: v_min_f16_e32 v0, v0, v5
2551 ; GFX940-NEXT: v_lshlrev_b32_e32 v0, s6, v0
2552 ; GFX940-NEXT: v_and_or_b32 v0, v1, s7, v0
2553 ; GFX940-NEXT: v_mov_b64_e32 v[2:3], v[0:1]
2554 ; GFX940-NEXT: buffer_wbl2 sc1
2555 ; GFX940-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[0:3], 0 offen sc0
2556 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2557 ; GFX940-NEXT: buffer_inv sc1
2558 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
2559 ; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2560 ; GFX940-NEXT: v_mov_b32_e32 v1, v2
2561 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
2562 ; GFX940-NEXT: s_cbranch_execnz .LBB10_1
2563 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
2564 ; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
2565 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, s6, v2
2566 ; GFX940-NEXT: s_setpc_b64 s[30:31]
2567 ;
2568 ; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
2569 ; GFX11: ; %bb.0:
2570 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2571 ; GFX11-NEXT: s_addk_i32 s16, 0x200
2572 ; GFX11-NEXT: v_max_f16_e32 v5, v0, v0
2573 ; GFX11-NEXT: s_and_b32 s4, s16, -4
2574 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
2575 ; GFX11-NEXT: v_mov_b32_e32 v4, s4
2576 ; GFX11-NEXT: s_and_b32 s4, s16, 3
2577 ; GFX11-NEXT: s_lshl_b32 s4, s4, 3
2578 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
2579 ; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
2580 ; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
2581 ; GFX11-NEXT: s_not_b32 s6, s5
2582 ; GFX11-NEXT: s_mov_b32 s5, 0
2583 ; GFX11-NEXT: .p2align 6
2584 ; GFX11-NEXT: .LBB10_1: ; %atomicrmw.start
2585 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
2586 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2587 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
2588 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2589 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2590 ; GFX11-NEXT: v_max_f16_e32 v0, v0, v0
2591 ; GFX11-NEXT: v_min_f16_e32 v0, v0, v5
2592 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2593 ; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
2594 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
2595 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2596 ; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
2597 ; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
2598 ; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
2599 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2600 ; GFX11-NEXT: buffer_gl1_inv
2601 ; GFX11-NEXT: buffer_gl0_inv
2602 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
2603 ; GFX11-NEXT: v_mov_b32_e32 v1, v2
2604 ; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
2605 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
2606 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
2607 ; GFX11-NEXT: s_cbranch_execnz .LBB10_1
2608 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
2609 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
2610 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2
2611 ; GFX11-NEXT: s_setpc_b64 s[30:31]
2612 ;
2613 ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
2614 ; GFX10: ; %bb.0:
2615 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2616 ; GFX10-NEXT: s_addk_i32 s20, 0x200
2617 ; GFX10-NEXT: v_max_f16_e32 v5, v0, v0
2618 ; GFX10-NEXT: s_and_b32 s4, s20, -4
2619 ; GFX10-NEXT: v_mov_b32_e32 v4, s4
2620 ; GFX10-NEXT: s_and_b32 s4, s20, 3
2621 ; GFX10-NEXT: s_lshl_b32 s4, s4, 3
2622 ; GFX10-NEXT: s_lshl_b32 s5, 0xffff, s4
2623 ; GFX10-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
2624 ; GFX10-NEXT: s_not_b32 s6, s5
2625 ; GFX10-NEXT: s_mov_b32 s5, 0
2626 ; GFX10-NEXT: .LBB10_1: ; %atomicrmw.start
2627 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
2628 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2629 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, s4, v1
2630 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2631 ; GFX10-NEXT: v_max_f16_e32 v0, v0, v0
2632 ; GFX10-NEXT: v_min_f16_e32 v0, v0, v5
2633 ; GFX10-NEXT: v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
2634 ; GFX10-NEXT: v_and_or_b32 v0, v1, s6, v0
2635 ; GFX10-NEXT: v_mov_b32_e32 v3, v1
2636 ; GFX10-NEXT: v_mov_b32_e32 v2, v0
2637 ; GFX10-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
2638 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2639 ; GFX10-NEXT: buffer_gl1_inv
2640 ; GFX10-NEXT: buffer_gl0_inv
2641 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
2642 ; GFX10-NEXT: v_mov_b32_e32 v1, v2
2643 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
2644 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
2645 ; GFX10-NEXT: s_cbranch_execnz .LBB10_1
2646 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
2647 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
2648 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, s4, v2
2649 ; GFX10-NEXT: s_setpc_b64 s[30:31]
2650 ;
2651 ; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
2652 ; GFX90A: ; %bb.0:
2653 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2654 ; GFX90A-NEXT: s_addk_i32 s20, 0x200
2655 ; GFX90A-NEXT: s_and_b32 s4, s20, -4
2656 ; GFX90A-NEXT: v_mov_b32_e32 v4, s4
2657 ; GFX90A-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
2658 ; GFX90A-NEXT: s_and_b32 s4, s20, 3
2659 ; GFX90A-NEXT: s_lshl_b32 s6, s4, 3
2660 ; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s6
2661 ; GFX90A-NEXT: s_not_b32 s7, s4
2662 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
2663 ; GFX90A-NEXT: v_max_f16_e32 v5, v0, v0
2664 ; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start
2665 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
2666 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2667 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s6, v1
2668 ; GFX90A-NEXT: v_max_f16_e32 v0, v0, v0
2669 ; GFX90A-NEXT: v_min_f16_e32 v0, v0, v5
2670 ; GFX90A-NEXT: v_lshlrev_b32_e32 v0, s6, v0
2671 ; GFX90A-NEXT: v_and_or_b32 v0, v1, s7, v0
2672 ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
2673 ; GFX90A-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
2674 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
2675 ; GFX90A-NEXT: buffer_wbinvl1
2676 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
2677 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2678 ; GFX90A-NEXT: v_mov_b32_e32 v1, v2
2679 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
2680 ; GFX90A-NEXT: s_cbranch_execnz .LBB10_1
2681 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
2682 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
2683 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s6, v2
2684 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
2685 ;
2686 ; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
2687 ; GFX908: ; %bb.0:
2688 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2689 ; GFX908-NEXT: s_addk_i32 s20, 0x200
2690 ; GFX908-NEXT: s_and_b32 s4, s20, -4
2691 ; GFX908-NEXT: v_mov_b32_e32 v4, s4
2692 ; GFX908-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
2693 ; GFX908-NEXT: s_and_b32 s4, s20, 3
2694 ; GFX908-NEXT: s_lshl_b32 s6, s4, 3
2695 ; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s6
2696 ; GFX908-NEXT: s_not_b32 s7, s4
2697 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
2698 ; GFX908-NEXT: v_max_f16_e32 v5, v0, v0
2699 ; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
2700 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
2701 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2702 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, s6, v1
2703 ; GFX908-NEXT: v_max_f16_e32 v0, v0, v0
2704 ; GFX908-NEXT: v_min_f16_e32 v0, v0, v5
2705 ; GFX908-NEXT: v_lshlrev_b32_e32 v0, s6, v0
2706 ; GFX908-NEXT: v_and_or_b32 v0, v1, s7, v0
2707 ; GFX908-NEXT: v_mov_b32_e32 v3, v1
2708 ; GFX908-NEXT: v_mov_b32_e32 v2, v0
2709 ; GFX908-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
2710 ; GFX908-NEXT: s_waitcnt vmcnt(0)
2711 ; GFX908-NEXT: buffer_wbinvl1
2712 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
2713 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2714 ; GFX908-NEXT: v_mov_b32_e32 v1, v2
2715 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
2716 ; GFX908-NEXT: s_cbranch_execnz .LBB10_1
2717 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
2718 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
2719 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, s6, v2
2720 ; GFX908-NEXT: s_setpc_b64 s[30:31]
2721 ;
2722 ; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
2723 ; GFX8: ; %bb.0:
2724 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2725 ; GFX8-NEXT: s_addk_i32 s20, 0x200
2726 ; GFX8-NEXT: s_and_b32 s4, s20, -4
2727 ; GFX8-NEXT: v_mov_b32_e32 v4, s4
2728 ; GFX8-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
2729 ; GFX8-NEXT: s_and_b32 s4, s20, 3
2730 ; GFX8-NEXT: s_lshl_b32 s6, s4, 3
2731 ; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s6
2732 ; GFX8-NEXT: s_not_b32 s7, s4
2733 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2734 ; GFX8-NEXT: v_max_f16_e32 v5, v0, v0
2735 ; GFX8-NEXT: .LBB10_1: ; %atomicrmw.start
2736 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2737 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2738 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, s6, v1
2739 ; GFX8-NEXT: v_max_f16_e32 v0, v0, v0
2740 ; GFX8-NEXT: v_min_f16_e32 v0, v0, v5
2741 ; GFX8-NEXT: v_and_b32_e32 v2, s7, v1
2742 ; GFX8-NEXT: v_lshlrev_b32_e32 v0, s6, v0
2743 ; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
2744 ; GFX8-NEXT: v_mov_b32_e32 v3, v1
2745 ; GFX8-NEXT: v_mov_b32_e32 v2, v0
2746 ; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
2747 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2748 ; GFX8-NEXT: buffer_wbinvl1
2749 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
2750 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2751 ; GFX8-NEXT: v_mov_b32_e32 v1, v2
2752 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2753 ; GFX8-NEXT: s_cbranch_execnz .LBB10_1
2754 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2755 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2756 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, s6, v2
2757 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2758 ;
2759 ; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
2760 ; GFX7: ; %bb.0:
2761 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2762 ; GFX7-NEXT: s_addk_i32 s20, 0x200
2763 ; GFX7-NEXT: s_and_b32 s4, s20, -4
2764 ; GFX7-NEXT: v_mov_b32_e32 v4, s4
2765 ; GFX7-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
2766 ; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
2767 ; GFX7-NEXT: s_and_b32 s4, s20, 3
2768 ; GFX7-NEXT: s_lshl_b32 s6, s4, 3
2769 ; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s6
2770 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v0
2771 ; GFX7-NEXT: s_not_b32 s7, s4
2772 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
2773 ; GFX7-NEXT: .LBB10_1: ; %atomicrmw.start
2774 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
2775 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2776 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v1
2777 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
2778 ; GFX7-NEXT: v_and_b32_e32 v2, s7, v1
2779 ; GFX7-NEXT: v_min_f32_e32 v0, v0, v5
2780 ; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
2781 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, s6, v0
2782 ; GFX7-NEXT: v_or_b32_e32 v0, v2, v0
2783 ; GFX7-NEXT: v_mov_b32_e32 v3, v1
2784 ; GFX7-NEXT: v_mov_b32_e32 v2, v0
2785 ; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
2786 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2787 ; GFX7-NEXT: buffer_wbinvl1
2788 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
2789 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2790 ; GFX7-NEXT: v_mov_b32_e32 v1, v2
2791 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
2792 ; GFX7-NEXT: s_cbranch_execnz .LBB10_1
2793 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
2794 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
2795 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v2
2796 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
2797 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2798 ;
2799 ; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
2800 ; GFX6: ; %bb.0:
2801 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2802 ; GFX6-NEXT: s_addk_i32 s20, 0x200
2803 ; GFX6-NEXT: s_and_b32 s4, s20, -4
2804 ; GFX6-NEXT: v_mov_b32_e32 v4, s4
2805 ; GFX6-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
2806 ; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
2807 ; GFX6-NEXT: s_and_b32 s4, s20, 3
2808 ; GFX6-NEXT: s_lshl_b32 s6, s4, 3
2809 ; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s6
2810 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v0
2811 ; GFX6-NEXT: s_not_b32 s7, s4
2812 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
2813 ; GFX6-NEXT: .LBB10_1: ; %atomicrmw.start
2814 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
2815 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2816 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v1
2817 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
2818 ; GFX6-NEXT: s_waitcnt expcnt(0)
2819 ; GFX6-NEXT: v_and_b32_e32 v2, s7, v1
2820 ; GFX6-NEXT: v_min_f32_e32 v0, v0, v5
2821 ; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
2822 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, s6, v0
2823 ; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
2824 ; GFX6-NEXT: v_mov_b32_e32 v3, v1
2825 ; GFX6-NEXT: v_mov_b32_e32 v2, v0
2826 ; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
2827 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2828 ; GFX6-NEXT: buffer_wbinvl1
2829 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
2830 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2831 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
2832 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
2833 ; GFX6-NEXT: s_cbranch_execnz .LBB10_1
2834 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
2835 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
2836 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v2
2837 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
2838 ; GFX6-NEXT: s_waitcnt expcnt(0)
2839 ; GFX6-NEXT: s_setpc_b64 s[30:31]
2840 %gep = getelementptr half, ptr addrspace(7) %ptr, i32 256
2841 %result = atomicrmw fmin ptr addrspace(7) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
2842 ret half %result
2843 }
2845 define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, half %val) #0 {
2846 ; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
2847 ; GFX12: ; %bb.0:
2848 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
2849 ; GFX12-NEXT: s_wait_expcnt 0x0
2850 ; GFX12-NEXT: s_wait_samplecnt 0x0
2851 ; GFX12-NEXT: s_wait_bvhcnt 0x0
2852 ; GFX12-NEXT: s_wait_kmcnt 0x0
2853 ; GFX12-NEXT: s_addk_co_i32 s16, 0x200
2854 ; GFX12-NEXT: v_max_num_f16_e32 v3, v0, v0
2855 ; GFX12-NEXT: s_wait_alu 0xfffe
2856 ; GFX12-NEXT: s_and_b32 s4, s16, -4
2857 ; GFX12-NEXT: s_wait_alu 0xfffe
2858 ; GFX12-NEXT: v_mov_b32_e32 v2, s4
2859 ; GFX12-NEXT: s_and_b32 s4, s16, 3
2860 ; GFX12-NEXT: s_wait_alu 0xfffe
2861 ; GFX12-NEXT: s_lshl_b32 s4, s4, 3
2862 ; GFX12-NEXT: s_wait_alu 0xfffe
2863 ; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
2864 ; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
2865 ; GFX12-NEXT: s_not_b32 s6, s5
2866 ; GFX12-NEXT: s_mov_b32 s5, 0
2867 ; GFX12-NEXT: .LBB11_1: ; %atomicrmw.start
2868 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
2869 ; GFX12-NEXT: s_wait_loadcnt 0x0
2870 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
2871 ; GFX12-NEXT: s_wait_storecnt 0x0
2872 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2873 ; GFX12-NEXT: v_max_num_f16_e32 v0, v0, v0
2874 ; GFX12-NEXT: v_min_num_f16_e32 v0, v0, v3
2875 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2876 ; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
2877 ; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
2878 ; GFX12-NEXT: s_wait_alu 0xfffe
2879 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2880 ; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
2881 ; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
2882 ; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
2883 ; GFX12-NEXT: s_wait_loadcnt 0x0
2884 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
2885 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
2886 ; GFX12-NEXT: v_mov_b32_e32 v1, v4
2887 ; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
2888 ; GFX12-NEXT: s_wait_alu 0xfffe
2889 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
2890 ; GFX12-NEXT: s_cbranch_execnz .LBB11_1
2891 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
2892 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
2893 ; GFX12-NEXT: s_setpc_b64 s[30:31]
2894 ;
2895 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
2896 ; GFX940: ; %bb.0:
2897 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2898 ; GFX940-NEXT: s_addk_i32 s16, 0x200
2899 ; GFX940-NEXT: s_and_b32 s4, s16, -4
2900 ; GFX940-NEXT: v_mov_b32_e32 v2, s4
2901 ; GFX940-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen
2902 ; GFX940-NEXT: s_and_b32 s4, s16, 3
2903 ; GFX940-NEXT: s_lshl_b32 s6, s4, 3
2904 ; GFX940-NEXT: s_lshl_b32 s4, 0xffff, s6
2905 ; GFX940-NEXT: s_not_b32 s7, s4
2906 ; GFX940-NEXT: s_mov_b64 s[4:5], 0
2907 ; GFX940-NEXT: v_max_f16_e32 v3, v0, v0
2908 ; GFX940-NEXT: .LBB11_1: ; %atomicrmw.start
2909 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
2910 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2911 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, s6, v1
2912 ; GFX940-NEXT: v_max_f16_e32 v0, v0, v0
2913 ; GFX940-NEXT: v_min_f16_e32 v0, v0, v3
2914 ; GFX940-NEXT: v_lshlrev_b32_e32 v0, s6, v0
2915 ; GFX940-NEXT: v_and_or_b32 v0, v1, s7, v0
2916 ; GFX940-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
2917 ; GFX940-NEXT: buffer_wbl2 sc1
2918 ; GFX940-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[0:3], 0 offen sc0
2919 ; GFX940-NEXT: s_waitcnt vmcnt(0)
2920 ; GFX940-NEXT: buffer_inv sc1
2921 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
2922 ; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2923 ; GFX940-NEXT: v_mov_b32_e32 v1, v4
2924 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
2925 ; GFX940-NEXT: s_cbranch_execnz .LBB11_1
2926 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
2927 ; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
2928 ; GFX940-NEXT: s_setpc_b64 s[30:31]
2929 ;
2930 ; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
2931 ; GFX11: ; %bb.0:
2932 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2933 ; GFX11-NEXT: s_addk_i32 s16, 0x200
2934 ; GFX11-NEXT: v_max_f16_e32 v3, v0, v0
2935 ; GFX11-NEXT: s_and_b32 s4, s16, -4
2936 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
2937 ; GFX11-NEXT: v_mov_b32_e32 v2, s4
2938 ; GFX11-NEXT: s_and_b32 s4, s16, 3
2939 ; GFX11-NEXT: s_lshl_b32 s4, s4, 3
2940 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
2941 ; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
2942 ; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
2943 ; GFX11-NEXT: s_not_b32 s6, s5
2944 ; GFX11-NEXT: s_mov_b32 s5, 0
2945 ; GFX11-NEXT: .p2align 6
2946 ; GFX11-NEXT: .LBB11_1: ; %atomicrmw.start
2947 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
2948 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2949 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
2950 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
2951 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2952 ; GFX11-NEXT: v_max_f16_e32 v0, v0, v0
2953 ; GFX11-NEXT: v_min_f16_e32 v0, v0, v3
2954 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2955 ; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
2956 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
2957 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2958 ; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
2959 ; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
2960 ; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
2961 ; GFX11-NEXT: s_waitcnt vmcnt(0)
2962 ; GFX11-NEXT: buffer_gl1_inv
2963 ; GFX11-NEXT: buffer_gl0_inv
2964 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
2965 ; GFX11-NEXT: v_mov_b32_e32 v1, v4
2966 ; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
2967 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
2968 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
2969 ; GFX11-NEXT: s_cbranch_execnz .LBB11_1
2970 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
2971 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
2972 ; GFX11-NEXT: s_setpc_b64 s[30:31]
2973 ;
2974 ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
2975 ; GFX10: ; %bb.0:
2976 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2977 ; GFX10-NEXT: s_addk_i32 s20, 0x200
2978 ; GFX10-NEXT: v_max_f16_e32 v3, v0, v0
2979 ; GFX10-NEXT: s_and_b32 s4, s20, -4
2980 ; GFX10-NEXT: v_mov_b32_e32 v2, s4
2981 ; GFX10-NEXT: s_and_b32 s4, s20, 3
2982 ; GFX10-NEXT: s_lshl_b32 s4, s4, 3
2983 ; GFX10-NEXT: s_lshl_b32 s5, 0xffff, s4
2984 ; GFX10-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
2985 ; GFX10-NEXT: s_not_b32 s6, s5
2986 ; GFX10-NEXT: s_mov_b32 s5, 0
2987 ; GFX10-NEXT: .LBB11_1: ; %atomicrmw.start
2988 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
2989 ; GFX10-NEXT: s_waitcnt vmcnt(0)
2990 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, s4, v1
2991 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
2992 ; GFX10-NEXT: v_max_f16_e32 v0, v0, v0
2993 ; GFX10-NEXT: v_min_f16_e32 v0, v0, v3
2994 ; GFX10-NEXT: v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
2995 ; GFX10-NEXT: v_and_or_b32 v0, v1, s6, v0
2996 ; GFX10-NEXT: v_mov_b32_e32 v5, v1
2997 ; GFX10-NEXT: v_mov_b32_e32 v4, v0
2998 ; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
2999 ; GFX10-NEXT: s_waitcnt vmcnt(0)
3000 ; GFX10-NEXT: buffer_gl1_inv
3001 ; GFX10-NEXT: buffer_gl0_inv
3002 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
3003 ; GFX10-NEXT: v_mov_b32_e32 v1, v4
3004 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
3005 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
3006 ; GFX10-NEXT: s_cbranch_execnz .LBB11_1
3007 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
3008 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
3009 ; GFX10-NEXT: s_setpc_b64 s[30:31]
3010 ;
3011 ; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
3012 ; GFX90A: ; %bb.0:
3013 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3014 ; GFX90A-NEXT: s_addk_i32 s20, 0x200
3015 ; GFX90A-NEXT: s_and_b32 s4, s20, -4
3016 ; GFX90A-NEXT: v_mov_b32_e32 v2, s4
3017 ; GFX90A-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
3018 ; GFX90A-NEXT: s_and_b32 s4, s20, 3
3019 ; GFX90A-NEXT: s_lshl_b32 s6, s4, 3
3020 ; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s6
3021 ; GFX90A-NEXT: s_not_b32 s7, s4
3022 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
3023 ; GFX90A-NEXT: v_max_f16_e32 v3, v0, v0
3024 ; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start
3025 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
3026 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3027 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s6, v1
3028 ; GFX90A-NEXT: v_max_f16_e32 v0, v0, v0
3029 ; GFX90A-NEXT: v_min_f16_e32 v0, v0, v3
3030 ; GFX90A-NEXT: v_lshlrev_b32_e32 v0, s6, v0
3031 ; GFX90A-NEXT: v_and_or_b32 v0, v1, s7, v0
3032 ; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
3033 ; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
3034 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3035 ; GFX90A-NEXT: buffer_wbinvl1
3036 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
3037 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3038 ; GFX90A-NEXT: v_mov_b32_e32 v1, v4
3039 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
3040 ; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
3041 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
3042 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
3043 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
3044 ;
3045 ; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
3046 ; GFX908: ; %bb.0:
3047 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3048 ; GFX908-NEXT: s_addk_i32 s20, 0x200
3049 ; GFX908-NEXT: s_and_b32 s4, s20, -4
3050 ; GFX908-NEXT: v_mov_b32_e32 v2, s4
3051 ; GFX908-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
3052 ; GFX908-NEXT: s_and_b32 s4, s20, 3
3053 ; GFX908-NEXT: s_lshl_b32 s6, s4, 3
3054 ; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s6
3055 ; GFX908-NEXT: s_not_b32 s7, s4
3056 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
3057 ; GFX908-NEXT: v_max_f16_e32 v3, v0, v0
3058 ; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
3059 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
3060 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3061 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, s6, v1
3062 ; GFX908-NEXT: v_max_f16_e32 v0, v0, v0
3063 ; GFX908-NEXT: v_min_f16_e32 v0, v0, v3
3064 ; GFX908-NEXT: v_lshlrev_b32_e32 v0, s6, v0
3065 ; GFX908-NEXT: v_and_or_b32 v0, v1, s7, v0
3066 ; GFX908-NEXT: v_mov_b32_e32 v5, v1
3067 ; GFX908-NEXT: v_mov_b32_e32 v4, v0
3068 ; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
3069 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3070 ; GFX908-NEXT: buffer_wbinvl1
3071 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
3072 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3073 ; GFX908-NEXT: v_mov_b32_e32 v1, v4
3074 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
3075 ; GFX908-NEXT: s_cbranch_execnz .LBB11_1
3076 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
3077 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
3078 ; GFX908-NEXT: s_setpc_b64 s[30:31]
3079 ;
3080 ; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
3081 ; GFX8: ; %bb.0:
3082 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3083 ; GFX8-NEXT: s_addk_i32 s20, 0x200
3084 ; GFX8-NEXT: s_and_b32 s4, s20, -4
3085 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
3086 ; GFX8-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
3087 ; GFX8-NEXT: s_and_b32 s4, s20, 3
3088 ; GFX8-NEXT: s_lshl_b32 s6, s4, 3
3089 ; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s6
3090 ; GFX8-NEXT: s_not_b32 s7, s4
3091 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
3092 ; GFX8-NEXT: v_max_f16_e32 v3, v0, v0
3093 ; GFX8-NEXT: .LBB11_1: ; %atomicrmw.start
3094 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3095 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3096 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, s6, v1
3097 ; GFX8-NEXT: v_max_f16_e32 v0, v0, v0
3098 ; GFX8-NEXT: v_min_f16_e32 v0, v0, v3
3099 ; GFX8-NEXT: v_and_b32_e32 v4, s7, v1
3100 ; GFX8-NEXT: v_lshlrev_b32_e32 v0, s6, v0
3101 ; GFX8-NEXT: v_or_b32_e32 v0, v4, v0
3102 ; GFX8-NEXT: v_mov_b32_e32 v5, v1
3103 ; GFX8-NEXT: v_mov_b32_e32 v4, v0
3104 ; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
3105 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3106 ; GFX8-NEXT: buffer_wbinvl1
3107 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
3108 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3109 ; GFX8-NEXT: v_mov_b32_e32 v1, v4
3110 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
3111 ; GFX8-NEXT: s_cbranch_execnz .LBB11_1
3112 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
3113 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
3114 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3115 ;
3116 ; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
3117 ; GFX7: ; %bb.0:
3118 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3119 ; GFX7-NEXT: s_addk_i32 s20, 0x200
3120 ; GFX7-NEXT: s_and_b32 s4, s20, -4
3121 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
3122 ; GFX7-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
3123 ; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
3124 ; GFX7-NEXT: s_and_b32 s4, s20, 3
3125 ; GFX7-NEXT: s_lshl_b32 s6, s4, 3
3126 ; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s6
3127 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v0
3128 ; GFX7-NEXT: s_not_b32 s7, s4
3129 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
3130 ; GFX7-NEXT: .LBB11_1: ; %atomicrmw.start
3131 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
3132 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3133 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v1
3134 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
3135 ; GFX7-NEXT: v_and_b32_e32 v4, s7, v1
3136 ; GFX7-NEXT: v_min_f32_e32 v0, v0, v3
3137 ; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
3138 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, s6, v0
3139 ; GFX7-NEXT: v_or_b32_e32 v0, v4, v0
3140 ; GFX7-NEXT: v_mov_b32_e32 v5, v1
3141 ; GFX7-NEXT: v_mov_b32_e32 v4, v0
3142 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
3143 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3144 ; GFX7-NEXT: buffer_wbinvl1
3145 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
3146 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3147 ; GFX7-NEXT: v_mov_b32_e32 v1, v4
3148 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
3149 ; GFX7-NEXT: s_cbranch_execnz .LBB11_1
3150 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
3151 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
3152 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3153 ;
3154 ; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
3155 ; GFX6: ; %bb.0:
3156 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3157 ; GFX6-NEXT: s_addk_i32 s20, 0x200
3158 ; GFX6-NEXT: s_and_b32 s4, s20, -4
3159 ; GFX6-NEXT: v_mov_b32_e32 v2, s4
3160 ; GFX6-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
3161 ; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
3162 ; GFX6-NEXT: s_and_b32 s4, s20, 3
3163 ; GFX6-NEXT: s_lshl_b32 s6, s4, 3
3164 ; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s6
3165 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v0
3166 ; GFX6-NEXT: s_not_b32 s7, s4
3167 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
3168 ; GFX6-NEXT: .LBB11_1: ; %atomicrmw.start
3169 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
3170 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3171 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v1
3172 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
3173 ; GFX6-NEXT: s_waitcnt expcnt(0)
3174 ; GFX6-NEXT: v_and_b32_e32 v4, s7, v1
3175 ; GFX6-NEXT: v_min_f32_e32 v0, v0, v3
3176 ; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
3177 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, s6, v0
3178 ; GFX6-NEXT: v_or_b32_e32 v0, v4, v0
3179 ; GFX6-NEXT: v_mov_b32_e32 v5, v1
3180 ; GFX6-NEXT: v_mov_b32_e32 v4, v0
3181 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
3182 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3183 ; GFX6-NEXT: buffer_wbinvl1
3184 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
3185 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3186 ; GFX6-NEXT: v_mov_b32_e32 v1, v4
3187 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
3188 ; GFX6-NEXT: s_cbranch_execnz .LBB11_1
3189 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
3190 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
3191 ; GFX6-NEXT: s_waitcnt expcnt(0)
3192 ; GFX6-NEXT: s_setpc_b64 s[30:31]
3193 %gep = getelementptr half, ptr addrspace(7) %ptr, i32 256
3194 %unused = atomicrmw fmin ptr addrspace(7) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
3195 ret void
3196 }
3198 define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, half %val) #0 {
3199 ; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
3200 ; GFX12: ; %bb.0:
3201 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
3202 ; GFX12-NEXT: s_wait_expcnt 0x0
3203 ; GFX12-NEXT: s_wait_samplecnt 0x0
3204 ; GFX12-NEXT: s_wait_bvhcnt 0x0
3205 ; GFX12-NEXT: s_wait_kmcnt 0x0
3206 ; GFX12-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
3207 ; GFX12-NEXT: s_mov_b32 s1, exec_lo
3208 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
3209 ; GFX12-NEXT: v_and_b32_e32 v6, 3, v4
3210 ; GFX12-NEXT: v_and_b32_e32 v8, -4, v4
3211 ; GFX12-NEXT: v_lshlrev_b32_e32 v7, 3, v6
3212 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3213 ; GFX12-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
3214 ; GFX12-NEXT: v_not_b32_e32 v9, v6
3215 ; GFX12-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
3216 ; GFX12-NEXT: v_readfirstlane_b32 s4, v0
3217 ; GFX12-NEXT: v_readfirstlane_b32 s5, v1
3218 ; GFX12-NEXT: v_readfirstlane_b32 s6, v2
3219 ; GFX12-NEXT: v_readfirstlane_b32 s7, v3
3220 ; GFX12-NEXT: s_wait_alu 0xf1ff
3221 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
3222 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
3223 ; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
3224 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
3225 ; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
3226 ; GFX12-NEXT: s_wait_alu 0xfffe
3227 ; GFX12-NEXT: s_and_saveexec_b32 s0, s0
3228 ; GFX12-NEXT: s_wait_loadcnt 0x0
3229 ; GFX12-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
3230 ; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
3231 ; GFX12-NEXT: s_cbranch_execnz .LBB12_1
3232 ; GFX12-NEXT: ; %bb.2:
3233 ; GFX12-NEXT: s_mov_b32 exec_lo, s1
3234 ; GFX12-NEXT: v_max_num_f16_e32 v10, v5, v5
3235 ; GFX12-NEXT: s_mov_b32 s1, 0
3236 ; GFX12-NEXT: .LBB12_3: ; %atomicrmw.start
3237 ; GFX12-NEXT: ; =>This Loop Header: Depth=1
3238 ; GFX12-NEXT: ; Child Loop BB12_4 Depth 2
3239 ; GFX12-NEXT: s_wait_loadcnt 0x0
3240 ; GFX12-NEXT: v_lshrrev_b32_e32 v4, v7, v6
3241 ; GFX12-NEXT: s_mov_b32 s2, exec_lo
3242 ; GFX12-NEXT: s_wait_storecnt 0x0
3243 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3244 ; GFX12-NEXT: v_max_num_f16_e32 v4, v4, v4
3245 ; GFX12-NEXT: v_min_num_f16_e32 v4, v4, v10
3246 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3247 ; GFX12-NEXT: v_and_b32_e32 v4, 0xffff, v4
3248 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, v7, v4
3249 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3250 ; GFX12-NEXT: v_and_or_b32 v5, v6, v9, v4
3251 ; GFX12-NEXT: v_mov_b32_e32 v4, v5
3252 ; GFX12-NEXT: v_mov_b32_e32 v5, v6
3253 ; GFX12-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
3254 ; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
3255 ; GFX12-NEXT: v_readfirstlane_b32 s4, v0
3256 ; GFX12-NEXT: v_readfirstlane_b32 s5, v1
3257 ; GFX12-NEXT: v_readfirstlane_b32 s6, v2
3258 ; GFX12-NEXT: v_readfirstlane_b32 s7, v3
3259 ; GFX12-NEXT: s_wait_alu 0xf1ff
3260 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
3261 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
3262 ; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
3263 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
3264 ; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
3265 ; GFX12-NEXT: s_wait_alu 0xfffe
3266 ; GFX12-NEXT: s_and_saveexec_b32 s0, s0
3267 ; GFX12-NEXT: s_wait_loadcnt 0x0
3268 ; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
3269 ; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
3270 ; GFX12-NEXT: s_cbranch_execnz .LBB12_4
3271 ; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
3272 ; GFX12-NEXT: s_mov_b32 exec_lo, s2
3273 ; GFX12-NEXT: s_wait_loadcnt 0x0
3274 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
3275 ; GFX12-NEXT: v_mov_b32_e32 v6, v4
3276 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
3277 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
3278 ; GFX12-NEXT: s_wait_alu 0xfffe
3279 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
3280 ; GFX12-NEXT: s_cbranch_execnz .LBB12_3
3281 ; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
3282 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
3283 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v7, v4
3284 ; GFX12-NEXT: s_setpc_b64 s[30:31]
3285 ;
3286 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
3287 ; GFX940: ; %bb.0:
3288 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3289 ; GFX940-NEXT: v_add_u32_e32 v4, 0x200, v4
3290 ; GFX940-NEXT: v_and_b32_e32 v9, -4, v4
3291 ; GFX940-NEXT: v_and_b32_e32 v4, 3, v4
3292 ; GFX940-NEXT: v_lshlrev_b32_e32 v8, 3, v4
3293 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
3294 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v8, s0
3295 ; GFX940-NEXT: v_not_b32_e32 v10, v4
3296 ; GFX940-NEXT: s_mov_b64 s[2:3], exec
3297 ; GFX940-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
3298 ; GFX940-NEXT: v_readfirstlane_b32 s4, v0
3299 ; GFX940-NEXT: v_readfirstlane_b32 s5, v1
3300 ; GFX940-NEXT: v_readfirstlane_b32 s6, v2
3301 ; GFX940-NEXT: v_readfirstlane_b32 s7, v3
3302 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
3303 ; GFX940-NEXT: s_nop 0
3304 ; GFX940-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
3305 ; GFX940-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
3306 ; GFX940-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
3307 ; GFX940-NEXT: buffer_load_dword v7, v9, s[4:7], 0 offen
3308 ; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
3309 ; GFX940-NEXT: s_cbranch_execnz .LBB12_1
3310 ; GFX940-NEXT: ; %bb.2:
3311 ; GFX940-NEXT: s_mov_b64 exec, s[2:3]
3312 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
3313 ; GFX940-NEXT: v_max_f16_e32 v11, v5, v5
3314 ; GFX940-NEXT: .LBB12_3: ; %atomicrmw.start
3315 ; GFX940-NEXT: ; =>This Loop Header: Depth=1
3316 ; GFX940-NEXT: ; Child Loop BB12_4 Depth 2
3317 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3318 ; GFX940-NEXT: v_lshrrev_b32_e32 v4, v8, v7
3319 ; GFX940-NEXT: v_max_f16_e32 v4, v4, v4
3320 ; GFX940-NEXT: v_min_f16_e32 v4, v4, v11
3321 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, v8, v4
3322 ; GFX940-NEXT: v_and_or_b32 v6, v7, v10, v4
3323 ; GFX940-NEXT: s_mov_b64 s[8:9], exec
3324 ; GFX940-NEXT: v_mov_b64_e32 v[4:5], v[6:7]
3325 ; GFX940-NEXT: buffer_wbl2 sc1
3326 ; GFX940-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
3327 ; GFX940-NEXT: ; => This Inner Loop Header: Depth=2
3328 ; GFX940-NEXT: v_readfirstlane_b32 s4, v0
3329 ; GFX940-NEXT: v_readfirstlane_b32 s5, v1
3330 ; GFX940-NEXT: v_readfirstlane_b32 s6, v2
3331 ; GFX940-NEXT: v_readfirstlane_b32 s7, v3
3332 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
3333 ; GFX940-NEXT: s_nop 0
3334 ; GFX940-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
3335 ; GFX940-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
3336 ; GFX940-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
3337 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3338 ; GFX940-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[4:7], 0 offen sc0
3339 ; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
3340 ; GFX940-NEXT: s_cbranch_execnz .LBB12_4
3341 ; GFX940-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
3342 ; GFX940-NEXT: s_mov_b64 exec, s[8:9]
3343 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3344 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
3345 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
3346 ; GFX940-NEXT: v_mov_b32_e32 v7, v4
3347 ; GFX940-NEXT: buffer_inv sc1
3348 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
3349 ; GFX940-NEXT: s_cbranch_execnz .LBB12_3
3350 ; GFX940-NEXT: ; %bb.6: ; %atomicrmw.end
3351 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
3352 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v8, v4
3353 ; GFX940-NEXT: s_setpc_b64 s[30:31]
3354 ;
3355 ; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
3356 ; GFX11: ; %bb.0:
3357 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3358 ; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
3359 ; GFX11-NEXT: s_mov_b32 s1, 0
3360 ; GFX11-NEXT: s_mov_b32 s2, exec_lo
3361 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
3362 ; GFX11-NEXT: v_and_b32_e32 v6, 3, v4
3363 ; GFX11-NEXT: v_and_b32_e32 v8, -4, v4
3364 ; GFX11-NEXT: v_lshlrev_b32_e32 v7, 3, v6
3365 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3366 ; GFX11-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
3367 ; GFX11-NEXT: v_not_b32_e32 v9, v6
3368 ; GFX11-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
3369 ; GFX11-NEXT: v_readfirstlane_b32 s4, v0
3370 ; GFX11-NEXT: v_readfirstlane_b32 s5, v1
3371 ; GFX11-NEXT: v_readfirstlane_b32 s6, v2
3372 ; GFX11-NEXT: v_readfirstlane_b32 s7, v3
3373 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
3374 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
3375 ; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
3376 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
3377 ; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
3378 ; GFX11-NEXT: s_and_saveexec_b32 s0, s0
3379 ; GFX11-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
3380 ; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
3381 ; GFX11-NEXT: s_cbranch_execnz .LBB12_1
3382 ; GFX11-NEXT: ; %bb.2:
3383 ; GFX11-NEXT: s_mov_b32 exec_lo, s2
3384 ; GFX11-NEXT: v_max_f16_e32 v10, v5, v5
3385 ; GFX11-NEXT: .p2align 6
3386 ; GFX11-NEXT: .LBB12_3: ; %atomicrmw.start
3387 ; GFX11-NEXT: ; =>This Loop Header: Depth=1
3388 ; GFX11-NEXT: ; Child Loop BB12_4 Depth 2
3389 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3390 ; GFX11-NEXT: v_lshrrev_b32_e32 v4, v7, v6
3391 ; GFX11-NEXT: s_mov_b32 s2, exec_lo
3392 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
3393 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3394 ; GFX11-NEXT: v_max_f16_e32 v4, v4, v4
3395 ; GFX11-NEXT: v_min_f16_e32 v4, v4, v10
3396 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3397 ; GFX11-NEXT: v_and_b32_e32 v4, 0xffff, v4
3398 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, v7, v4
3399 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3400 ; GFX11-NEXT: v_and_or_b32 v5, v6, v9, v4
3401 ; GFX11-NEXT: v_mov_b32_e32 v4, v5
3402 ; GFX11-NEXT: v_mov_b32_e32 v5, v6
3403 ; GFX11-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
3404 ; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
3405 ; GFX11-NEXT: v_readfirstlane_b32 s4, v0
3406 ; GFX11-NEXT: v_readfirstlane_b32 s5, v1
3407 ; GFX11-NEXT: v_readfirstlane_b32 s6, v2
3408 ; GFX11-NEXT: v_readfirstlane_b32 s7, v3
3409 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
3410 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
3411 ; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
3412 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
3413 ; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
3414 ; GFX11-NEXT: s_and_saveexec_b32 s0, s0
3415 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3416 ; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
3417 ; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
3418 ; GFX11-NEXT: s_cbranch_execnz .LBB12_4
3419 ; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
3420 ; GFX11-NEXT: s_mov_b32 exec_lo, s2
3421 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3422 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
3423 ; GFX11-NEXT: v_mov_b32_e32 v6, v4
3424 ; GFX11-NEXT: buffer_gl1_inv
3425 ; GFX11-NEXT: buffer_gl0_inv
3426 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
3427 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3428 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
3429 ; GFX11-NEXT: s_cbranch_execnz .LBB12_3
3430 ; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
3431 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
3432 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v7, v4
3433 ; GFX11-NEXT: s_setpc_b64 s[30:31]
3434 ;
3435 ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
3436 ; GFX10: ; %bb.0:
3437 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3438 ; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
3439 ; GFX10-NEXT: s_mov_b32 s5, 0
3440 ; GFX10-NEXT: s_mov_b32 s6, exec_lo
3441 ; GFX10-NEXT: v_and_b32_e32 v6, 3, v4
3442 ; GFX10-NEXT: v_and_b32_e32 v8, -4, v4
3443 ; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v6
3444 ; GFX10-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
3445 ; GFX10-NEXT: v_not_b32_e32 v9, v6
3446 ; GFX10-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
3447 ; GFX10-NEXT: v_readfirstlane_b32 s8, v0
3448 ; GFX10-NEXT: v_readfirstlane_b32 s9, v1
3449 ; GFX10-NEXT: v_readfirstlane_b32 s10, v2
3450 ; GFX10-NEXT: v_readfirstlane_b32 s11, v3
3451 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
3452 ; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
3453 ; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
3454 ; GFX10-NEXT: s_and_saveexec_b32 s4, s4
3455 ; GFX10-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
3456 ; GFX10-NEXT: s_waitcnt_depctr 0xffe3
3457 ; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
3458 ; GFX10-NEXT: s_cbranch_execnz .LBB12_1
3459 ; GFX10-NEXT: ; %bb.2:
3460 ; GFX10-NEXT: s_mov_b32 exec_lo, s6
3461 ; GFX10-NEXT: v_max_f16_e32 v10, v5, v5
3462 ; GFX10-NEXT: .LBB12_3: ; %atomicrmw.start
3463 ; GFX10-NEXT: ; =>This Loop Header: Depth=1
3464 ; GFX10-NEXT: ; Child Loop BB12_4 Depth 2
3465 ; GFX10-NEXT: s_waitcnt vmcnt(0)
3466 ; GFX10-NEXT: v_lshrrev_b32_e32 v4, v7, v6
3467 ; GFX10-NEXT: s_mov_b32 s6, exec_lo
3468 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
3469 ; GFX10-NEXT: v_max_f16_e32 v4, v4, v4
3470 ; GFX10-NEXT: v_min_f16_e32 v4, v4, v10
3471 ; GFX10-NEXT: v_lshlrev_b32_sdwa v4, v7, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
3472 ; GFX10-NEXT: v_and_or_b32 v5, v6, v9, v4
3473 ; GFX10-NEXT: v_mov_b32_e32 v4, v5
3474 ; GFX10-NEXT: v_mov_b32_e32 v5, v6
3475 ; GFX10-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
3476 ; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
3477 ; GFX10-NEXT: v_readfirstlane_b32 s8, v0
3478 ; GFX10-NEXT: v_readfirstlane_b32 s9, v1
3479 ; GFX10-NEXT: v_readfirstlane_b32 s10, v2
3480 ; GFX10-NEXT: v_readfirstlane_b32 s11, v3
3481 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
3482 ; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
3483 ; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
3484 ; GFX10-NEXT: s_and_saveexec_b32 s4, s4
3485 ; GFX10-NEXT: s_waitcnt vmcnt(0)
3486 ; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
3487 ; GFX10-NEXT: s_waitcnt_depctr 0xffe3
3488 ; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
3489 ; GFX10-NEXT: s_cbranch_execnz .LBB12_4
3490 ; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
3491 ; GFX10-NEXT: s_mov_b32 exec_lo, s6
3492 ; GFX10-NEXT: s_waitcnt vmcnt(0)
3493 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
3494 ; GFX10-NEXT: v_mov_b32_e32 v6, v4
3495 ; GFX10-NEXT: buffer_gl1_inv
3496 ; GFX10-NEXT: buffer_gl0_inv
3497 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
3498 ; GFX10-NEXT: s_waitcnt_depctr 0xffe3
3499 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
3500 ; GFX10-NEXT: s_cbranch_execnz .LBB12_3
3501 ; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
3502 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
3503 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v7, v4
3504 ; GFX10-NEXT: s_setpc_b64 s[30:31]
3505 ;
3506 ; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
3507 ; GFX90A: ; %bb.0:
3508 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3509 ; GFX90A-NEXT: v_add_u32_e32 v4, 0x200, v4
3510 ; GFX90A-NEXT: v_and_b32_e32 v9, -4, v4
3511 ; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
3512 ; GFX90A-NEXT: v_lshlrev_b32_e32 v8, 3, v4
3513 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
3514 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v8, s4
3515 ; GFX90A-NEXT: v_not_b32_e32 v10, v4
3516 ; GFX90A-NEXT: s_mov_b64 s[6:7], exec
3517 ; GFX90A-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
3518 ; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
3519 ; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
3520 ; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
3521 ; GFX90A-NEXT: v_readfirstlane_b32 s11, v3
3522 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
3523 ; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
3524 ; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
3525 ; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
3526 ; GFX90A-NEXT: s_nop 0
3527 ; GFX90A-NEXT: buffer_load_dword v7, v9, s[8:11], 0 offen
3528 ; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
3529 ; GFX90A-NEXT: s_cbranch_execnz .LBB12_1
3530 ; GFX90A-NEXT: ; %bb.2:
3531 ; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
3532 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
3533 ; GFX90A-NEXT: v_max_f16_e32 v11, v5, v5
3534 ; GFX90A-NEXT: .LBB12_3: ; %atomicrmw.start
3535 ; GFX90A-NEXT: ; =>This Loop Header: Depth=1
3536 ; GFX90A-NEXT: ; Child Loop BB12_4 Depth 2
3537 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3538 ; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v8, v7
3539 ; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
3540 ; GFX90A-NEXT: v_min_f16_e32 v4, v4, v11
3541 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v8, v4
3542 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v10, v4
3543 ; GFX90A-NEXT: s_mov_b64 s[12:13], exec
3544 ; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[6:7] op_sel:[0,1]
3545 ; GFX90A-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
3546 ; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
3547 ; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
3548 ; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
3549 ; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
3550 ; GFX90A-NEXT: v_readfirstlane_b32 s11, v3
3551 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
3552 ; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
3553 ; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
3554 ; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
3555 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3556 ; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[8:11], 0 offen glc
3557 ; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
3558 ; GFX90A-NEXT: s_cbranch_execnz .LBB12_4
3559 ; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
3560 ; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
3561 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
3562 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
3563 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
3564 ; GFX90A-NEXT: v_mov_b32_e32 v7, v4
3565 ; GFX90A-NEXT: buffer_wbinvl1
3566 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
3567 ; GFX90A-NEXT: s_cbranch_execnz .LBB12_3
3568 ; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.end
3569 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
3570 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v8, v4
3571 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
3572 ;
3573 ; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
3574 ; GFX908: ; %bb.0:
3575 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3576 ; GFX908-NEXT: v_add_u32_e32 v4, 0x200, v4
3577 ; GFX908-NEXT: v_and_b32_e32 v8, -4, v4
3578 ; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
3579 ; GFX908-NEXT: v_lshlrev_b32_e32 v7, 3, v4
3580 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
3581 ; GFX908-NEXT: v_lshlrev_b32_e64 v4, v7, s4
3582 ; GFX908-NEXT: v_not_b32_e32 v9, v4
3583 ; GFX908-NEXT: s_mov_b64 s[6:7], exec
3584 ; GFX908-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
3585 ; GFX908-NEXT: v_readfirstlane_b32 s8, v0
3586 ; GFX908-NEXT: v_readfirstlane_b32 s9, v1
3587 ; GFX908-NEXT: v_readfirstlane_b32 s10, v2
3588 ; GFX908-NEXT: v_readfirstlane_b32 s11, v3
3589 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
3590 ; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
3591 ; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
3592 ; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
3593 ; GFX908-NEXT: s_nop 0
3594 ; GFX908-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
3595 ; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
3596 ; GFX908-NEXT: s_cbranch_execnz .LBB12_1
3597 ; GFX908-NEXT: ; %bb.2:
3598 ; GFX908-NEXT: s_mov_b64 exec, s[6:7]
3599 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
3600 ; GFX908-NEXT: v_max_f16_e32 v10, v5, v5
3601 ; GFX908-NEXT: .LBB12_3: ; %atomicrmw.start
3602 ; GFX908-NEXT: ; =>This Loop Header: Depth=1
3603 ; GFX908-NEXT: ; Child Loop BB12_4 Depth 2
3604 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3605 ; GFX908-NEXT: v_lshrrev_b32_e32 v4, v7, v6
3606 ; GFX908-NEXT: v_max_f16_e32 v4, v4, v4
3607 ; GFX908-NEXT: v_min_f16_e32 v4, v4, v10
3608 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, v7, v4
3609 ; GFX908-NEXT: v_and_or_b32 v5, v6, v9, v4
3610 ; GFX908-NEXT: v_mov_b32_e32 v4, v5
3611 ; GFX908-NEXT: s_mov_b64 s[12:13], exec
3612 ; GFX908-NEXT: v_mov_b32_e32 v5, v6
3613 ; GFX908-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
3614 ; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
3615 ; GFX908-NEXT: v_readfirstlane_b32 s8, v0
3616 ; GFX908-NEXT: v_readfirstlane_b32 s9, v1
3617 ; GFX908-NEXT: v_readfirstlane_b32 s10, v2
3618 ; GFX908-NEXT: v_readfirstlane_b32 s11, v3
3619 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
3620 ; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
3621 ; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
3622 ; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
3623 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3624 ; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
3625 ; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
3626 ; GFX908-NEXT: s_cbranch_execnz .LBB12_4
3627 ; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
3628 ; GFX908-NEXT: s_mov_b64 exec, s[12:13]
3629 ; GFX908-NEXT: s_waitcnt vmcnt(0)
3630 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
3631 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
3632 ; GFX908-NEXT: v_mov_b32_e32 v6, v4
3633 ; GFX908-NEXT: buffer_wbinvl1
3634 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
3635 ; GFX908-NEXT: s_cbranch_execnz .LBB12_3
3636 ; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
3637 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
3638 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v7, v4
3639 ; GFX908-NEXT: s_setpc_b64 s[30:31]
3640 ;
3641 ; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
3642 ; GFX8: ; %bb.0:
3643 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3644 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x200, v4
3645 ; GFX8-NEXT: v_and_b32_e32 v8, -4, v4
3646 ; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
3647 ; GFX8-NEXT: v_lshlrev_b32_e32 v7, 3, v4
3648 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
3649 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v7, s4
3650 ; GFX8-NEXT: v_not_b32_e32 v9, v4
3651 ; GFX8-NEXT: s_mov_b64 s[6:7], exec
3652 ; GFX8-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
3653 ; GFX8-NEXT: v_readfirstlane_b32 s8, v0
3654 ; GFX8-NEXT: v_readfirstlane_b32 s9, v1
3655 ; GFX8-NEXT: v_readfirstlane_b32 s10, v2
3656 ; GFX8-NEXT: v_readfirstlane_b32 s11, v3
3657 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
3658 ; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
3659 ; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
3660 ; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
3661 ; GFX8-NEXT: s_nop 0
3662 ; GFX8-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
3663 ; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
3664 ; GFX8-NEXT: s_cbranch_execnz .LBB12_1
3665 ; GFX8-NEXT: ; %bb.2:
3666 ; GFX8-NEXT: s_mov_b64 exec, s[6:7]
3667 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
3668 ; GFX8-NEXT: v_max_f16_e32 v10, v5, v5
3669 ; GFX8-NEXT: .LBB12_3: ; %atomicrmw.start
3670 ; GFX8-NEXT: ; =>This Loop Header: Depth=1
3671 ; GFX8-NEXT: ; Child Loop BB12_4 Depth 2
3672 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3673 ; GFX8-NEXT: v_lshrrev_b32_e32 v4, v7, v6
3674 ; GFX8-NEXT: v_max_f16_e32 v4, v4, v4
3675 ; GFX8-NEXT: v_min_f16_e32 v4, v4, v10
3676 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, v7, v4
3677 ; GFX8-NEXT: v_and_b32_e32 v5, v6, v9
3678 ; GFX8-NEXT: v_or_b32_e32 v5, v5, v4
3679 ; GFX8-NEXT: v_mov_b32_e32 v4, v5
3680 ; GFX8-NEXT: s_mov_b64 s[12:13], exec
3681 ; GFX8-NEXT: v_mov_b32_e32 v5, v6
3682 ; GFX8-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
3683 ; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
3684 ; GFX8-NEXT: v_readfirstlane_b32 s8, v0
3685 ; GFX8-NEXT: v_readfirstlane_b32 s9, v1
3686 ; GFX8-NEXT: v_readfirstlane_b32 s10, v2
3687 ; GFX8-NEXT: v_readfirstlane_b32 s11, v3
3688 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
3689 ; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
3690 ; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
3691 ; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
3692 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3693 ; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
3694 ; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
3695 ; GFX8-NEXT: s_cbranch_execnz .LBB12_4
3696 ; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
3697 ; GFX8-NEXT: s_mov_b64 exec, s[12:13]
3698 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3699 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
3700 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
3701 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
3702 ; GFX8-NEXT: buffer_wbinvl1
3703 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
3704 ; GFX8-NEXT: s_cbranch_execnz .LBB12_3
3705 ; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
3706 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
3707 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v7, v4
3708 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3709 ;
3710 ; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
3711 ; GFX7: ; %bb.0:
3712 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3713 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x200, v4
3714 ; GFX7-NEXT: v_and_b32_e32 v8, -4, v4
3715 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
3716 ; GFX7-NEXT: v_lshlrev_b32_e32 v7, 3, v4
3717 ; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v7
3718 ; GFX7-NEXT: v_not_b32_e32 v9, v4
3719 ; GFX7-NEXT: s_mov_b64 s[6:7], exec
3720 ; GFX7-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
3721 ; GFX7-NEXT: v_readfirstlane_b32 s8, v0
3722 ; GFX7-NEXT: v_readfirstlane_b32 s9, v1
3723 ; GFX7-NEXT: v_readfirstlane_b32 s10, v2
3724 ; GFX7-NEXT: v_readfirstlane_b32 s11, v3
3725 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
3726 ; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
3727 ; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
3728 ; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
3729 ; GFX7-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
3730 ; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
3731 ; GFX7-NEXT: s_cbranch_execnz .LBB12_1
3732 ; GFX7-NEXT: ; %bb.2:
3733 ; GFX7-NEXT: s_mov_b64 exec, s[6:7]
3734 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v5
3735 ; GFX7-NEXT: s_mov_b64 s[6:7], 0
3736 ; GFX7-NEXT: v_cvt_f32_f16_e32 v10, v4
3737 ; GFX7-NEXT: .LBB12_3: ; %atomicrmw.start
3738 ; GFX7-NEXT: ; =>This Loop Header: Depth=1
3739 ; GFX7-NEXT: ; Child Loop BB12_4 Depth 2
3740 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3741 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, v7, v6
3742 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v4
3743 ; GFX7-NEXT: v_and_b32_e32 v5, v6, v9
3744 ; GFX7-NEXT: s_mov_b64 s[12:13], exec
3745 ; GFX7-NEXT: v_min_f32_e32 v4, v4, v10
3746 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
3747 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, v7, v4
3748 ; GFX7-NEXT: v_or_b32_e32 v5, v5, v4
3749 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
3750 ; GFX7-NEXT: v_mov_b32_e32 v5, v6
3751 ; GFX7-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
3752 ; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
3753 ; GFX7-NEXT: v_readfirstlane_b32 s8, v0
3754 ; GFX7-NEXT: v_readfirstlane_b32 s9, v1
3755 ; GFX7-NEXT: v_readfirstlane_b32 s10, v2
3756 ; GFX7-NEXT: v_readfirstlane_b32 s11, v3
3757 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
3758 ; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
3759 ; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
3760 ; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
3761 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3762 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
3763 ; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
3764 ; GFX7-NEXT: s_cbranch_execnz .LBB12_4
3765 ; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
3766 ; GFX7-NEXT: s_mov_b64 exec, s[12:13]
3767 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3768 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
3769 ; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
3770 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
3771 ; GFX7-NEXT: buffer_wbinvl1
3772 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
3773 ; GFX7-NEXT: s_cbranch_execnz .LBB12_3
3774 ; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end
3775 ; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
3776 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v7, v4
3777 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
3778 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3779 ;
3780 ; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
3781 ; GFX6: ; %bb.0:
3782 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3783 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x200, v4
3784 ; GFX6-NEXT: v_and_b32_e32 v8, -4, v4
3785 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
3786 ; GFX6-NEXT: v_lshlrev_b32_e32 v7, 3, v4
3787 ; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v7
3788 ; GFX6-NEXT: v_not_b32_e32 v9, v4
3789 ; GFX6-NEXT: s_mov_b64 s[6:7], exec
3790 ; GFX6-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
3791 ; GFX6-NEXT: v_readfirstlane_b32 s8, v0
3792 ; GFX6-NEXT: v_readfirstlane_b32 s9, v1
3793 ; GFX6-NEXT: v_readfirstlane_b32 s10, v2
3794 ; GFX6-NEXT: v_readfirstlane_b32 s11, v3
3795 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
3796 ; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
3797 ; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
3798 ; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
3799 ; GFX6-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
3800 ; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
3801 ; GFX6-NEXT: s_cbranch_execnz .LBB12_1
3802 ; GFX6-NEXT: ; %bb.2:
3803 ; GFX6-NEXT: s_mov_b64 exec, s[6:7]
3804 ; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v5
3805 ; GFX6-NEXT: s_mov_b64 s[6:7], 0
3806 ; GFX6-NEXT: v_cvt_f32_f16_e32 v10, v4
3807 ; GFX6-NEXT: .LBB12_3: ; %atomicrmw.start
3808 ; GFX6-NEXT: ; =>This Loop Header: Depth=1
3809 ; GFX6-NEXT: ; Child Loop BB12_4 Depth 2
3810 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
3811 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, v7, v6
3812 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v4
3813 ; GFX6-NEXT: v_and_b32_e32 v5, v6, v9
3814 ; GFX6-NEXT: s_mov_b64 s[12:13], exec
3815 ; GFX6-NEXT: v_min_f32_e32 v4, v4, v10
3816 ; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
3817 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, v7, v4
3818 ; GFX6-NEXT: v_or_b32_e32 v5, v5, v4
3819 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
3820 ; GFX6-NEXT: v_mov_b32_e32 v5, v6
3821 ; GFX6-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
3822 ; GFX6-NEXT: ; => This Inner Loop Header: Depth=2
3823 ; GFX6-NEXT: v_readfirstlane_b32 s8, v0
3824 ; GFX6-NEXT: v_readfirstlane_b32 s9, v1
3825 ; GFX6-NEXT: v_readfirstlane_b32 s10, v2
3826 ; GFX6-NEXT: v_readfirstlane_b32 s11, v3
3827 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
3828 ; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
3829 ; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
3830 ; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
3831 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
3832 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
3833 ; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
3834 ; GFX6-NEXT: s_cbranch_execnz .LBB12_4
3835 ; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
3836 ; GFX6-NEXT: s_mov_b64 exec, s[12:13]
3837 ; GFX6-NEXT: s_waitcnt vmcnt(0)
3838 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
3839 ; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
3840 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
3841 ; GFX6-NEXT: buffer_wbinvl1
3842 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
3843 ; GFX6-NEXT: s_cbranch_execnz .LBB12_3
3844 ; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end
3845 ; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
3846 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v7, v4
3847 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
3848 ; GFX6-NEXT: s_waitcnt expcnt(0)
3849 ; GFX6-NEXT: s_setpc_b64 s[30:31]
3850 %gep = getelementptr half, ptr addrspace(7) %ptr, i32 256
3851 %result = atomicrmw fmin ptr addrspace(7) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
3852 ret half %result
3853 }
3855 ; --------------------------------------------------------------------
3856 ; bfloat
3857 ; --------------------------------------------------------------------
3859 define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
3860 ; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
3861 ; GFX12: ; %bb.0:
3862 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
3863 ; GFX12-NEXT: s_wait_expcnt 0x0
3864 ; GFX12-NEXT: s_wait_samplecnt 0x0
3865 ; GFX12-NEXT: s_wait_bvhcnt 0x0
3866 ; GFX12-NEXT: s_wait_kmcnt 0x0
3867 ; GFX12-NEXT: s_addk_co_i32 s16, 0x200
3868 ; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v0
3869 ; GFX12-NEXT: s_wait_alu 0xfffe
3870 ; GFX12-NEXT: s_and_b32 s4, s16, -4
3871 ; GFX12-NEXT: s_wait_alu 0xfffe
3872 ; GFX12-NEXT: v_mov_b32_e32 v4, s4
3873 ; GFX12-NEXT: s_and_b32 s4, s16, 3
3874 ; GFX12-NEXT: s_wait_alu 0xfffe
3875 ; GFX12-NEXT: s_lshl_b32 s4, s4, 3
3876 ; GFX12-NEXT: s_wait_alu 0xfffe
3877 ; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
3878 ; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
3879 ; GFX12-NEXT: s_not_b32 s6, s5
3880 ; GFX12-NEXT: s_mov_b32 s5, 0
3881 ; GFX12-NEXT: .LBB13_1: ; %atomicrmw.start
3882 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
3883 ; GFX12-NEXT: s_wait_loadcnt 0x0
3884 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
3885 ; GFX12-NEXT: s_wait_storecnt 0x0
3886 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3887 ; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
3888 ; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v5
3889 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
3890 ; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
3891 ; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v0
3892 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
3893 ; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
3894 ; GFX12-NEXT: s_wait_alu 0xfffd
3895 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3896 ; GFX12-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
3897 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
3898 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
3899 ; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
3900 ; GFX12-NEXT: s_wait_alu 0xfffe
3901 ; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
3902 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
3903 ; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
3904 ; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
3905 ; GFX12-NEXT: s_wait_loadcnt 0x0
3906 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
3907 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
3908 ; GFX12-NEXT: v_mov_b32_e32 v1, v2
3909 ; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
3910 ; GFX12-NEXT: s_wait_alu 0xfffe
3911 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
3912 ; GFX12-NEXT: s_cbranch_execnz .LBB13_1
3913 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
3914 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
3915 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2
3916 ; GFX12-NEXT: s_setpc_b64 s[30:31]
3917 ;
3918 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
3919 ; GFX940: ; %bb.0:
3920 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3921 ; GFX940-NEXT: s_addk_i32 s16, 0x200
3922 ; GFX940-NEXT: s_and_b32 s4, s16, -4
3923 ; GFX940-NEXT: v_mov_b32_e32 v4, s4
3924 ; GFX940-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen
3925 ; GFX940-NEXT: s_and_b32 s4, s16, 3
3926 ; GFX940-NEXT: s_lshl_b32 s6, s4, 3
3927 ; GFX940-NEXT: s_lshl_b32 s4, 0xffff, s6
3928 ; GFX940-NEXT: s_not_b32 s7, s4
3929 ; GFX940-NEXT: s_mov_b64 s[4:5], 0
3930 ; GFX940-NEXT: v_lshlrev_b32_e32 v5, 16, v0
3931 ; GFX940-NEXT: s_movk_i32 s8, 0x7fff
3932 ; GFX940-NEXT: .LBB13_1: ; %atomicrmw.start
3933 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
3934 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3935 ; GFX940-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
3936 ; GFX940-NEXT: buffer_wbl2 sc1
3937 ; GFX940-NEXT: v_min_f32_e32 v0, v0, v5
3938 ; GFX940-NEXT: v_bfe_u32 v2, v0, 16, 1
3939 ; GFX940-NEXT: v_or_b32_e32 v3, 0x400000, v0
3940 ; GFX940-NEXT: v_add3_u32 v2, v2, v0, s8
3941 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
3942 ; GFX940-NEXT: s_nop 1
3943 ; GFX940-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
3944 ; GFX940-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
3945 ; GFX940-NEXT: v_and_or_b32 v0, v1, s7, v0
3946 ; GFX940-NEXT: v_mov_b64_e32 v[2:3], v[0:1]
3947 ; GFX940-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[0:3], 0 offen sc0
3948 ; GFX940-NEXT: s_waitcnt vmcnt(0)
3949 ; GFX940-NEXT: buffer_inv sc1
3950 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
3951 ; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3952 ; GFX940-NEXT: v_mov_b32_e32 v1, v2
3953 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
3954 ; GFX940-NEXT: s_cbranch_execnz .LBB13_1
3955 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
3956 ; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
3957 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, s6, v2
3958 ; GFX940-NEXT: s_setpc_b64 s[30:31]
3959 ;
3960 ; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
3961 ; GFX11: ; %bb.0:
3962 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3963 ; GFX11-NEXT: s_addk_i32 s16, 0x200
3964 ; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0
3965 ; GFX11-NEXT: s_and_b32 s4, s16, -4
3966 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
3967 ; GFX11-NEXT: v_mov_b32_e32 v4, s4
3968 ; GFX11-NEXT: s_and_b32 s4, s16, 3
3969 ; GFX11-NEXT: s_lshl_b32 s4, s4, 3
3970 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
3971 ; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
3972 ; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
3973 ; GFX11-NEXT: s_not_b32 s6, s5
3974 ; GFX11-NEXT: s_mov_b32 s5, 0
3975 ; GFX11-NEXT: .p2align 6
3976 ; GFX11-NEXT: .LBB13_1: ; %atomicrmw.start
3977 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
3978 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3979 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
3980 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
3981 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3982 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
3983 ; GFX11-NEXT: v_min_f32_e32 v0, v0, v5
3984 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
3985 ; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1
3986 ; GFX11-NEXT: v_or_b32_e32 v3, 0x400000, v0
3987 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
3988 ; GFX11-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
3989 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3990 ; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
3991 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
3992 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
3993 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
3994 ; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
3995 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
3996 ; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
3997 ; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
3998 ; GFX11-NEXT: s_waitcnt vmcnt(0)
3999 ; GFX11-NEXT: buffer_gl1_inv
4000 ; GFX11-NEXT: buffer_gl0_inv
4001 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
4002 ; GFX11-NEXT: v_mov_b32_e32 v1, v2
4003 ; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
4004 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4005 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
4006 ; GFX11-NEXT: s_cbranch_execnz .LBB13_1
4007 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
4008 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
4009 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2
4010 ; GFX11-NEXT: s_setpc_b64 s[30:31]
4011 ;
4012 ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
4013 ; GFX10: ; %bb.0:
4014 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4015 ; GFX10-NEXT: s_addk_i32 s20, 0x200
4016 ; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v0
4017 ; GFX10-NEXT: s_and_b32 s4, s20, -4
4018 ; GFX10-NEXT: v_mov_b32_e32 v4, s4
4019 ; GFX10-NEXT: s_and_b32 s4, s20, 3
4020 ; GFX10-NEXT: s_lshl_b32 s4, s4, 3
4021 ; GFX10-NEXT: s_lshl_b32 s5, 0xffff, s4
4022 ; GFX10-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
4023 ; GFX10-NEXT: s_not_b32 s6, s5
4024 ; GFX10-NEXT: s_mov_b32 s5, 0
4025 ; GFX10-NEXT: .LBB13_1: ; %atomicrmw.start
4026 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
4027 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4028 ; GFX10-NEXT: v_lshrrev_b32_sdwa v0, s4, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4029 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
4030 ; GFX10-NEXT: v_min_f32_e32 v0, v0, v5
4031 ; GFX10-NEXT: v_bfe_u32 v2, v0, 16, 1
4032 ; GFX10-NEXT: v_or_b32_e32 v3, 0x400000, v0
4033 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
4034 ; GFX10-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
4035 ; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
4036 ; GFX10-NEXT: v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4037 ; GFX10-NEXT: v_and_or_b32 v0, v1, s6, v0
4038 ; GFX10-NEXT: v_mov_b32_e32 v3, v1
4039 ; GFX10-NEXT: v_mov_b32_e32 v2, v0
4040 ; GFX10-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
4041 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4042 ; GFX10-NEXT: buffer_gl1_inv
4043 ; GFX10-NEXT: buffer_gl0_inv
4044 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
4045 ; GFX10-NEXT: v_mov_b32_e32 v1, v2
4046 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
4047 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
4048 ; GFX10-NEXT: s_cbranch_execnz .LBB13_1
4049 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
4050 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
4051 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, s4, v2
4052 ; GFX10-NEXT: s_setpc_b64 s[30:31]
4053 ;
4054 ; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
4055 ; GFX90A: ; %bb.0:
4056 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4057 ; GFX90A-NEXT: s_addk_i32 s20, 0x200
4058 ; GFX90A-NEXT: s_and_b32 s4, s20, -4
4059 ; GFX90A-NEXT: v_mov_b32_e32 v4, s4
4060 ; GFX90A-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
4061 ; GFX90A-NEXT: s_and_b32 s4, s20, 3
4062 ; GFX90A-NEXT: s_lshl_b32 s6, s4, 3
4063 ; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s6
4064 ; GFX90A-NEXT: s_not_b32 s7, s4
4065 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
4066 ; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v0
4067 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
4068 ; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
4069 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
4070 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4071 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4072 ; GFX90A-NEXT: v_min_f32_e32 v0, v0, v5
4073 ; GFX90A-NEXT: v_bfe_u32 v2, v0, 16, 1
4074 ; GFX90A-NEXT: v_or_b32_e32 v3, 0x400000, v0
4075 ; GFX90A-NEXT: v_add3_u32 v2, v2, v0, s8
4076 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
4077 ; GFX90A-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
4078 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4079 ; GFX90A-NEXT: v_and_or_b32 v0, v1, s7, v0
4080 ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
4081 ; GFX90A-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
4082 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4083 ; GFX90A-NEXT: buffer_wbinvl1
4084 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
4085 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4086 ; GFX90A-NEXT: v_mov_b32_e32 v1, v2
4087 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
4088 ; GFX90A-NEXT: s_cbranch_execnz .LBB13_1
4089 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
4090 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
4091 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s6, v2
4092 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
4093 ;
4094 ; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
4095 ; GFX908: ; %bb.0:
4096 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4097 ; GFX908-NEXT: s_addk_i32 s20, 0x200
4098 ; GFX908-NEXT: s_and_b32 s4, s20, -4
4099 ; GFX908-NEXT: v_mov_b32_e32 v4, s4
4100 ; GFX908-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
4101 ; GFX908-NEXT: s_and_b32 s4, s20, 3
4102 ; GFX908-NEXT: s_lshl_b32 s6, s4, 3
4103 ; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s6
4104 ; GFX908-NEXT: s_not_b32 s7, s4
4105 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
4106 ; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v0
4107 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
4108 ; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
4109 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
4110 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4111 ; GFX908-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4112 ; GFX908-NEXT: v_min_f32_e32 v0, v0, v5
4113 ; GFX908-NEXT: v_bfe_u32 v2, v0, 16, 1
4114 ; GFX908-NEXT: v_or_b32_e32 v3, 0x400000, v0
4115 ; GFX908-NEXT: v_add3_u32 v2, v2, v0, s8
4116 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
4117 ; GFX908-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
4118 ; GFX908-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4119 ; GFX908-NEXT: v_and_or_b32 v0, v1, s7, v0
4120 ; GFX908-NEXT: v_mov_b32_e32 v3, v1
4121 ; GFX908-NEXT: v_mov_b32_e32 v2, v0
4122 ; GFX908-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
4123 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4124 ; GFX908-NEXT: buffer_wbinvl1
4125 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
4126 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4127 ; GFX908-NEXT: v_mov_b32_e32 v1, v2
4128 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
4129 ; GFX908-NEXT: s_cbranch_execnz .LBB13_1
4130 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
4131 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
4132 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, s6, v2
4133 ; GFX908-NEXT: s_setpc_b64 s[30:31]
4134 ;
4135 ; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
4136 ; GFX8: ; %bb.0:
4137 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4138 ; GFX8-NEXT: s_addk_i32 s20, 0x200
4139 ; GFX8-NEXT: s_and_b32 s4, s20, -4
4140 ; GFX8-NEXT: v_mov_b32_e32 v4, s4
4141 ; GFX8-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
4142 ; GFX8-NEXT: s_and_b32 s4, s20, 3
4143 ; GFX8-NEXT: s_lshl_b32 s6, s4, 3
4144 ; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s6
4145 ; GFX8-NEXT: s_not_b32 s7, s4
4146 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
4147 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0
4148 ; GFX8-NEXT: .LBB13_1: ; %atomicrmw.start
4149 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4150 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
4151 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4152 ; GFX8-NEXT: v_lshrrev_b32_sdwa v3, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4153 ; GFX8-NEXT: v_min_f32_e32 v3, v3, v5
4154 ; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
4155 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
4156 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
4157 ; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
4158 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
4159 ; GFX8-NEXT: v_cndmask_b32_e32 v3, v6, v7, vcc
4160 ; GFX8-NEXT: v_and_b32_e32 v2, s7, v1
4161 ; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4162 ; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
4163 ; GFX8-NEXT: v_mov_b32_e32 v3, v1
4164 ; GFX8-NEXT: v_mov_b32_e32 v2, v0
4165 ; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
4166 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4167 ; GFX8-NEXT: buffer_wbinvl1
4168 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
4169 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4170 ; GFX8-NEXT: v_mov_b32_e32 v1, v2
4171 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
4172 ; GFX8-NEXT: s_cbranch_execnz .LBB13_1
4173 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4174 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
4175 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, s6, v2
4176 ; GFX8-NEXT: s_setpc_b64 s[30:31]
4177 ;
4178 ; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
4179 ; GFX7: ; %bb.0:
4180 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4181 ; GFX7-NEXT: s_addk_i32 s20, 0x200
4182 ; GFX7-NEXT: s_and_b32 s4, s20, -4
4183 ; GFX7-NEXT: v_mov_b32_e32 v4, s4
4184 ; GFX7-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
4185 ; GFX7-NEXT: s_and_b32 s4, s20, 3
4186 ; GFX7-NEXT: s_lshl_b32 s6, s4, 3
4187 ; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s6
4188 ; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
4189 ; GFX7-NEXT: s_not_b32 s7, s4
4190 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
4191 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
4192 ; GFX7-NEXT: .LBB13_1: ; %atomicrmw.start
4193 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4194 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4195 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v1
4196 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
4197 ; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
4198 ; GFX7-NEXT: v_min_f32_e32 v0, v0, v5
4199 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
4200 ; GFX7-NEXT: v_and_b32_e32 v2, s7, v1
4201 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, s6, v0
4202 ; GFX7-NEXT: v_or_b32_e32 v0, v2, v0
4203 ; GFX7-NEXT: v_mov_b32_e32 v3, v1
4204 ; GFX7-NEXT: v_mov_b32_e32 v2, v0
4205 ; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
4206 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4207 ; GFX7-NEXT: buffer_wbinvl1
4208 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
4209 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4210 ; GFX7-NEXT: v_mov_b32_e32 v1, v2
4211 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
4212 ; GFX7-NEXT: s_cbranch_execnz .LBB13_1
4213 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4214 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
4215 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v2
4216 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
4217 ; GFX7-NEXT: s_setpc_b64 s[30:31]
4218 ;
4219 ; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
4220 ; GFX6: ; %bb.0:
4221 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4222 ; GFX6-NEXT: s_addk_i32 s20, 0x200
4223 ; GFX6-NEXT: s_and_b32 s4, s20, -4
4224 ; GFX6-NEXT: v_mov_b32_e32 v4, s4
4225 ; GFX6-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
4226 ; GFX6-NEXT: s_and_b32 s4, s20, 3
4227 ; GFX6-NEXT: s_lshl_b32 s6, s4, 3
4228 ; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s6
4229 ; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
4230 ; GFX6-NEXT: s_not_b32 s7, s4
4231 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
4232 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
4233 ; GFX6-NEXT: .LBB13_1: ; %atomicrmw.start
4234 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
4235 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4236 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v1
4237 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
4238 ; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
4239 ; GFX6-NEXT: v_min_f32_e32 v0, v0, v5
4240 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
4241 ; GFX6-NEXT: s_waitcnt expcnt(0)
4242 ; GFX6-NEXT: v_and_b32_e32 v2, s7, v1
4243 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, s6, v0
4244 ; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
4245 ; GFX6-NEXT: v_mov_b32_e32 v3, v1
4246 ; GFX6-NEXT: v_mov_b32_e32 v2, v0
4247 ; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
4248 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4249 ; GFX6-NEXT: buffer_wbinvl1
4250 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
4251 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4252 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
4253 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
4254 ; GFX6-NEXT: s_cbranch_execnz .LBB13_1
4255 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
4256 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
4257 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v2
4258 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
4259 ; GFX6-NEXT: s_waitcnt expcnt(0)
4260 ; GFX6-NEXT: s_setpc_b64 s[30:31]
4261 %gep = getelementptr bfloat, ptr addrspace(7) %ptr, i32 256
4262 %result = atomicrmw fmin ptr addrspace(7) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
4263 ret bfloat %result
4264 }
4266 define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
4267 ; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
4268 ; GFX12: ; %bb.0:
4269 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
4270 ; GFX12-NEXT: s_wait_expcnt 0x0
4271 ; GFX12-NEXT: s_wait_samplecnt 0x0
4272 ; GFX12-NEXT: s_wait_bvhcnt 0x0
4273 ; GFX12-NEXT: s_wait_kmcnt 0x0
4274 ; GFX12-NEXT: s_addk_co_i32 s16, 0x200
4275 ; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v0
4276 ; GFX12-NEXT: s_wait_alu 0xfffe
4277 ; GFX12-NEXT: s_and_b32 s4, s16, -4
4278 ; GFX12-NEXT: s_wait_alu 0xfffe
4279 ; GFX12-NEXT: v_mov_b32_e32 v2, s4
4280 ; GFX12-NEXT: s_and_b32 s4, s16, 3
4281 ; GFX12-NEXT: s_wait_alu 0xfffe
4282 ; GFX12-NEXT: s_lshl_b32 s4, s4, 3
4283 ; GFX12-NEXT: s_wait_alu 0xfffe
4284 ; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
4285 ; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
4286 ; GFX12-NEXT: s_not_b32 s6, s5
4287 ; GFX12-NEXT: s_mov_b32 s5, 0
4288 ; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
4289 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
4290 ; GFX12-NEXT: s_wait_loadcnt 0x0
4291 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
4292 ; GFX12-NEXT: s_wait_storecnt 0x0
4293 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4294 ; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
4295 ; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v3
4296 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
4297 ; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
4298 ; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
4299 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
4300 ; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
4301 ; GFX12-NEXT: s_wait_alu 0xfffd
4302 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4303 ; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
4304 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
4305 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
4306 ; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
4307 ; GFX12-NEXT: s_wait_alu 0xfffe
4308 ; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
4309 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
4310 ; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
4311 ; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
4312 ; GFX12-NEXT: s_wait_loadcnt 0x0
4313 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
4314 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
4315 ; GFX12-NEXT: v_mov_b32_e32 v1, v4
4316 ; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
4317 ; GFX12-NEXT: s_wait_alu 0xfffe
4318 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
4319 ; GFX12-NEXT: s_cbranch_execnz .LBB14_1
4320 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
4321 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
4322 ; GFX12-NEXT: s_setpc_b64 s[30:31]
4323 ;
4324 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
4325 ; GFX940: ; %bb.0:
4326 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4327 ; GFX940-NEXT: s_addk_i32 s16, 0x200
4328 ; GFX940-NEXT: s_and_b32 s4, s16, -4
4329 ; GFX940-NEXT: v_mov_b32_e32 v2, s4
4330 ; GFX940-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen
4331 ; GFX940-NEXT: s_and_b32 s4, s16, 3
4332 ; GFX940-NEXT: s_lshl_b32 s6, s4, 3
4333 ; GFX940-NEXT: s_lshl_b32 s4, 0xffff, s6
4334 ; GFX940-NEXT: s_not_b32 s7, s4
4335 ; GFX940-NEXT: s_mov_b64 s[4:5], 0
4336 ; GFX940-NEXT: v_lshlrev_b32_e32 v3, 16, v0
4337 ; GFX940-NEXT: s_movk_i32 s8, 0x7fff
4338 ; GFX940-NEXT: .LBB14_1: ; %atomicrmw.start
4339 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
4340 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4341 ; GFX940-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4342 ; GFX940-NEXT: buffer_wbl2 sc1
4343 ; GFX940-NEXT: v_min_f32_e32 v0, v0, v3
4344 ; GFX940-NEXT: v_bfe_u32 v4, v0, 16, 1
4345 ; GFX940-NEXT: v_or_b32_e32 v5, 0x400000, v0
4346 ; GFX940-NEXT: v_add3_u32 v4, v4, v0, s8
4347 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
4348 ; GFX940-NEXT: s_nop 1
4349 ; GFX940-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
4350 ; GFX940-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4351 ; GFX940-NEXT: v_and_or_b32 v0, v1, s7, v0
4352 ; GFX940-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
4353 ; GFX940-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[0:3], 0 offen sc0
4354 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4355 ; GFX940-NEXT: buffer_inv sc1
4356 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
4357 ; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4358 ; GFX940-NEXT: v_mov_b32_e32 v1, v4
4359 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
4360 ; GFX940-NEXT: s_cbranch_execnz .LBB14_1
4361 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
4362 ; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
4363 ; GFX940-NEXT: s_setpc_b64 s[30:31]
4364 ;
4365 ; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
4366 ; GFX11: ; %bb.0:
4367 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4368 ; GFX11-NEXT: s_addk_i32 s16, 0x200
4369 ; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0
4370 ; GFX11-NEXT: s_and_b32 s4, s16, -4
4371 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
4372 ; GFX11-NEXT: v_mov_b32_e32 v2, s4
4373 ; GFX11-NEXT: s_and_b32 s4, s16, 3
4374 ; GFX11-NEXT: s_lshl_b32 s4, s4, 3
4375 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4376 ; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
4377 ; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
4378 ; GFX11-NEXT: s_not_b32 s6, s5
4379 ; GFX11-NEXT: s_mov_b32 s5, 0
4380 ; GFX11-NEXT: .p2align 6
4381 ; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
4382 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
4383 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4384 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
4385 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
4386 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4387 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
4388 ; GFX11-NEXT: v_min_f32_e32 v0, v0, v3
4389 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
4390 ; GFX11-NEXT: v_bfe_u32 v4, v0, 16, 1
4391 ; GFX11-NEXT: v_or_b32_e32 v5, 0x400000, v0
4392 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
4393 ; GFX11-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
4394 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4395 ; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
4396 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
4397 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4398 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
4399 ; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
4400 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
4401 ; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
4402 ; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
4403 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4404 ; GFX11-NEXT: buffer_gl1_inv
4405 ; GFX11-NEXT: buffer_gl0_inv
4406 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
4407 ; GFX11-NEXT: v_mov_b32_e32 v1, v4
4408 ; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
4409 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4410 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
4411 ; GFX11-NEXT: s_cbranch_execnz .LBB14_1
4412 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
4413 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
4414 ; GFX11-NEXT: s_setpc_b64 s[30:31]
4415 ;
4416 ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
4417 ; GFX10: ; %bb.0:
4418 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4419 ; GFX10-NEXT: s_addk_i32 s20, 0x200
4420 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v0
4421 ; GFX10-NEXT: s_and_b32 s4, s20, -4
4422 ; GFX10-NEXT: v_mov_b32_e32 v2, s4
4423 ; GFX10-NEXT: s_and_b32 s4, s20, 3
4424 ; GFX10-NEXT: s_lshl_b32 s4, s4, 3
4425 ; GFX10-NEXT: s_lshl_b32 s5, 0xffff, s4
4426 ; GFX10-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
4427 ; GFX10-NEXT: s_not_b32 s6, s5
4428 ; GFX10-NEXT: s_mov_b32 s5, 0
4429 ; GFX10-NEXT: .LBB14_1: ; %atomicrmw.start
4430 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
4431 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4432 ; GFX10-NEXT: v_lshrrev_b32_sdwa v0, s4, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4433 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
4434 ; GFX10-NEXT: v_min_f32_e32 v0, v0, v3
4435 ; GFX10-NEXT: v_bfe_u32 v4, v0, 16, 1
4436 ; GFX10-NEXT: v_or_b32_e32 v5, 0x400000, v0
4437 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
4438 ; GFX10-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
4439 ; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
4440 ; GFX10-NEXT: v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4441 ; GFX10-NEXT: v_and_or_b32 v0, v1, s6, v0
4442 ; GFX10-NEXT: v_mov_b32_e32 v5, v1
4443 ; GFX10-NEXT: v_mov_b32_e32 v4, v0
4444 ; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
4445 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4446 ; GFX10-NEXT: buffer_gl1_inv
4447 ; GFX10-NEXT: buffer_gl0_inv
4448 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
4449 ; GFX10-NEXT: v_mov_b32_e32 v1, v4
4450 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
4451 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
4452 ; GFX10-NEXT: s_cbranch_execnz .LBB14_1
4453 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
4454 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
4455 ; GFX10-NEXT: s_setpc_b64 s[30:31]
4456 ;
4457 ; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
4458 ; GFX90A: ; %bb.0:
4459 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4460 ; GFX90A-NEXT: s_addk_i32 s20, 0x200
4461 ; GFX90A-NEXT: s_and_b32 s4, s20, -4
4462 ; GFX90A-NEXT: v_mov_b32_e32 v2, s4
4463 ; GFX90A-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
4464 ; GFX90A-NEXT: s_and_b32 s4, s20, 3
4465 ; GFX90A-NEXT: s_lshl_b32 s6, s4, 3
4466 ; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s6
4467 ; GFX90A-NEXT: s_not_b32 s7, s4
4468 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
4469 ; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
4470 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
4471 ; GFX90A-NEXT: .LBB14_1: ; %atomicrmw.start
4472 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
4473 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4474 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4475 ; GFX90A-NEXT: v_min_f32_e32 v0, v0, v3
4476 ; GFX90A-NEXT: v_bfe_u32 v4, v0, 16, 1
4477 ; GFX90A-NEXT: v_or_b32_e32 v5, 0x400000, v0
4478 ; GFX90A-NEXT: v_add3_u32 v4, v4, v0, s8
4479 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
4480 ; GFX90A-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
4481 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4482 ; GFX90A-NEXT: v_and_or_b32 v0, v1, s7, v0
4483 ; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
4484 ; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
4485 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
4486 ; GFX90A-NEXT: buffer_wbinvl1
4487 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
4488 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4489 ; GFX90A-NEXT: v_mov_b32_e32 v1, v4
4490 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
4491 ; GFX90A-NEXT: s_cbranch_execnz .LBB14_1
4492 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
4493 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
4494 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
4495 ;
4496 ; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
4497 ; GFX908: ; %bb.0:
4498 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4499 ; GFX908-NEXT: s_addk_i32 s20, 0x200
4500 ; GFX908-NEXT: s_and_b32 s4, s20, -4
4501 ; GFX908-NEXT: v_mov_b32_e32 v2, s4
4502 ; GFX908-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
4503 ; GFX908-NEXT: s_and_b32 s4, s20, 3
4504 ; GFX908-NEXT: s_lshl_b32 s6, s4, 3
4505 ; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s6
4506 ; GFX908-NEXT: s_not_b32 s7, s4
4507 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
4508 ; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v0
4509 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
4510 ; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
4511 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
4512 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4513 ; GFX908-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4514 ; GFX908-NEXT: v_min_f32_e32 v0, v0, v3
4515 ; GFX908-NEXT: v_bfe_u32 v4, v0, 16, 1
4516 ; GFX908-NEXT: v_or_b32_e32 v5, 0x400000, v0
4517 ; GFX908-NEXT: v_add3_u32 v4, v4, v0, s8
4518 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
4519 ; GFX908-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
4520 ; GFX908-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4521 ; GFX908-NEXT: v_and_or_b32 v0, v1, s7, v0
4522 ; GFX908-NEXT: v_mov_b32_e32 v5, v1
4523 ; GFX908-NEXT: v_mov_b32_e32 v4, v0
4524 ; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
4525 ; GFX908-NEXT: s_waitcnt vmcnt(0)
4526 ; GFX908-NEXT: buffer_wbinvl1
4527 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
4528 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4529 ; GFX908-NEXT: v_mov_b32_e32 v1, v4
4530 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
4531 ; GFX908-NEXT: s_cbranch_execnz .LBB14_1
4532 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
4533 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
4534 ; GFX908-NEXT: s_setpc_b64 s[30:31]
4535 ;
4536 ; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
4537 ; GFX8: ; %bb.0:
4538 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4539 ; GFX8-NEXT: s_addk_i32 s20, 0x200
4540 ; GFX8-NEXT: s_and_b32 s4, s20, -4
4541 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
4542 ; GFX8-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
4543 ; GFX8-NEXT: s_and_b32 s4, s20, 3
4544 ; GFX8-NEXT: s_lshl_b32 s6, s4, 3
4545 ; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s6
4546 ; GFX8-NEXT: s_not_b32 s7, s4
4547 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
4548 ; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v0
4549 ; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
4550 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4551 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
4552 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4553 ; GFX8-NEXT: v_lshrrev_b32_sdwa v5, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4554 ; GFX8-NEXT: v_min_f32_e32 v5, v5, v3
4555 ; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
4556 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
4557 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
4558 ; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
4559 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
4560 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
4561 ; GFX8-NEXT: v_and_b32_e32 v4, s7, v1
4562 ; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4563 ; GFX8-NEXT: v_or_b32_e32 v0, v4, v0
4564 ; GFX8-NEXT: v_mov_b32_e32 v5, v1
4565 ; GFX8-NEXT: v_mov_b32_e32 v4, v0
4566 ; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
4567 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4568 ; GFX8-NEXT: buffer_wbinvl1
4569 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
4570 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4571 ; GFX8-NEXT: v_mov_b32_e32 v1, v4
4572 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
4573 ; GFX8-NEXT: s_cbranch_execnz .LBB14_1
4574 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4575 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
4576 ; GFX8-NEXT: s_setpc_b64 s[30:31]
4577 ;
4578 ; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
4579 ; GFX7: ; %bb.0:
4580 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4581 ; GFX7-NEXT: s_addk_i32 s20, 0x200
4582 ; GFX7-NEXT: s_and_b32 s4, s20, -4
4583 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
4584 ; GFX7-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
4585 ; GFX7-NEXT: s_and_b32 s4, s20, 3
4586 ; GFX7-NEXT: s_lshl_b32 s6, s4, 3
4587 ; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s6
4588 ; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
4589 ; GFX7-NEXT: s_not_b32 s7, s4
4590 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
4591 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
4592 ; GFX7-NEXT: .LBB14_1: ; %atomicrmw.start
4593 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4594 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4595 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v1
4596 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
4597 ; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
4598 ; GFX7-NEXT: v_min_f32_e32 v0, v0, v3
4599 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
4600 ; GFX7-NEXT: v_and_b32_e32 v4, s7, v1
4601 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, s6, v0
4602 ; GFX7-NEXT: v_or_b32_e32 v0, v4, v0
4603 ; GFX7-NEXT: v_mov_b32_e32 v5, v1
4604 ; GFX7-NEXT: v_mov_b32_e32 v4, v0
4605 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
4606 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4607 ; GFX7-NEXT: buffer_wbinvl1
4608 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
4609 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4610 ; GFX7-NEXT: v_mov_b32_e32 v1, v4
4611 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
4612 ; GFX7-NEXT: s_cbranch_execnz .LBB14_1
4613 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4614 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
4615 ; GFX7-NEXT: s_setpc_b64 s[30:31]
4616 ;
4617 ; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
4618 ; GFX6: ; %bb.0:
4619 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4620 ; GFX6-NEXT: s_addk_i32 s20, 0x200
4621 ; GFX6-NEXT: s_and_b32 s4, s20, -4
4622 ; GFX6-NEXT: v_mov_b32_e32 v2, s4
4623 ; GFX6-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
4624 ; GFX6-NEXT: s_and_b32 s4, s20, 3
4625 ; GFX6-NEXT: s_lshl_b32 s6, s4, 3
4626 ; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s6
4627 ; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
4628 ; GFX6-NEXT: s_not_b32 s7, s4
4629 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
4630 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
4631 ; GFX6-NEXT: .LBB14_1: ; %atomicrmw.start
4632 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
4633 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4634 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v1
4635 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
4636 ; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
4637 ; GFX6-NEXT: v_min_f32_e32 v0, v0, v3
4638 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
4639 ; GFX6-NEXT: s_waitcnt expcnt(0)
4640 ; GFX6-NEXT: v_and_b32_e32 v4, s7, v1
4641 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, s6, v0
4642 ; GFX6-NEXT: v_or_b32_e32 v0, v4, v0
4643 ; GFX6-NEXT: v_mov_b32_e32 v5, v1
4644 ; GFX6-NEXT: v_mov_b32_e32 v4, v0
4645 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
4646 ; GFX6-NEXT: s_waitcnt vmcnt(0)
4647 ; GFX6-NEXT: buffer_wbinvl1
4648 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
4649 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4650 ; GFX6-NEXT: v_mov_b32_e32 v1, v4
4651 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
4652 ; GFX6-NEXT: s_cbranch_execnz .LBB14_1
4653 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
4654 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
4655 ; GFX6-NEXT: s_waitcnt expcnt(0)
4656 ; GFX6-NEXT: s_setpc_b64 s[30:31]
4657 %gep = getelementptr bfloat, ptr addrspace(7) %ptr, i32 256
4658 %unused = atomicrmw fmin ptr addrspace(7) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
4659 ret void
4660 }
4662 define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, bfloat %val) #0 {
4663 ; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
4664 ; GFX12: ; %bb.0:
4665 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
4666 ; GFX12-NEXT: s_wait_expcnt 0x0
4667 ; GFX12-NEXT: s_wait_samplecnt 0x0
4668 ; GFX12-NEXT: s_wait_bvhcnt 0x0
4669 ; GFX12-NEXT: s_wait_kmcnt 0x0
4670 ; GFX12-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
4671 ; GFX12-NEXT: s_mov_b32 s1, exec_lo
4672 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
4673 ; GFX12-NEXT: v_and_b32_e32 v6, 3, v4
4674 ; GFX12-NEXT: v_and_b32_e32 v8, -4, v4
4675 ; GFX12-NEXT: v_lshlrev_b32_e32 v7, 3, v6
4676 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4677 ; GFX12-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
4678 ; GFX12-NEXT: v_not_b32_e32 v9, v6
4679 ; GFX12-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
4680 ; GFX12-NEXT: v_readfirstlane_b32 s4, v0
4681 ; GFX12-NEXT: v_readfirstlane_b32 s5, v1
4682 ; GFX12-NEXT: v_readfirstlane_b32 s6, v2
4683 ; GFX12-NEXT: v_readfirstlane_b32 s7, v3
4684 ; GFX12-NEXT: s_wait_alu 0xf1ff
4685 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
4686 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
4687 ; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
4688 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
4689 ; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
4690 ; GFX12-NEXT: s_wait_alu 0xfffe
4691 ; GFX12-NEXT: s_and_saveexec_b32 s0, s0
4692 ; GFX12-NEXT: s_wait_loadcnt 0x0
4693 ; GFX12-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
4694 ; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
4695 ; GFX12-NEXT: s_cbranch_execnz .LBB15_1
4696 ; GFX12-NEXT: ; %bb.2:
4697 ; GFX12-NEXT: s_mov_b32 exec_lo, s1
4698 ; GFX12-NEXT: v_lshlrev_b32_e32 v10, 16, v5
4699 ; GFX12-NEXT: s_mov_b32 s1, 0
4700 ; GFX12-NEXT: .LBB15_3: ; %atomicrmw.start
4701 ; GFX12-NEXT: ; =>This Loop Header: Depth=1
4702 ; GFX12-NEXT: ; Child Loop BB15_4 Depth 2
4703 ; GFX12-NEXT: s_wait_loadcnt 0x0
4704 ; GFX12-NEXT: v_lshrrev_b32_e32 v4, v7, v6
4705 ; GFX12-NEXT: s_mov_b32 s2, exec_lo
4706 ; GFX12-NEXT: s_wait_storecnt 0x0
4707 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4708 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v4
4709 ; GFX12-NEXT: v_min_num_f32_e32 v4, v4, v10
4710 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
4711 ; GFX12-NEXT: v_bfe_u32 v5, v4, 16, 1
4712 ; GFX12-NEXT: v_or_b32_e32 v11, 0x400000, v4
4713 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
4714 ; GFX12-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
4715 ; GFX12-NEXT: s_wait_alu 0xfffd
4716 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4717 ; GFX12-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
4718 ; GFX12-NEXT: v_lshrrev_b32_e32 v4, 16, v4
4719 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4720 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, v7, v4
4721 ; GFX12-NEXT: v_and_or_b32 v5, v6, v9, v4
4722 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
4723 ; GFX12-NEXT: v_mov_b32_e32 v4, v5
4724 ; GFX12-NEXT: v_mov_b32_e32 v5, v6
4725 ; GFX12-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
4726 ; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
4727 ; GFX12-NEXT: v_readfirstlane_b32 s4, v0
4728 ; GFX12-NEXT: v_readfirstlane_b32 s5, v1
4729 ; GFX12-NEXT: v_readfirstlane_b32 s6, v2
4730 ; GFX12-NEXT: v_readfirstlane_b32 s7, v3
4731 ; GFX12-NEXT: s_wait_alu 0xf1ff
4732 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
4733 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
4734 ; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
4735 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
4736 ; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
4737 ; GFX12-NEXT: s_wait_alu 0xfffe
4738 ; GFX12-NEXT: s_and_saveexec_b32 s0, s0
4739 ; GFX12-NEXT: s_wait_loadcnt 0x0
4740 ; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
4741 ; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
4742 ; GFX12-NEXT: s_cbranch_execnz .LBB15_4
4743 ; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
4744 ; GFX12-NEXT: s_mov_b32 exec_lo, s2
4745 ; GFX12-NEXT: s_wait_loadcnt 0x0
4746 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
4747 ; GFX12-NEXT: v_mov_b32_e32 v6, v4
4748 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
4749 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
4750 ; GFX12-NEXT: s_wait_alu 0xfffe
4751 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
4752 ; GFX12-NEXT: s_cbranch_execnz .LBB15_3
4753 ; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
4754 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
4755 ; GFX12-NEXT: v_lshrrev_b32_e32 v0, v7, v4
4756 ; GFX12-NEXT: s_setpc_b64 s[30:31]
4757 ;
4758 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
4759 ; GFX940: ; %bb.0:
4760 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4761 ; GFX940-NEXT: v_add_u32_e32 v4, 0x200, v4
4762 ; GFX940-NEXT: v_and_b32_e32 v9, -4, v4
4763 ; GFX940-NEXT: v_and_b32_e32 v4, 3, v4
4764 ; GFX940-NEXT: v_lshlrev_b32_e32 v8, 3, v4
4765 ; GFX940-NEXT: s_mov_b32 s0, 0xffff
4766 ; GFX940-NEXT: v_lshlrev_b32_e64 v4, v8, s0
4767 ; GFX940-NEXT: v_not_b32_e32 v10, v4
4768 ; GFX940-NEXT: s_mov_b64 s[2:3], exec
4769 ; GFX940-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
4770 ; GFX940-NEXT: v_readfirstlane_b32 s4, v0
4771 ; GFX940-NEXT: v_readfirstlane_b32 s5, v1
4772 ; GFX940-NEXT: v_readfirstlane_b32 s6, v2
4773 ; GFX940-NEXT: v_readfirstlane_b32 s7, v3
4774 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
4775 ; GFX940-NEXT: s_nop 0
4776 ; GFX940-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
4777 ; GFX940-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
4778 ; GFX940-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
4779 ; GFX940-NEXT: buffer_load_dword v7, v9, s[4:7], 0 offen
4780 ; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
4781 ; GFX940-NEXT: s_cbranch_execnz .LBB15_1
4782 ; GFX940-NEXT: ; %bb.2:
4783 ; GFX940-NEXT: s_mov_b64 exec, s[2:3]
4784 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
4785 ; GFX940-NEXT: v_lshlrev_b32_e32 v11, 16, v5
4786 ; GFX940-NEXT: s_movk_i32 s10, 0x7fff
4787 ; GFX940-NEXT: .LBB15_3: ; %atomicrmw.start
4788 ; GFX940-NEXT: ; =>This Loop Header: Depth=1
4789 ; GFX940-NEXT: ; Child Loop BB15_4 Depth 2
4790 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4791 ; GFX940-NEXT: v_lshrrev_b32_sdwa v4, v8, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4792 ; GFX940-NEXT: s_mov_b64 s[8:9], exec
4793 ; GFX940-NEXT: v_min_f32_e32 v4, v4, v11
4794 ; GFX940-NEXT: v_bfe_u32 v5, v4, 16, 1
4795 ; GFX940-NEXT: v_add3_u32 v5, v5, v4, s10
4796 ; GFX940-NEXT: v_or_b32_e32 v6, 0x400000, v4
4797 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
4798 ; GFX940-NEXT: buffer_wbl2 sc1
4799 ; GFX940-NEXT: s_nop 0
4800 ; GFX940-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
4801 ; GFX940-NEXT: v_lshlrev_b32_sdwa v4, v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4802 ; GFX940-NEXT: v_and_or_b32 v6, v7, v10, v4
4803 ; GFX940-NEXT: v_mov_b64_e32 v[4:5], v[6:7]
4804 ; GFX940-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
4805 ; GFX940-NEXT: ; => This Inner Loop Header: Depth=2
4806 ; GFX940-NEXT: v_readfirstlane_b32 s4, v0
4807 ; GFX940-NEXT: v_readfirstlane_b32 s5, v1
4808 ; GFX940-NEXT: v_readfirstlane_b32 s6, v2
4809 ; GFX940-NEXT: v_readfirstlane_b32 s7, v3
4810 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
4811 ; GFX940-NEXT: s_nop 0
4812 ; GFX940-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
4813 ; GFX940-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
4814 ; GFX940-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
4815 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4816 ; GFX940-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[4:7], 0 offen sc0
4817 ; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
4818 ; GFX940-NEXT: s_cbranch_execnz .LBB15_4
4819 ; GFX940-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
4820 ; GFX940-NEXT: s_mov_b64 exec, s[8:9]
4821 ; GFX940-NEXT: s_waitcnt vmcnt(0)
4822 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
4823 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
4824 ; GFX940-NEXT: v_mov_b32_e32 v7, v4
4825 ; GFX940-NEXT: buffer_inv sc1
4826 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
4827 ; GFX940-NEXT: s_cbranch_execnz .LBB15_3
4828 ; GFX940-NEXT: ; %bb.6: ; %atomicrmw.end
4829 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
4830 ; GFX940-NEXT: v_lshrrev_b32_e32 v0, v8, v4
4831 ; GFX940-NEXT: s_setpc_b64 s[30:31]
4832 ;
4833 ; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
4834 ; GFX11: ; %bb.0:
4835 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4836 ; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
4837 ; GFX11-NEXT: s_mov_b32 s1, 0
4838 ; GFX11-NEXT: s_mov_b32 s2, exec_lo
4839 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
4840 ; GFX11-NEXT: v_and_b32_e32 v6, 3, v4
4841 ; GFX11-NEXT: v_and_b32_e32 v8, -4, v4
4842 ; GFX11-NEXT: v_lshlrev_b32_e32 v7, 3, v6
4843 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4844 ; GFX11-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
4845 ; GFX11-NEXT: v_not_b32_e32 v9, v6
4846 ; GFX11-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
4847 ; GFX11-NEXT: v_readfirstlane_b32 s4, v0
4848 ; GFX11-NEXT: v_readfirstlane_b32 s5, v1
4849 ; GFX11-NEXT: v_readfirstlane_b32 s6, v2
4850 ; GFX11-NEXT: v_readfirstlane_b32 s7, v3
4851 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
4852 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
4853 ; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
4854 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
4855 ; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
4856 ; GFX11-NEXT: s_and_saveexec_b32 s0, s0
4857 ; GFX11-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
4858 ; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
4859 ; GFX11-NEXT: s_cbranch_execnz .LBB15_1
4860 ; GFX11-NEXT: ; %bb.2:
4861 ; GFX11-NEXT: s_mov_b32 exec_lo, s2
4862 ; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v5
4863 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
4864 ; GFX11-NEXT: .p2align 6
4865 ; GFX11-NEXT: .LBB15_3: ; %atomicrmw.start
4866 ; GFX11-NEXT: ; =>This Loop Header: Depth=1
4867 ; GFX11-NEXT: ; Child Loop BB15_4 Depth 2
4868 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4869 ; GFX11-NEXT: v_lshrrev_b32_e32 v4, v7, v6
4870 ; GFX11-NEXT: s_mov_b32 s2, exec_lo
4871 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
4872 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4873 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
4874 ; GFX11-NEXT: v_min_f32_e32 v4, v4, v10
4875 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
4876 ; GFX11-NEXT: v_bfe_u32 v5, v4, 16, 1
4877 ; GFX11-NEXT: v_or_b32_e32 v11, 0x400000, v4
4878 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
4879 ; GFX11-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
4880 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4881 ; GFX11-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
4882 ; GFX11-NEXT: v_lshrrev_b32_e32 v4, 16, v4
4883 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
4884 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, v7, v4
4885 ; GFX11-NEXT: v_and_or_b32 v5, v6, v9, v4
4886 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
4887 ; GFX11-NEXT: v_mov_b32_e32 v4, v5
4888 ; GFX11-NEXT: v_mov_b32_e32 v5, v6
4889 ; GFX11-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
4890 ; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
4891 ; GFX11-NEXT: v_readfirstlane_b32 s4, v0
4892 ; GFX11-NEXT: v_readfirstlane_b32 s5, v1
4893 ; GFX11-NEXT: v_readfirstlane_b32 s6, v2
4894 ; GFX11-NEXT: v_readfirstlane_b32 s7, v3
4895 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
4896 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
4897 ; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
4898 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
4899 ; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
4900 ; GFX11-NEXT: s_and_saveexec_b32 s0, s0
4901 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4902 ; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
4903 ; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
4904 ; GFX11-NEXT: s_cbranch_execnz .LBB15_4
4905 ; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
4906 ; GFX11-NEXT: s_mov_b32 exec_lo, s2
4907 ; GFX11-NEXT: s_waitcnt vmcnt(0)
4908 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
4909 ; GFX11-NEXT: v_mov_b32_e32 v6, v4
4910 ; GFX11-NEXT: buffer_gl1_inv
4911 ; GFX11-NEXT: buffer_gl0_inv
4912 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
4913 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
4914 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
4915 ; GFX11-NEXT: s_cbranch_execnz .LBB15_3
4916 ; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
4917 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
4918 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
4919 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, v7, v4
4920 ; GFX11-NEXT: s_setpc_b64 s[30:31]
4921 ;
4922 ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
4923 ; GFX10: ; %bb.0:
4924 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4925 ; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
4926 ; GFX10-NEXT: s_mov_b32 s5, 0
4927 ; GFX10-NEXT: s_mov_b32 s6, exec_lo
4928 ; GFX10-NEXT: v_and_b32_e32 v6, 3, v4
4929 ; GFX10-NEXT: v_and_b32_e32 v8, -4, v4
4930 ; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v6
4931 ; GFX10-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
4932 ; GFX10-NEXT: v_not_b32_e32 v9, v6
4933 ; GFX10-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
4934 ; GFX10-NEXT: v_readfirstlane_b32 s8, v0
4935 ; GFX10-NEXT: v_readfirstlane_b32 s9, v1
4936 ; GFX10-NEXT: v_readfirstlane_b32 s10, v2
4937 ; GFX10-NEXT: v_readfirstlane_b32 s11, v3
4938 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
4939 ; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
4940 ; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
4941 ; GFX10-NEXT: s_and_saveexec_b32 s4, s4
4942 ; GFX10-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
4943 ; GFX10-NEXT: s_waitcnt_depctr 0xffe3
4944 ; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
4945 ; GFX10-NEXT: s_cbranch_execnz .LBB15_1
4946 ; GFX10-NEXT: ; %bb.2:
4947 ; GFX10-NEXT: s_mov_b32 exec_lo, s6
4948 ; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5
4949 ; GFX10-NEXT: .LBB15_3: ; %atomicrmw.start
4950 ; GFX10-NEXT: ; =>This Loop Header: Depth=1
4951 ; GFX10-NEXT: ; Child Loop BB15_4 Depth 2
4952 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4953 ; GFX10-NEXT: v_lshrrev_b32_sdwa v4, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
4954 ; GFX10-NEXT: s_mov_b32 s6, exec_lo
4955 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
4956 ; GFX10-NEXT: v_min_f32_e32 v4, v4, v10
4957 ; GFX10-NEXT: v_bfe_u32 v5, v4, 16, 1
4958 ; GFX10-NEXT: v_or_b32_e32 v11, 0x400000, v4
4959 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
4960 ; GFX10-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
4961 ; GFX10-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
4962 ; GFX10-NEXT: v_lshlrev_b32_sdwa v4, v7, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
4963 ; GFX10-NEXT: v_and_or_b32 v5, v6, v9, v4
4964 ; GFX10-NEXT: v_mov_b32_e32 v4, v5
4965 ; GFX10-NEXT: v_mov_b32_e32 v5, v6
4966 ; GFX10-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
4967 ; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
4968 ; GFX10-NEXT: v_readfirstlane_b32 s8, v0
4969 ; GFX10-NEXT: v_readfirstlane_b32 s9, v1
4970 ; GFX10-NEXT: v_readfirstlane_b32 s10, v2
4971 ; GFX10-NEXT: v_readfirstlane_b32 s11, v3
4972 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
4973 ; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
4974 ; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
4975 ; GFX10-NEXT: s_and_saveexec_b32 s4, s4
4976 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4977 ; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
4978 ; GFX10-NEXT: s_waitcnt_depctr 0xffe3
4979 ; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
4980 ; GFX10-NEXT: s_cbranch_execnz .LBB15_4
4981 ; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
4982 ; GFX10-NEXT: s_mov_b32 exec_lo, s6
4983 ; GFX10-NEXT: s_waitcnt vmcnt(0)
4984 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
4985 ; GFX10-NEXT: v_mov_b32_e32 v6, v4
4986 ; GFX10-NEXT: buffer_gl1_inv
4987 ; GFX10-NEXT: buffer_gl0_inv
4988 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
4989 ; GFX10-NEXT: s_waitcnt_depctr 0xffe3
4990 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
4991 ; GFX10-NEXT: s_cbranch_execnz .LBB15_3
4992 ; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
4993 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
4994 ; GFX10-NEXT: v_lshrrev_b32_e32 v0, v7, v4
4995 ; GFX10-NEXT: s_setpc_b64 s[30:31]
4996 ;
4997 ; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
4998 ; GFX90A: ; %bb.0:
4999 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5000 ; GFX90A-NEXT: v_add_u32_e32 v4, 0x200, v4
5001 ; GFX90A-NEXT: v_and_b32_e32 v9, -4, v4
5002 ; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
5003 ; GFX90A-NEXT: v_lshlrev_b32_e32 v8, 3, v4
5004 ; GFX90A-NEXT: s_mov_b32 s4, 0xffff
5005 ; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v8, s4
5006 ; GFX90A-NEXT: v_not_b32_e32 v10, v4
5007 ; GFX90A-NEXT: s_mov_b64 s[6:7], exec
5008 ; GFX90A-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
5009 ; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
5010 ; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
5011 ; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
5012 ; GFX90A-NEXT: v_readfirstlane_b32 s11, v3
5013 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
5014 ; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
5015 ; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
5016 ; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
5017 ; GFX90A-NEXT: s_nop 0
5018 ; GFX90A-NEXT: buffer_load_dword v7, v9, s[8:11], 0 offen
5019 ; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
5020 ; GFX90A-NEXT: s_cbranch_execnz .LBB15_1
5021 ; GFX90A-NEXT: ; %bb.2:
5022 ; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
5023 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
5024 ; GFX90A-NEXT: v_lshlrev_b32_e32 v11, 16, v5
5025 ; GFX90A-NEXT: s_movk_i32 s14, 0x7fff
5026 ; GFX90A-NEXT: .LBB15_3: ; %atomicrmw.start
5027 ; GFX90A-NEXT: ; =>This Loop Header: Depth=1
5028 ; GFX90A-NEXT: ; Child Loop BB15_4 Depth 2
5029 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
5030 ; GFX90A-NEXT: v_lshrrev_b32_sdwa v4, v8, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
5031 ; GFX90A-NEXT: v_min_f32_e32 v4, v4, v11
5032 ; GFX90A-NEXT: v_bfe_u32 v5, v4, 16, 1
5033 ; GFX90A-NEXT: v_add3_u32 v5, v5, v4, s14
5034 ; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v4
5035 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
5036 ; GFX90A-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
5037 ; GFX90A-NEXT: v_lshlrev_b32_sdwa v4, v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
5038 ; GFX90A-NEXT: v_and_or_b32 v6, v7, v10, v4
5039 ; GFX90A-NEXT: s_mov_b64 s[12:13], exec
5040 ; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[6:7] op_sel:[0,1]
5041 ; GFX90A-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
5042 ; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
5043 ; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
5044 ; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
5045 ; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
5046 ; GFX90A-NEXT: v_readfirstlane_b32 s11, v3
5047 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
5048 ; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
5049 ; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
5050 ; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
5051 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
5052 ; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[8:11], 0 offen glc
5053 ; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
5054 ; GFX90A-NEXT: s_cbranch_execnz .LBB15_4
5055 ; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
5056 ; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
5057 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
5058 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
5059 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
5060 ; GFX90A-NEXT: v_mov_b32_e32 v7, v4
5061 ; GFX90A-NEXT: buffer_wbinvl1
5062 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
5063 ; GFX90A-NEXT: s_cbranch_execnz .LBB15_3
5064 ; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.end
5065 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
5066 ; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v8, v4
5067 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
5068 ;
5069 ; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
5070 ; GFX908: ; %bb.0:
5071 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5072 ; GFX908-NEXT: v_add_u32_e32 v4, 0x200, v4
5073 ; GFX908-NEXT: v_and_b32_e32 v8, -4, v4
5074 ; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
5075 ; GFX908-NEXT: v_lshlrev_b32_e32 v7, 3, v4
5076 ; GFX908-NEXT: s_mov_b32 s4, 0xffff
5077 ; GFX908-NEXT: v_lshlrev_b32_e64 v4, v7, s4
5078 ; GFX908-NEXT: v_not_b32_e32 v9, v4
5079 ; GFX908-NEXT: s_mov_b64 s[6:7], exec
5080 ; GFX908-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
5081 ; GFX908-NEXT: v_readfirstlane_b32 s8, v0
5082 ; GFX908-NEXT: v_readfirstlane_b32 s9, v1
5083 ; GFX908-NEXT: v_readfirstlane_b32 s10, v2
5084 ; GFX908-NEXT: v_readfirstlane_b32 s11, v3
5085 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
5086 ; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
5087 ; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
5088 ; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
5089 ; GFX908-NEXT: s_nop 0
5090 ; GFX908-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
5091 ; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
5092 ; GFX908-NEXT: s_cbranch_execnz .LBB15_1
5093 ; GFX908-NEXT: ; %bb.2:
5094 ; GFX908-NEXT: s_mov_b64 exec, s[6:7]
5095 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
5096 ; GFX908-NEXT: v_lshlrev_b32_e32 v10, 16, v5
5097 ; GFX908-NEXT: s_movk_i32 s14, 0x7fff
5098 ; GFX908-NEXT: .LBB15_3: ; %atomicrmw.start
5099 ; GFX908-NEXT: ; =>This Loop Header: Depth=1
5100 ; GFX908-NEXT: ; Child Loop BB15_4 Depth 2
5101 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5102 ; GFX908-NEXT: v_lshrrev_b32_sdwa v4, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
5103 ; GFX908-NEXT: v_min_f32_e32 v4, v4, v10
5104 ; GFX908-NEXT: v_bfe_u32 v5, v4, 16, 1
5105 ; GFX908-NEXT: v_add3_u32 v5, v5, v4, s14
5106 ; GFX908-NEXT: v_or_b32_e32 v11, 0x400000, v4
5107 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
5108 ; GFX908-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc
5109 ; GFX908-NEXT: v_lshlrev_b32_sdwa v4, v7, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
5110 ; GFX908-NEXT: v_and_or_b32 v5, v6, v9, v4
5111 ; GFX908-NEXT: v_mov_b32_e32 v4, v5
5112 ; GFX908-NEXT: s_mov_b64 s[12:13], exec
5113 ; GFX908-NEXT: v_mov_b32_e32 v5, v6
5114 ; GFX908-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
5115 ; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
5116 ; GFX908-NEXT: v_readfirstlane_b32 s8, v0
5117 ; GFX908-NEXT: v_readfirstlane_b32 s9, v1
5118 ; GFX908-NEXT: v_readfirstlane_b32 s10, v2
5119 ; GFX908-NEXT: v_readfirstlane_b32 s11, v3
5120 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
5121 ; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
5122 ; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
5123 ; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
5124 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5125 ; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
5126 ; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
5127 ; GFX908-NEXT: s_cbranch_execnz .LBB15_4
5128 ; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
5129 ; GFX908-NEXT: s_mov_b64 exec, s[12:13]
5130 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5131 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
5132 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
5133 ; GFX908-NEXT: v_mov_b32_e32 v6, v4
5134 ; GFX908-NEXT: buffer_wbinvl1
5135 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
5136 ; GFX908-NEXT: s_cbranch_execnz .LBB15_3
5137 ; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
5138 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
5139 ; GFX908-NEXT: v_lshrrev_b32_e32 v0, v7, v4
5140 ; GFX908-NEXT: s_setpc_b64 s[30:31]
5141 ;
5142 ; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
5143 ; GFX8: ; %bb.0:
5144 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5145 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x200, v4
5146 ; GFX8-NEXT: v_and_b32_e32 v8, -4, v4
5147 ; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
5148 ; GFX8-NEXT: v_lshlrev_b32_e32 v7, 3, v4
5149 ; GFX8-NEXT: s_mov_b32 s4, 0xffff
5150 ; GFX8-NEXT: v_lshlrev_b32_e64 v4, v7, s4
5151 ; GFX8-NEXT: v_not_b32_e32 v9, v4
5152 ; GFX8-NEXT: s_mov_b64 s[6:7], exec
5153 ; GFX8-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
5154 ; GFX8-NEXT: v_readfirstlane_b32 s8, v0
5155 ; GFX8-NEXT: v_readfirstlane_b32 s9, v1
5156 ; GFX8-NEXT: v_readfirstlane_b32 s10, v2
5157 ; GFX8-NEXT: v_readfirstlane_b32 s11, v3
5158 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
5159 ; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
5160 ; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
5161 ; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
5162 ; GFX8-NEXT: s_nop 0
5163 ; GFX8-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
5164 ; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
5165 ; GFX8-NEXT: s_cbranch_execnz .LBB15_1
5166 ; GFX8-NEXT: ; %bb.2:
5167 ; GFX8-NEXT: s_mov_b64 exec, s[6:7]
5168 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
5169 ; GFX8-NEXT: v_lshlrev_b32_e32 v10, 16, v5
5170 ; GFX8-NEXT: .LBB15_3: ; %atomicrmw.start
5171 ; GFX8-NEXT: ; =>This Loop Header: Depth=1
5172 ; GFX8-NEXT: ; Child Loop BB15_4 Depth 2
5173 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5174 ; GFX8-NEXT: v_lshrrev_b32_sdwa v4, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
5175 ; GFX8-NEXT: v_min_f32_e32 v4, v4, v10
5176 ; GFX8-NEXT: v_bfe_u32 v5, v4, 16, 1
5177 ; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v4
5178 ; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
5179 ; GFX8-NEXT: v_or_b32_e32 v11, 0x400000, v4
5180 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
5181 ; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc
5182 ; GFX8-NEXT: v_lshlrev_b32_sdwa v4, v7, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
5183 ; GFX8-NEXT: v_and_b32_e32 v5, v6, v9
5184 ; GFX8-NEXT: v_or_b32_e32 v5, v5, v4
5185 ; GFX8-NEXT: v_mov_b32_e32 v4, v5
5186 ; GFX8-NEXT: s_mov_b64 s[12:13], exec
5187 ; GFX8-NEXT: v_mov_b32_e32 v5, v6
5188 ; GFX8-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
5189 ; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
5190 ; GFX8-NEXT: v_readfirstlane_b32 s8, v0
5191 ; GFX8-NEXT: v_readfirstlane_b32 s9, v1
5192 ; GFX8-NEXT: v_readfirstlane_b32 s10, v2
5193 ; GFX8-NEXT: v_readfirstlane_b32 s11, v3
5194 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
5195 ; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
5196 ; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
5197 ; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
5198 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5199 ; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
5200 ; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
5201 ; GFX8-NEXT: s_cbranch_execnz .LBB15_4
5202 ; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
5203 ; GFX8-NEXT: s_mov_b64 exec, s[12:13]
5204 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5205 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
5206 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
5207 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
5208 ; GFX8-NEXT: buffer_wbinvl1
5209 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
5210 ; GFX8-NEXT: s_cbranch_execnz .LBB15_3
5211 ; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
5212 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
5213 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, v7, v4
5214 ; GFX8-NEXT: s_setpc_b64 s[30:31]
5215 ;
5216 ; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
5217 ; GFX7: ; %bb.0:
5218 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5219 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x200, v4
5220 ; GFX7-NEXT: v_and_b32_e32 v8, -4, v4
5221 ; GFX7-NEXT: v_and_b32_e32 v4, 3, v4
5222 ; GFX7-NEXT: v_lshlrev_b32_e32 v7, 3, v4
5223 ; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v7
5224 ; GFX7-NEXT: v_not_b32_e32 v9, v4
5225 ; GFX7-NEXT: s_mov_b64 s[6:7], exec
5226 ; GFX7-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
5227 ; GFX7-NEXT: v_readfirstlane_b32 s8, v0
5228 ; GFX7-NEXT: v_readfirstlane_b32 s9, v1
5229 ; GFX7-NEXT: v_readfirstlane_b32 s10, v2
5230 ; GFX7-NEXT: v_readfirstlane_b32 s11, v3
5231 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
5232 ; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
5233 ; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
5234 ; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
5235 ; GFX7-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
5236 ; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
5237 ; GFX7-NEXT: s_cbranch_execnz .LBB15_1
5238 ; GFX7-NEXT: ; %bb.2:
5239 ; GFX7-NEXT: s_mov_b64 exec, s[6:7]
5240 ; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v5
5241 ; GFX7-NEXT: s_mov_b64 s[6:7], 0
5242 ; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v4
5243 ; GFX7-NEXT: .LBB15_3: ; %atomicrmw.start
5244 ; GFX7-NEXT: ; =>This Loop Header: Depth=1
5245 ; GFX7-NEXT: ; Child Loop BB15_4 Depth 2
5246 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5247 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, v7, v6
5248 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
5249 ; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
5250 ; GFX7-NEXT: v_min_f32_e32 v4, v4, v10
5251 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
5252 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, v7, v4
5253 ; GFX7-NEXT: v_and_b32_e32 v5, v6, v9
5254 ; GFX7-NEXT: v_or_b32_e32 v5, v5, v4
5255 ; GFX7-NEXT: v_mov_b32_e32 v4, v5
5256 ; GFX7-NEXT: s_mov_b64 s[12:13], exec
5257 ; GFX7-NEXT: v_mov_b32_e32 v5, v6
5258 ; GFX7-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
5259 ; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
5260 ; GFX7-NEXT: v_readfirstlane_b32 s8, v0
5261 ; GFX7-NEXT: v_readfirstlane_b32 s9, v1
5262 ; GFX7-NEXT: v_readfirstlane_b32 s10, v2
5263 ; GFX7-NEXT: v_readfirstlane_b32 s11, v3
5264 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
5265 ; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
5266 ; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
5267 ; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
5268 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5269 ; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
5270 ; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
5271 ; GFX7-NEXT: s_cbranch_execnz .LBB15_4
5272 ; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
5273 ; GFX7-NEXT: s_mov_b64 exec, s[12:13]
5274 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5275 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
5276 ; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
5277 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
5278 ; GFX7-NEXT: buffer_wbinvl1
5279 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
5280 ; GFX7-NEXT: s_cbranch_execnz .LBB15_3
5281 ; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end
5282 ; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
5283 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, v7, v4
5284 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
5285 ; GFX7-NEXT: s_setpc_b64 s[30:31]
5286 ;
5287 ; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
5288 ; GFX6: ; %bb.0:
5289 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5290 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x200, v4
5291 ; GFX6-NEXT: v_and_b32_e32 v8, -4, v4
5292 ; GFX6-NEXT: v_and_b32_e32 v4, 3, v4
5293 ; GFX6-NEXT: v_lshlrev_b32_e32 v7, 3, v4
5294 ; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v7
5295 ; GFX6-NEXT: v_not_b32_e32 v9, v4
5296 ; GFX6-NEXT: s_mov_b64 s[6:7], exec
5297 ; GFX6-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
5298 ; GFX6-NEXT: v_readfirstlane_b32 s8, v0
5299 ; GFX6-NEXT: v_readfirstlane_b32 s9, v1
5300 ; GFX6-NEXT: v_readfirstlane_b32 s10, v2
5301 ; GFX6-NEXT: v_readfirstlane_b32 s11, v3
5302 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
5303 ; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
5304 ; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
5305 ; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
5306 ; GFX6-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
5307 ; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
5308 ; GFX6-NEXT: s_cbranch_execnz .LBB15_1
5309 ; GFX6-NEXT: ; %bb.2:
5310 ; GFX6-NEXT: s_mov_b64 exec, s[6:7]
5311 ; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v5
5312 ; GFX6-NEXT: s_mov_b64 s[6:7], 0
5313 ; GFX6-NEXT: v_and_b32_e32 v10, 0xffff0000, v4
5314 ; GFX6-NEXT: .LBB15_3: ; %atomicrmw.start
5315 ; GFX6-NEXT: ; =>This Loop Header: Depth=1
5316 ; GFX6-NEXT: ; Child Loop BB15_4 Depth 2
5317 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
5318 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, v7, v6
5319 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
5320 ; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
5321 ; GFX6-NEXT: v_min_f32_e32 v4, v4, v10
5322 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
5323 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, v7, v4
5324 ; GFX6-NEXT: v_and_b32_e32 v5, v6, v9
5325 ; GFX6-NEXT: v_or_b32_e32 v5, v5, v4
5326 ; GFX6-NEXT: v_mov_b32_e32 v4, v5
5327 ; GFX6-NEXT: s_mov_b64 s[12:13], exec
5328 ; GFX6-NEXT: v_mov_b32_e32 v5, v6
5329 ; GFX6-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
5330 ; GFX6-NEXT: ; => This Inner Loop Header: Depth=2
5331 ; GFX6-NEXT: v_readfirstlane_b32 s8, v0
5332 ; GFX6-NEXT: v_readfirstlane_b32 s9, v1
5333 ; GFX6-NEXT: v_readfirstlane_b32 s10, v2
5334 ; GFX6-NEXT: v_readfirstlane_b32 s11, v3
5335 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
5336 ; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
5337 ; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
5338 ; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
5339 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
5340 ; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
5341 ; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
5342 ; GFX6-NEXT: s_cbranch_execnz .LBB15_4
5343 ; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
5344 ; GFX6-NEXT: s_mov_b64 exec, s[12:13]
5345 ; GFX6-NEXT: s_waitcnt vmcnt(0)
5346 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
5347 ; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
5348 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
5349 ; GFX6-NEXT: buffer_wbinvl1
5350 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
5351 ; GFX6-NEXT: s_cbranch_execnz .LBB15_3
5352 ; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end
5353 ; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
5354 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, v7, v4
5355 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
5356 ; GFX6-NEXT: s_waitcnt expcnt(0)
5357 ; GFX6-NEXT: s_setpc_b64 s[30:31]
5358 %gep = getelementptr bfloat, ptr addrspace(7) %ptr, i32 256
5359 %result = atomicrmw fmin ptr addrspace(7) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
5360 ret bfloat %result
5361 }
5363 ; --------------------------------------------------------------------
5364 ; <2 x half>
5365 ; --------------------------------------------------------------------
5367 define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, <2 x half> %val) #0 {
5368 ; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
5369 ; GFX12: ; %bb.0:
5370 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
5371 ; GFX12-NEXT: s_wait_expcnt 0x0
5372 ; GFX12-NEXT: s_wait_samplecnt 0x0
5373 ; GFX12-NEXT: s_wait_bvhcnt 0x0
5374 ; GFX12-NEXT: s_wait_kmcnt 0x0
5375 ; GFX12-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
5376 ; GFX12-NEXT: s_add_co_i32 s4, s16, 0x400
5377 ; GFX12-NEXT: s_wait_alu 0xfffe
5378 ; GFX12-NEXT: v_mov_b32_e32 v3, s4
5379 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
5380 ; GFX12-NEXT: v_pk_max_num_f16 v2, v1, v1
5381 ; GFX12-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen offset:1024
5382 ; GFX12-NEXT: s_mov_b32 s4, 0
5383 ; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start
5384 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
5385 ; GFX12-NEXT: s_wait_loadcnt 0x0
5386 ; GFX12-NEXT: v_mov_b32_e32 v5, v0
5387 ; GFX12-NEXT: s_wait_storecnt 0x0
5388 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5389 ; GFX12-NEXT: v_pk_max_num_f16 v0, v5, v5
5390 ; GFX12-NEXT: v_pk_min_num_f16 v4, v0, v2
5391 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
5392 ; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
5393 ; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], null offen th:TH_ATOMIC_RETURN
5394 ; GFX12-NEXT: s_wait_loadcnt 0x0
5395 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
5396 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
5397 ; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
5398 ; GFX12-NEXT: s_wait_alu 0xfffe
5399 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
5400 ; GFX12-NEXT: s_cbranch_execnz .LBB16_1
5401 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
5402 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
5403 ; GFX12-NEXT: s_setpc_b64 s[30:31]
5404 ;
5405 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
5406 ; GFX940: ; %bb.0:
5407 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5408 ; GFX940-NEXT: v_mov_b32_e32 v1, v0
5409 ; GFX940-NEXT: v_mov_b32_e32 v0, s16
5410 ; GFX940-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
5411 ; GFX940-NEXT: s_add_i32 s6, s16, 0x400
5412 ; GFX940-NEXT: s_mov_b64 s[4:5], 0
5413 ; GFX940-NEXT: v_pk_max_f16 v2, v1, v1
5414 ; GFX940-NEXT: v_mov_b32_e32 v3, s6
5415 ; GFX940-NEXT: .LBB16_1: ; %atomicrmw.start
5416 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
5417 ; GFX940-NEXT: s_waitcnt vmcnt(0)
5418 ; GFX940-NEXT: v_mov_b32_e32 v5, v0
5419 ; GFX940-NEXT: v_pk_max_f16 v0, v5, v5
5420 ; GFX940-NEXT: buffer_wbl2 sc1
5421 ; GFX940-NEXT: v_pk_min_f16 v4, v0, v2
5422 ; GFX940-NEXT: s_nop 0
5423 ; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
5424 ; GFX940-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen sc0
5425 ; GFX940-NEXT: s_waitcnt vmcnt(0)
5426 ; GFX940-NEXT: buffer_inv sc1
5427 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
5428 ; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5429 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
5430 ; GFX940-NEXT: s_cbranch_execnz .LBB16_1
5431 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
5432 ; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
5433 ; GFX940-NEXT: s_setpc_b64 s[30:31]
5434 ;
5435 ; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
5436 ; GFX11: ; %bb.0:
5437 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5438 ; GFX11-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
5439 ; GFX11-NEXT: s_add_i32 s4, s16, 0x400
5440 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
5441 ; GFX11-NEXT: v_mov_b32_e32 v3, s4
5442 ; GFX11-NEXT: v_pk_max_f16 v2, v1, v1
5443 ; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
5444 ; GFX11-NEXT: s_mov_b32 s4, 0
5445 ; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start
5446 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
5447 ; GFX11-NEXT: s_waitcnt vmcnt(0)
5448 ; GFX11-NEXT: v_mov_b32_e32 v5, v0
5449 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
5450 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5451 ; GFX11-NEXT: v_pk_max_f16 v0, v5, v5
5452 ; GFX11-NEXT: v_pk_min_f16 v4, v0, v2
5453 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
5454 ; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
5455 ; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen glc
5456 ; GFX11-NEXT: s_waitcnt vmcnt(0)
5457 ; GFX11-NEXT: buffer_gl1_inv
5458 ; GFX11-NEXT: buffer_gl0_inv
5459 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
5460 ; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
5461 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
5462 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
5463 ; GFX11-NEXT: s_cbranch_execnz .LBB16_1
5464 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
5465 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
5466 ; GFX11-NEXT: s_setpc_b64 s[30:31]
5467 ;
5468 ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
5469 ; GFX10: ; %bb.0:
5470 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5471 ; GFX10-NEXT: v_mov_b32_e32 v1, v0
5472 ; GFX10-NEXT: v_mov_b32_e32 v0, s20
5473 ; GFX10-NEXT: s_add_i32 s4, s20, 0x400
5474 ; GFX10-NEXT: v_mov_b32_e32 v3, s4
5475 ; GFX10-NEXT: v_pk_max_f16 v2, v1, v1
5476 ; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
5477 ; GFX10-NEXT: s_mov_b32 s4, 0
5478 ; GFX10-NEXT: .LBB16_1: ; %atomicrmw.start
5479 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
5480 ; GFX10-NEXT: s_waitcnt vmcnt(0)
5481 ; GFX10-NEXT: v_mov_b32_e32 v5, v0
5482 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
5483 ; GFX10-NEXT: v_pk_max_f16 v0, v5, v5
5484 ; GFX10-NEXT: v_pk_min_f16 v4, v0, v2
5485 ; GFX10-NEXT: v_mov_b32_e32 v0, v4
5486 ; GFX10-NEXT: v_mov_b32_e32 v1, v5
5487 ; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
5488 ; GFX10-NEXT: s_waitcnt vmcnt(0)
5489 ; GFX10-NEXT: buffer_gl1_inv
5490 ; GFX10-NEXT: buffer_gl0_inv
5491 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
5492 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
5493 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
5494 ; GFX10-NEXT: s_cbranch_execnz .LBB16_1
5495 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
5496 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
5497 ; GFX10-NEXT: s_setpc_b64 s[30:31]
5498 ;
5499 ; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
5500 ; GFX90A: ; %bb.0:
5501 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5502 ; GFX90A-NEXT: v_mov_b32_e32 v1, v0
5503 ; GFX90A-NEXT: v_mov_b32_e32 v0, s20
5504 ; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
5505 ; GFX90A-NEXT: s_add_i32 s6, s20, 0x400
5506 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
5507 ; GFX90A-NEXT: v_pk_max_f16 v2, v1, v1
5508 ; GFX90A-NEXT: v_mov_b32_e32 v3, s6
5509 ; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
5510 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
5511 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
5512 ; GFX90A-NEXT: v_mov_b32_e32 v5, v0
5513 ; GFX90A-NEXT: v_pk_max_f16 v0, v5, v5
5514 ; GFX90A-NEXT: v_pk_min_f16 v4, v0, v2
5515 ; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
5516 ; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
5517 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
5518 ; GFX90A-NEXT: buffer_wbinvl1
5519 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
5520 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5521 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
5522 ; GFX90A-NEXT: s_cbranch_execnz .LBB16_1
5523 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
5524 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
5525 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
5526 ;
5527 ; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
5528 ; GFX908: ; %bb.0:
5529 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5530 ; GFX908-NEXT: v_mov_b32_e32 v1, v0
5531 ; GFX908-NEXT: v_mov_b32_e32 v0, s20
5532 ; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
5533 ; GFX908-NEXT: s_add_i32 s6, s20, 0x400
5534 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
5535 ; GFX908-NEXT: v_pk_max_f16 v2, v1, v1
5536 ; GFX908-NEXT: v_mov_b32_e32 v3, s6
5537 ; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
5538 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
5539 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5540 ; GFX908-NEXT: v_mov_b32_e32 v5, v0
5541 ; GFX908-NEXT: v_pk_max_f16 v0, v5, v5
5542 ; GFX908-NEXT: v_pk_min_f16 v4, v0, v2
5543 ; GFX908-NEXT: v_mov_b32_e32 v0, v4
5544 ; GFX908-NEXT: v_mov_b32_e32 v1, v5
5545 ; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
5546 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5547 ; GFX908-NEXT: buffer_wbinvl1
5548 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
5549 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5550 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
5551 ; GFX908-NEXT: s_cbranch_execnz .LBB16_1
5552 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
5553 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
5554 ; GFX908-NEXT: s_setpc_b64 s[30:31]
5555 ;
5556 ; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
5557 ; GFX8: ; %bb.0:
5558 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5559 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
5560 ; GFX8-NEXT: v_mov_b32_e32 v0, s20
5561 ; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
5562 ; GFX8-NEXT: s_add_i32 s6, s20, 0x400
5563 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
5564 ; GFX8-NEXT: v_max_f16_sdwa v2, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
5565 ; GFX8-NEXT: v_max_f16_e32 v3, v1, v1
5566 ; GFX8-NEXT: v_mov_b32_e32 v4, s6
5567 ; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start
5568 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5569 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5570 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
5571 ; GFX8-NEXT: v_max_f16_sdwa v0, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
5572 ; GFX8-NEXT: v_max_f16_e32 v1, v6, v6
5573 ; GFX8-NEXT: v_min_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
5574 ; GFX8-NEXT: v_min_f16_e32 v1, v1, v3
5575 ; GFX8-NEXT: v_or_b32_e32 v5, v1, v0
5576 ; GFX8-NEXT: v_mov_b32_e32 v0, v5
5577 ; GFX8-NEXT: v_mov_b32_e32 v1, v6
5578 ; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen glc
5579 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5580 ; GFX8-NEXT: buffer_wbinvl1
5581 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
5582 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5583 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
5584 ; GFX8-NEXT: s_cbranch_execnz .LBB16_1
5585 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5586 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
5587 ; GFX8-NEXT: s_setpc_b64 s[30:31]
5588 ;
5589 ; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
5590 ; GFX7: ; %bb.0:
5591 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5592 ; GFX7-NEXT: v_mov_b32_e32 v2, s20
5593 ; GFX7-NEXT: buffer_load_dword v3, v2, s[16:19], 0 offen offset:1024
5594 ; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
5595 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v0
5596 ; GFX7-NEXT: s_add_i32 s6, s20, 0x400
5597 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
5598 ; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v1
5599 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5600 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v3
5601 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v3
5602 ; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
5603 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v4
5604 ; GFX7-NEXT: v_mov_b32_e32 v4, s6
5605 ; GFX7-NEXT: .LBB16_1: ; %atomicrmw.start
5606 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5607 ; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
5608 ; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
5609 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v1
5610 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v0
5611 ; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
5612 ; GFX7-NEXT: v_min_f32_e32 v5, v5, v2
5613 ; GFX7-NEXT: v_min_f32_e32 v6, v6, v3
5614 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
5615 ; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v6
5616 ; GFX7-NEXT: v_or_b32_e32 v6, v0, v1
5617 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v5
5618 ; GFX7-NEXT: v_or_b32_e32 v5, v7, v0
5619 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
5620 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
5621 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v4, s[16:19], 0 offen glc
5622 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5623 ; GFX7-NEXT: buffer_wbinvl1
5624 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v7
5625 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v7
5626 ; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
5627 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
5628 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5629 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
5630 ; GFX7-NEXT: s_cbranch_execnz .LBB16_1
5631 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5632 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
5633 ; GFX7-NEXT: s_setpc_b64 s[30:31]
5634 ;
5635 ; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
5636 ; GFX6: ; %bb.0:
5637 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5638 ; GFX6-NEXT: v_mov_b32_e32 v2, s20
5639 ; GFX6-NEXT: buffer_load_dword v3, v2, s[16:19], 0 offen offset:1024
5640 ; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
5641 ; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v0
5642 ; GFX6-NEXT: s_add_i32 s6, s20, 0x400
5643 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
5644 ; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v1
5645 ; GFX6-NEXT: s_waitcnt vmcnt(0)
5646 ; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v3
5647 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v3
5648 ; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
5649 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v4
5650 ; GFX6-NEXT: v_mov_b32_e32 v4, s6
5651 ; GFX6-NEXT: .LBB16_1: ; %atomicrmw.start
5652 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
5653 ; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
5654 ; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
5655 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v1
5656 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v0
5657 ; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
5658 ; GFX6-NEXT: v_min_f32_e32 v5, v5, v2
5659 ; GFX6-NEXT: v_min_f32_e32 v6, v6, v3
5660 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
5661 ; GFX6-NEXT: s_waitcnt expcnt(0)
5662 ; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v6
5663 ; GFX6-NEXT: v_or_b32_e32 v6, v0, v1
5664 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v5
5665 ; GFX6-NEXT: v_or_b32_e32 v5, v7, v0
5666 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
5667 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
5668 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v4, s[16:19], 0 offen glc
5669 ; GFX6-NEXT: s_waitcnt vmcnt(0)
5670 ; GFX6-NEXT: buffer_wbinvl1
5671 ; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v7
5672 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v7
5673 ; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
5674 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
5675 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5676 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
5677 ; GFX6-NEXT: s_cbranch_execnz .LBB16_1
5678 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
5679 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
5680 ; GFX6-NEXT: s_waitcnt expcnt(0)
5681 ; GFX6-NEXT: s_setpc_b64 s[30:31]
5682 %gep = getelementptr <2 x half>, ptr addrspace(7) %ptr, i32 256
5683 %result = atomicrmw fmin ptr addrspace(7) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
5684 ret <2 x half> %result
5685 }
5687 define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, <2 x half> %val) #0 {
5688 ; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
5689 ; GFX12: ; %bb.0:
5690 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
5691 ; GFX12-NEXT: s_wait_expcnt 0x0
5692 ; GFX12-NEXT: s_wait_samplecnt 0x0
5693 ; GFX12-NEXT: s_wait_bvhcnt 0x0
5694 ; GFX12-NEXT: s_wait_kmcnt 0x0
5695 ; GFX12-NEXT: v_mov_b32_e32 v1, s16
5696 ; GFX12-NEXT: s_add_co_i32 s4, s16, 0x400
5697 ; GFX12-NEXT: v_pk_max_num_f16 v2, v0, v0
5698 ; GFX12-NEXT: s_wait_alu 0xfffe
5699 ; GFX12-NEXT: v_mov_b32_e32 v3, s4
5700 ; GFX12-NEXT: s_mov_b32 s4, 0
5701 ; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null offen offset:1024
5702 ; GFX12-NEXT: .LBB17_1: ; %atomicrmw.start
5703 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
5704 ; GFX12-NEXT: s_wait_loadcnt 0x0
5705 ; GFX12-NEXT: v_pk_max_num_f16 v0, v1, v1
5706 ; GFX12-NEXT: s_wait_storecnt 0x0
5707 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5708 ; GFX12-NEXT: v_pk_min_num_f16 v0, v0, v2
5709 ; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
5710 ; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen th:TH_ATOMIC_RETURN
5711 ; GFX12-NEXT: s_wait_loadcnt 0x0
5712 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
5713 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
5714 ; GFX12-NEXT: v_mov_b32_e32 v1, v4
5715 ; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
5716 ; GFX12-NEXT: s_wait_alu 0xfffe
5717 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
5718 ; GFX12-NEXT: s_cbranch_execnz .LBB17_1
5719 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
5720 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
5721 ; GFX12-NEXT: s_setpc_b64 s[30:31]
5722 ;
5723 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
5724 ; GFX940: ; %bb.0:
5725 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5726 ; GFX940-NEXT: v_mov_b32_e32 v1, s16
5727 ; GFX940-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen offset:1024
5728 ; GFX940-NEXT: s_add_i32 s6, s16, 0x400
5729 ; GFX940-NEXT: s_mov_b64 s[4:5], 0
5730 ; GFX940-NEXT: v_pk_max_f16 v2, v0, v0
5731 ; GFX940-NEXT: v_mov_b32_e32 v3, s6
5732 ; GFX940-NEXT: .LBB17_1: ; %atomicrmw.start
5733 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
5734 ; GFX940-NEXT: s_waitcnt vmcnt(0)
5735 ; GFX940-NEXT: v_pk_max_f16 v0, v1, v1
5736 ; GFX940-NEXT: buffer_wbl2 sc1
5737 ; GFX940-NEXT: v_pk_min_f16 v0, v0, v2
5738 ; GFX940-NEXT: s_nop 0
5739 ; GFX940-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
5740 ; GFX940-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[0:3], 0 offen sc0
5741 ; GFX940-NEXT: s_waitcnt vmcnt(0)
5742 ; GFX940-NEXT: buffer_inv sc1
5743 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
5744 ; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5745 ; GFX940-NEXT: v_mov_b32_e32 v1, v4
5746 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
5747 ; GFX940-NEXT: s_cbranch_execnz .LBB17_1
5748 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
5749 ; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
5750 ; GFX940-NEXT: s_setpc_b64 s[30:31]
5751 ;
5752 ; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
5753 ; GFX11: ; %bb.0:
5754 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5755 ; GFX11-NEXT: v_mov_b32_e32 v1, s16
5756 ; GFX11-NEXT: s_add_i32 s4, s16, 0x400
5757 ; GFX11-NEXT: v_pk_max_f16 v2, v0, v0
5758 ; GFX11-NEXT: v_mov_b32_e32 v3, s4
5759 ; GFX11-NEXT: s_mov_b32 s4, 0
5760 ; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:1024
5761 ; GFX11-NEXT: .LBB17_1: ; %atomicrmw.start
5762 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
5763 ; GFX11-NEXT: s_waitcnt vmcnt(0)
5764 ; GFX11-NEXT: v_pk_max_f16 v0, v1, v1
5765 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
5766 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
5767 ; GFX11-NEXT: v_pk_min_f16 v0, v0, v2
5768 ; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
5769 ; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen glc
5770 ; GFX11-NEXT: s_waitcnt vmcnt(0)
5771 ; GFX11-NEXT: buffer_gl1_inv
5772 ; GFX11-NEXT: buffer_gl0_inv
5773 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
5774 ; GFX11-NEXT: v_mov_b32_e32 v1, v4
5775 ; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
5776 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
5777 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
5778 ; GFX11-NEXT: s_cbranch_execnz .LBB17_1
5779 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
5780 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
5781 ; GFX11-NEXT: s_setpc_b64 s[30:31]
5782 ;
5783 ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
5784 ; GFX10: ; %bb.0:
5785 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5786 ; GFX10-NEXT: v_mov_b32_e32 v1, s20
5787 ; GFX10-NEXT: s_add_i32 s4, s20, 0x400
5788 ; GFX10-NEXT: v_pk_max_f16 v2, v0, v0
5789 ; GFX10-NEXT: v_mov_b32_e32 v3, s4
5790 ; GFX10-NEXT: s_mov_b32 s4, 0
5791 ; GFX10-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
5792 ; GFX10-NEXT: .LBB17_1: ; %atomicrmw.start
5793 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
5794 ; GFX10-NEXT: s_waitcnt vmcnt(0)
5795 ; GFX10-NEXT: v_pk_max_f16 v0, v1, v1
5796 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
5797 ; GFX10-NEXT: v_pk_min_f16 v0, v0, v2
5798 ; GFX10-NEXT: v_mov_b32_e32 v5, v1
5799 ; GFX10-NEXT: v_mov_b32_e32 v4, v0
5800 ; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen glc
5801 ; GFX10-NEXT: s_waitcnt vmcnt(0)
5802 ; GFX10-NEXT: buffer_gl1_inv
5803 ; GFX10-NEXT: buffer_gl0_inv
5804 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
5805 ; GFX10-NEXT: v_mov_b32_e32 v1, v4
5806 ; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
5807 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
5808 ; GFX10-NEXT: s_cbranch_execnz .LBB17_1
5809 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
5810 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
5811 ; GFX10-NEXT: s_setpc_b64 s[30:31]
5812 ;
5813 ; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
5814 ; GFX90A: ; %bb.0:
5815 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5816 ; GFX90A-NEXT: v_mov_b32_e32 v1, s20
5817 ; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
5818 ; GFX90A-NEXT: s_add_i32 s6, s20, 0x400
5819 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
5820 ; GFX90A-NEXT: v_pk_max_f16 v2, v0, v0
5821 ; GFX90A-NEXT: v_mov_b32_e32 v3, s6
5822 ; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
5823 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
5824 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
5825 ; GFX90A-NEXT: v_pk_max_f16 v0, v1, v1
5826 ; GFX90A-NEXT: v_pk_min_f16 v0, v0, v2
5827 ; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
5828 ; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen glc
5829 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
5830 ; GFX90A-NEXT: buffer_wbinvl1
5831 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
5832 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5833 ; GFX90A-NEXT: v_mov_b32_e32 v1, v4
5834 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
5835 ; GFX90A-NEXT: s_cbranch_execnz .LBB17_1
5836 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
5837 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
5838 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
5839 ;
5840 ; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
5841 ; GFX908: ; %bb.0:
5842 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5843 ; GFX908-NEXT: v_mov_b32_e32 v1, s20
5844 ; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
5845 ; GFX908-NEXT: s_add_i32 s6, s20, 0x400
5846 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
5847 ; GFX908-NEXT: v_pk_max_f16 v2, v0, v0
5848 ; GFX908-NEXT: v_mov_b32_e32 v3, s6
5849 ; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
5850 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
5851 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5852 ; GFX908-NEXT: v_pk_max_f16 v0, v1, v1
5853 ; GFX908-NEXT: v_pk_min_f16 v0, v0, v2
5854 ; GFX908-NEXT: v_mov_b32_e32 v5, v1
5855 ; GFX908-NEXT: v_mov_b32_e32 v4, v0
5856 ; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen glc
5857 ; GFX908-NEXT: s_waitcnt vmcnt(0)
5858 ; GFX908-NEXT: buffer_wbinvl1
5859 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
5860 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5861 ; GFX908-NEXT: v_mov_b32_e32 v1, v4
5862 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
5863 ; GFX908-NEXT: s_cbranch_execnz .LBB17_1
5864 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
5865 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
5866 ; GFX908-NEXT: s_setpc_b64 s[30:31]
5867 ;
5868 ; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
5869 ; GFX8: ; %bb.0:
5870 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5871 ; GFX8-NEXT: v_mov_b32_e32 v1, s20
5872 ; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
5873 ; GFX8-NEXT: s_add_i32 s6, s20, 0x400
5874 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
5875 ; GFX8-NEXT: v_max_f16_sdwa v2, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
5876 ; GFX8-NEXT: v_max_f16_e32 v3, v0, v0
5877 ; GFX8-NEXT: v_mov_b32_e32 v4, s6
5878 ; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
5879 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5880 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5881 ; GFX8-NEXT: v_max_f16_sdwa v0, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
5882 ; GFX8-NEXT: v_max_f16_e32 v5, v1, v1
5883 ; GFX8-NEXT: v_min_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
5884 ; GFX8-NEXT: v_min_f16_e32 v5, v5, v3
5885 ; GFX8-NEXT: v_or_b32_e32 v0, v5, v0
5886 ; GFX8-NEXT: v_mov_b32_e32 v6, v1
5887 ; GFX8-NEXT: v_mov_b32_e32 v5, v0
5888 ; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen glc
5889 ; GFX8-NEXT: s_waitcnt vmcnt(0)
5890 ; GFX8-NEXT: buffer_wbinvl1
5891 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v1
5892 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5893 ; GFX8-NEXT: v_mov_b32_e32 v1, v5
5894 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
5895 ; GFX8-NEXT: s_cbranch_execnz .LBB17_1
5896 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5897 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
5898 ; GFX8-NEXT: s_setpc_b64 s[30:31]
5899 ;
5900 ; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
5901 ; GFX7: ; %bb.0:
5902 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5903 ; GFX7-NEXT: v_mov_b32_e32 v2, s20
5904 ; GFX7-NEXT: buffer_load_dword v2, v2, s[16:19], 0 offen offset:1024
5905 ; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
5906 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v0
5907 ; GFX7-NEXT: s_add_i32 s6, s20, 0x400
5908 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
5909 ; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v1
5910 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5911 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
5912 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v2
5913 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v1
5914 ; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v5
5915 ; GFX7-NEXT: v_mov_b32_e32 v2, s6
5916 ; GFX7-NEXT: .LBB17_1: ; %atomicrmw.start
5917 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5918 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
5919 ; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
5920 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v4
5921 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v3
5922 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
5923 ; GFX7-NEXT: v_min_f32_e32 v5, v5, v0
5924 ; GFX7-NEXT: v_min_f32_e32 v6, v6, v1
5925 ; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v5
5926 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6
5927 ; GFX7-NEXT: v_or_b32_e32 v5, v3, v4
5928 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v7
5929 ; GFX7-NEXT: v_or_b32_e32 v4, v6, v3
5930 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
5931 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
5932 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v2, s[16:19], 0 offen glc
5933 ; GFX7-NEXT: s_waitcnt vmcnt(0)
5934 ; GFX7-NEXT: buffer_wbinvl1
5935 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
5936 ; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v6
5937 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v4
5938 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
5939 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5940 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
5941 ; GFX7-NEXT: s_cbranch_execnz .LBB17_1
5942 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5943 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
5944 ; GFX7-NEXT: s_setpc_b64 s[30:31]
5945 ;
5946 ; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
5947 ; GFX6: ; %bb.0:
5948 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5949 ; GFX6-NEXT: v_mov_b32_e32 v2, s20
5950 ; GFX6-NEXT: buffer_load_dword v2, v2, s[16:19], 0 offen offset:1024
5951 ; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
5952 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v0
5953 ; GFX6-NEXT: s_add_i32 s6, s20, 0x400
5954 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
5955 ; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v1
5956 ; GFX6-NEXT: s_waitcnt vmcnt(0)
5957 ; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v2
5958 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v2
5959 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v1
5960 ; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v5
5961 ; GFX6-NEXT: v_mov_b32_e32 v2, s6
5962 ; GFX6-NEXT: .LBB17_1: ; %atomicrmw.start
5963 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
5964 ; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
5965 ; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
5966 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v4
5967 ; GFX6-NEXT: s_waitcnt expcnt(0)
5968 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v3
5969 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
5970 ; GFX6-NEXT: v_min_f32_e32 v5, v5, v0
5971 ; GFX6-NEXT: v_min_f32_e32 v6, v6, v1
5972 ; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v5
5973 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
5974 ; GFX6-NEXT: v_or_b32_e32 v5, v3, v4
5975 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v7
5976 ; GFX6-NEXT: v_or_b32_e32 v4, v6, v3
5977 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
5978 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
5979 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v2, s[16:19], 0 offen glc
5980 ; GFX6-NEXT: s_waitcnt vmcnt(0)
5981 ; GFX6-NEXT: buffer_wbinvl1
5982 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
5983 ; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v6
5984 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v4
5985 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
5986 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5987 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
5988 ; GFX6-NEXT: s_cbranch_execnz .LBB17_1
5989 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
5990 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
5991 ; GFX6-NEXT: s_waitcnt expcnt(0)
5992 ; GFX6-NEXT: s_setpc_b64 s[30:31]
5993 %gep = getelementptr <2 x half>, ptr addrspace(7) %ptr, i32 256
5994 %unused = atomicrmw fmin ptr addrspace(7) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
5995 ret void
5996 }
5998 define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, <2 x half> %val) #0 {
5999 ; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
6000 ; GFX12: ; %bb.0:
6001 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
6002 ; GFX12-NEXT: s_wait_expcnt 0x0
6003 ; GFX12-NEXT: s_wait_samplecnt 0x0
6004 ; GFX12-NEXT: s_wait_bvhcnt 0x0
6005 ; GFX12-NEXT: s_wait_kmcnt 0x0
6006 ; GFX12-NEXT: v_add_nc_u32_e32 v7, 0x400, v4
6007 ; GFX12-NEXT: s_mov_b32 s1, exec_lo
6008 ; GFX12-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
6009 ; GFX12-NEXT: v_readfirstlane_b32 s4, v0
6010 ; GFX12-NEXT: v_readfirstlane_b32 s5, v1
6011 ; GFX12-NEXT: v_readfirstlane_b32 s6, v2
6012 ; GFX12-NEXT: v_readfirstlane_b32 s7, v3
6013 ; GFX12-NEXT: s_wait_alu 0xf1ff
6014 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
6015 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
6016 ; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
6017 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
6018 ; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
6019 ; GFX12-NEXT: s_wait_alu 0xfffe
6020 ; GFX12-NEXT: s_and_saveexec_b32 s0, s0
6021 ; GFX12-NEXT: s_wait_loadcnt 0x0
6022 ; GFX12-NEXT: buffer_load_b32 v6, v4, s[4:7], null offen offset:1024
6023 ; GFX12-NEXT: ; implicit-def: $vgpr4
6024 ; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
6025 ; GFX12-NEXT: s_cbranch_execnz .LBB18_1
6026 ; GFX12-NEXT: ; %bb.2:
6027 ; GFX12-NEXT: s_mov_b32 exec_lo, s1
6028 ; GFX12-NEXT: v_pk_max_num_f16 v8, v5, v5
6029 ; GFX12-NEXT: s_mov_b32 s1, 0
6030 ; GFX12-NEXT: .LBB18_3: ; %atomicrmw.start
6031 ; GFX12-NEXT: ; =>This Loop Header: Depth=1
6032 ; GFX12-NEXT: ; Child Loop BB18_4 Depth 2
6033 ; GFX12-NEXT: s_wait_loadcnt 0x0
6034 ; GFX12-NEXT: v_pk_max_num_f16 v4, v6, v6
6035 ; GFX12-NEXT: s_mov_b32 s2, exec_lo
6036 ; GFX12-NEXT: s_wait_storecnt 0x0
6037 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6038 ; GFX12-NEXT: v_pk_min_num_f16 v5, v4, v8
6039 ; GFX12-NEXT: v_mov_b32_e32 v4, v5
6040 ; GFX12-NEXT: v_mov_b32_e32 v5, v6
6041 ; GFX12-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
6042 ; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
6043 ; GFX12-NEXT: v_readfirstlane_b32 s4, v0
6044 ; GFX12-NEXT: v_readfirstlane_b32 s5, v1
6045 ; GFX12-NEXT: v_readfirstlane_b32 s6, v2
6046 ; GFX12-NEXT: v_readfirstlane_b32 s7, v3
6047 ; GFX12-NEXT: s_wait_alu 0xf1ff
6048 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
6049 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
6050 ; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
6051 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
6052 ; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
6053 ; GFX12-NEXT: s_wait_alu 0xfffe
6054 ; GFX12-NEXT: s_and_saveexec_b32 s0, s0
6055 ; GFX12-NEXT: s_wait_loadcnt 0x0
6056 ; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v7, s[4:7], null offen th:TH_ATOMIC_RETURN
6057 ; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
6058 ; GFX12-NEXT: s_cbranch_execnz .LBB18_4
6059 ; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
6060 ; GFX12-NEXT: s_mov_b32 exec_lo, s2
6061 ; GFX12-NEXT: s_wait_loadcnt 0x0
6062 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
6063 ; GFX12-NEXT: v_mov_b32_e32 v6, v4
6064 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
6065 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
6066 ; GFX12-NEXT: s_wait_alu 0xfffe
6067 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
6068 ; GFX12-NEXT: s_cbranch_execnz .LBB18_3
6069 ; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
6070 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
6071 ; GFX12-NEXT: v_mov_b32_e32 v0, v4
6072 ; GFX12-NEXT: s_setpc_b64 s[30:31]
6073 ;
6074 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
6075 ; GFX940: ; %bb.0:
6076 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6077 ; GFX940-NEXT: v_add_u32_e32 v8, 0x400, v4
6078 ; GFX940-NEXT: s_mov_b64 s[2:3], exec
6079 ; GFX940-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
6080 ; GFX940-NEXT: v_readfirstlane_b32 s4, v0
6081 ; GFX940-NEXT: v_readfirstlane_b32 s5, v1
6082 ; GFX940-NEXT: v_readfirstlane_b32 s6, v2
6083 ; GFX940-NEXT: v_readfirstlane_b32 s7, v3
6084 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
6085 ; GFX940-NEXT: s_nop 0
6086 ; GFX940-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
6087 ; GFX940-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
6088 ; GFX940-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
6089 ; GFX940-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
6090 ; GFX940-NEXT: ; implicit-def: $vgpr4
6091 ; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
6092 ; GFX940-NEXT: s_cbranch_execnz .LBB18_1
6093 ; GFX940-NEXT: ; %bb.2:
6094 ; GFX940-NEXT: s_mov_b64 exec, s[2:3]
6095 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
6096 ; GFX940-NEXT: v_pk_max_f16 v9, v5, v5
6097 ; GFX940-NEXT: .LBB18_3: ; %atomicrmw.start
6098 ; GFX940-NEXT: ; =>This Loop Header: Depth=1
6099 ; GFX940-NEXT: ; Child Loop BB18_4 Depth 2
6100 ; GFX940-NEXT: s_waitcnt vmcnt(0)
6101 ; GFX940-NEXT: v_pk_max_f16 v4, v7, v7
6102 ; GFX940-NEXT: s_mov_b64 s[8:9], exec
6103 ; GFX940-NEXT: v_pk_min_f16 v6, v4, v9
6104 ; GFX940-NEXT: buffer_wbl2 sc1
6105 ; GFX940-NEXT: v_mov_b64_e32 v[4:5], v[6:7]
6106 ; GFX940-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
6107 ; GFX940-NEXT: ; => This Inner Loop Header: Depth=2
6108 ; GFX940-NEXT: v_readfirstlane_b32 s4, v0
6109 ; GFX940-NEXT: v_readfirstlane_b32 s5, v1
6110 ; GFX940-NEXT: v_readfirstlane_b32 s6, v2
6111 ; GFX940-NEXT: v_readfirstlane_b32 s7, v3
6112 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
6113 ; GFX940-NEXT: s_nop 0
6114 ; GFX940-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
6115 ; GFX940-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
6116 ; GFX940-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
6117 ; GFX940-NEXT: s_waitcnt vmcnt(0)
6118 ; GFX940-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen sc0
6119 ; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
6120 ; GFX940-NEXT: s_cbranch_execnz .LBB18_4
6121 ; GFX940-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
6122 ; GFX940-NEXT: s_mov_b64 exec, s[8:9]
6123 ; GFX940-NEXT: s_waitcnt vmcnt(0)
6124 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
6125 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
6126 ; GFX940-NEXT: v_mov_b32_e32 v7, v4
6127 ; GFX940-NEXT: buffer_inv sc1
6128 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
6129 ; GFX940-NEXT: s_cbranch_execnz .LBB18_3
6130 ; GFX940-NEXT: ; %bb.6: ; %atomicrmw.end
6131 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
6132 ; GFX940-NEXT: v_mov_b32_e32 v0, v4
6133 ; GFX940-NEXT: s_setpc_b64 s[30:31]
6134 ;
6135 ; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
6136 ; GFX11: ; %bb.0:
6137 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6138 ; GFX11-NEXT: v_add_nc_u32_e32 v7, 0x400, v4
6139 ; GFX11-NEXT: s_mov_b32 s1, 0
6140 ; GFX11-NEXT: s_mov_b32 s2, exec_lo
6141 ; GFX11-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
6142 ; GFX11-NEXT: v_readfirstlane_b32 s4, v0
6143 ; GFX11-NEXT: v_readfirstlane_b32 s5, v1
6144 ; GFX11-NEXT: v_readfirstlane_b32 s6, v2
6145 ; GFX11-NEXT: v_readfirstlane_b32 s7, v3
6146 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
6147 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
6148 ; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
6149 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
6150 ; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
6151 ; GFX11-NEXT: s_and_saveexec_b32 s0, s0
6152 ; GFX11-NEXT: buffer_load_b32 v6, v4, s[4:7], 0 offen offset:1024
6153 ; GFX11-NEXT: ; implicit-def: $vgpr4
6154 ; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
6155 ; GFX11-NEXT: s_cbranch_execnz .LBB18_1
6156 ; GFX11-NEXT: ; %bb.2:
6157 ; GFX11-NEXT: s_mov_b32 exec_lo, s2
6158 ; GFX11-NEXT: v_pk_max_f16 v8, v5, v5
6159 ; GFX11-NEXT: .p2align 6
6160 ; GFX11-NEXT: .LBB18_3: ; %atomicrmw.start
6161 ; GFX11-NEXT: ; =>This Loop Header: Depth=1
6162 ; GFX11-NEXT: ; Child Loop BB18_4 Depth 2
6163 ; GFX11-NEXT: s_waitcnt vmcnt(0)
6164 ; GFX11-NEXT: v_pk_max_f16 v4, v6, v6
6165 ; GFX11-NEXT: s_mov_b32 s2, exec_lo
6166 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
6167 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6168 ; GFX11-NEXT: v_pk_min_f16 v5, v4, v8
6169 ; GFX11-NEXT: v_mov_b32_e32 v4, v5
6170 ; GFX11-NEXT: v_mov_b32_e32 v5, v6
6171 ; GFX11-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
6172 ; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
6173 ; GFX11-NEXT: v_readfirstlane_b32 s4, v0
6174 ; GFX11-NEXT: v_readfirstlane_b32 s5, v1
6175 ; GFX11-NEXT: v_readfirstlane_b32 s6, v2
6176 ; GFX11-NEXT: v_readfirstlane_b32 s7, v3
6177 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
6178 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
6179 ; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
6180 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
6181 ; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
6182 ; GFX11-NEXT: s_and_saveexec_b32 s0, s0
6183 ; GFX11-NEXT: s_waitcnt vmcnt(0)
6184 ; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v7, s[4:7], 0 offen glc
6185 ; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
6186 ; GFX11-NEXT: s_cbranch_execnz .LBB18_4
6187 ; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
6188 ; GFX11-NEXT: s_mov_b32 exec_lo, s2
6189 ; GFX11-NEXT: s_waitcnt vmcnt(0)
6190 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
6191 ; GFX11-NEXT: v_mov_b32_e32 v6, v4
6192 ; GFX11-NEXT: buffer_gl1_inv
6193 ; GFX11-NEXT: buffer_gl0_inv
6194 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
6195 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
6196 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
6197 ; GFX11-NEXT: s_cbranch_execnz .LBB18_3
6198 ; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
6199 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
6200 ; GFX11-NEXT: v_mov_b32_e32 v0, v4
6201 ; GFX11-NEXT: s_setpc_b64 s[30:31]
6202 ;
6203 ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
6204 ; GFX10: ; %bb.0:
6205 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6206 ; GFX10-NEXT: v_add_nc_u32_e32 v7, 0x400, v4
6207 ; GFX10-NEXT: s_mov_b32 s5, 0
6208 ; GFX10-NEXT: s_mov_b32 s6, exec_lo
6209 ; GFX10-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
6210 ; GFX10-NEXT: v_readfirstlane_b32 s8, v0
6211 ; GFX10-NEXT: v_readfirstlane_b32 s9, v1
6212 ; GFX10-NEXT: v_readfirstlane_b32 s10, v2
6213 ; GFX10-NEXT: v_readfirstlane_b32 s11, v3
6214 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
6215 ; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
6216 ; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
6217 ; GFX10-NEXT: s_and_saveexec_b32 s4, s4
6218 ; GFX10-NEXT: buffer_load_dword v6, v4, s[8:11], 0 offen offset:1024
6219 ; GFX10-NEXT: ; implicit-def: $vgpr4
6220 ; GFX10-NEXT: s_waitcnt_depctr 0xffe3
6221 ; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
6222 ; GFX10-NEXT: s_cbranch_execnz .LBB18_1
6223 ; GFX10-NEXT: ; %bb.2:
6224 ; GFX10-NEXT: s_mov_b32 exec_lo, s6
6225 ; GFX10-NEXT: v_pk_max_f16 v8, v5, v5
6226 ; GFX10-NEXT: .LBB18_3: ; %atomicrmw.start
6227 ; GFX10-NEXT: ; =>This Loop Header: Depth=1
6228 ; GFX10-NEXT: ; Child Loop BB18_4 Depth 2
6229 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6230 ; GFX10-NEXT: v_pk_max_f16 v4, v6, v6
6231 ; GFX10-NEXT: s_mov_b32 s6, exec_lo
6232 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
6233 ; GFX10-NEXT: v_pk_min_f16 v5, v4, v8
6234 ; GFX10-NEXT: v_mov_b32_e32 v4, v5
6235 ; GFX10-NEXT: v_mov_b32_e32 v5, v6
6236 ; GFX10-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
6237 ; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
6238 ; GFX10-NEXT: v_readfirstlane_b32 s8, v0
6239 ; GFX10-NEXT: v_readfirstlane_b32 s9, v1
6240 ; GFX10-NEXT: v_readfirstlane_b32 s10, v2
6241 ; GFX10-NEXT: v_readfirstlane_b32 s11, v3
6242 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
6243 ; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
6244 ; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
6245 ; GFX10-NEXT: s_and_saveexec_b32 s4, s4
6246 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6247 ; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v7, s[8:11], 0 offen glc
6248 ; GFX10-NEXT: s_waitcnt_depctr 0xffe3
6249 ; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
6250 ; GFX10-NEXT: s_cbranch_execnz .LBB18_4
6251 ; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
6252 ; GFX10-NEXT: s_mov_b32 exec_lo, s6
6253 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6254 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
6255 ; GFX10-NEXT: v_mov_b32_e32 v6, v4
6256 ; GFX10-NEXT: buffer_gl1_inv
6257 ; GFX10-NEXT: buffer_gl0_inv
6258 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
6259 ; GFX10-NEXT: s_waitcnt_depctr 0xffe3
6260 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
6261 ; GFX10-NEXT: s_cbranch_execnz .LBB18_3
6262 ; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
6263 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
6264 ; GFX10-NEXT: v_mov_b32_e32 v0, v4
6265 ; GFX10-NEXT: s_setpc_b64 s[30:31]
6266 ;
6267 ; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
6268 ; GFX90A: ; %bb.0:
6269 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6270 ; GFX90A-NEXT: v_add_u32_e32 v8, 0x400, v4
6271 ; GFX90A-NEXT: s_mov_b64 s[6:7], exec
6272 ; GFX90A-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
6273 ; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
6274 ; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
6275 ; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
6276 ; GFX90A-NEXT: v_readfirstlane_b32 s11, v3
6277 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
6278 ; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
6279 ; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
6280 ; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
6281 ; GFX90A-NEXT: s_nop 0
6282 ; GFX90A-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
6283 ; GFX90A-NEXT: ; implicit-def: $vgpr4
6284 ; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
6285 ; GFX90A-NEXT: s_cbranch_execnz .LBB18_1
6286 ; GFX90A-NEXT: ; %bb.2:
6287 ; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
6288 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
6289 ; GFX90A-NEXT: v_pk_max_f16 v9, v5, v5
6290 ; GFX90A-NEXT: .LBB18_3: ; %atomicrmw.start
6291 ; GFX90A-NEXT: ; =>This Loop Header: Depth=1
6292 ; GFX90A-NEXT: ; Child Loop BB18_4 Depth 2
6293 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6294 ; GFX90A-NEXT: v_pk_max_f16 v4, v7, v7
6295 ; GFX90A-NEXT: v_pk_min_f16 v6, v4, v9
6296 ; GFX90A-NEXT: s_mov_b64 s[12:13], exec
6297 ; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[6:7] op_sel:[0,1]
6298 ; GFX90A-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
6299 ; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
6300 ; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
6301 ; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
6302 ; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
6303 ; GFX90A-NEXT: v_readfirstlane_b32 s11, v3
6304 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
6305 ; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
6306 ; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
6307 ; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
6308 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6309 ; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
6310 ; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
6311 ; GFX90A-NEXT: s_cbranch_execnz .LBB18_4
6312 ; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
6313 ; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
6314 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6315 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
6316 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
6317 ; GFX90A-NEXT: v_mov_b32_e32 v7, v4
6318 ; GFX90A-NEXT: buffer_wbinvl1
6319 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
6320 ; GFX90A-NEXT: s_cbranch_execnz .LBB18_3
6321 ; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.end
6322 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
6323 ; GFX90A-NEXT: v_mov_b32_e32 v0, v4
6324 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
6325 ;
6326 ; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
6327 ; GFX908: ; %bb.0:
6328 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6329 ; GFX908-NEXT: v_add_u32_e32 v7, 0x400, v4
6330 ; GFX908-NEXT: s_mov_b64 s[6:7], exec
6331 ; GFX908-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
6332 ; GFX908-NEXT: v_readfirstlane_b32 s8, v0
6333 ; GFX908-NEXT: v_readfirstlane_b32 s9, v1
6334 ; GFX908-NEXT: v_readfirstlane_b32 s10, v2
6335 ; GFX908-NEXT: v_readfirstlane_b32 s11, v3
6336 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
6337 ; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
6338 ; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
6339 ; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
6340 ; GFX908-NEXT: s_nop 0
6341 ; GFX908-NEXT: buffer_load_dword v6, v4, s[8:11], 0 offen offset:1024
6342 ; GFX908-NEXT: ; implicit-def: $vgpr4
6343 ; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
6344 ; GFX908-NEXT: s_cbranch_execnz .LBB18_1
6345 ; GFX908-NEXT: ; %bb.2:
6346 ; GFX908-NEXT: s_mov_b64 exec, s[6:7]
6347 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
6348 ; GFX908-NEXT: v_pk_max_f16 v8, v5, v5
6349 ; GFX908-NEXT: .LBB18_3: ; %atomicrmw.start
6350 ; GFX908-NEXT: ; =>This Loop Header: Depth=1
6351 ; GFX908-NEXT: ; Child Loop BB18_4 Depth 2
6352 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6353 ; GFX908-NEXT: v_pk_max_f16 v4, v6, v6
6354 ; GFX908-NEXT: v_pk_min_f16 v5, v4, v8
6355 ; GFX908-NEXT: v_mov_b32_e32 v4, v5
6356 ; GFX908-NEXT: s_mov_b64 s[12:13], exec
6357 ; GFX908-NEXT: v_mov_b32_e32 v5, v6
6358 ; GFX908-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
6359 ; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
6360 ; GFX908-NEXT: v_readfirstlane_b32 s8, v0
6361 ; GFX908-NEXT: v_readfirstlane_b32 s9, v1
6362 ; GFX908-NEXT: v_readfirstlane_b32 s10, v2
6363 ; GFX908-NEXT: v_readfirstlane_b32 s11, v3
6364 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
6365 ; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
6366 ; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
6367 ; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
6368 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6369 ; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v7, s[8:11], 0 offen glc
6370 ; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
6371 ; GFX908-NEXT: s_cbranch_execnz .LBB18_4
6372 ; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
6373 ; GFX908-NEXT: s_mov_b64 exec, s[12:13]
6374 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6375 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
6376 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
6377 ; GFX908-NEXT: v_mov_b32_e32 v6, v4
6378 ; GFX908-NEXT: buffer_wbinvl1
6379 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
6380 ; GFX908-NEXT: s_cbranch_execnz .LBB18_3
6381 ; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
6382 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
6383 ; GFX908-NEXT: v_mov_b32_e32 v0, v4
6384 ; GFX908-NEXT: s_setpc_b64 s[30:31]
6385 ;
6386 ; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
6387 ; GFX8: ; %bb.0:
6388 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6389 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x400, v4
6390 ; GFX8-NEXT: s_mov_b64 s[6:7], exec
6391 ; GFX8-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
6392 ; GFX8-NEXT: v_readfirstlane_b32 s8, v0
6393 ; GFX8-NEXT: v_readfirstlane_b32 s9, v1
6394 ; GFX8-NEXT: v_readfirstlane_b32 s10, v2
6395 ; GFX8-NEXT: v_readfirstlane_b32 s11, v3
6396 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
6397 ; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
6398 ; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
6399 ; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
6400 ; GFX8-NEXT: s_nop 0
6401 ; GFX8-NEXT: buffer_load_dword v6, v4, s[8:11], 0 offen offset:1024
6402 ; GFX8-NEXT: ; implicit-def: $vgpr4
6403 ; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
6404 ; GFX8-NEXT: s_cbranch_execnz .LBB18_1
6405 ; GFX8-NEXT: ; %bb.2:
6406 ; GFX8-NEXT: s_mov_b64 exec, s[6:7]
6407 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
6408 ; GFX8-NEXT: v_max_f16_sdwa v8, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
6409 ; GFX8-NEXT: v_max_f16_e32 v9, v5, v5
6410 ; GFX8-NEXT: .LBB18_3: ; %atomicrmw.start
6411 ; GFX8-NEXT: ; =>This Loop Header: Depth=1
6412 ; GFX8-NEXT: ; Child Loop BB18_4 Depth 2
6413 ; GFX8-NEXT: s_waitcnt vmcnt(0)
6414 ; GFX8-NEXT: v_max_f16_sdwa v4, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
6415 ; GFX8-NEXT: v_max_f16_e32 v5, v6, v6
6416 ; GFX8-NEXT: v_min_f16_sdwa v4, v4, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
6417 ; GFX8-NEXT: v_min_f16_e32 v5, v5, v9
6418 ; GFX8-NEXT: v_or_b32_e32 v5, v5, v4
6419 ; GFX8-NEXT: v_mov_b32_e32 v4, v5
6420 ; GFX8-NEXT: s_mov_b64 s[12:13], exec
6421 ; GFX8-NEXT: v_mov_b32_e32 v5, v6
6422 ; GFX8-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
6423 ; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
6424 ; GFX8-NEXT: v_readfirstlane_b32 s8, v0
6425 ; GFX8-NEXT: v_readfirstlane_b32 s9, v1
6426 ; GFX8-NEXT: v_readfirstlane_b32 s10, v2
6427 ; GFX8-NEXT: v_readfirstlane_b32 s11, v3
6428 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
6429 ; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
6430 ; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
6431 ; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
6432 ; GFX8-NEXT: s_waitcnt vmcnt(0)
6433 ; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v7, s[8:11], 0 offen glc
6434 ; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
6435 ; GFX8-NEXT: s_cbranch_execnz .LBB18_4
6436 ; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
6437 ; GFX8-NEXT: s_mov_b64 exec, s[12:13]
6438 ; GFX8-NEXT: s_waitcnt vmcnt(0)
6439 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
6440 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
6441 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
6442 ; GFX8-NEXT: buffer_wbinvl1
6443 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
6444 ; GFX8-NEXT: s_cbranch_execnz .LBB18_3
6445 ; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
6446 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
6447 ; GFX8-NEXT: v_mov_b32_e32 v0, v4
6448 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6449 ;
6450 ; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
6451 ; GFX7: ; %bb.0:
6452 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6453 ; GFX7-NEXT: v_add_i32_e32 v9, vcc, 0x400, v4
6454 ; GFX7-NEXT: s_mov_b64 s[6:7], exec
6455 ; GFX7-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
6456 ; GFX7-NEXT: v_readfirstlane_b32 s8, v0
6457 ; GFX7-NEXT: v_readfirstlane_b32 s9, v1
6458 ; GFX7-NEXT: v_readfirstlane_b32 s10, v2
6459 ; GFX7-NEXT: v_readfirstlane_b32 s11, v3
6460 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
6461 ; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
6462 ; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
6463 ; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
6464 ; GFX7-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
6465 ; GFX7-NEXT: ; implicit-def: $vgpr4
6466 ; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
6467 ; GFX7-NEXT: s_cbranch_execnz .LBB18_1
6468 ; GFX7-NEXT: ; %bb.2:
6469 ; GFX7-NEXT: s_mov_b64 exec, s[6:7]
6470 ; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v6
6471 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v5
6472 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6473 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v7
6474 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v7
6475 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
6476 ; GFX7-NEXT: v_cvt_f32_f16_e32 v10, v6
6477 ; GFX7-NEXT: v_cvt_f32_f16_e32 v11, v8
6478 ; GFX7-NEXT: s_mov_b64 s[6:7], 0
6479 ; GFX7-NEXT: .LBB18_3: ; %atomicrmw.start
6480 ; GFX7-NEXT: ; =>This Loop Header: Depth=1
6481 ; GFX7-NEXT: ; Child Loop BB18_4 Depth 2
6482 ; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
6483 ; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v4
6484 ; GFX7-NEXT: s_mov_b64 s[12:13], exec
6485 ; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v5
6486 ; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v4
6487 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
6488 ; GFX7-NEXT: v_min_f32_e32 v6, v6, v10
6489 ; GFX7-NEXT: v_min_f32_e32 v7, v7, v11
6490 ; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v6
6491 ; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v7
6492 ; GFX7-NEXT: v_or_b32_e32 v6, v4, v5
6493 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v8
6494 ; GFX7-NEXT: v_or_b32_e32 v5, v7, v4
6495 ; GFX7-NEXT: v_mov_b32_e32 v8, v6
6496 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
6497 ; GFX7-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
6498 ; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
6499 ; GFX7-NEXT: v_readfirstlane_b32 s8, v0
6500 ; GFX7-NEXT: v_readfirstlane_b32 s9, v1
6501 ; GFX7-NEXT: v_readfirstlane_b32 s10, v2
6502 ; GFX7-NEXT: v_readfirstlane_b32 s11, v3
6503 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
6504 ; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
6505 ; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
6506 ; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
6507 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6508 ; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v9, s[8:11], 0 offen glc
6509 ; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
6510 ; GFX7-NEXT: s_cbranch_execnz .LBB18_4
6511 ; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
6512 ; GFX7-NEXT: s_mov_b64 exec, s[12:13]
6513 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6514 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v7
6515 ; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v7
6516 ; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
6517 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
6518 ; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
6519 ; GFX7-NEXT: buffer_wbinvl1
6520 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
6521 ; GFX7-NEXT: s_cbranch_execnz .LBB18_3
6522 ; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end
6523 ; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
6524 ; GFX7-NEXT: v_mov_b32_e32 v0, v4
6525 ; GFX7-NEXT: v_mov_b32_e32 v1, v5
6526 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6527 ;
6528 ; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
6529 ; GFX6: ; %bb.0:
6530 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6531 ; GFX6-NEXT: v_add_i32_e32 v9, vcc, 0x400, v4
6532 ; GFX6-NEXT: s_mov_b64 s[6:7], exec
6533 ; GFX6-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
6534 ; GFX6-NEXT: v_readfirstlane_b32 s8, v0
6535 ; GFX6-NEXT: v_readfirstlane_b32 s9, v1
6536 ; GFX6-NEXT: v_readfirstlane_b32 s10, v2
6537 ; GFX6-NEXT: v_readfirstlane_b32 s11, v3
6538 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
6539 ; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
6540 ; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
6541 ; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
6542 ; GFX6-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
6543 ; GFX6-NEXT: ; implicit-def: $vgpr4
6544 ; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
6545 ; GFX6-NEXT: s_cbranch_execnz .LBB18_1
6546 ; GFX6-NEXT: ; %bb.2:
6547 ; GFX6-NEXT: s_mov_b64 exec, s[6:7]
6548 ; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
6549 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v5
6550 ; GFX6-NEXT: s_waitcnt vmcnt(0)
6551 ; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v7
6552 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v7
6553 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
6554 ; GFX6-NEXT: v_cvt_f32_f16_e32 v10, v6
6555 ; GFX6-NEXT: v_cvt_f32_f16_e32 v11, v8
6556 ; GFX6-NEXT: s_mov_b64 s[6:7], 0
6557 ; GFX6-NEXT: .LBB18_3: ; %atomicrmw.start
6558 ; GFX6-NEXT: ; =>This Loop Header: Depth=1
6559 ; GFX6-NEXT: ; Child Loop BB18_4 Depth 2
6560 ; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
6561 ; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
6562 ; GFX6-NEXT: s_mov_b64 s[12:13], exec
6563 ; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v5
6564 ; GFX6-NEXT: s_waitcnt expcnt(0)
6565 ; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v4
6566 ; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
6567 ; GFX6-NEXT: v_min_f32_e32 v6, v6, v10
6568 ; GFX6-NEXT: v_min_f32_e32 v7, v7, v11
6569 ; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v6
6570 ; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v7
6571 ; GFX6-NEXT: v_or_b32_e32 v6, v4, v5
6572 ; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v8
6573 ; GFX6-NEXT: v_or_b32_e32 v5, v7, v4
6574 ; GFX6-NEXT: v_mov_b32_e32 v8, v6
6575 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
6576 ; GFX6-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
6577 ; GFX6-NEXT: ; => This Inner Loop Header: Depth=2
6578 ; GFX6-NEXT: v_readfirstlane_b32 s8, v0
6579 ; GFX6-NEXT: v_readfirstlane_b32 s9, v1
6580 ; GFX6-NEXT: v_readfirstlane_b32 s10, v2
6581 ; GFX6-NEXT: v_readfirstlane_b32 s11, v3
6582 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
6583 ; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
6584 ; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
6585 ; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
6586 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
6587 ; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v9, s[8:11], 0 offen glc
6588 ; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
6589 ; GFX6-NEXT: s_cbranch_execnz .LBB18_4
6590 ; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
6591 ; GFX6-NEXT: s_mov_b64 exec, s[12:13]
6592 ; GFX6-NEXT: s_waitcnt vmcnt(0)
6593 ; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v7
6594 ; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v7
6595 ; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
6596 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
6597 ; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
6598 ; GFX6-NEXT: buffer_wbinvl1
6599 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
6600 ; GFX6-NEXT: s_cbranch_execnz .LBB18_3
6601 ; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end
6602 ; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
6603 ; GFX6-NEXT: v_mov_b32_e32 v0, v4
6604 ; GFX6-NEXT: v_mov_b32_e32 v1, v5
6605 ; GFX6-NEXT: s_waitcnt expcnt(0)
6606 ; GFX6-NEXT: s_setpc_b64 s[30:31]
6607 %gep = getelementptr <2 x half>, ptr addrspace(7) %ptr, i32 256
6608 %result = atomicrmw fmin ptr addrspace(7) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
6609 ret <2 x half> %result
6610 }
6612 ; --------------------------------------------------------------------
6613 ; <2 x bfloat>
6614 ; --------------------------------------------------------------------
6616 define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, <2 x bfloat> %val) #0 {
6617 ; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
6618 ; GFX12: ; %bb.0:
6619 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
6620 ; GFX12-NEXT: s_wait_expcnt 0x0
6621 ; GFX12-NEXT: s_wait_samplecnt 0x0
6622 ; GFX12-NEXT: s_wait_bvhcnt 0x0
6623 ; GFX12-NEXT: s_wait_kmcnt 0x0
6624 ; GFX12-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
6625 ; GFX12-NEXT: s_add_co_i32 s4, s16, 0x400
6626 ; GFX12-NEXT: s_mov_b32 s5, 0
6627 ; GFX12-NEXT: s_wait_alu 0xfffe
6628 ; GFX12-NEXT: v_mov_b32_e32 v4, s4
6629 ; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v1
6630 ; GFX12-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen offset:1024
6631 ; GFX12-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
6632 ; GFX12-NEXT: .LBB19_1: ; %atomicrmw.start
6633 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
6634 ; GFX12-NEXT: s_wait_loadcnt 0x0
6635 ; GFX12-NEXT: v_mov_b32_e32 v6, v0
6636 ; GFX12-NEXT: s_wait_storecnt 0x0
6637 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6638 ; GFX12-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
6639 ; GFX12-NEXT: v_min_num_f32_e32 v1, v1, v3
6640 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
6641 ; GFX12-NEXT: v_bfe_u32 v7, v1, 16, 1
6642 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v1
6643 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
6644 ; GFX12-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
6645 ; GFX12-NEXT: s_wait_alu 0xfffd
6646 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6647 ; GFX12-NEXT: v_dual_cndmask_b32 v1, v7, v9 :: v_dual_lshlrev_b32 v0, 16, v6
6648 ; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v2
6649 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
6650 ; GFX12-NEXT: v_bfe_u32 v5, v0, 16, 1
6651 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v0
6652 ; GFX12-NEXT: v_cmp_u_f32_e64 s4, v0, v0
6653 ; GFX12-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
6654 ; GFX12-NEXT: s_wait_alu 0xf1ff
6655 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6656 ; GFX12-NEXT: v_cndmask_b32_e64 v0, v5, v8, s4
6657 ; GFX12-NEXT: v_perm_b32 v5, v1, v0, 0x7060302
6658 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
6659 ; GFX12-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
6660 ; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
6661 ; GFX12-NEXT: s_wait_loadcnt 0x0
6662 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
6663 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
6664 ; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
6665 ; GFX12-NEXT: s_wait_alu 0xfffe
6666 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
6667 ; GFX12-NEXT: s_cbranch_execnz .LBB19_1
6668 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
6669 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
6670 ; GFX12-NEXT: s_setpc_b64 s[30:31]
6671 ;
6672 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
6673 ; GFX940: ; %bb.0:
6674 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6675 ; GFX940-NEXT: v_mov_b32_e32 v1, v0
6676 ; GFX940-NEXT: v_mov_b32_e32 v0, s16
6677 ; GFX940-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
6678 ; GFX940-NEXT: s_add_i32 s4, s16, 0x400
6679 ; GFX940-NEXT: s_mov_b64 s[6:7], 0
6680 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v1
6681 ; GFX940-NEXT: s_movk_i32 s8, 0x7fff
6682 ; GFX940-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
6683 ; GFX940-NEXT: s_mov_b32 s9, 0x7060302
6684 ; GFX940-NEXT: v_mov_b32_e32 v4, s4
6685 ; GFX940-NEXT: .LBB19_1: ; %atomicrmw.start
6686 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
6687 ; GFX940-NEXT: s_waitcnt vmcnt(0)
6688 ; GFX940-NEXT: v_mov_b32_e32 v7, v0
6689 ; GFX940-NEXT: v_lshlrev_b32_e32 v0, 16, v7
6690 ; GFX940-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
6691 ; GFX940-NEXT: v_min_f32_e32 v0, v0, v2
6692 ; GFX940-NEXT: v_min_f32_e32 v1, v1, v3
6693 ; GFX940-NEXT: v_bfe_u32 v5, v0, 16, 1
6694 ; GFX940-NEXT: v_bfe_u32 v8, v1, 16, 1
6695 ; GFX940-NEXT: v_or_b32_e32 v6, 0x400000, v0
6696 ; GFX940-NEXT: v_or_b32_e32 v9, 0x400000, v1
6697 ; GFX940-NEXT: v_add3_u32 v5, v5, v0, s8
6698 ; GFX940-NEXT: v_add3_u32 v8, v8, v1, s8
6699 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
6700 ; GFX940-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
6701 ; GFX940-NEXT: buffer_wbl2 sc1
6702 ; GFX940-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
6703 ; GFX940-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]
6704 ; GFX940-NEXT: v_perm_b32 v6, v1, v0, s9
6705 ; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[6:7]
6706 ; GFX940-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[0:3], 0 offen sc0
6707 ; GFX940-NEXT: s_waitcnt vmcnt(0)
6708 ; GFX940-NEXT: buffer_inv sc1
6709 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
6710 ; GFX940-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
6711 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[6:7]
6712 ; GFX940-NEXT: s_cbranch_execnz .LBB19_1
6713 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
6714 ; GFX940-NEXT: s_or_b64 exec, exec, s[6:7]
6715 ; GFX940-NEXT: s_setpc_b64 s[30:31]
6716 ;
6717 ; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
6718 ; GFX11: ; %bb.0:
6719 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6720 ; GFX11-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
6721 ; GFX11-NEXT: s_add_i32 s4, s16, 0x400
6722 ; GFX11-NEXT: s_mov_b32 s5, 0
6723 ; GFX11-NEXT: v_mov_b32_e32 v4, s4
6724 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
6725 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v1
6726 ; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
6727 ; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
6728 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
6729 ; GFX11-NEXT: .p2align 6
6730 ; GFX11-NEXT: .LBB19_1: ; %atomicrmw.start
6731 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
6732 ; GFX11-NEXT: s_waitcnt vmcnt(0)
6733 ; GFX11-NEXT: v_mov_b32_e32 v6, v0
6734 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
6735 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6736 ; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
6737 ; GFX11-NEXT: v_min_f32_e32 v1, v1, v3
6738 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
6739 ; GFX11-NEXT: v_bfe_u32 v7, v1, 16, 1
6740 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v1
6741 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
6742 ; GFX11-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
6743 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6744 ; GFX11-NEXT: v_dual_cndmask_b32 v1, v7, v9 :: v_dual_lshlrev_b32 v0, 16, v6
6745 ; GFX11-NEXT: v_min_f32_e32 v0, v0, v2
6746 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
6747 ; GFX11-NEXT: v_bfe_u32 v5, v0, 16, 1
6748 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v0
6749 ; GFX11-NEXT: v_cmp_u_f32_e64 s4, v0, v0
6750 ; GFX11-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
6751 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
6752 ; GFX11-NEXT: v_cndmask_b32_e64 v0, v5, v8, s4
6753 ; GFX11-NEXT: v_perm_b32 v5, v1, v0, 0x7060302
6754 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
6755 ; GFX11-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
6756 ; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen glc
6757 ; GFX11-NEXT: s_waitcnt vmcnt(0)
6758 ; GFX11-NEXT: buffer_gl1_inv
6759 ; GFX11-NEXT: buffer_gl0_inv
6760 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
6761 ; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
6762 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
6763 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
6764 ; GFX11-NEXT: s_cbranch_execnz .LBB19_1
6765 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
6766 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
6767 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
6768 ; GFX11-NEXT: s_setpc_b64 s[30:31]
6769 ;
6770 ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
6771 ; GFX10: ; %bb.0:
6772 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6773 ; GFX10-NEXT: v_mov_b32_e32 v1, v0
6774 ; GFX10-NEXT: v_mov_b32_e32 v0, s20
6775 ; GFX10-NEXT: s_add_i32 s4, s20, 0x400
6776 ; GFX10-NEXT: s_mov_b32 s5, 0
6777 ; GFX10-NEXT: v_mov_b32_e32 v4, s4
6778 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
6779 ; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
6780 ; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
6781 ; GFX10-NEXT: .LBB19_1: ; %atomicrmw.start
6782 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
6783 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6784 ; GFX10-NEXT: v_mov_b32_e32 v6, v0
6785 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
6786 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v6
6787 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
6788 ; GFX10-NEXT: v_min_f32_e32 v0, v0, v2
6789 ; GFX10-NEXT: v_min_f32_e32 v1, v1, v3
6790 ; GFX10-NEXT: v_bfe_u32 v5, v0, 16, 1
6791 ; GFX10-NEXT: v_bfe_u32 v7, v1, 16, 1
6792 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
6793 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v1
6794 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
6795 ; GFX10-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
6796 ; GFX10-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
6797 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
6798 ; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
6799 ; GFX10-NEXT: v_cndmask_b32_e64 v0, v5, v8, s4
6800 ; GFX10-NEXT: v_perm_b32 v5, v1, v0, 0x7060302
6801 ; GFX10-NEXT: v_mov_b32_e32 v0, v5
6802 ; GFX10-NEXT: v_mov_b32_e32 v1, v6
6803 ; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen glc
6804 ; GFX10-NEXT: s_waitcnt vmcnt(0)
6805 ; GFX10-NEXT: buffer_gl1_inv
6806 ; GFX10-NEXT: buffer_gl0_inv
6807 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
6808 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
6809 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
6810 ; GFX10-NEXT: s_cbranch_execnz .LBB19_1
6811 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
6812 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
6813 ; GFX10-NEXT: s_setpc_b64 s[30:31]
6814 ;
6815 ; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
6816 ; GFX90A: ; %bb.0:
6817 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6818 ; GFX90A-NEXT: v_mov_b32_e32 v1, v0
6819 ; GFX90A-NEXT: v_mov_b32_e32 v0, s20
6820 ; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
6821 ; GFX90A-NEXT: s_add_i32 s4, s20, 0x400
6822 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
6823 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
6824 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
6825 ; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
6826 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
6827 ; GFX90A-NEXT: v_mov_b32_e32 v4, s4
6828 ; GFX90A-NEXT: .LBB19_1: ; %atomicrmw.start
6829 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
6830 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6831 ; GFX90A-NEXT: v_mov_b32_e32 v7, v0
6832 ; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v7
6833 ; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
6834 ; GFX90A-NEXT: v_min_f32_e32 v0, v0, v2
6835 ; GFX90A-NEXT: v_min_f32_e32 v1, v1, v3
6836 ; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
6837 ; GFX90A-NEXT: v_bfe_u32 v8, v1, 16, 1
6838 ; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
6839 ; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v1
6840 ; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s8
6841 ; GFX90A-NEXT: v_add3_u32 v8, v8, v1, s8
6842 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
6843 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
6844 ; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]
6845 ; GFX90A-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
6846 ; GFX90A-NEXT: v_perm_b32 v6, v1, v0, s9
6847 ; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[0,1]
6848 ; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen glc
6849 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
6850 ; GFX90A-NEXT: buffer_wbinvl1
6851 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
6852 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
6853 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
6854 ; GFX90A-NEXT: s_cbranch_execnz .LBB19_1
6855 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
6856 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
6857 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
6858 ;
6859 ; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
6860 ; GFX908: ; %bb.0:
6861 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6862 ; GFX908-NEXT: v_mov_b32_e32 v1, v0
6863 ; GFX908-NEXT: v_mov_b32_e32 v0, s20
6864 ; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
6865 ; GFX908-NEXT: s_add_i32 s4, s20, 0x400
6866 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
6867 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v1
6868 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
6869 ; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
6870 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
6871 ; GFX908-NEXT: v_mov_b32_e32 v4, s4
6872 ; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
6873 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
6874 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6875 ; GFX908-NEXT: v_mov_b32_e32 v6, v0
6876 ; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v6
6877 ; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
6878 ; GFX908-NEXT: v_min_f32_e32 v0, v0, v2
6879 ; GFX908-NEXT: v_min_f32_e32 v1, v1, v3
6880 ; GFX908-NEXT: v_bfe_u32 v5, v0, 16, 1
6881 ; GFX908-NEXT: v_bfe_u32 v8, v1, 16, 1
6882 ; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v0
6883 ; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v1
6884 ; GFX908-NEXT: v_add3_u32 v5, v5, v0, s8
6885 ; GFX908-NEXT: v_add3_u32 v8, v8, v1, s8
6886 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
6887 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
6888 ; GFX908-NEXT: v_cndmask_b32_e64 v0, v5, v7, s[4:5]
6889 ; GFX908-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
6890 ; GFX908-NEXT: v_perm_b32 v5, v1, v0, s9
6891 ; GFX908-NEXT: v_mov_b32_e32 v0, v5
6892 ; GFX908-NEXT: v_mov_b32_e32 v1, v6
6893 ; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen glc
6894 ; GFX908-NEXT: s_waitcnt vmcnt(0)
6895 ; GFX908-NEXT: buffer_wbinvl1
6896 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
6897 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
6898 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
6899 ; GFX908-NEXT: s_cbranch_execnz .LBB19_1
6900 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
6901 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
6902 ; GFX908-NEXT: s_setpc_b64 s[30:31]
6903 ;
6904 ; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
6905 ; GFX8: ; %bb.0:
6906 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6907 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
6908 ; GFX8-NEXT: v_mov_b32_e32 v0, s20
6909 ; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
6910 ; GFX8-NEXT: s_add_i32 s4, s20, 0x400
6911 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
6912 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
6913 ; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
6914 ; GFX8-NEXT: v_mov_b32_e32 v4, s4
6915 ; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
6916 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6917 ; GFX8-NEXT: s_waitcnt vmcnt(0)
6918 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
6919 ; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
6920 ; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
6921 ; GFX8-NEXT: v_min_f32_e32 v0, v0, v2
6922 ; GFX8-NEXT: v_min_f32_e32 v1, v1, v3
6923 ; GFX8-NEXT: v_bfe_u32 v5, v0, 16, 1
6924 ; GFX8-NEXT: v_bfe_u32 v8, v1, 16, 1
6925 ; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v0
6926 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v1
6927 ; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
6928 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
6929 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v1
6930 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
6931 ; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v0
6932 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
6933 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
6934 ; GFX8-NEXT: v_cndmask_b32_e64 v0, v5, v7, s[4:5]
6935 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
6936 ; GFX8-NEXT: v_alignbit_b32 v5, v1, v0, 16
6937 ; GFX8-NEXT: v_mov_b32_e32 v0, v5
6938 ; GFX8-NEXT: v_mov_b32_e32 v1, v6
6939 ; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen glc
6940 ; GFX8-NEXT: s_waitcnt vmcnt(0)
6941 ; GFX8-NEXT: buffer_wbinvl1
6942 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
6943 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
6944 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
6945 ; GFX8-NEXT: s_cbranch_execnz .LBB19_1
6946 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6947 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
6948 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6949 ;
6950 ; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
6951 ; GFX7: ; %bb.0:
6952 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6953 ; GFX7-NEXT: v_mov_b32_e32 v2, s20
6954 ; GFX7-NEXT: buffer_load_dword v4, v2, s[16:19], 0 offen offset:1024
6955 ; GFX7-NEXT: s_add_i32 s6, s20, 0x400
6956 ; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
6957 ; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
6958 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
6959 ; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
6960 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
6961 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6962 ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
6963 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v4
6964 ; GFX7-NEXT: v_mov_b32_e32 v4, s6
6965 ; GFX7-NEXT: .LBB19_1: ; %atomicrmw.start
6966 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
6967 ; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
6968 ; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
6969 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
6970 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
6971 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
6972 ; GFX7-NEXT: v_min_f32_e32 v5, v5, v2
6973 ; GFX7-NEXT: v_min_f32_e32 v6, v6, v3
6974 ; GFX7-NEXT: v_alignbit_b32 v1, v1, v0, 16
6975 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v5
6976 ; GFX7-NEXT: v_alignbit_b32 v0, v0, v6, 16
6977 ; GFX7-NEXT: v_mov_b32_e32 v6, v1
6978 ; GFX7-NEXT: v_mov_b32_e32 v5, v0
6979 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen glc
6980 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6981 ; GFX7-NEXT: buffer_wbinvl1
6982 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v1
6983 ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v5
6984 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6985 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v5
6986 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
6987 ; GFX7-NEXT: s_cbranch_execnz .LBB19_1
6988 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6989 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
6990 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6991 ;
6992 ; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
6993 ; GFX6: ; %bb.0:
6994 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6995 ; GFX6-NEXT: v_mov_b32_e32 v2, s20
6996 ; GFX6-NEXT: buffer_load_dword v4, v2, s[16:19], 0 offen offset:1024
6997 ; GFX6-NEXT: s_add_i32 s6, s20, 0x400
6998 ; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
6999 ; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
7000 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
7001 ; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
7002 ; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
7003 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7004 ; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
7005 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v4
7006 ; GFX6-NEXT: v_mov_b32_e32 v4, s6
7007 ; GFX6-NEXT: .LBB19_1: ; %atomicrmw.start
7008 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
7009 ; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
7010 ; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
7011 ; GFX6-NEXT: s_waitcnt expcnt(0)
7012 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
7013 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
7014 ; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
7015 ; GFX6-NEXT: v_min_f32_e32 v5, v5, v2
7016 ; GFX6-NEXT: v_min_f32_e32 v6, v6, v3
7017 ; GFX6-NEXT: v_alignbit_b32 v1, v1, v0, 16
7018 ; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v5
7019 ; GFX6-NEXT: v_alignbit_b32 v0, v0, v6, 16
7020 ; GFX6-NEXT: v_mov_b32_e32 v6, v1
7021 ; GFX6-NEXT: v_mov_b32_e32 v5, v0
7022 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen glc
7023 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7024 ; GFX6-NEXT: buffer_wbinvl1
7025 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v1
7026 ; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v5
7027 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7028 ; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v5
7029 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
7030 ; GFX6-NEXT: s_cbranch_execnz .LBB19_1
7031 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
7032 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
7033 ; GFX6-NEXT: s_waitcnt expcnt(0)
7034 ; GFX6-NEXT: s_setpc_b64 s[30:31]
7035 %gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
7036 %result = atomicrmw fmin ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
7037 ret <2 x bfloat> %result
7038 }
7040 define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, <2 x bfloat> %val) #0 {
7041 ; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
7042 ; GFX12: ; %bb.0:
7043 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
7044 ; GFX12-NEXT: s_wait_expcnt 0x0
7045 ; GFX12-NEXT: s_wait_samplecnt 0x0
7046 ; GFX12-NEXT: s_wait_bvhcnt 0x0
7047 ; GFX12-NEXT: s_wait_kmcnt 0x0
7048 ; GFX12-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_lshlrev_b32 v2, 16, v0
7049 ; GFX12-NEXT: s_add_co_i32 s4, s16, 0x400
7050 ; GFX12-NEXT: s_wait_alu 0xfffe
7051 ; GFX12-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_and_b32 v3, 0xffff0000, v0
7052 ; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null offen offset:1024
7053 ; GFX12-NEXT: s_mov_b32 s5, 0
7054 ; GFX12-NEXT: .LBB20_1: ; %atomicrmw.start
7055 ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
7056 ; GFX12-NEXT: s_wait_loadcnt 0x0
7057 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
7058 ; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v1
7059 ; GFX12-NEXT: s_wait_storecnt 0x0
7060 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7061 ; GFX12-NEXT: v_dual_min_num_f32 v5, v5, v3 :: v_dual_min_num_f32 v0, v0, v2
7062 ; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
7063 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
7064 ; GFX12-NEXT: v_bfe_u32 v6, v0, 16, 1
7065 ; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v0
7066 ; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v5
7067 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
7068 ; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
7069 ; GFX12-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
7070 ; GFX12-NEXT: v_cmp_u_f32_e64 s4, v0, v0
7071 ; GFX12-NEXT: s_wait_alu 0xfffd
7072 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
7073 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
7074 ; GFX12-NEXT: s_wait_alu 0xf1ff
7075 ; GFX12-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
7076 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7077 ; GFX12-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
7078 ; GFX12-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
7079 ; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
7080 ; GFX12-NEXT: s_wait_loadcnt 0x0
7081 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
7082 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
7083 ; GFX12-NEXT: v_mov_b32_e32 v1, v5
7084 ; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
7085 ; GFX12-NEXT: s_wait_alu 0xfffe
7086 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
7087 ; GFX12-NEXT: s_cbranch_execnz .LBB20_1
7088 ; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
7089 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
7090 ; GFX12-NEXT: s_setpc_b64 s[30:31]
7091 ;
7092 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
7093 ; GFX940: ; %bb.0:
7094 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7095 ; GFX940-NEXT: v_mov_b32_e32 v1, s16
7096 ; GFX940-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen offset:1024
7097 ; GFX940-NEXT: s_add_i32 s4, s16, 0x400
7098 ; GFX940-NEXT: s_mov_b64 s[6:7], 0
7099 ; GFX940-NEXT: v_lshlrev_b32_e32 v2, 16, v0
7100 ; GFX940-NEXT: s_movk_i32 s8, 0x7fff
7101 ; GFX940-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
7102 ; GFX940-NEXT: s_mov_b32 s9, 0x7060302
7103 ; GFX940-NEXT: v_mov_b32_e32 v4, s4
7104 ; GFX940-NEXT: .LBB20_1: ; %atomicrmw.start
7105 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
7106 ; GFX940-NEXT: s_waitcnt vmcnt(0)
7107 ; GFX940-NEXT: v_lshlrev_b32_e32 v0, 16, v1
7108 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
7109 ; GFX940-NEXT: v_min_f32_e32 v0, v0, v2
7110 ; GFX940-NEXT: v_min_f32_e32 v5, v5, v3
7111 ; GFX940-NEXT: v_bfe_u32 v6, v0, 16, 1
7112 ; GFX940-NEXT: v_bfe_u32 v8, v5, 16, 1
7113 ; GFX940-NEXT: v_or_b32_e32 v7, 0x400000, v0
7114 ; GFX940-NEXT: v_or_b32_e32 v9, 0x400000, v5
7115 ; GFX940-NEXT: v_add3_u32 v6, v6, v0, s8
7116 ; GFX940-NEXT: v_add3_u32 v8, v8, v5, s8
7117 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
7118 ; GFX940-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
7119 ; GFX940-NEXT: buffer_wbl2 sc1
7120 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
7121 ; GFX940-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
7122 ; GFX940-NEXT: v_perm_b32 v0, v5, v0, s9
7123 ; GFX940-NEXT: v_mov_b64_e32 v[6:7], v[0:1]
7124 ; GFX940-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[0:3], 0 offen sc0
7125 ; GFX940-NEXT: s_waitcnt vmcnt(0)
7126 ; GFX940-NEXT: buffer_inv sc1
7127 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v6, v1
7128 ; GFX940-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
7129 ; GFX940-NEXT: v_mov_b32_e32 v1, v6
7130 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[6:7]
7131 ; GFX940-NEXT: s_cbranch_execnz .LBB20_1
7132 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
7133 ; GFX940-NEXT: s_or_b64 exec, exec, s[6:7]
7134 ; GFX940-NEXT: s_setpc_b64 s[30:31]
7135 ;
7136 ; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
7137 ; GFX11: ; %bb.0:
7138 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7139 ; GFX11-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_lshlrev_b32 v2, 16, v0
7140 ; GFX11-NEXT: s_add_i32 s4, s16, 0x400
7141 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
7142 ; GFX11-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_and_b32 v3, 0xffff0000, v0
7143 ; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:1024
7144 ; GFX11-NEXT: s_mov_b32 s5, 0
7145 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
7146 ; GFX11-NEXT: .p2align 6
7147 ; GFX11-NEXT: .LBB20_1: ; %atomicrmw.start
7148 ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
7149 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7150 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
7151 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v1
7152 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
7153 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7154 ; GFX11-NEXT: v_dual_min_f32 v5, v5, v3 :: v_dual_min_f32 v0, v0, v2
7155 ; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
7156 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
7157 ; GFX11-NEXT: v_bfe_u32 v6, v0, 16, 1
7158 ; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v0
7159 ; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v5
7160 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
7161 ; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
7162 ; GFX11-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
7163 ; GFX11-NEXT: v_cmp_u_f32_e64 s4, v0, v0
7164 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
7165 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
7166 ; GFX11-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
7167 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7168 ; GFX11-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
7169 ; GFX11-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
7170 ; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen glc
7171 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7172 ; GFX11-NEXT: buffer_gl1_inv
7173 ; GFX11-NEXT: buffer_gl0_inv
7174 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
7175 ; GFX11-NEXT: v_mov_b32_e32 v1, v5
7176 ; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
7177 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
7178 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
7179 ; GFX11-NEXT: s_cbranch_execnz .LBB20_1
7180 ; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
7181 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
7182 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
7183 ; GFX11-NEXT: s_setpc_b64 s[30:31]
7184 ;
7185 ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
7186 ; GFX10: ; %bb.0:
7187 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7188 ; GFX10-NEXT: v_mov_b32_e32 v1, s20
7189 ; GFX10-NEXT: s_add_i32 s4, s20, 0x400
7190 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
7191 ; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
7192 ; GFX10-NEXT: v_mov_b32_e32 v4, s4
7193 ; GFX10-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
7194 ; GFX10-NEXT: s_mov_b32 s5, 0
7195 ; GFX10-NEXT: .LBB20_1: ; %atomicrmw.start
7196 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
7197 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7198 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v1
7199 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
7200 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
7201 ; GFX10-NEXT: v_min_f32_e32 v0, v0, v2
7202 ; GFX10-NEXT: v_min_f32_e32 v5, v5, v3
7203 ; GFX10-NEXT: v_bfe_u32 v6, v0, 16, 1
7204 ; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
7205 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
7206 ; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
7207 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
7208 ; GFX10-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
7209 ; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
7210 ; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
7211 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
7212 ; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
7213 ; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
7214 ; GFX10-NEXT: v_mov_b32_e32 v6, v1
7215 ; GFX10-NEXT: v_mov_b32_e32 v5, v0
7216 ; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen glc
7217 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7218 ; GFX10-NEXT: buffer_gl1_inv
7219 ; GFX10-NEXT: buffer_gl0_inv
7220 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
7221 ; GFX10-NEXT: v_mov_b32_e32 v1, v5
7222 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
7223 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
7224 ; GFX10-NEXT: s_cbranch_execnz .LBB20_1
7225 ; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
7226 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
7227 ; GFX10-NEXT: s_setpc_b64 s[30:31]
7228 ;
7229 ; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
7230 ; GFX90A: ; %bb.0:
7231 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7232 ; GFX90A-NEXT: v_mov_b32_e32 v1, s20
7233 ; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
7234 ; GFX90A-NEXT: s_add_i32 s4, s20, 0x400
7235 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
7236 ; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
7237 ; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
7238 ; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
7239 ; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
7240 ; GFX90A-NEXT: v_mov_b32_e32 v4, s4
7241 ; GFX90A-NEXT: .LBB20_1: ; %atomicrmw.start
7242 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
7243 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7244 ; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
7245 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
7246 ; GFX90A-NEXT: v_min_f32_e32 v0, v0, v2
7247 ; GFX90A-NEXT: v_min_f32_e32 v5, v5, v3
7248 ; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
7249 ; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
7250 ; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
7251 ; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
7252 ; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
7253 ; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
7254 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
7255 ; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
7256 ; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
7257 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
7258 ; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
7259 ; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[0:1], v[0:1] op_sel:[0,1]
7260 ; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[16:19], 0 offen glc
7261 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7262 ; GFX90A-NEXT: buffer_wbinvl1
7263 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v1
7264 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
7265 ; GFX90A-NEXT: v_mov_b32_e32 v1, v6
7266 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
7267 ; GFX90A-NEXT: s_cbranch_execnz .LBB20_1
7268 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
7269 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
7270 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
7271 ;
7272 ; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
7273 ; GFX908: ; %bb.0:
7274 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7275 ; GFX908-NEXT: v_mov_b32_e32 v1, s20
7276 ; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
7277 ; GFX908-NEXT: s_add_i32 s4, s20, 0x400
7278 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
7279 ; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
7280 ; GFX908-NEXT: s_movk_i32 s8, 0x7fff
7281 ; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
7282 ; GFX908-NEXT: s_mov_b32 s9, 0x7060302
7283 ; GFX908-NEXT: v_mov_b32_e32 v4, s4
7284 ; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
7285 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
7286 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7287 ; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v1
7288 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
7289 ; GFX908-NEXT: v_min_f32_e32 v0, v0, v2
7290 ; GFX908-NEXT: v_min_f32_e32 v5, v5, v3
7291 ; GFX908-NEXT: v_bfe_u32 v6, v0, 16, 1
7292 ; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
7293 ; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v0
7294 ; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
7295 ; GFX908-NEXT: v_add3_u32 v6, v6, v0, s8
7296 ; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
7297 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
7298 ; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
7299 ; GFX908-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
7300 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
7301 ; GFX908-NEXT: v_perm_b32 v0, v5, v0, s9
7302 ; GFX908-NEXT: v_mov_b32_e32 v6, v1
7303 ; GFX908-NEXT: v_mov_b32_e32 v5, v0
7304 ; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen glc
7305 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7306 ; GFX908-NEXT: buffer_wbinvl1
7307 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v1
7308 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
7309 ; GFX908-NEXT: v_mov_b32_e32 v1, v5
7310 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
7311 ; GFX908-NEXT: s_cbranch_execnz .LBB20_1
7312 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
7313 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
7314 ; GFX908-NEXT: s_setpc_b64 s[30:31]
7315 ;
7316 ; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
7317 ; GFX8: ; %bb.0:
7318 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7319 ; GFX8-NEXT: v_mov_b32_e32 v1, s20
7320 ; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
7321 ; GFX8-NEXT: s_add_i32 s4, s20, 0x400
7322 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
7323 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
7324 ; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
7325 ; GFX8-NEXT: v_mov_b32_e32 v4, s4
7326 ; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
7327 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7328 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7329 ; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v1
7330 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
7331 ; GFX8-NEXT: v_min_f32_e32 v0, v0, v2
7332 ; GFX8-NEXT: v_min_f32_e32 v5, v5, v3
7333 ; GFX8-NEXT: v_bfe_u32 v6, v0, 16, 1
7334 ; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
7335 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v0
7336 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
7337 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
7338 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
7339 ; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
7340 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
7341 ; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v0
7342 ; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
7343 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
7344 ; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
7345 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
7346 ; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
7347 ; GFX8-NEXT: v_mov_b32_e32 v6, v1
7348 ; GFX8-NEXT: v_mov_b32_e32 v5, v0
7349 ; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen glc
7350 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7351 ; GFX8-NEXT: buffer_wbinvl1
7352 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v1
7353 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
7354 ; GFX8-NEXT: v_mov_b32_e32 v1, v5
7355 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
7356 ; GFX8-NEXT: s_cbranch_execnz .LBB20_1
7357 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7358 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
7359 ; GFX8-NEXT: s_setpc_b64 s[30:31]
7360 ;
7361 ; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
7362 ; GFX7: ; %bb.0:
7363 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7364 ; GFX7-NEXT: v_mov_b32_e32 v2, s20
7365 ; GFX7-NEXT: buffer_load_dword v2, v2, s[16:19], 0 offen offset:1024
7366 ; GFX7-NEXT: s_add_i32 s6, s20, 0x400
7367 ; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
7368 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v0
7369 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
7370 ; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
7371 ; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
7372 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7373 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
7374 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
7375 ; GFX7-NEXT: v_mov_b32_e32 v2, s6
7376 ; GFX7-NEXT: .LBB20_1: ; %atomicrmw.start
7377 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7378 ; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
7379 ; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
7380 ; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
7381 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
7382 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
7383 ; GFX7-NEXT: v_min_f32_e32 v5, v5, v0
7384 ; GFX7-NEXT: v_min_f32_e32 v6, v6, v1
7385 ; GFX7-NEXT: v_alignbit_b32 v4, v4, v3, 16
7386 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
7387 ; GFX7-NEXT: v_alignbit_b32 v3, v3, v6, 16
7388 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
7389 ; GFX7-NEXT: v_mov_b32_e32 v5, v3
7390 ; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
7391 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7392 ; GFX7-NEXT: buffer_wbinvl1
7393 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
7394 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
7395 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7396 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
7397 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
7398 ; GFX7-NEXT: s_cbranch_execnz .LBB20_1
7399 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
7400 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
7401 ; GFX7-NEXT: s_setpc_b64 s[30:31]
7402 ;
7403 ; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
7404 ; GFX6: ; %bb.0:
7405 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7406 ; GFX6-NEXT: v_mov_b32_e32 v2, s20
7407 ; GFX6-NEXT: buffer_load_dword v2, v2, s[16:19], 0 offen offset:1024
7408 ; GFX6-NEXT: s_add_i32 s6, s20, 0x400
7409 ; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
7410 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v0
7411 ; GFX6-NEXT: s_mov_b64 s[4:5], 0
7412 ; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
7413 ; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
7414 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7415 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
7416 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
7417 ; GFX6-NEXT: v_mov_b32_e32 v2, s6
7418 ; GFX6-NEXT: .LBB20_1: ; %atomicrmw.start
7419 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
7420 ; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
7421 ; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
7422 ; GFX6-NEXT: s_waitcnt expcnt(0)
7423 ; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
7424 ; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
7425 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
7426 ; GFX6-NEXT: v_min_f32_e32 v5, v5, v0
7427 ; GFX6-NEXT: v_min_f32_e32 v6, v6, v1
7428 ; GFX6-NEXT: v_alignbit_b32 v4, v4, v3, 16
7429 ; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
7430 ; GFX6-NEXT: v_alignbit_b32 v3, v3, v6, 16
7431 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
7432 ; GFX6-NEXT: v_mov_b32_e32 v5, v3
7433 ; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
7434 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7435 ; GFX6-NEXT: buffer_wbinvl1
7436 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
7437 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
7438 ; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7439 ; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
7440 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
7441 ; GFX6-NEXT: s_cbranch_execnz .LBB20_1
7442 ; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
7443 ; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
7444 ; GFX6-NEXT: s_waitcnt expcnt(0)
7445 ; GFX6-NEXT: s_setpc_b64 s[30:31]
7446 %gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
7447 %unused = atomicrmw fmin ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
7448 ret void
7449 }
7451 define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, <2 x bfloat> %val) #0 {
7452 ; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
7453 ; GFX12: ; %bb.0:
7454 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
7455 ; GFX12-NEXT: s_wait_expcnt 0x0
7456 ; GFX12-NEXT: s_wait_samplecnt 0x0
7457 ; GFX12-NEXT: s_wait_bvhcnt 0x0
7458 ; GFX12-NEXT: s_wait_kmcnt 0x0
7459 ; GFX12-NEXT: v_add_nc_u32_e32 v7, 0x400, v4
7460 ; GFX12-NEXT: s_mov_b32 s1, exec_lo
7461 ; GFX12-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
7462 ; GFX12-NEXT: v_readfirstlane_b32 s4, v0
7463 ; GFX12-NEXT: v_readfirstlane_b32 s5, v1
7464 ; GFX12-NEXT: v_readfirstlane_b32 s6, v2
7465 ; GFX12-NEXT: v_readfirstlane_b32 s7, v3
7466 ; GFX12-NEXT: s_wait_alu 0xf1ff
7467 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
7468 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
7469 ; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
7470 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
7471 ; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
7472 ; GFX12-NEXT: s_wait_alu 0xfffe
7473 ; GFX12-NEXT: s_and_saveexec_b32 s0, s0
7474 ; GFX12-NEXT: s_wait_loadcnt 0x0
7475 ; GFX12-NEXT: buffer_load_b32 v6, v4, s[4:7], null offen offset:1024
7476 ; GFX12-NEXT: ; implicit-def: $vgpr4
7477 ; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
7478 ; GFX12-NEXT: s_cbranch_execnz .LBB21_1
7479 ; GFX12-NEXT: ; %bb.2:
7480 ; GFX12-NEXT: s_mov_b32 exec_lo, s1
7481 ; GFX12-NEXT: v_lshlrev_b32_e32 v8, 16, v5
7482 ; GFX12-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
7483 ; GFX12-NEXT: s_mov_b32 s1, 0
7484 ; GFX12-NEXT: .LBB21_3: ; %atomicrmw.start
7485 ; GFX12-NEXT: ; =>This Loop Header: Depth=1
7486 ; GFX12-NEXT: ; Child Loop BB21_4 Depth 2
7487 ; GFX12-NEXT: s_wait_loadcnt 0x0
7488 ; GFX12-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
7489 ; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v6
7490 ; GFX12-NEXT: s_mov_b32 s2, exec_lo
7491 ; GFX12-NEXT: s_wait_storecnt 0x0
7492 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7493 ; GFX12-NEXT: v_dual_min_num_f32 v5, v5, v9 :: v_dual_min_num_f32 v4, v4, v8
7494 ; GFX12-NEXT: v_bfe_u32 v11, v5, 16, 1
7495 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
7496 ; GFX12-NEXT: v_bfe_u32 v10, v4, 16, 1
7497 ; GFX12-NEXT: v_or_b32_e32 v12, 0x400000, v4
7498 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
7499 ; GFX12-NEXT: v_or_b32_e32 v13, 0x400000, v5
7500 ; GFX12-NEXT: v_add3_u32 v11, v11, v5, 0x7fff
7501 ; GFX12-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
7502 ; GFX12-NEXT: s_wait_alu 0xfffd
7503 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
7504 ; GFX12-NEXT: v_cndmask_b32_e32 v4, v10, v12, vcc_lo
7505 ; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
7506 ; GFX12-NEXT: s_wait_alu 0xfffd
7507 ; GFX12-NEXT: v_cndmask_b32_e32 v5, v11, v13, vcc_lo
7508 ; GFX12-NEXT: v_perm_b32 v5, v5, v4, 0x7060302
7509 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
7510 ; GFX12-NEXT: v_mov_b32_e32 v4, v5
7511 ; GFX12-NEXT: v_mov_b32_e32 v5, v6
7512 ; GFX12-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
7513 ; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
7514 ; GFX12-NEXT: v_readfirstlane_b32 s4, v0
7515 ; GFX12-NEXT: v_readfirstlane_b32 s5, v1
7516 ; GFX12-NEXT: v_readfirstlane_b32 s6, v2
7517 ; GFX12-NEXT: v_readfirstlane_b32 s7, v3
7518 ; GFX12-NEXT: s_wait_alu 0xf1ff
7519 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
7520 ; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
7521 ; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
7522 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
7523 ; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
7524 ; GFX12-NEXT: s_wait_alu 0xfffe
7525 ; GFX12-NEXT: s_and_saveexec_b32 s0, s0
7526 ; GFX12-NEXT: s_wait_loadcnt 0x0
7527 ; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v7, s[4:7], null offen th:TH_ATOMIC_RETURN
7528 ; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
7529 ; GFX12-NEXT: s_cbranch_execnz .LBB21_4
7530 ; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
7531 ; GFX12-NEXT: s_mov_b32 exec_lo, s2
7532 ; GFX12-NEXT: s_wait_loadcnt 0x0
7533 ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
7534 ; GFX12-NEXT: v_mov_b32_e32 v6, v4
7535 ; GFX12-NEXT: global_inv scope:SCOPE_DEV
7536 ; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
7537 ; GFX12-NEXT: s_wait_alu 0xfffe
7538 ; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
7539 ; GFX12-NEXT: s_cbranch_execnz .LBB21_3
7540 ; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
7541 ; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
7542 ; GFX12-NEXT: v_mov_b32_e32 v0, v4
7543 ; GFX12-NEXT: s_setpc_b64 s[30:31]
7544 ;
7545 ; GFX940-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
7546 ; GFX940: ; %bb.0:
7547 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7548 ; GFX940-NEXT: v_add_u32_e32 v8, 0x400, v4
7549 ; GFX940-NEXT: s_mov_b64 s[2:3], exec
7550 ; GFX940-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
7551 ; GFX940-NEXT: v_readfirstlane_b32 s4, v0
7552 ; GFX940-NEXT: v_readfirstlane_b32 s5, v1
7553 ; GFX940-NEXT: v_readfirstlane_b32 s6, v2
7554 ; GFX940-NEXT: v_readfirstlane_b32 s7, v3
7555 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
7556 ; GFX940-NEXT: s_nop 0
7557 ; GFX940-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
7558 ; GFX940-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
7559 ; GFX940-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
7560 ; GFX940-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
7561 ; GFX940-NEXT: ; implicit-def: $vgpr4
7562 ; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
7563 ; GFX940-NEXT: s_cbranch_execnz .LBB21_1
7564 ; GFX940-NEXT: ; %bb.2:
7565 ; GFX940-NEXT: s_mov_b64 exec, s[2:3]
7566 ; GFX940-NEXT: s_mov_b64 s[2:3], 0
7567 ; GFX940-NEXT: v_lshlrev_b32_e32 v9, 16, v5
7568 ; GFX940-NEXT: s_movk_i32 s10, 0x7fff
7569 ; GFX940-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
7570 ; GFX940-NEXT: s_mov_b32 s11, 0x7060302
7571 ; GFX940-NEXT: .LBB21_3: ; %atomicrmw.start
7572 ; GFX940-NEXT: ; =>This Loop Header: Depth=1
7573 ; GFX940-NEXT: ; Child Loop BB21_4 Depth 2
7574 ; GFX940-NEXT: s_waitcnt vmcnt(0)
7575 ; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v7
7576 ; GFX940-NEXT: v_min_f32_e32 v4, v4, v9
7577 ; GFX940-NEXT: v_bfe_u32 v5, v4, 16, 1
7578 ; GFX940-NEXT: v_add3_u32 v5, v5, v4, s10
7579 ; GFX940-NEXT: v_or_b32_e32 v6, 0x400000, v4
7580 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
7581 ; GFX940-NEXT: s_mov_b64 s[8:9], exec
7582 ; GFX940-NEXT: buffer_wbl2 sc1
7583 ; GFX940-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
7584 ; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
7585 ; GFX940-NEXT: v_min_f32_e32 v5, v5, v10
7586 ; GFX940-NEXT: v_bfe_u32 v6, v5, 16, 1
7587 ; GFX940-NEXT: v_add3_u32 v6, v6, v5, s10
7588 ; GFX940-NEXT: v_or_b32_e32 v11, 0x400000, v5
7589 ; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
7590 ; GFX940-NEXT: s_nop 1
7591 ; GFX940-NEXT: v_cndmask_b32_e32 v5, v6, v11, vcc
7592 ; GFX940-NEXT: v_perm_b32 v6, v5, v4, s11
7593 ; GFX940-NEXT: v_mov_b64_e32 v[4:5], v[6:7]
7594 ; GFX940-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
7595 ; GFX940-NEXT: ; => This Inner Loop Header: Depth=2
7596 ; GFX940-NEXT: v_readfirstlane_b32 s4, v0
7597 ; GFX940-NEXT: v_readfirstlane_b32 s5, v1
7598 ; GFX940-NEXT: v_readfirstlane_b32 s6, v2
7599 ; GFX940-NEXT: v_readfirstlane_b32 s7, v3
7600 ; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
7601 ; GFX940-NEXT: s_nop 0
7602 ; GFX940-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
7603 ; GFX940-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
7604 ; GFX940-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
7605 ; GFX940-NEXT: s_waitcnt vmcnt(0)
7606 ; GFX940-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen sc0
7607 ; GFX940-NEXT: s_xor_b64 exec, exec, s[0:1]
7608 ; GFX940-NEXT: s_cbranch_execnz .LBB21_4
7609 ; GFX940-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
7610 ; GFX940-NEXT: s_mov_b64 exec, s[8:9]
7611 ; GFX940-NEXT: s_waitcnt vmcnt(0)
7612 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
7613 ; GFX940-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
7614 ; GFX940-NEXT: v_mov_b32_e32 v7, v4
7615 ; GFX940-NEXT: buffer_inv sc1
7616 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[2:3]
7617 ; GFX940-NEXT: s_cbranch_execnz .LBB21_3
7618 ; GFX940-NEXT: ; %bb.6: ; %atomicrmw.end
7619 ; GFX940-NEXT: s_or_b64 exec, exec, s[2:3]
7620 ; GFX940-NEXT: v_mov_b32_e32 v0, v4
7621 ; GFX940-NEXT: s_setpc_b64 s[30:31]
7622 ;
7623 ; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
7624 ; GFX11: ; %bb.0:
7625 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7626 ; GFX11-NEXT: v_add_nc_u32_e32 v7, 0x400, v4
7627 ; GFX11-NEXT: s_mov_b32 s1, 0
7628 ; GFX11-NEXT: s_mov_b32 s2, exec_lo
7629 ; GFX11-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
7630 ; GFX11-NEXT: v_readfirstlane_b32 s4, v0
7631 ; GFX11-NEXT: v_readfirstlane_b32 s5, v1
7632 ; GFX11-NEXT: v_readfirstlane_b32 s6, v2
7633 ; GFX11-NEXT: v_readfirstlane_b32 s7, v3
7634 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
7635 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
7636 ; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
7637 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
7638 ; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
7639 ; GFX11-NEXT: s_and_saveexec_b32 s0, s0
7640 ; GFX11-NEXT: buffer_load_b32 v6, v4, s[4:7], 0 offen offset:1024
7641 ; GFX11-NEXT: ; implicit-def: $vgpr4
7642 ; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
7643 ; GFX11-NEXT: s_cbranch_execnz .LBB21_1
7644 ; GFX11-NEXT: ; %bb.2:
7645 ; GFX11-NEXT: s_mov_b32 exec_lo, s2
7646 ; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v5
7647 ; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
7648 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
7649 ; GFX11-NEXT: .p2align 6
7650 ; GFX11-NEXT: .LBB21_3: ; %atomicrmw.start
7651 ; GFX11-NEXT: ; =>This Loop Header: Depth=1
7652 ; GFX11-NEXT: ; Child Loop BB21_4 Depth 2
7653 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7654 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
7655 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v6
7656 ; GFX11-NEXT: s_mov_b32 s2, exec_lo
7657 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
7658 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7659 ; GFX11-NEXT: v_dual_min_f32 v5, v5, v9 :: v_dual_min_f32 v4, v4, v8
7660 ; GFX11-NEXT: v_bfe_u32 v11, v5, 16, 1
7661 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
7662 ; GFX11-NEXT: v_bfe_u32 v10, v4, 16, 1
7663 ; GFX11-NEXT: v_or_b32_e32 v12, 0x400000, v4
7664 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
7665 ; GFX11-NEXT: v_or_b32_e32 v13, 0x400000, v5
7666 ; GFX11-NEXT: v_add3_u32 v11, v11, v5, 0x7fff
7667 ; GFX11-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
7668 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
7669 ; GFX11-NEXT: v_cndmask_b32_e32 v4, v10, v12, vcc_lo
7670 ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
7671 ; GFX11-NEXT: v_cndmask_b32_e32 v5, v11, v13, vcc_lo
7672 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
7673 ; GFX11-NEXT: v_perm_b32 v5, v5, v4, 0x7060302
7674 ; GFX11-NEXT: v_mov_b32_e32 v4, v5
7675 ; GFX11-NEXT: v_mov_b32_e32 v5, v6
7676 ; GFX11-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
7677 ; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
7678 ; GFX11-NEXT: v_readfirstlane_b32 s4, v0
7679 ; GFX11-NEXT: v_readfirstlane_b32 s5, v1
7680 ; GFX11-NEXT: v_readfirstlane_b32 s6, v2
7681 ; GFX11-NEXT: v_readfirstlane_b32 s7, v3
7682 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
7683 ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
7684 ; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
7685 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
7686 ; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
7687 ; GFX11-NEXT: s_and_saveexec_b32 s0, s0
7688 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7689 ; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v7, s[4:7], 0 offen glc
7690 ; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
7691 ; GFX11-NEXT: s_cbranch_execnz .LBB21_4
7692 ; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
7693 ; GFX11-NEXT: s_mov_b32 exec_lo, s2
7694 ; GFX11-NEXT: s_waitcnt vmcnt(0)
7695 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
7696 ; GFX11-NEXT: v_mov_b32_e32 v6, v4
7697 ; GFX11-NEXT: buffer_gl1_inv
7698 ; GFX11-NEXT: buffer_gl0_inv
7699 ; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
7700 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
7701 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
7702 ; GFX11-NEXT: s_cbranch_execnz .LBB21_3
7703 ; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
7704 ; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
7705 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
7706 ; GFX11-NEXT: v_mov_b32_e32 v0, v4
7707 ; GFX11-NEXT: s_setpc_b64 s[30:31]
7708 ;
7709 ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
7710 ; GFX10: ; %bb.0:
7711 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7712 ; GFX10-NEXT: v_add_nc_u32_e32 v7, 0x400, v4
7713 ; GFX10-NEXT: s_mov_b32 s5, 0
7714 ; GFX10-NEXT: s_mov_b32 s6, exec_lo
7715 ; GFX10-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
7716 ; GFX10-NEXT: v_readfirstlane_b32 s8, v0
7717 ; GFX10-NEXT: v_readfirstlane_b32 s9, v1
7718 ; GFX10-NEXT: v_readfirstlane_b32 s10, v2
7719 ; GFX10-NEXT: v_readfirstlane_b32 s11, v3
7720 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
7721 ; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
7722 ; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
7723 ; GFX10-NEXT: s_and_saveexec_b32 s4, s4
7724 ; GFX10-NEXT: buffer_load_dword v6, v4, s[8:11], 0 offen offset:1024
7725 ; GFX10-NEXT: ; implicit-def: $vgpr4
7726 ; GFX10-NEXT: s_waitcnt_depctr 0xffe3
7727 ; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
7728 ; GFX10-NEXT: s_cbranch_execnz .LBB21_1
7729 ; GFX10-NEXT: ; %bb.2:
7730 ; GFX10-NEXT: s_mov_b32 exec_lo, s6
7731 ; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v5
7732 ; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
7733 ; GFX10-NEXT: .LBB21_3: ; %atomicrmw.start
7734 ; GFX10-NEXT: ; =>This Loop Header: Depth=1
7735 ; GFX10-NEXT: ; Child Loop BB21_4 Depth 2
7736 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7737 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v6
7738 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
7739 ; GFX10-NEXT: s_mov_b32 s6, exec_lo
7740 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
7741 ; GFX10-NEXT: v_min_f32_e32 v4, v4, v8
7742 ; GFX10-NEXT: v_min_f32_e32 v5, v5, v9
7743 ; GFX10-NEXT: v_bfe_u32 v10, v4, 16, 1
7744 ; GFX10-NEXT: v_bfe_u32 v11, v5, 16, 1
7745 ; GFX10-NEXT: v_or_b32_e32 v12, 0x400000, v4
7746 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
7747 ; GFX10-NEXT: v_or_b32_e32 v13, 0x400000, v5
7748 ; GFX10-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
7749 ; GFX10-NEXT: v_add3_u32 v11, v11, v5, 0x7fff
7750 ; GFX10-NEXT: v_cndmask_b32_e32 v4, v10, v12, vcc_lo
7751 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
7752 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v11, v13, vcc_lo
7753 ; GFX10-NEXT: v_perm_b32 v5, v5, v4, 0x7060302
7754 ; GFX10-NEXT: v_mov_b32_e32 v4, v5
7755 ; GFX10-NEXT: v_mov_b32_e32 v5, v6
7756 ; GFX10-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
7757 ; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
7758 ; GFX10-NEXT: v_readfirstlane_b32 s8, v0
7759 ; GFX10-NEXT: v_readfirstlane_b32 s9, v1
7760 ; GFX10-NEXT: v_readfirstlane_b32 s10, v2
7761 ; GFX10-NEXT: v_readfirstlane_b32 s11, v3
7762 ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
7763 ; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
7764 ; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
7765 ; GFX10-NEXT: s_and_saveexec_b32 s4, s4
7766 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7767 ; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v7, s[8:11], 0 offen glc
7768 ; GFX10-NEXT: s_waitcnt_depctr 0xffe3
7769 ; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
7770 ; GFX10-NEXT: s_cbranch_execnz .LBB21_4
7771 ; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
7772 ; GFX10-NEXT: s_mov_b32 exec_lo, s6
7773 ; GFX10-NEXT: s_waitcnt vmcnt(0)
7774 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
7775 ; GFX10-NEXT: v_mov_b32_e32 v6, v4
7776 ; GFX10-NEXT: buffer_gl1_inv
7777 ; GFX10-NEXT: buffer_gl0_inv
7778 ; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
7779 ; GFX10-NEXT: s_waitcnt_depctr 0xffe3
7780 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
7781 ; GFX10-NEXT: s_cbranch_execnz .LBB21_3
7782 ; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
7783 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
7784 ; GFX10-NEXT: v_mov_b32_e32 v0, v4
7785 ; GFX10-NEXT: s_setpc_b64 s[30:31]
7786 ;
7787 ; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
7788 ; GFX90A: ; %bb.0:
7789 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7790 ; GFX90A-NEXT: v_add_u32_e32 v8, 0x400, v4
7791 ; GFX90A-NEXT: s_mov_b64 s[6:7], exec
7792 ; GFX90A-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
7793 ; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
7794 ; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
7795 ; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
7796 ; GFX90A-NEXT: v_readfirstlane_b32 s11, v3
7797 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
7798 ; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
7799 ; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
7800 ; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
7801 ; GFX90A-NEXT: s_nop 0
7802 ; GFX90A-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
7803 ; GFX90A-NEXT: ; implicit-def: $vgpr4
7804 ; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
7805 ; GFX90A-NEXT: s_cbranch_execnz .LBB21_1
7806 ; GFX90A-NEXT: ; %bb.2:
7807 ; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
7808 ; GFX90A-NEXT: s_mov_b64 s[6:7], 0
7809 ; GFX90A-NEXT: v_lshlrev_b32_e32 v9, 16, v5
7810 ; GFX90A-NEXT: s_movk_i32 s14, 0x7fff
7811 ; GFX90A-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
7812 ; GFX90A-NEXT: s_mov_b32 s15, 0x7060302
7813 ; GFX90A-NEXT: .LBB21_3: ; %atomicrmw.start
7814 ; GFX90A-NEXT: ; =>This Loop Header: Depth=1
7815 ; GFX90A-NEXT: ; Child Loop BB21_4 Depth 2
7816 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7817 ; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v7
7818 ; GFX90A-NEXT: v_min_f32_e32 v4, v4, v9
7819 ; GFX90A-NEXT: v_bfe_u32 v5, v4, 16, 1
7820 ; GFX90A-NEXT: v_add3_u32 v5, v5, v4, s14
7821 ; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v4
7822 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
7823 ; GFX90A-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
7824 ; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
7825 ; GFX90A-NEXT: v_min_f32_e32 v5, v5, v10
7826 ; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
7827 ; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s14
7828 ; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v5
7829 ; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
7830 ; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v11, vcc
7831 ; GFX90A-NEXT: v_perm_b32 v6, v5, v4, s15
7832 ; GFX90A-NEXT: s_mov_b64 s[12:13], exec
7833 ; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[6:7] op_sel:[0,1]
7834 ; GFX90A-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
7835 ; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
7836 ; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
7837 ; GFX90A-NEXT: v_readfirstlane_b32 s9, v1
7838 ; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
7839 ; GFX90A-NEXT: v_readfirstlane_b32 s11, v3
7840 ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
7841 ; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
7842 ; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
7843 ; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
7844 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7845 ; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
7846 ; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
7847 ; GFX90A-NEXT: s_cbranch_execnz .LBB21_4
7848 ; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
7849 ; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
7850 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
7851 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
7852 ; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
7853 ; GFX90A-NEXT: v_mov_b32_e32 v7, v4
7854 ; GFX90A-NEXT: buffer_wbinvl1
7855 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
7856 ; GFX90A-NEXT: s_cbranch_execnz .LBB21_3
7857 ; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.end
7858 ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
7859 ; GFX90A-NEXT: v_mov_b32_e32 v0, v4
7860 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
7861 ;
7862 ; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
7863 ; GFX908: ; %bb.0:
7864 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7865 ; GFX908-NEXT: v_add_u32_e32 v7, 0x400, v4
7866 ; GFX908-NEXT: s_mov_b64 s[6:7], exec
7867 ; GFX908-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
7868 ; GFX908-NEXT: v_readfirstlane_b32 s8, v0
7869 ; GFX908-NEXT: v_readfirstlane_b32 s9, v1
7870 ; GFX908-NEXT: v_readfirstlane_b32 s10, v2
7871 ; GFX908-NEXT: v_readfirstlane_b32 s11, v3
7872 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
7873 ; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
7874 ; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
7875 ; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
7876 ; GFX908-NEXT: s_nop 0
7877 ; GFX908-NEXT: buffer_load_dword v6, v4, s[8:11], 0 offen offset:1024
7878 ; GFX908-NEXT: ; implicit-def: $vgpr4
7879 ; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
7880 ; GFX908-NEXT: s_cbranch_execnz .LBB21_1
7881 ; GFX908-NEXT: ; %bb.2:
7882 ; GFX908-NEXT: s_mov_b64 exec, s[6:7]
7883 ; GFX908-NEXT: s_mov_b64 s[6:7], 0
7884 ; GFX908-NEXT: v_lshlrev_b32_e32 v8, 16, v5
7885 ; GFX908-NEXT: s_movk_i32 s14, 0x7fff
7886 ; GFX908-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
7887 ; GFX908-NEXT: s_mov_b32 s15, 0x7060302
7888 ; GFX908-NEXT: .LBB21_3: ; %atomicrmw.start
7889 ; GFX908-NEXT: ; =>This Loop Header: Depth=1
7890 ; GFX908-NEXT: ; Child Loop BB21_4 Depth 2
7891 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7892 ; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v6
7893 ; GFX908-NEXT: v_min_f32_e32 v4, v4, v8
7894 ; GFX908-NEXT: v_bfe_u32 v5, v4, 16, 1
7895 ; GFX908-NEXT: v_add3_u32 v5, v5, v4, s14
7896 ; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v4
7897 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
7898 ; GFX908-NEXT: v_cndmask_b32_e32 v4, v5, v10, vcc
7899 ; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
7900 ; GFX908-NEXT: v_min_f32_e32 v5, v5, v9
7901 ; GFX908-NEXT: v_bfe_u32 v10, v5, 16, 1
7902 ; GFX908-NEXT: v_add3_u32 v10, v10, v5, s14
7903 ; GFX908-NEXT: v_or_b32_e32 v11, 0x400000, v5
7904 ; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
7905 ; GFX908-NEXT: v_cndmask_b32_e32 v5, v10, v11, vcc
7906 ; GFX908-NEXT: v_perm_b32 v5, v5, v4, s15
7907 ; GFX908-NEXT: v_mov_b32_e32 v4, v5
7908 ; GFX908-NEXT: s_mov_b64 s[12:13], exec
7909 ; GFX908-NEXT: v_mov_b32_e32 v5, v6
7910 ; GFX908-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
7911 ; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
7912 ; GFX908-NEXT: v_readfirstlane_b32 s8, v0
7913 ; GFX908-NEXT: v_readfirstlane_b32 s9, v1
7914 ; GFX908-NEXT: v_readfirstlane_b32 s10, v2
7915 ; GFX908-NEXT: v_readfirstlane_b32 s11, v3
7916 ; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
7917 ; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
7918 ; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
7919 ; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
7920 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7921 ; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v7, s[8:11], 0 offen glc
7922 ; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
7923 ; GFX908-NEXT: s_cbranch_execnz .LBB21_4
7924 ; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
7925 ; GFX908-NEXT: s_mov_b64 exec, s[12:13]
7926 ; GFX908-NEXT: s_waitcnt vmcnt(0)
7927 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
7928 ; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
7929 ; GFX908-NEXT: v_mov_b32_e32 v6, v4
7930 ; GFX908-NEXT: buffer_wbinvl1
7931 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
7932 ; GFX908-NEXT: s_cbranch_execnz .LBB21_3
7933 ; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
7934 ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
7935 ; GFX908-NEXT: v_mov_b32_e32 v0, v4
7936 ; GFX908-NEXT: s_setpc_b64 s[30:31]
7937 ;
7938 ; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
7939 ; GFX8: ; %bb.0:
7940 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7941 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x400, v4
7942 ; GFX8-NEXT: s_mov_b64 s[6:7], exec
7943 ; GFX8-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
7944 ; GFX8-NEXT: v_readfirstlane_b32 s8, v0
7945 ; GFX8-NEXT: v_readfirstlane_b32 s9, v1
7946 ; GFX8-NEXT: v_readfirstlane_b32 s10, v2
7947 ; GFX8-NEXT: v_readfirstlane_b32 s11, v3
7948 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
7949 ; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
7950 ; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
7951 ; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
7952 ; GFX8-NEXT: s_nop 0
7953 ; GFX8-NEXT: buffer_load_dword v6, v4, s[8:11], 0 offen offset:1024
7954 ; GFX8-NEXT: ; implicit-def: $vgpr4
7955 ; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
7956 ; GFX8-NEXT: s_cbranch_execnz .LBB21_1
7957 ; GFX8-NEXT: ; %bb.2:
7958 ; GFX8-NEXT: s_mov_b64 exec, s[6:7]
7959 ; GFX8-NEXT: s_mov_b64 s[6:7], 0
7960 ; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v5
7961 ; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
7962 ; GFX8-NEXT: .LBB21_3: ; %atomicrmw.start
7963 ; GFX8-NEXT: ; =>This Loop Header: Depth=1
7964 ; GFX8-NEXT: ; Child Loop BB21_4 Depth 2
7965 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7966 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v6
7967 ; GFX8-NEXT: v_min_f32_e32 v4, v4, v8
7968 ; GFX8-NEXT: v_bfe_u32 v5, v4, 16, 1
7969 ; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v4
7970 ; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
7971 ; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v4
7972 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
7973 ; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v10, vcc
7974 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
7975 ; GFX8-NEXT: v_min_f32_e32 v5, v5, v9
7976 ; GFX8-NEXT: v_bfe_u32 v10, v5, 16, 1
7977 ; GFX8-NEXT: v_add_u32_e32 v10, vcc, v10, v5
7978 ; GFX8-NEXT: v_add_u32_e32 v10, vcc, 0x7fff, v10
7979 ; GFX8-NEXT: v_or_b32_e32 v11, 0x400000, v5
7980 ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
7981 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v10, v11, vcc
7982 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
7983 ; GFX8-NEXT: v_alignbit_b32 v5, v5, v4, 16
7984 ; GFX8-NEXT: v_mov_b32_e32 v4, v5
7985 ; GFX8-NEXT: s_mov_b64 s[12:13], exec
7986 ; GFX8-NEXT: v_mov_b32_e32 v5, v6
7987 ; GFX8-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
7988 ; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
7989 ; GFX8-NEXT: v_readfirstlane_b32 s8, v0
7990 ; GFX8-NEXT: v_readfirstlane_b32 s9, v1
7991 ; GFX8-NEXT: v_readfirstlane_b32 s10, v2
7992 ; GFX8-NEXT: v_readfirstlane_b32 s11, v3
7993 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
7994 ; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
7995 ; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
7996 ; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
7997 ; GFX8-NEXT: s_waitcnt vmcnt(0)
7998 ; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v7, s[8:11], 0 offen glc
7999 ; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
8000 ; GFX8-NEXT: s_cbranch_execnz .LBB21_4
8001 ; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
8002 ; GFX8-NEXT: s_mov_b64 exec, s[12:13]
8003 ; GFX8-NEXT: s_waitcnt vmcnt(0)
8004 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
8005 ; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
8006 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
8007 ; GFX8-NEXT: buffer_wbinvl1
8008 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
8009 ; GFX8-NEXT: s_cbranch_execnz .LBB21_3
8010 ; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
8011 ; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
8012 ; GFX8-NEXT: v_mov_b32_e32 v0, v4
8013 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8014 ;
8015 ; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
8016 ; GFX7: ; %bb.0:
8017 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8018 ; GFX7-NEXT: v_add_i32_e32 v8, vcc, 0x400, v4
8019 ; GFX7-NEXT: s_mov_b64 s[6:7], exec
8020 ; GFX7-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
8021 ; GFX7-NEXT: v_readfirstlane_b32 s8, v0
8022 ; GFX7-NEXT: v_readfirstlane_b32 s9, v1
8023 ; GFX7-NEXT: v_readfirstlane_b32 s10, v2
8024 ; GFX7-NEXT: v_readfirstlane_b32 s11, v3
8025 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
8026 ; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
8027 ; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
8028 ; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
8029 ; GFX7-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
8030 ; GFX7-NEXT: ; implicit-def: $vgpr4
8031 ; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
8032 ; GFX7-NEXT: s_cbranch_execnz .LBB21_1
8033 ; GFX7-NEXT: ; %bb.2:
8034 ; GFX7-NEXT: s_mov_b64 exec, s[6:7]
8035 ; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
8036 ; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
8037 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8038 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v7
8039 ; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
8040 ; GFX7-NEXT: s_mov_b64 s[6:7], 0
8041 ; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
8042 ; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
8043 ; GFX7-NEXT: .LBB21_3: ; %atomicrmw.start
8044 ; GFX7-NEXT: ; =>This Loop Header: Depth=1
8045 ; GFX7-NEXT: ; Child Loop BB21_4 Depth 2
8046 ; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v4
8047 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
8048 ; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v7
8049 ; GFX7-NEXT: v_min_f32_e32 v4, v4, v9
8050 ; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v6
8051 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
8052 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
8053 ; GFX7-NEXT: v_min_f32_e32 v7, v7, v10
8054 ; GFX7-NEXT: v_alignbit_b32 v5, v5, v6, 16
8055 ; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
8056 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
8057 ; GFX7-NEXT: s_mov_b64 s[12:13], exec
8058 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
8059 ; GFX7-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
8060 ; GFX7-NEXT: ; => This Inner Loop Header: Depth=2
8061 ; GFX7-NEXT: v_readfirstlane_b32 s8, v0
8062 ; GFX7-NEXT: v_readfirstlane_b32 s9, v1
8063 ; GFX7-NEXT: v_readfirstlane_b32 s10, v2
8064 ; GFX7-NEXT: v_readfirstlane_b32 s11, v3
8065 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
8066 ; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
8067 ; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
8068 ; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
8069 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8070 ; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v8, s[8:11], 0 offen glc
8071 ; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]
8072 ; GFX7-NEXT: s_cbranch_execnz .LBB21_4
8073 ; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
8074 ; GFX7-NEXT: s_mov_b64 exec, s[12:13]
8075 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8076 ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
8077 ; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
8078 ; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
8079 ; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v6
8080 ; GFX7-NEXT: buffer_wbinvl1
8081 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
8082 ; GFX7-NEXT: s_cbranch_execnz .LBB21_3
8083 ; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end
8084 ; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
8085 ; GFX7-NEXT: v_mov_b32_e32 v0, v7
8086 ; GFX7-NEXT: v_mov_b32_e32 v1, v4
8087 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8088 ;
8089 ; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
8090 ; GFX6: ; %bb.0:
8091 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8092 ; GFX6-NEXT: v_add_i32_e32 v8, vcc, 0x400, v4
8093 ; GFX6-NEXT: s_mov_b64 s[6:7], exec
8094 ; GFX6-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
8095 ; GFX6-NEXT: v_readfirstlane_b32 s8, v0
8096 ; GFX6-NEXT: v_readfirstlane_b32 s9, v1
8097 ; GFX6-NEXT: v_readfirstlane_b32 s10, v2
8098 ; GFX6-NEXT: v_readfirstlane_b32 s11, v3
8099 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
8100 ; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
8101 ; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
8102 ; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
8103 ; GFX6-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
8104 ; GFX6-NEXT: ; implicit-def: $vgpr4
8105 ; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
8106 ; GFX6-NEXT: s_cbranch_execnz .LBB21_1
8107 ; GFX6-NEXT: ; %bb.2:
8108 ; GFX6-NEXT: s_mov_b64 exec, s[6:7]
8109 ; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v6
8110 ; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v5
8111 ; GFX6-NEXT: s_waitcnt vmcnt(0)
8112 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v7
8113 ; GFX6-NEXT: v_lshlrev_b32_e32 v7, 16, v7
8114 ; GFX6-NEXT: s_mov_b64 s[6:7], 0
8115 ; GFX6-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
8116 ; GFX6-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
8117 ; GFX6-NEXT: .LBB21_3: ; %atomicrmw.start
8118 ; GFX6-NEXT: ; =>This Loop Header: Depth=1
8119 ; GFX6-NEXT: ; Child Loop BB21_4 Depth 2
8120 ; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v4
8121 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
8122 ; GFX6-NEXT: v_mul_f32_e32 v6, 1.0, v7
8123 ; GFX6-NEXT: v_min_f32_e32 v4, v4, v9
8124 ; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v6
8125 ; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v5
8126 ; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
8127 ; GFX6-NEXT: v_min_f32_e32 v7, v7, v10
8128 ; GFX6-NEXT: v_alignbit_b32 v5, v5, v6, 16
8129 ; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
8130 ; GFX6-NEXT: v_mov_b32_e32 v7, v5
8131 ; GFX6-NEXT: s_mov_b64 s[12:13], exec
8132 ; GFX6-NEXT: v_mov_b32_e32 v6, v4
8133 ; GFX6-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
8134 ; GFX6-NEXT: ; => This Inner Loop Header: Depth=2
8135 ; GFX6-NEXT: v_readfirstlane_b32 s8, v0
8136 ; GFX6-NEXT: v_readfirstlane_b32 s9, v1
8137 ; GFX6-NEXT: v_readfirstlane_b32 s10, v2
8138 ; GFX6-NEXT: v_readfirstlane_b32 s11, v3
8139 ; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
8140 ; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
8141 ; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
8142 ; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
8143 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
8144 ; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v8, s[8:11], 0 offen glc
8145 ; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]
8146 ; GFX6-NEXT: s_cbranch_execnz .LBB21_4
8147 ; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
8148 ; GFX6-NEXT: s_mov_b64 exec, s[12:13]
8149 ; GFX6-NEXT: s_waitcnt vmcnt(0)
8150 ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v5
8151 ; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
8152 ; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
8153 ; GFX6-NEXT: s_waitcnt expcnt(0)
8154 ; GFX6-NEXT: v_lshlrev_b32_e32 v7, 16, v6
8155 ; GFX6-NEXT: buffer_wbinvl1
8156 ; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]
8157 ; GFX6-NEXT: s_cbranch_execnz .LBB21_3
8158 ; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end
8159 ; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
8160 ; GFX6-NEXT: v_mov_b32_e32 v0, v7
8161 ; GFX6-NEXT: v_mov_b32_e32 v1, v4
8162 ; GFX6-NEXT: s_setpc_b64 s[30:31]
8163 %gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 256
8164 %result = atomicrmw fmin ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
8165 ret <2 x bfloat> %result
8166 }
8168 ; --------------------------------------------------------------------
8169 ; misc
8170 ; --------------------------------------------------------------------
8172 define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, float %val) #0 {
8173 ; GFX12-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
8174 ; GFX12: ; %bb.0:
8175 ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
8176 ; GFX12-NEXT: s_wait_expcnt 0x0
8177 ; GFX12-NEXT: s_wait_samplecnt 0x0
8178 ; GFX12-NEXT: s_wait_bvhcnt 0x0
8179 ; GFX12-NEXT: s_wait_kmcnt 0x0
8180 ; GFX12-NEXT: v_mov_b32_e32 v1, s16
8181 ; GFX12-NEXT: global_wb scope:SCOPE_SYS
8182 ; GFX12-NEXT: s_wait_storecnt 0x0
8183 ; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
8184 ; GFX12-NEXT: s_wait_loadcnt 0x0
8185 ; GFX12-NEXT: global_inv scope:SCOPE_SYS
8186 ; GFX12-NEXT: s_setpc_b64 s[30:31]
8187 ;
8188 ; GFX940-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
8189 ; GFX940: ; %bb.0:
8190 ; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8191 ; GFX940-NEXT: v_mov_b32_e32 v1, v0
8192 ; GFX940-NEXT: v_mov_b32_e32 v0, s16
8193 ; GFX940-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
8194 ; GFX940-NEXT: s_add_i32 s6, s16, 0x400
8195 ; GFX940-NEXT: s_mov_b64 s[4:5], 0
8196 ; GFX940-NEXT: v_max_f32_e32 v2, v1, v1
8197 ; GFX940-NEXT: v_mov_b32_e32 v3, s6
8198 ; GFX940-NEXT: .LBB22_1: ; %atomicrmw.start
8199 ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
8200 ; GFX940-NEXT: s_waitcnt vmcnt(0)
8201 ; GFX940-NEXT: v_mov_b32_e32 v5, v0
8202 ; GFX940-NEXT: v_max_f32_e32 v0, v5, v5
8203 ; GFX940-NEXT: v_min_f32_e32 v4, v0, v2
8204 ; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
8205 ; GFX940-NEXT: buffer_wbl2 sc0 sc1
8206 ; GFX940-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen sc0
8207 ; GFX940-NEXT: s_waitcnt vmcnt(0)
8208 ; GFX940-NEXT: buffer_inv sc0 sc1
8209 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
8210 ; GFX940-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8211 ; GFX940-NEXT: s_andn2_b64 exec, exec, s[4:5]
8212 ; GFX940-NEXT: s_cbranch_execnz .LBB22_1
8213 ; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
8214 ; GFX940-NEXT: s_or_b64 exec, exec, s[4:5]
8215 ; GFX940-NEXT: s_setpc_b64 s[30:31]
8216 ;
8217 ; GFX11-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
8218 ; GFX11: ; %bb.0:
8219 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8220 ; GFX11-NEXT: v_mov_b32_e32 v1, s16
8221 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
8222 ; GFX11-NEXT: buffer_atomic_min_f32 v0, v1, s[0:3], 0 offen offset:1024 glc
8223 ; GFX11-NEXT: s_waitcnt vmcnt(0)
8224 ; GFX11-NEXT: buffer_gl1_inv
8225 ; GFX11-NEXT: buffer_gl0_inv
8226 ; GFX11-NEXT: s_setpc_b64 s[30:31]
8227 ;
8228 ; GFX10-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
8229 ; GFX10: ; %bb.0:
8230 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8231 ; GFX10-NEXT: v_mov_b32_e32 v1, s20
8232 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
8233 ; GFX10-NEXT: buffer_atomic_fmin v0, v1, s[16:19], 0 offen offset:1024 glc
8234 ; GFX10-NEXT: s_waitcnt vmcnt(0)
8235 ; GFX10-NEXT: buffer_gl1_inv
8236 ; GFX10-NEXT: buffer_gl0_inv
8237 ; GFX10-NEXT: s_setpc_b64 s[30:31]
8238 ;
8239 ; GFX90A-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
8240 ; GFX90A: ; %bb.0:
8241 ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8242 ; GFX90A-NEXT: v_mov_b32_e32 v1, v0
8243 ; GFX90A-NEXT: v_mov_b32_e32 v0, s20
8244 ; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
8245 ; GFX90A-NEXT: s_add_i32 s6, s20, 0x400
8246 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
8247 ; GFX90A-NEXT: v_max_f32_e32 v2, v1, v1
8248 ; GFX90A-NEXT: v_mov_b32_e32 v3, s6
8249 ; GFX90A-NEXT: .LBB22_1: ; %atomicrmw.start
8250 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
8251 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
8252 ; GFX90A-NEXT: v_mov_b32_e32 v5, v0
8253 ; GFX90A-NEXT: v_max_f32_e32 v0, v5, v5
8254 ; GFX90A-NEXT: v_min_f32_e32 v4, v0, v2
8255 ; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
8256 ; GFX90A-NEXT: buffer_wbl2
8257 ; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
8258 ; GFX90A-NEXT: s_waitcnt vmcnt(0)
8259 ; GFX90A-NEXT: buffer_invl2
8260 ; GFX90A-NEXT: buffer_wbinvl1
8261 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
8262 ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8263 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
8264 ; GFX90A-NEXT: s_cbranch_execnz .LBB22_1
8265 ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
8266 ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
8267 ; GFX90A-NEXT: s_setpc_b64 s[30:31]
8268 ;
8269 ; GFX908-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
8270 ; GFX908: ; %bb.0:
8271 ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8272 ; GFX908-NEXT: v_mov_b32_e32 v1, v0
8273 ; GFX908-NEXT: v_mov_b32_e32 v0, s20
8274 ; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
8275 ; GFX908-NEXT: s_add_i32 s6, s20, 0x400
8276 ; GFX908-NEXT: s_mov_b64 s[4:5], 0
8277 ; GFX908-NEXT: v_max_f32_e32 v2, v1, v1
8278 ; GFX908-NEXT: v_mov_b32_e32 v3, s6
8279 ; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
8280 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
8281 ; GFX908-NEXT: s_waitcnt vmcnt(0)
8282 ; GFX908-NEXT: v_mov_b32_e32 v5, v0
8283 ; GFX908-NEXT: v_max_f32_e32 v0, v5, v5
8284 ; GFX908-NEXT: v_min_f32_e32 v4, v0, v2
8285 ; GFX908-NEXT: v_mov_b32_e32 v0, v4
8286 ; GFX908-NEXT: v_mov_b32_e32 v1, v5
8287 ; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
8288 ; GFX908-NEXT: s_waitcnt vmcnt(0)
8289 ; GFX908-NEXT: buffer_wbinvl1
8290 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
8291 ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8292 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
8293 ; GFX908-NEXT: s_cbranch_execnz .LBB22_1
8294 ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
8295 ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
8296 ; GFX908-NEXT: s_setpc_b64 s[30:31]
8297 ;
8298 ; GFX8-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
8299 ; GFX8: ; %bb.0:
8300 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8301 ; GFX8-NEXT: v_mov_b32_e32 v1, v0
8302 ; GFX8-NEXT: v_mov_b32_e32 v0, s20
8303 ; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
8304 ; GFX8-NEXT: s_add_i32 s6, s20, 0x400
8305 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
8306 ; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v1
8307 ; GFX8-NEXT: v_mov_b32_e32 v3, s6
8308 ; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
8309 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
8310 ; GFX8-NEXT: s_waitcnt vmcnt(0)
8311 ; GFX8-NEXT: v_mov_b32_e32 v5, v0
8312 ; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v5
8313 ; GFX8-NEXT: v_min_f32_e32 v4, v0, v2
8314 ; GFX8-NEXT: v_mov_b32_e32 v0, v4
8315 ; GFX8-NEXT: v_mov_b32_e32 v1, v5
8316 ; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
8317 ; GFX8-NEXT: s_waitcnt vmcnt(0)
8318 ; GFX8-NEXT: buffer_wbinvl1
8319 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
8320 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8321 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
8322 ; GFX8-NEXT: s_cbranch_execnz .LBB22_1
8323 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
8324 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
8325 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8326 ;
8327 ; GFX7-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
8328 ; GFX7: ; %bb.0:
8329 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8330 ; GFX7-NEXT: v_mov_b32_e32 v1, s20
8331 ; GFX7-NEXT: buffer_atomic_fmin v0, v1, s[16:19], 0 offen offset:1024 glc
8332 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8333 ; GFX7-NEXT: buffer_wbinvl1
8334 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8335 ;
8336 ; GFX6-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
8337 ; GFX6: ; %bb.0:
8338 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8339 ; GFX6-NEXT: v_mov_b32_e32 v1, s20
8340 ; GFX6-NEXT: buffer_atomic_fmin v0, v1, s[16:19], 0 offen offset:1024 glc
8341 ; GFX6-NEXT: s_waitcnt vmcnt(0)
8342 ; GFX6-NEXT: buffer_wbinvl1
8343 ; GFX6-NEXT: s_waitcnt expcnt(0)
8344 ; GFX6-NEXT: s_setpc_b64 s[30:31]
8345 %gep = getelementptr float, ptr addrspace(7) %ptr, i32 256
8346 %result = atomicrmw fmin ptr addrspace(7) %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !0
8347 ret float %result
8348 }
8350 attributes #0 = { nounwind }
8352 !0 = !{}