| blob | fe47461ebf95690f56f35a71e9369b267c4d2dc0 |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc -mtriple=amdgcn -mcpu=bonaire < %s | FileCheck -check-prefix=GFX7 %s
3 ; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=GFX8 %s
4 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9 %s
6 ; ---------------------------------------------------------------------
7 ; atomicrmw xchg
8 ; ---------------------------------------------------------------------
10 define void @flat_atomic_xchg_i64_noret(ptr %ptr, i64 %in) {
11 ; GFX7-LABEL: flat_atomic_xchg_i64_noret:
12 ; GFX7: ; %bb.0:
13 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14 ; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
15 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
16 ; GFX7-NEXT: buffer_wbinvl1_vol
17 ; GFX7-NEXT: s_setpc_b64 s[30:31]
18 ;
19 ; GFX8-LABEL: flat_atomic_xchg_i64_noret:
20 ; GFX8: ; %bb.0:
21 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
22 ; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
23 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
24 ; GFX8-NEXT: buffer_wbinvl1_vol
25 ; GFX8-NEXT: s_setpc_b64 s[30:31]
26 ;
27 ; GFX9-LABEL: flat_atomic_xchg_i64_noret:
28 ; GFX9: ; %bb.0:
29 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
30 ; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
31 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
32 ; GFX9-NEXT: buffer_wbinvl1_vol
33 ; GFX9-NEXT: s_setpc_b64 s[30:31]
34 %tmp0 = atomicrmw xchg ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
35 ret void
36 }
38 define void @flat_atomic_xchg_i64_noret_offset(ptr %out, i64 %in) {
39 ; GFX7-LABEL: flat_atomic_xchg_i64_noret_offset:
40 ; GFX7: ; %bb.0:
41 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
42 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
43 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
44 ; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
45 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
46 ; GFX7-NEXT: buffer_wbinvl1_vol
47 ; GFX7-NEXT: s_setpc_b64 s[30:31]
48 ;
49 ; GFX8-LABEL: flat_atomic_xchg_i64_noret_offset:
50 ; GFX8: ; %bb.0:
51 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
52 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
53 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
54 ; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
55 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
56 ; GFX8-NEXT: buffer_wbinvl1_vol
57 ; GFX8-NEXT: s_setpc_b64 s[30:31]
58 ;
59 ; GFX9-LABEL: flat_atomic_xchg_i64_noret_offset:
60 ; GFX9: ; %bb.0:
61 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
62 ; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] offset:32
63 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
64 ; GFX9-NEXT: buffer_wbinvl1_vol
65 ; GFX9-NEXT: s_setpc_b64 s[30:31]
66 %gep = getelementptr i64, ptr %out, i64 4
67 %tmp0 = atomicrmw xchg ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
68 ret void
69 }
71 define i64 @flat_atomic_xchg_i64_ret(ptr %ptr, i64 %in) {
72 ; GFX7-LABEL: flat_atomic_xchg_i64_ret:
73 ; GFX7: ; %bb.0:
74 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
75 ; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
76 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
77 ; GFX7-NEXT: buffer_wbinvl1_vol
78 ; GFX7-NEXT: s_setpc_b64 s[30:31]
79 ;
80 ; GFX8-LABEL: flat_atomic_xchg_i64_ret:
81 ; GFX8: ; %bb.0:
82 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
83 ; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
84 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
85 ; GFX8-NEXT: buffer_wbinvl1_vol
86 ; GFX8-NEXT: s_setpc_b64 s[30:31]
87 ;
88 ; GFX9-LABEL: flat_atomic_xchg_i64_ret:
89 ; GFX9: ; %bb.0:
90 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
91 ; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
92 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
93 ; GFX9-NEXT: buffer_wbinvl1_vol
94 ; GFX9-NEXT: s_setpc_b64 s[30:31]
95 %result = atomicrmw xchg ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
96 ret i64 %result
97 }
99 define i64 @flat_atomic_xchg_i64_ret_offset(ptr %out, i64 %in) {
100 ; GFX7-LABEL: flat_atomic_xchg_i64_ret_offset:
101 ; GFX7: ; %bb.0:
102 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
103 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
104 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
105 ; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
106 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
107 ; GFX7-NEXT: buffer_wbinvl1_vol
108 ; GFX7-NEXT: s_setpc_b64 s[30:31]
109 ;
110 ; GFX8-LABEL: flat_atomic_xchg_i64_ret_offset:
111 ; GFX8: ; %bb.0:
112 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
113 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
114 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
115 ; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
116 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
117 ; GFX8-NEXT: buffer_wbinvl1_vol
118 ; GFX8-NEXT: s_setpc_b64 s[30:31]
119 ;
120 ; GFX9-LABEL: flat_atomic_xchg_i64_ret_offset:
121 ; GFX9: ; %bb.0:
122 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
123 ; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
124 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
125 ; GFX9-NEXT: buffer_wbinvl1_vol
126 ; GFX9-NEXT: s_setpc_b64 s[30:31]
127 %gep = getelementptr i64, ptr %out, i64 4
128 %result = atomicrmw xchg ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
129 ret i64 %result
130 }
132 define amdgpu_gfx void @flat_atomic_xchg_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
133 ; GFX7-LABEL: flat_atomic_xchg_i64_noret_scalar:
134 ; GFX7: ; %bb.0:
135 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
136 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
137 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
138 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
139 ; GFX7-NEXT: v_mov_b32_e32 v3, s5
140 ; GFX7-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]
141 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
142 ; GFX7-NEXT: buffer_wbinvl1_vol
143 ; GFX7-NEXT: s_setpc_b64 s[30:31]
144 ;
145 ; GFX8-LABEL: flat_atomic_xchg_i64_noret_scalar:
146 ; GFX8: ; %bb.0:
147 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
148 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
149 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
150 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
151 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
152 ; GFX8-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]
153 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
154 ; GFX8-NEXT: buffer_wbinvl1_vol
155 ; GFX8-NEXT: s_setpc_b64 s[30:31]
156 ;
157 ; GFX9-LABEL: flat_atomic_xchg_i64_noret_scalar:
158 ; GFX9: ; %bb.0:
159 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
160 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
161 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
162 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
163 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
164 ; GFX9-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]
165 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
166 ; GFX9-NEXT: buffer_wbinvl1_vol
167 ; GFX9-NEXT: s_setpc_b64 s[30:31]
168 %tmp0 = atomicrmw xchg ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
169 ret void
170 }
172 define amdgpu_gfx void @flat_atomic_xchg_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
173 ; GFX7-LABEL: flat_atomic_xchg_i64_noret_offset_scalar:
174 ; GFX7: ; %bb.0:
175 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
176 ; GFX7-NEXT: s_add_u32 s34, s4, 32
177 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
178 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
179 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
180 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
181 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
182 ; GFX7-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]
183 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
184 ; GFX7-NEXT: buffer_wbinvl1_vol
185 ; GFX7-NEXT: s_setpc_b64 s[30:31]
186 ;
187 ; GFX8-LABEL: flat_atomic_xchg_i64_noret_offset_scalar:
188 ; GFX8: ; %bb.0:
189 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
190 ; GFX8-NEXT: s_add_u32 s34, s4, 32
191 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
192 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
193 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
194 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
195 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
196 ; GFX8-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]
197 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
198 ; GFX8-NEXT: buffer_wbinvl1_vol
199 ; GFX8-NEXT: s_setpc_b64 s[30:31]
200 ;
201 ; GFX9-LABEL: flat_atomic_xchg_i64_noret_offset_scalar:
202 ; GFX9: ; %bb.0:
203 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
204 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
205 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
206 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
207 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
208 ; GFX9-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1] offset:32
209 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
210 ; GFX9-NEXT: buffer_wbinvl1_vol
211 ; GFX9-NEXT: s_setpc_b64 s[30:31]
212 %gep = getelementptr i64, ptr %out, i64 4
213 %tmp0 = atomicrmw xchg ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
214 ret void
215 }
217 define amdgpu_gfx i64 @flat_atomic_xchg_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
218 ; GFX7-LABEL: flat_atomic_xchg_i64_ret_scalar:
219 ; GFX7: ; %bb.0:
220 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
221 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
222 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
223 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
224 ; GFX7-NEXT: v_mov_b32_e32 v3, s5
225 ; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc
226 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
227 ; GFX7-NEXT: buffer_wbinvl1_vol
228 ; GFX7-NEXT: s_setpc_b64 s[30:31]
229 ;
230 ; GFX8-LABEL: flat_atomic_xchg_i64_ret_scalar:
231 ; GFX8: ; %bb.0:
232 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
233 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
234 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
235 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
236 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
237 ; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc
238 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
239 ; GFX8-NEXT: buffer_wbinvl1_vol
240 ; GFX8-NEXT: s_setpc_b64 s[30:31]
241 ;
242 ; GFX9-LABEL: flat_atomic_xchg_i64_ret_scalar:
243 ; GFX9: ; %bb.0:
244 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
245 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
246 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
247 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
248 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
249 ; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc
250 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
251 ; GFX9-NEXT: buffer_wbinvl1_vol
252 ; GFX9-NEXT: s_setpc_b64 s[30:31]
253 %result = atomicrmw xchg ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
254 ret i64 %result
255 }
257 define amdgpu_gfx i64 @flat_atomic_xchg_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
258 ; GFX7-LABEL: flat_atomic_xchg_i64_ret_offset_scalar:
259 ; GFX7: ; %bb.0:
260 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
261 ; GFX7-NEXT: s_add_u32 s34, s4, 32
262 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
263 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
264 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
265 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
266 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
267 ; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc
268 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
269 ; GFX7-NEXT: buffer_wbinvl1_vol
270 ; GFX7-NEXT: s_setpc_b64 s[30:31]
271 ;
272 ; GFX8-LABEL: flat_atomic_xchg_i64_ret_offset_scalar:
273 ; GFX8: ; %bb.0:
274 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
275 ; GFX8-NEXT: s_add_u32 s34, s4, 32
276 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
277 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
278 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
279 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
280 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
281 ; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc
282 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
283 ; GFX8-NEXT: buffer_wbinvl1_vol
284 ; GFX8-NEXT: s_setpc_b64 s[30:31]
285 ;
286 ; GFX9-LABEL: flat_atomic_xchg_i64_ret_offset_scalar:
287 ; GFX9: ; %bb.0:
288 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
289 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
290 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
291 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
292 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
293 ; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] offset:32 glc
294 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
295 ; GFX9-NEXT: buffer_wbinvl1_vol
296 ; GFX9-NEXT: s_setpc_b64 s[30:31]
297 %gep = getelementptr i64, ptr %out, i64 4
298 %result = atomicrmw xchg ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
299 ret i64 %result
300 }
302 define void @flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
303 ; GFX7-LABEL: flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory:
304 ; GFX7: ; %bb.0:
305 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
306 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
307 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
308 ; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
309 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
310 ; GFX7-NEXT: buffer_wbinvl1_vol
311 ; GFX7-NEXT: s_setpc_b64 s[30:31]
312 ;
313 ; GFX8-LABEL: flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory:
314 ; GFX8: ; %bb.0:
315 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
316 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
317 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
318 ; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
319 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
320 ; GFX8-NEXT: buffer_wbinvl1_vol
321 ; GFX8-NEXT: s_setpc_b64 s[30:31]
322 ;
323 ; GFX9-LABEL: flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory:
324 ; GFX9: ; %bb.0:
325 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
326 ; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] offset:32
327 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
328 ; GFX9-NEXT: buffer_wbinvl1_vol
329 ; GFX9-NEXT: s_setpc_b64 s[30:31]
330 %gep = getelementptr i64, ptr %out, i64 4
331 %tmp0 = atomicrmw xchg ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
332 ret void
333 }
335 define i64 @flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
336 ; GFX7-LABEL: flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory:
337 ; GFX7: ; %bb.0:
338 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
339 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
340 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
341 ; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
342 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
343 ; GFX7-NEXT: buffer_wbinvl1_vol
344 ; GFX7-NEXT: s_setpc_b64 s[30:31]
345 ;
346 ; GFX8-LABEL: flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory:
347 ; GFX8: ; %bb.0:
348 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
349 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
350 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
351 ; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
352 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
353 ; GFX8-NEXT: buffer_wbinvl1_vol
354 ; GFX8-NEXT: s_setpc_b64 s[30:31]
355 ;
356 ; GFX9-LABEL: flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory:
357 ; GFX9: ; %bb.0:
358 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
359 ; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
360 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
361 ; GFX9-NEXT: buffer_wbinvl1_vol
362 ; GFX9-NEXT: s_setpc_b64 s[30:31]
363 %gep = getelementptr i64, ptr %out, i64 4
364 %result = atomicrmw xchg ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
365 ret i64 %result
366 }
368 ; ---------------------------------------------------------------------
369 ; atomicrmw xchg f64
370 ; ---------------------------------------------------------------------
372 define void @flat_atomic_xchg_f64_noret(ptr %ptr, double %in) {
373 ; GFX7-LABEL: flat_atomic_xchg_f64_noret:
374 ; GFX7: ; %bb.0:
375 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
376 ; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
377 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
378 ; GFX7-NEXT: buffer_wbinvl1_vol
379 ; GFX7-NEXT: s_setpc_b64 s[30:31]
380 ;
381 ; GFX8-LABEL: flat_atomic_xchg_f64_noret:
382 ; GFX8: ; %bb.0:
383 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
384 ; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
385 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
386 ; GFX8-NEXT: buffer_wbinvl1_vol
387 ; GFX8-NEXT: s_setpc_b64 s[30:31]
388 ;
389 ; GFX9-LABEL: flat_atomic_xchg_f64_noret:
390 ; GFX9: ; %bb.0:
391 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
392 ; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
393 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
394 ; GFX9-NEXT: buffer_wbinvl1_vol
395 ; GFX9-NEXT: s_setpc_b64 s[30:31]
396 %tmp0 = atomicrmw xchg ptr %ptr, double %in seq_cst, !noalias.addrspace !1
397 ret void
398 }
400 define void @flat_atomic_xchg_f64_noret_offset(ptr %out, double %in) {
401 ; GFX7-LABEL: flat_atomic_xchg_f64_noret_offset:
402 ; GFX7: ; %bb.0:
403 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
404 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
405 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
406 ; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
407 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
408 ; GFX7-NEXT: buffer_wbinvl1_vol
409 ; GFX7-NEXT: s_setpc_b64 s[30:31]
410 ;
411 ; GFX8-LABEL: flat_atomic_xchg_f64_noret_offset:
412 ; GFX8: ; %bb.0:
413 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
414 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
415 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
416 ; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
417 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
418 ; GFX8-NEXT: buffer_wbinvl1_vol
419 ; GFX8-NEXT: s_setpc_b64 s[30:31]
420 ;
421 ; GFX9-LABEL: flat_atomic_xchg_f64_noret_offset:
422 ; GFX9: ; %bb.0:
423 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
424 ; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] offset:32
425 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
426 ; GFX9-NEXT: buffer_wbinvl1_vol
427 ; GFX9-NEXT: s_setpc_b64 s[30:31]
428 %gep = getelementptr double, ptr %out, i32 4
429 %tmp0 = atomicrmw xchg ptr %gep, double %in seq_cst, !noalias.addrspace !1
430 ret void
431 }
433 define double @flat_atomic_xchg_f64_ret(ptr %ptr, double %in) {
434 ; GFX7-LABEL: flat_atomic_xchg_f64_ret:
435 ; GFX7: ; %bb.0:
436 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
437 ; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
438 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
439 ; GFX7-NEXT: buffer_wbinvl1_vol
440 ; GFX7-NEXT: s_setpc_b64 s[30:31]
441 ;
442 ; GFX8-LABEL: flat_atomic_xchg_f64_ret:
443 ; GFX8: ; %bb.0:
444 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
445 ; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
446 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
447 ; GFX8-NEXT: buffer_wbinvl1_vol
448 ; GFX8-NEXT: s_setpc_b64 s[30:31]
449 ;
450 ; GFX9-LABEL: flat_atomic_xchg_f64_ret:
451 ; GFX9: ; %bb.0:
452 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
453 ; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
454 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
455 ; GFX9-NEXT: buffer_wbinvl1_vol
456 ; GFX9-NEXT: s_setpc_b64 s[30:31]
457 %result = atomicrmw xchg ptr %ptr, double %in seq_cst, !noalias.addrspace !1
458 ret double %result
459 }
461 define double @flat_atomic_xchg_f64_ret_offset(ptr %out, double %in) {
462 ; GFX7-LABEL: flat_atomic_xchg_f64_ret_offset:
463 ; GFX7: ; %bb.0:
464 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
465 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
466 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
467 ; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
468 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
469 ; GFX7-NEXT: buffer_wbinvl1_vol
470 ; GFX7-NEXT: s_setpc_b64 s[30:31]
471 ;
472 ; GFX8-LABEL: flat_atomic_xchg_f64_ret_offset:
473 ; GFX8: ; %bb.0:
474 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
475 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
476 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
477 ; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
478 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
479 ; GFX8-NEXT: buffer_wbinvl1_vol
480 ; GFX8-NEXT: s_setpc_b64 s[30:31]
481 ;
482 ; GFX9-LABEL: flat_atomic_xchg_f64_ret_offset:
483 ; GFX9: ; %bb.0:
484 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
485 ; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
486 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
487 ; GFX9-NEXT: buffer_wbinvl1_vol
488 ; GFX9-NEXT: s_setpc_b64 s[30:31]
489 %gep = getelementptr double, ptr %out, i32 4
490 %result = atomicrmw xchg ptr %gep, double %in seq_cst, !noalias.addrspace !1
491 ret double %result
492 }
494 define amdgpu_gfx void @flat_atomic_xchg_f64_noret_scalar(ptr inreg %ptr, double inreg %in) {
495 ; GFX7-LABEL: flat_atomic_xchg_f64_noret_scalar:
496 ; GFX7: ; %bb.0:
497 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
498 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
499 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
500 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
501 ; GFX7-NEXT: v_mov_b32_e32 v3, s5
502 ; GFX7-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]
503 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
504 ; GFX7-NEXT: buffer_wbinvl1_vol
505 ; GFX7-NEXT: s_setpc_b64 s[30:31]
506 ;
507 ; GFX8-LABEL: flat_atomic_xchg_f64_noret_scalar:
508 ; GFX8: ; %bb.0:
509 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
510 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
511 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
512 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
513 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
514 ; GFX8-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]
515 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
516 ; GFX8-NEXT: buffer_wbinvl1_vol
517 ; GFX8-NEXT: s_setpc_b64 s[30:31]
518 ;
519 ; GFX9-LABEL: flat_atomic_xchg_f64_noret_scalar:
520 ; GFX9: ; %bb.0:
521 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
522 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
523 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
524 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
525 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
526 ; GFX9-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]
527 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
528 ; GFX9-NEXT: buffer_wbinvl1_vol
529 ; GFX9-NEXT: s_setpc_b64 s[30:31]
530 %tmp0 = atomicrmw xchg ptr %ptr, double %in seq_cst, !noalias.addrspace !1
531 ret void
532 }
534 define amdgpu_gfx void @flat_atomic_xchg_f64_noret_offset_scalar(ptr inreg %out, double inreg %in) {
535 ; GFX7-LABEL: flat_atomic_xchg_f64_noret_offset_scalar:
536 ; GFX7: ; %bb.0:
537 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
538 ; GFX7-NEXT: s_add_u32 s34, s4, 32
539 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
540 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
541 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
542 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
543 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
544 ; GFX7-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]
545 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
546 ; GFX7-NEXT: buffer_wbinvl1_vol
547 ; GFX7-NEXT: s_setpc_b64 s[30:31]
548 ;
549 ; GFX8-LABEL: flat_atomic_xchg_f64_noret_offset_scalar:
550 ; GFX8: ; %bb.0:
551 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
552 ; GFX8-NEXT: s_add_u32 s34, s4, 32
553 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
554 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
555 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
556 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
557 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
558 ; GFX8-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]
559 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
560 ; GFX8-NEXT: buffer_wbinvl1_vol
561 ; GFX8-NEXT: s_setpc_b64 s[30:31]
562 ;
563 ; GFX9-LABEL: flat_atomic_xchg_f64_noret_offset_scalar:
564 ; GFX9: ; %bb.0:
565 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
566 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
567 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
568 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
569 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
570 ; GFX9-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1] offset:32
571 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
572 ; GFX9-NEXT: buffer_wbinvl1_vol
573 ; GFX9-NEXT: s_setpc_b64 s[30:31]
574 %gep = getelementptr double, ptr %out, i32 4
575 %tmp0 = atomicrmw xchg ptr %gep, double %in seq_cst, !noalias.addrspace !1
576 ret void
577 }
579 define amdgpu_gfx double @flat_atomic_xchg_f64_ret_scalar(ptr inreg %ptr, double inreg %in) {
580 ; GFX7-LABEL: flat_atomic_xchg_f64_ret_scalar:
581 ; GFX7: ; %bb.0:
582 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
583 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
584 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
585 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
586 ; GFX7-NEXT: v_mov_b32_e32 v3, s5
587 ; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc
588 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
589 ; GFX7-NEXT: buffer_wbinvl1_vol
590 ; GFX7-NEXT: s_setpc_b64 s[30:31]
591 ;
592 ; GFX8-LABEL: flat_atomic_xchg_f64_ret_scalar:
593 ; GFX8: ; %bb.0:
594 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
595 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
596 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
597 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
598 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
599 ; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc
600 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
601 ; GFX8-NEXT: buffer_wbinvl1_vol
602 ; GFX8-NEXT: s_setpc_b64 s[30:31]
603 ;
604 ; GFX9-LABEL: flat_atomic_xchg_f64_ret_scalar:
605 ; GFX9: ; %bb.0:
606 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
607 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
608 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
609 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
610 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
611 ; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc
612 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
613 ; GFX9-NEXT: buffer_wbinvl1_vol
614 ; GFX9-NEXT: s_setpc_b64 s[30:31]
615 %result = atomicrmw xchg ptr %ptr, double %in seq_cst, !noalias.addrspace !1
616 ret double %result
617 }
619 define amdgpu_gfx double @flat_atomic_xchg_f64_ret_offset_scalar(ptr inreg %out, double inreg %in) {
620 ; GFX7-LABEL: flat_atomic_xchg_f64_ret_offset_scalar:
621 ; GFX7: ; %bb.0:
622 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
623 ; GFX7-NEXT: s_add_u32 s34, s4, 32
624 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
625 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
626 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
627 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
628 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
629 ; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc
630 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
631 ; GFX7-NEXT: buffer_wbinvl1_vol
632 ; GFX7-NEXT: s_setpc_b64 s[30:31]
633 ;
634 ; GFX8-LABEL: flat_atomic_xchg_f64_ret_offset_scalar:
635 ; GFX8: ; %bb.0:
636 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
637 ; GFX8-NEXT: s_add_u32 s34, s4, 32
638 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
639 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
640 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
641 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
642 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
643 ; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc
644 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
645 ; GFX8-NEXT: buffer_wbinvl1_vol
646 ; GFX8-NEXT: s_setpc_b64 s[30:31]
647 ;
648 ; GFX9-LABEL: flat_atomic_xchg_f64_ret_offset_scalar:
649 ; GFX9: ; %bb.0:
650 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
651 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
652 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
653 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
654 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
655 ; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] offset:32 glc
656 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
657 ; GFX9-NEXT: buffer_wbinvl1_vol
658 ; GFX9-NEXT: s_setpc_b64 s[30:31]
659 %gep = getelementptr double, ptr %out, i32 4
660 %result = atomicrmw xchg ptr %gep, double %in seq_cst, !noalias.addrspace !1
661 ret double %result
662 }
664 define void @flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory(ptr %out, double %in) {
665 ; GFX7-LABEL: flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory:
666 ; GFX7: ; %bb.0:
667 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
668 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
669 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
670 ; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
671 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
672 ; GFX7-NEXT: buffer_wbinvl1_vol
673 ; GFX7-NEXT: s_setpc_b64 s[30:31]
674 ;
675 ; GFX8-LABEL: flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory:
676 ; GFX8: ; %bb.0:
677 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
678 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
679 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
680 ; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
681 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
682 ; GFX8-NEXT: buffer_wbinvl1_vol
683 ; GFX8-NEXT: s_setpc_b64 s[30:31]
684 ;
685 ; GFX9-LABEL: flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory:
686 ; GFX9: ; %bb.0:
687 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
688 ; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] offset:32
689 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
690 ; GFX9-NEXT: buffer_wbinvl1_vol
691 ; GFX9-NEXT: s_setpc_b64 s[30:31]
692 %gep = getelementptr double, ptr %out, i64 4
693 %tmp0 = atomicrmw xchg ptr %gep, double %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
694 ret void
695 }
697 define double @flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory(ptr %out, double %in) {
698 ; GFX7-LABEL: flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory:
699 ; GFX7: ; %bb.0:
700 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
701 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
702 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
703 ; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
704 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
705 ; GFX7-NEXT: buffer_wbinvl1_vol
706 ; GFX7-NEXT: s_setpc_b64 s[30:31]
707 ;
708 ; GFX8-LABEL: flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory:
709 ; GFX8: ; %bb.0:
710 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
711 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
712 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
713 ; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
714 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
715 ; GFX8-NEXT: buffer_wbinvl1_vol
716 ; GFX8-NEXT: s_setpc_b64 s[30:31]
717 ;
718 ; GFX9-LABEL: flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory:
719 ; GFX9: ; %bb.0:
720 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
721 ; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
722 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
723 ; GFX9-NEXT: buffer_wbinvl1_vol
724 ; GFX9-NEXT: s_setpc_b64 s[30:31]
725 %gep = getelementptr double, ptr %out, i64 4
726 %result = atomicrmw xchg ptr %gep, double %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
727 ret double %result
728 }
730 ; ---------------------------------------------------------------------
731 ; atomicrmw add
732 ; ---------------------------------------------------------------------
734 define void @flat_atomic_add_i64_noret(ptr %ptr, i64 %in) {
735 ; GFX7-LABEL: flat_atomic_add_i64_noret:
736 ; GFX7: ; %bb.0:
737 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
738 ; GFX7-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
739 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
740 ; GFX7-NEXT: buffer_wbinvl1_vol
741 ; GFX7-NEXT: s_setpc_b64 s[30:31]
742 ;
743 ; GFX8-LABEL: flat_atomic_add_i64_noret:
744 ; GFX8: ; %bb.0:
745 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
746 ; GFX8-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
747 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
748 ; GFX8-NEXT: buffer_wbinvl1_vol
749 ; GFX8-NEXT: s_setpc_b64 s[30:31]
750 ;
751 ; GFX9-LABEL: flat_atomic_add_i64_noret:
752 ; GFX9: ; %bb.0:
753 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
754 ; GFX9-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
755 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
756 ; GFX9-NEXT: buffer_wbinvl1_vol
757 ; GFX9-NEXT: s_setpc_b64 s[30:31]
758 %tmp0 = atomicrmw add ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
759 ret void
760 }
762 define void @flat_atomic_add_i64_noret_offset(ptr %out, i64 %in) {
763 ; GFX7-LABEL: flat_atomic_add_i64_noret_offset:
764 ; GFX7: ; %bb.0:
765 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
766 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
767 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
768 ; GFX7-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
769 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
770 ; GFX7-NEXT: buffer_wbinvl1_vol
771 ; GFX7-NEXT: s_setpc_b64 s[30:31]
772 ;
773 ; GFX8-LABEL: flat_atomic_add_i64_noret_offset:
774 ; GFX8: ; %bb.0:
775 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
776 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
777 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
778 ; GFX8-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
779 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
780 ; GFX8-NEXT: buffer_wbinvl1_vol
781 ; GFX8-NEXT: s_setpc_b64 s[30:31]
782 ;
783 ; GFX9-LABEL: flat_atomic_add_i64_noret_offset:
784 ; GFX9: ; %bb.0:
785 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
786 ; GFX9-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] offset:32
787 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
788 ; GFX9-NEXT: buffer_wbinvl1_vol
789 ; GFX9-NEXT: s_setpc_b64 s[30:31]
790 %gep = getelementptr i64, ptr %out, i64 4
791 %tmp0 = atomicrmw add ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
792 ret void
793 }
795 define i64 @flat_atomic_add_i64_ret(ptr %ptr, i64 %in) {
796 ; GFX7-LABEL: flat_atomic_add_i64_ret:
797 ; GFX7: ; %bb.0:
798 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
799 ; GFX7-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
800 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
801 ; GFX7-NEXT: buffer_wbinvl1_vol
802 ; GFX7-NEXT: s_setpc_b64 s[30:31]
803 ;
804 ; GFX8-LABEL: flat_atomic_add_i64_ret:
805 ; GFX8: ; %bb.0:
806 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
807 ; GFX8-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
808 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
809 ; GFX8-NEXT: buffer_wbinvl1_vol
810 ; GFX8-NEXT: s_setpc_b64 s[30:31]
811 ;
812 ; GFX9-LABEL: flat_atomic_add_i64_ret:
813 ; GFX9: ; %bb.0:
814 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
815 ; GFX9-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
816 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
817 ; GFX9-NEXT: buffer_wbinvl1_vol
818 ; GFX9-NEXT: s_setpc_b64 s[30:31]
819 %result = atomicrmw add ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
820 ret i64 %result
821 }
823 define i64 @flat_atomic_add_i64_ret_offset(ptr %out, i64 %in) {
824 ; GFX7-LABEL: flat_atomic_add_i64_ret_offset:
825 ; GFX7: ; %bb.0:
826 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
827 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
828 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
829 ; GFX7-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
830 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
831 ; GFX7-NEXT: buffer_wbinvl1_vol
832 ; GFX7-NEXT: s_setpc_b64 s[30:31]
833 ;
834 ; GFX8-LABEL: flat_atomic_add_i64_ret_offset:
835 ; GFX8: ; %bb.0:
836 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
837 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
838 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
839 ; GFX8-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
840 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
841 ; GFX8-NEXT: buffer_wbinvl1_vol
842 ; GFX8-NEXT: s_setpc_b64 s[30:31]
843 ;
844 ; GFX9-LABEL: flat_atomic_add_i64_ret_offset:
845 ; GFX9: ; %bb.0:
846 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
847 ; GFX9-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
848 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
849 ; GFX9-NEXT: buffer_wbinvl1_vol
850 ; GFX9-NEXT: s_setpc_b64 s[30:31]
851 %gep = getelementptr i64, ptr %out, i64 4
852 %result = atomicrmw add ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
853 ret i64 %result
854 }
856 define amdgpu_gfx void @flat_atomic_add_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
857 ; GFX7-LABEL: flat_atomic_add_i64_noret_scalar:
858 ; GFX7: ; %bb.0:
859 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
860 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
861 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
862 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
863 ; GFX7-NEXT: v_mov_b32_e32 v3, s5
864 ; GFX7-NEXT: flat_atomic_add_x2 v[2:3], v[0:1]
865 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
866 ; GFX7-NEXT: buffer_wbinvl1_vol
867 ; GFX7-NEXT: s_setpc_b64 s[30:31]
868 ;
869 ; GFX8-LABEL: flat_atomic_add_i64_noret_scalar:
870 ; GFX8: ; %bb.0:
871 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
872 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
873 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
874 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
875 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
876 ; GFX8-NEXT: flat_atomic_add_x2 v[2:3], v[0:1]
877 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
878 ; GFX8-NEXT: buffer_wbinvl1_vol
879 ; GFX8-NEXT: s_setpc_b64 s[30:31]
880 ;
881 ; GFX9-LABEL: flat_atomic_add_i64_noret_scalar:
882 ; GFX9: ; %bb.0:
883 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
884 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
885 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
886 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
887 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
888 ; GFX9-NEXT: flat_atomic_add_x2 v[2:3], v[0:1]
889 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
890 ; GFX9-NEXT: buffer_wbinvl1_vol
891 ; GFX9-NEXT: s_setpc_b64 s[30:31]
892 %tmp0 = atomicrmw add ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
893 ret void
894 }
896 define amdgpu_gfx void @flat_atomic_add_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
897 ; GFX7-LABEL: flat_atomic_add_i64_noret_offset_scalar:
898 ; GFX7: ; %bb.0:
899 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
900 ; GFX7-NEXT: s_add_u32 s34, s4, 32
901 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
902 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
903 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
904 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
905 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
906 ; GFX7-NEXT: flat_atomic_add_x2 v[2:3], v[0:1]
907 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
908 ; GFX7-NEXT: buffer_wbinvl1_vol
909 ; GFX7-NEXT: s_setpc_b64 s[30:31]
910 ;
911 ; GFX8-LABEL: flat_atomic_add_i64_noret_offset_scalar:
912 ; GFX8: ; %bb.0:
913 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
914 ; GFX8-NEXT: s_add_u32 s34, s4, 32
915 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
916 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
917 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
918 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
919 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
920 ; GFX8-NEXT: flat_atomic_add_x2 v[2:3], v[0:1]
921 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
922 ; GFX8-NEXT: buffer_wbinvl1_vol
923 ; GFX8-NEXT: s_setpc_b64 s[30:31]
924 ;
925 ; GFX9-LABEL: flat_atomic_add_i64_noret_offset_scalar:
926 ; GFX9: ; %bb.0:
927 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
928 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
929 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
930 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
931 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
932 ; GFX9-NEXT: flat_atomic_add_x2 v[2:3], v[0:1] offset:32
933 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
934 ; GFX9-NEXT: buffer_wbinvl1_vol
935 ; GFX9-NEXT: s_setpc_b64 s[30:31]
936 %gep = getelementptr i64, ptr %out, i64 4
937 %tmp0 = atomicrmw add ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
938 ret void
939 }
941 define amdgpu_gfx i64 @flat_atomic_add_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
942 ; GFX7-LABEL: flat_atomic_add_i64_ret_scalar:
943 ; GFX7: ; %bb.0:
944 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
945 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
946 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
947 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
948 ; GFX7-NEXT: v_mov_b32_e32 v3, s5
949 ; GFX7-NEXT: flat_atomic_add_x2 v[0:1], v[2:3], v[0:1] glc
950 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
951 ; GFX7-NEXT: buffer_wbinvl1_vol
952 ; GFX7-NEXT: s_setpc_b64 s[30:31]
953 ;
954 ; GFX8-LABEL: flat_atomic_add_i64_ret_scalar:
955 ; GFX8: ; %bb.0:
956 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
957 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
958 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
959 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
960 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
961 ; GFX8-NEXT: flat_atomic_add_x2 v[0:1], v[2:3], v[0:1] glc
962 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
963 ; GFX8-NEXT: buffer_wbinvl1_vol
964 ; GFX8-NEXT: s_setpc_b64 s[30:31]
965 ;
966 ; GFX9-LABEL: flat_atomic_add_i64_ret_scalar:
967 ; GFX9: ; %bb.0:
968 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
969 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
970 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
971 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
972 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
973 ; GFX9-NEXT: flat_atomic_add_x2 v[0:1], v[2:3], v[0:1] glc
974 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
975 ; GFX9-NEXT: buffer_wbinvl1_vol
976 ; GFX9-NEXT: s_setpc_b64 s[30:31]
977 %result = atomicrmw add ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
978 ret i64 %result
979 }
981 define amdgpu_gfx i64 @flat_atomic_add_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
982 ; GFX7-LABEL: flat_atomic_add_i64_ret_offset_scalar:
983 ; GFX7: ; %bb.0:
984 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
985 ; GFX7-NEXT: s_add_u32 s34, s4, 32
986 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
987 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
988 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
989 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
990 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
991 ; GFX7-NEXT: flat_atomic_add_x2 v[0:1], v[2:3], v[0:1] glc
992 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
993 ; GFX7-NEXT: buffer_wbinvl1_vol
994 ; GFX7-NEXT: s_setpc_b64 s[30:31]
995 ;
996 ; GFX8-LABEL: flat_atomic_add_i64_ret_offset_scalar:
997 ; GFX8: ; %bb.0:
998 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
999 ; GFX8-NEXT: s_add_u32 s34, s4, 32
1000 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
1001 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
1002 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
1003 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
1004 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
1005 ; GFX8-NEXT: flat_atomic_add_x2 v[0:1], v[2:3], v[0:1] glc
1006 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1007 ; GFX8-NEXT: buffer_wbinvl1_vol
1008 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1009 ;
1010 ; GFX9-LABEL: flat_atomic_add_i64_ret_offset_scalar:
1011 ; GFX9: ; %bb.0:
1012 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1013 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
1014 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
1015 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
1016 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
1017 ; GFX9-NEXT: flat_atomic_add_x2 v[0:1], v[2:3], v[0:1] offset:32 glc
1018 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1019 ; GFX9-NEXT: buffer_wbinvl1_vol
1020 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1021 %gep = getelementptr i64, ptr %out, i64 4
1022 %result = atomicrmw add ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
1023 ret i64 %result
1024 }
1026 define void @flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
1027 ; GFX7-LABEL: flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory:
1028 ; GFX7: ; %bb.0:
1029 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1030 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
1031 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1032 ; GFX7-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
1033 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1034 ; GFX7-NEXT: buffer_wbinvl1_vol
1035 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1036 ;
1037 ; GFX8-LABEL: flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory:
1038 ; GFX8: ; %bb.0:
1039 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1040 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
1041 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1042 ; GFX8-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
1043 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1044 ; GFX8-NEXT: buffer_wbinvl1_vol
1045 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1046 ;
1047 ; GFX9-LABEL: flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory:
1048 ; GFX9: ; %bb.0:
1049 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1050 ; GFX9-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] offset:32
1051 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1052 ; GFX9-NEXT: buffer_wbinvl1_vol
1053 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1054 %gep = getelementptr i64, ptr %out, i64 4
1055 %tmp0 = atomicrmw add ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
1056 ret void
1057 }
1059 define i64 @flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
1060 ; GFX7-LABEL: flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory:
1061 ; GFX7: ; %bb.0:
1062 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1063 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
1064 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1065 ; GFX7-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
1066 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1067 ; GFX7-NEXT: buffer_wbinvl1_vol
1068 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1069 ;
1070 ; GFX8-LABEL: flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory:
1071 ; GFX8: ; %bb.0:
1072 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1073 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
1074 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1075 ; GFX8-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
1076 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1077 ; GFX8-NEXT: buffer_wbinvl1_vol
1078 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1079 ;
1080 ; GFX9-LABEL: flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory:
1081 ; GFX9: ; %bb.0:
1082 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1083 ; GFX9-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
1084 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1085 ; GFX9-NEXT: buffer_wbinvl1_vol
1086 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1087 %gep = getelementptr i64, ptr %out, i64 4
1088 %result = atomicrmw add ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
1089 ret i64 %result
1090 }
1092 ; ---------------------------------------------------------------------
1093 ; atomicrmw sub
1094 ; ---------------------------------------------------------------------
1096 define void @flat_atomic_sub_i64_noret(ptr %ptr, i64 %in) {
1097 ; GFX7-LABEL: flat_atomic_sub_i64_noret:
1098 ; GFX7: ; %bb.0:
1099 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1100 ; GFX7-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
1101 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1102 ; GFX7-NEXT: buffer_wbinvl1_vol
1103 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1104 ;
1105 ; GFX8-LABEL: flat_atomic_sub_i64_noret:
1106 ; GFX8: ; %bb.0:
1107 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1108 ; GFX8-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
1109 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1110 ; GFX8-NEXT: buffer_wbinvl1_vol
1111 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1112 ;
1113 ; GFX9-LABEL: flat_atomic_sub_i64_noret:
1114 ; GFX9: ; %bb.0:
1115 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1116 ; GFX9-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
1117 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1118 ; GFX9-NEXT: buffer_wbinvl1_vol
1119 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1120 %tmp0 = atomicrmw sub ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
1121 ret void
1122 }
1124 define void @flat_atomic_sub_i64_noret_offset(ptr %out, i64 %in) {
1125 ; GFX7-LABEL: flat_atomic_sub_i64_noret_offset:
1126 ; GFX7: ; %bb.0:
1127 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1128 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
1129 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1130 ; GFX7-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
1131 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1132 ; GFX7-NEXT: buffer_wbinvl1_vol
1133 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1134 ;
1135 ; GFX8-LABEL: flat_atomic_sub_i64_noret_offset:
1136 ; GFX8: ; %bb.0:
1137 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1138 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
1139 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1140 ; GFX8-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
1141 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1142 ; GFX8-NEXT: buffer_wbinvl1_vol
1143 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1144 ;
1145 ; GFX9-LABEL: flat_atomic_sub_i64_noret_offset:
1146 ; GFX9: ; %bb.0:
1147 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1148 ; GFX9-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3] offset:32
1149 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1150 ; GFX9-NEXT: buffer_wbinvl1_vol
1151 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1152 %gep = getelementptr i64, ptr %out, i64 4
1153 %tmp0 = atomicrmw sub ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
1154 ret void
1155 }
1157 define i64 @flat_atomic_sub_i64_ret(ptr %ptr, i64 %in) {
1158 ; GFX7-LABEL: flat_atomic_sub_i64_ret:
1159 ; GFX7: ; %bb.0:
1160 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1161 ; GFX7-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
1162 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1163 ; GFX7-NEXT: buffer_wbinvl1_vol
1164 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1165 ;
1166 ; GFX8-LABEL: flat_atomic_sub_i64_ret:
1167 ; GFX8: ; %bb.0:
1168 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1169 ; GFX8-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
1170 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1171 ; GFX8-NEXT: buffer_wbinvl1_vol
1172 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1173 ;
1174 ; GFX9-LABEL: flat_atomic_sub_i64_ret:
1175 ; GFX9: ; %bb.0:
1176 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1177 ; GFX9-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
1178 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1179 ; GFX9-NEXT: buffer_wbinvl1_vol
1180 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1181 %result = atomicrmw sub ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
1182 ret i64 %result
1183 }
1185 define i64 @flat_atomic_sub_i64_ret_offset(ptr %out, i64 %in) {
1186 ; GFX7-LABEL: flat_atomic_sub_i64_ret_offset:
1187 ; GFX7: ; %bb.0:
1188 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1189 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
1190 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1191 ; GFX7-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
1192 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1193 ; GFX7-NEXT: buffer_wbinvl1_vol
1194 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1195 ;
1196 ; GFX8-LABEL: flat_atomic_sub_i64_ret_offset:
1197 ; GFX8: ; %bb.0:
1198 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1199 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
1200 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1201 ; GFX8-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
1202 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1203 ; GFX8-NEXT: buffer_wbinvl1_vol
1204 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1205 ;
1206 ; GFX9-LABEL: flat_atomic_sub_i64_ret_offset:
1207 ; GFX9: ; %bb.0:
1208 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1209 ; GFX9-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
1210 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1211 ; GFX9-NEXT: buffer_wbinvl1_vol
1212 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1213 %gep = getelementptr i64, ptr %out, i64 4
1214 %result = atomicrmw sub ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
1215 ret i64 %result
1216 }
1218 define amdgpu_gfx void @flat_atomic_sub_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
1219 ; GFX7-LABEL: flat_atomic_sub_i64_noret_scalar:
1220 ; GFX7: ; %bb.0:
1221 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1222 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1223 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1224 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
1225 ; GFX7-NEXT: v_mov_b32_e32 v3, s5
1226 ; GFX7-NEXT: flat_atomic_sub_x2 v[2:3], v[0:1]
1227 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1228 ; GFX7-NEXT: buffer_wbinvl1_vol
1229 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1230 ;
1231 ; GFX8-LABEL: flat_atomic_sub_i64_noret_scalar:
1232 ; GFX8: ; %bb.0:
1233 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1234 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
1235 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
1236 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
1237 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
1238 ; GFX8-NEXT: flat_atomic_sub_x2 v[2:3], v[0:1]
1239 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1240 ; GFX8-NEXT: buffer_wbinvl1_vol
1241 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1242 ;
1243 ; GFX9-LABEL: flat_atomic_sub_i64_noret_scalar:
1244 ; GFX9: ; %bb.0:
1245 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1246 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
1247 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
1248 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
1249 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
1250 ; GFX9-NEXT: flat_atomic_sub_x2 v[2:3], v[0:1]
1251 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1252 ; GFX9-NEXT: buffer_wbinvl1_vol
1253 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1254 %tmp0 = atomicrmw sub ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
1255 ret void
1256 }
1258 define amdgpu_gfx void @flat_atomic_sub_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
1259 ; GFX7-LABEL: flat_atomic_sub_i64_noret_offset_scalar:
1260 ; GFX7: ; %bb.0:
1261 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1262 ; GFX7-NEXT: s_add_u32 s34, s4, 32
1263 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
1264 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
1265 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1266 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1267 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
1268 ; GFX7-NEXT: flat_atomic_sub_x2 v[2:3], v[0:1]
1269 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1270 ; GFX7-NEXT: buffer_wbinvl1_vol
1271 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1272 ;
1273 ; GFX8-LABEL: flat_atomic_sub_i64_noret_offset_scalar:
1274 ; GFX8: ; %bb.0:
1275 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1276 ; GFX8-NEXT: s_add_u32 s34, s4, 32
1277 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
1278 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
1279 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
1280 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
1281 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
1282 ; GFX8-NEXT: flat_atomic_sub_x2 v[2:3], v[0:1]
1283 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1284 ; GFX8-NEXT: buffer_wbinvl1_vol
1285 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1286 ;
1287 ; GFX9-LABEL: flat_atomic_sub_i64_noret_offset_scalar:
1288 ; GFX9: ; %bb.0:
1289 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1290 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
1291 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
1292 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
1293 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
1294 ; GFX9-NEXT: flat_atomic_sub_x2 v[2:3], v[0:1] offset:32
1295 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1296 ; GFX9-NEXT: buffer_wbinvl1_vol
1297 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1298 %gep = getelementptr i64, ptr %out, i64 4
1299 %tmp0 = atomicrmw sub ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
1300 ret void
1301 }
1303 define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
1304 ; GFX7-LABEL: flat_atomic_sub_i64_ret_scalar:
1305 ; GFX7: ; %bb.0:
1306 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1307 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1308 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1309 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
1310 ; GFX7-NEXT: v_mov_b32_e32 v3, s5
1311 ; GFX7-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3], v[0:1] glc
1312 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1313 ; GFX7-NEXT: buffer_wbinvl1_vol
1314 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1315 ;
1316 ; GFX8-LABEL: flat_atomic_sub_i64_ret_scalar:
1317 ; GFX8: ; %bb.0:
1318 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1319 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
1320 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
1321 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
1322 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
1323 ; GFX8-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3], v[0:1] glc
1324 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1325 ; GFX8-NEXT: buffer_wbinvl1_vol
1326 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1327 ;
1328 ; GFX9-LABEL: flat_atomic_sub_i64_ret_scalar:
1329 ; GFX9: ; %bb.0:
1330 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1331 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
1332 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
1333 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
1334 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
1335 ; GFX9-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3], v[0:1] glc
1336 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1337 ; GFX9-NEXT: buffer_wbinvl1_vol
1338 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1339 %result = atomicrmw sub ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
1340 ret i64 %result
1341 }
1343 define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
1344 ; GFX7-LABEL: flat_atomic_sub_i64_ret_offset_scalar:
1345 ; GFX7: ; %bb.0:
1346 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1347 ; GFX7-NEXT: s_add_u32 s34, s4, 32
1348 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
1349 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
1350 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1351 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1352 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
1353 ; GFX7-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3], v[0:1] glc
1354 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1355 ; GFX7-NEXT: buffer_wbinvl1_vol
1356 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1357 ;
1358 ; GFX8-LABEL: flat_atomic_sub_i64_ret_offset_scalar:
1359 ; GFX8: ; %bb.0:
1360 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1361 ; GFX8-NEXT: s_add_u32 s34, s4, 32
1362 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
1363 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
1364 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
1365 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
1366 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
1367 ; GFX8-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3], v[0:1] glc
1368 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1369 ; GFX8-NEXT: buffer_wbinvl1_vol
1370 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1371 ;
1372 ; GFX9-LABEL: flat_atomic_sub_i64_ret_offset_scalar:
1373 ; GFX9: ; %bb.0:
1374 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1375 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
1376 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
1377 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
1378 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
1379 ; GFX9-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3], v[0:1] offset:32 glc
1380 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1381 ; GFX9-NEXT: buffer_wbinvl1_vol
1382 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1383 %gep = getelementptr i64, ptr %out, i64 4
1384 %result = atomicrmw sub ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
1385 ret i64 %result
1386 }
1388 define void @flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
1389 ; GFX7-LABEL: flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory:
1390 ; GFX7: ; %bb.0:
1391 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1392 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
1393 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1394 ; GFX7-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
1395 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1396 ; GFX7-NEXT: buffer_wbinvl1_vol
1397 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1398 ;
1399 ; GFX8-LABEL: flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory:
1400 ; GFX8: ; %bb.0:
1401 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1402 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
1403 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1404 ; GFX8-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
1405 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1406 ; GFX8-NEXT: buffer_wbinvl1_vol
1407 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1408 ;
1409 ; GFX9-LABEL: flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory:
1410 ; GFX9: ; %bb.0:
1411 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1412 ; GFX9-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3] offset:32
1413 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1414 ; GFX9-NEXT: buffer_wbinvl1_vol
1415 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1416 %gep = getelementptr i64, ptr %out, i64 4
1417 %tmp0 = atomicrmw sub ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
1418 ret void
1419 }
1421 define i64 @flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
1422 ; GFX7-LABEL: flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory:
1423 ; GFX7: ; %bb.0:
1424 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1425 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
1426 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1427 ; GFX7-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
1428 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1429 ; GFX7-NEXT: buffer_wbinvl1_vol
1430 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1431 ;
1432 ; GFX8-LABEL: flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory:
1433 ; GFX8: ; %bb.0:
1434 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1435 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
1436 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1437 ; GFX8-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
1438 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1439 ; GFX8-NEXT: buffer_wbinvl1_vol
1440 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1441 ;
1442 ; GFX9-LABEL: flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory:
1443 ; GFX9: ; %bb.0:
1444 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1445 ; GFX9-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
1446 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1447 ; GFX9-NEXT: buffer_wbinvl1_vol
1448 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1449 %gep = getelementptr i64, ptr %out, i64 4
1450 %result = atomicrmw sub ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
1451 ret i64 %result
1452 }
1454 ; ---------------------------------------------------------------------
1455 ; atomicrmw and
1456 ; ---------------------------------------------------------------------
1458 define void @flat_atomic_and_i64_noret(ptr %ptr, i64 %in) {
1459 ; GFX7-LABEL: flat_atomic_and_i64_noret:
1460 ; GFX7: ; %bb.0:
1461 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1462 ; GFX7-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
1463 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1464 ; GFX7-NEXT: buffer_wbinvl1_vol
1465 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1466 ;
1467 ; GFX8-LABEL: flat_atomic_and_i64_noret:
1468 ; GFX8: ; %bb.0:
1469 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1470 ; GFX8-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
1471 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1472 ; GFX8-NEXT: buffer_wbinvl1_vol
1473 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1474 ;
1475 ; GFX9-LABEL: flat_atomic_and_i64_noret:
1476 ; GFX9: ; %bb.0:
1477 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1478 ; GFX9-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
1479 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1480 ; GFX9-NEXT: buffer_wbinvl1_vol
1481 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1482 %tmp0 = atomicrmw and ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
1483 ret void
1484 }
1486 define void @flat_atomic_and_i64_noret_offset(ptr %out, i64 %in) {
1487 ; GFX7-LABEL: flat_atomic_and_i64_noret_offset:
1488 ; GFX7: ; %bb.0:
1489 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1490 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
1491 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1492 ; GFX7-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
1493 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1494 ; GFX7-NEXT: buffer_wbinvl1_vol
1495 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1496 ;
1497 ; GFX8-LABEL: flat_atomic_and_i64_noret_offset:
1498 ; GFX8: ; %bb.0:
1499 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1500 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
1501 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1502 ; GFX8-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
1503 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1504 ; GFX8-NEXT: buffer_wbinvl1_vol
1505 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1506 ;
1507 ; GFX9-LABEL: flat_atomic_and_i64_noret_offset:
1508 ; GFX9: ; %bb.0:
1509 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1510 ; GFX9-NEXT: flat_atomic_and_x2 v[0:1], v[2:3] offset:32
1511 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1512 ; GFX9-NEXT: buffer_wbinvl1_vol
1513 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1514 %gep = getelementptr i64, ptr %out, i64 4
1515 %tmp0 = atomicrmw and ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
1516 ret void
1517 }
1519 define i64 @flat_atomic_and_i64_ret(ptr %ptr, i64 %in) {
1520 ; GFX7-LABEL: flat_atomic_and_i64_ret:
1521 ; GFX7: ; %bb.0:
1522 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1523 ; GFX7-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
1524 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1525 ; GFX7-NEXT: buffer_wbinvl1_vol
1526 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1527 ;
1528 ; GFX8-LABEL: flat_atomic_and_i64_ret:
1529 ; GFX8: ; %bb.0:
1530 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1531 ; GFX8-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
1532 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1533 ; GFX8-NEXT: buffer_wbinvl1_vol
1534 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1535 ;
1536 ; GFX9-LABEL: flat_atomic_and_i64_ret:
1537 ; GFX9: ; %bb.0:
1538 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1539 ; GFX9-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
1540 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1541 ; GFX9-NEXT: buffer_wbinvl1_vol
1542 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1543 %result = atomicrmw and ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
1544 ret i64 %result
1545 }
1547 define i64 @flat_atomic_and_i64_ret_offset(ptr %out, i64 %in) {
1548 ; GFX7-LABEL: flat_atomic_and_i64_ret_offset:
1549 ; GFX7: ; %bb.0:
1550 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1551 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
1552 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1553 ; GFX7-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
1554 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1555 ; GFX7-NEXT: buffer_wbinvl1_vol
1556 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1557 ;
1558 ; GFX8-LABEL: flat_atomic_and_i64_ret_offset:
1559 ; GFX8: ; %bb.0:
1560 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1561 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
1562 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1563 ; GFX8-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
1564 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1565 ; GFX8-NEXT: buffer_wbinvl1_vol
1566 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1567 ;
1568 ; GFX9-LABEL: flat_atomic_and_i64_ret_offset:
1569 ; GFX9: ; %bb.0:
1570 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1571 ; GFX9-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
1572 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1573 ; GFX9-NEXT: buffer_wbinvl1_vol
1574 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1575 %gep = getelementptr i64, ptr %out, i64 4
1576 %result = atomicrmw and ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
1577 ret i64 %result
1578 }
1580 define amdgpu_gfx void @flat_atomic_and_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
1581 ; GFX7-LABEL: flat_atomic_and_i64_noret_scalar:
1582 ; GFX7: ; %bb.0:
1583 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1584 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1585 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1586 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
1587 ; GFX7-NEXT: v_mov_b32_e32 v3, s5
1588 ; GFX7-NEXT: flat_atomic_and_x2 v[2:3], v[0:1]
1589 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1590 ; GFX7-NEXT: buffer_wbinvl1_vol
1591 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1592 ;
1593 ; GFX8-LABEL: flat_atomic_and_i64_noret_scalar:
1594 ; GFX8: ; %bb.0:
1595 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1596 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
1597 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
1598 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
1599 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
1600 ; GFX8-NEXT: flat_atomic_and_x2 v[2:3], v[0:1]
1601 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1602 ; GFX8-NEXT: buffer_wbinvl1_vol
1603 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1604 ;
1605 ; GFX9-LABEL: flat_atomic_and_i64_noret_scalar:
1606 ; GFX9: ; %bb.0:
1607 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1608 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
1609 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
1610 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
1611 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
1612 ; GFX9-NEXT: flat_atomic_and_x2 v[2:3], v[0:1]
1613 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1614 ; GFX9-NEXT: buffer_wbinvl1_vol
1615 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1616 %tmp0 = atomicrmw and ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
1617 ret void
1618 }
1620 define amdgpu_gfx void @flat_atomic_and_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
1621 ; GFX7-LABEL: flat_atomic_and_i64_noret_offset_scalar:
1622 ; GFX7: ; %bb.0:
1623 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1624 ; GFX7-NEXT: s_add_u32 s34, s4, 32
1625 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
1626 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
1627 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1628 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1629 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
1630 ; GFX7-NEXT: flat_atomic_and_x2 v[2:3], v[0:1]
1631 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1632 ; GFX7-NEXT: buffer_wbinvl1_vol
1633 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1634 ;
1635 ; GFX8-LABEL: flat_atomic_and_i64_noret_offset_scalar:
1636 ; GFX8: ; %bb.0:
1637 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1638 ; GFX8-NEXT: s_add_u32 s34, s4, 32
1639 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
1640 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
1641 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
1642 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
1643 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
1644 ; GFX8-NEXT: flat_atomic_and_x2 v[2:3], v[0:1]
1645 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1646 ; GFX8-NEXT: buffer_wbinvl1_vol
1647 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1648 ;
1649 ; GFX9-LABEL: flat_atomic_and_i64_noret_offset_scalar:
1650 ; GFX9: ; %bb.0:
1651 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1652 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
1653 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
1654 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
1655 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
1656 ; GFX9-NEXT: flat_atomic_and_x2 v[2:3], v[0:1] offset:32
1657 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1658 ; GFX9-NEXT: buffer_wbinvl1_vol
1659 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1660 %gep = getelementptr i64, ptr %out, i64 4
1661 %tmp0 = atomicrmw and ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
1662 ret void
1663 }
1665 define amdgpu_gfx i64 @flat_atomic_and_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
1666 ; GFX7-LABEL: flat_atomic_and_i64_ret_scalar:
1667 ; GFX7: ; %bb.0:
1668 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1669 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1670 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1671 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
1672 ; GFX7-NEXT: v_mov_b32_e32 v3, s5
1673 ; GFX7-NEXT: flat_atomic_and_x2 v[0:1], v[2:3], v[0:1] glc
1674 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1675 ; GFX7-NEXT: buffer_wbinvl1_vol
1676 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1677 ;
1678 ; GFX8-LABEL: flat_atomic_and_i64_ret_scalar:
1679 ; GFX8: ; %bb.0:
1680 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1681 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
1682 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
1683 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
1684 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
1685 ; GFX8-NEXT: flat_atomic_and_x2 v[0:1], v[2:3], v[0:1] glc
1686 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1687 ; GFX8-NEXT: buffer_wbinvl1_vol
1688 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1689 ;
1690 ; GFX9-LABEL: flat_atomic_and_i64_ret_scalar:
1691 ; GFX9: ; %bb.0:
1692 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1693 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
1694 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
1695 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
1696 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
1697 ; GFX9-NEXT: flat_atomic_and_x2 v[0:1], v[2:3], v[0:1] glc
1698 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1699 ; GFX9-NEXT: buffer_wbinvl1_vol
1700 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1701 %result = atomicrmw and ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
1702 ret i64 %result
1703 }
1705 define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
1706 ; GFX7-LABEL: flat_atomic_and_i64_ret_offset_scalar:
1707 ; GFX7: ; %bb.0:
1708 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1709 ; GFX7-NEXT: s_add_u32 s34, s4, 32
1710 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
1711 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
1712 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1713 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1714 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
1715 ; GFX7-NEXT: flat_atomic_and_x2 v[0:1], v[2:3], v[0:1] glc
1716 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1717 ; GFX7-NEXT: buffer_wbinvl1_vol
1718 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1719 ;
1720 ; GFX8-LABEL: flat_atomic_and_i64_ret_offset_scalar:
1721 ; GFX8: ; %bb.0:
1722 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1723 ; GFX8-NEXT: s_add_u32 s34, s4, 32
1724 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
1725 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
1726 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
1727 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
1728 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
1729 ; GFX8-NEXT: flat_atomic_and_x2 v[0:1], v[2:3], v[0:1] glc
1730 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1731 ; GFX8-NEXT: buffer_wbinvl1_vol
1732 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1733 ;
1734 ; GFX9-LABEL: flat_atomic_and_i64_ret_offset_scalar:
1735 ; GFX9: ; %bb.0:
1736 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1737 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
1738 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
1739 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
1740 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
1741 ; GFX9-NEXT: flat_atomic_and_x2 v[0:1], v[2:3], v[0:1] offset:32 glc
1742 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1743 ; GFX9-NEXT: buffer_wbinvl1_vol
1744 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1745 %gep = getelementptr i64, ptr %out, i64 4
1746 %result = atomicrmw and ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
1747 ret i64 %result
1748 }
1750 define void @flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
1751 ; GFX7-LABEL: flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory:
1752 ; GFX7: ; %bb.0:
1753 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1754 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
1755 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1756 ; GFX7-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
1757 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1758 ; GFX7-NEXT: buffer_wbinvl1_vol
1759 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1760 ;
1761 ; GFX8-LABEL: flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory:
1762 ; GFX8: ; %bb.0:
1763 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1764 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
1765 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1766 ; GFX8-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
1767 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1768 ; GFX8-NEXT: buffer_wbinvl1_vol
1769 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1770 ;
1771 ; GFX9-LABEL: flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory:
1772 ; GFX9: ; %bb.0:
1773 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1774 ; GFX9-NEXT: flat_atomic_and_x2 v[0:1], v[2:3] offset:32
1775 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1776 ; GFX9-NEXT: buffer_wbinvl1_vol
1777 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1778 %gep = getelementptr i64, ptr %out, i64 4
1779 %tmp0 = atomicrmw and ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
1780 ret void
1781 }
1783 define i64 @flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
1784 ; GFX7-LABEL: flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory:
1785 ; GFX7: ; %bb.0:
1786 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1787 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
1788 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1789 ; GFX7-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
1790 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1791 ; GFX7-NEXT: buffer_wbinvl1_vol
1792 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1793 ;
1794 ; GFX8-LABEL: flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory:
1795 ; GFX8: ; %bb.0:
1796 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1797 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
1798 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1799 ; GFX8-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
1800 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1801 ; GFX8-NEXT: buffer_wbinvl1_vol
1802 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1803 ;
1804 ; GFX9-LABEL: flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory:
1805 ; GFX9: ; %bb.0:
1806 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1807 ; GFX9-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
1808 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1809 ; GFX9-NEXT: buffer_wbinvl1_vol
1810 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1811 %gep = getelementptr i64, ptr %out, i64 4
1812 %result = atomicrmw and ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
1813 ret i64 %result
1814 }
1816 ; ---------------------------------------------------------------------
1817 ; atomicrmw nand
1818 ; ---------------------------------------------------------------------
1820 define void @flat_atomic_nand_i64_noret(ptr %ptr, i64 %in) {
1821 ; GFX7-LABEL: flat_atomic_nand_i64_noret:
1822 ; GFX7: ; %bb.0:
1823 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1824 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 4, v0
1825 ; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
1826 ; GFX7-NEXT: flat_load_dword v6, v[0:1]
1827 ; GFX7-NEXT: flat_load_dword v7, v[4:5]
1828 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
1829 ; GFX7-NEXT: .LBB50_1: ; %atomicrmw.start
1830 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
1831 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1832 ; GFX7-NEXT: v_and_b32_e32 v4, v7, v3
1833 ; GFX7-NEXT: v_and_b32_e32 v8, v6, v2
1834 ; GFX7-NEXT: v_not_b32_e32 v5, v4
1835 ; GFX7-NEXT: v_not_b32_e32 v4, v8
1836 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
1837 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1838 ; GFX7-NEXT: buffer_wbinvl1_vol
1839 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
1840 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
1841 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1842 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
1843 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
1844 ; GFX7-NEXT: s_cbranch_execnz .LBB50_1
1845 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
1846 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
1847 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1848 ;
1849 ; GFX8-LABEL: flat_atomic_nand_i64_noret:
1850 ; GFX8: ; %bb.0:
1851 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1852 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v0
1853 ; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
1854 ; GFX8-NEXT: flat_load_dword v6, v[0:1]
1855 ; GFX8-NEXT: flat_load_dword v7, v[4:5]
1856 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
1857 ; GFX8-NEXT: .LBB50_1: ; %atomicrmw.start
1858 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
1859 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1860 ; GFX8-NEXT: v_and_b32_e32 v4, v7, v3
1861 ; GFX8-NEXT: v_and_b32_e32 v8, v6, v2
1862 ; GFX8-NEXT: v_not_b32_e32 v5, v4
1863 ; GFX8-NEXT: v_not_b32_e32 v4, v8
1864 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
1865 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1866 ; GFX8-NEXT: buffer_wbinvl1_vol
1867 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
1868 ; GFX8-NEXT: v_mov_b32_e32 v7, v5
1869 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1870 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
1871 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1872 ; GFX8-NEXT: s_cbranch_execnz .LBB50_1
1873 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1874 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1875 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1876 ;
1877 ; GFX9-LABEL: flat_atomic_nand_i64_noret:
1878 ; GFX9: ; %bb.0:
1879 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1880 ; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
1881 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
1882 ; GFX9-NEXT: .LBB50_1: ; %atomicrmw.start
1883 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
1884 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1885 ; GFX9-NEXT: v_and_b32_e32 v4, v7, v3
1886 ; GFX9-NEXT: v_and_b32_e32 v8, v6, v2
1887 ; GFX9-NEXT: v_not_b32_e32 v5, v4
1888 ; GFX9-NEXT: v_not_b32_e32 v4, v8
1889 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
1890 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1891 ; GFX9-NEXT: buffer_wbinvl1_vol
1892 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
1893 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
1894 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1895 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
1896 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
1897 ; GFX9-NEXT: s_cbranch_execnz .LBB50_1
1898 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
1899 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
1900 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1901 %tmp0 = atomicrmw nand ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
1902 ret void
1903 }
1905 define void @flat_atomic_nand_i64_noret_offset(ptr %out, i64 %in) {
1906 ; GFX7-LABEL: flat_atomic_nand_i64_noret_offset:
1907 ; GFX7: ; %bb.0:
1908 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1909 ; GFX7-NEXT: v_add_i32_e32 v8, vcc, 32, v0
1910 ; GFX7-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
1911 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v0
1912 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1913 ; GFX7-NEXT: flat_load_dword v7, v[0:1]
1914 ; GFX7-NEXT: flat_load_dword v6, v[8:9]
1915 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
1916 ; GFX7-NEXT: .LBB51_1: ; %atomicrmw.start
1917 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
1918 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1919 ; GFX7-NEXT: v_and_b32_e32 v0, v7, v3
1920 ; GFX7-NEXT: v_and_b32_e32 v1, v6, v2
1921 ; GFX7-NEXT: v_not_b32_e32 v5, v0
1922 ; GFX7-NEXT: v_not_b32_e32 v4, v1
1923 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
1924 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1925 ; GFX7-NEXT: buffer_wbinvl1_vol
1926 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
1927 ; GFX7-NEXT: v_mov_b32_e32 v7, v1
1928 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1929 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
1930 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
1931 ; GFX7-NEXT: s_cbranch_execnz .LBB51_1
1932 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
1933 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
1934 ; GFX7-NEXT: s_setpc_b64 s[30:31]
1935 ;
1936 ; GFX8-LABEL: flat_atomic_nand_i64_noret_offset:
1937 ; GFX8: ; %bb.0:
1938 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1939 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 32, v0
1940 ; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
1941 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v0
1942 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1943 ; GFX8-NEXT: flat_load_dword v7, v[0:1]
1944 ; GFX8-NEXT: flat_load_dword v6, v[8:9]
1945 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
1946 ; GFX8-NEXT: .LBB51_1: ; %atomicrmw.start
1947 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
1948 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1949 ; GFX8-NEXT: v_and_b32_e32 v0, v7, v3
1950 ; GFX8-NEXT: v_and_b32_e32 v1, v6, v2
1951 ; GFX8-NEXT: v_not_b32_e32 v5, v0
1952 ; GFX8-NEXT: v_not_b32_e32 v4, v1
1953 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
1954 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1955 ; GFX8-NEXT: buffer_wbinvl1_vol
1956 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
1957 ; GFX8-NEXT: v_mov_b32_e32 v7, v1
1958 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1959 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
1960 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
1961 ; GFX8-NEXT: s_cbranch_execnz .LBB51_1
1962 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
1963 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
1964 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1965 ;
1966 ; GFX9-LABEL: flat_atomic_nand_i64_noret_offset:
1967 ; GFX9: ; %bb.0:
1968 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1969 ; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
1970 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
1971 ; GFX9-NEXT: .LBB51_1: ; %atomicrmw.start
1972 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
1973 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1974 ; GFX9-NEXT: v_and_b32_e32 v4, v7, v3
1975 ; GFX9-NEXT: v_and_b32_e32 v8, v6, v2
1976 ; GFX9-NEXT: v_not_b32_e32 v5, v4
1977 ; GFX9-NEXT: v_not_b32_e32 v4, v8
1978 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
1979 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1980 ; GFX9-NEXT: buffer_wbinvl1_vol
1981 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
1982 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
1983 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
1984 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
1985 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
1986 ; GFX9-NEXT: s_cbranch_execnz .LBB51_1
1987 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
1988 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
1989 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1990 %gep = getelementptr i64, ptr %out, i64 4
1991 %tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
1992 ret void
1993 }
1995 define i64 @flat_atomic_nand_i64_ret(ptr %ptr, i64 %in) {
1996 ; GFX7-LABEL: flat_atomic_nand_i64_ret:
1997 ; GFX7: ; %bb.0:
1998 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1999 ; GFX7-NEXT: v_add_i32_e32 v5, vcc, 4, v0
2000 ; GFX7-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
2001 ; GFX7-NEXT: flat_load_dword v4, v[0:1]
2002 ; GFX7-NEXT: flat_load_dword v5, v[5:6]
2003 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
2004 ; GFX7-NEXT: .LBB52_1: ; %atomicrmw.start
2005 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
2006 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2007 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
2008 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
2009 ; GFX7-NEXT: v_and_b32_e32 v4, v7, v3
2010 ; GFX7-NEXT: v_and_b32_e32 v8, v6, v2
2011 ; GFX7-NEXT: v_not_b32_e32 v5, v4
2012 ; GFX7-NEXT: v_not_b32_e32 v4, v8
2013 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
2014 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2015 ; GFX7-NEXT: buffer_wbinvl1_vol
2016 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
2017 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2018 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
2019 ; GFX7-NEXT: s_cbranch_execnz .LBB52_1
2020 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
2021 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
2022 ; GFX7-NEXT: v_mov_b32_e32 v0, v4
2023 ; GFX7-NEXT: v_mov_b32_e32 v1, v5
2024 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2025 ;
2026 ; GFX8-LABEL: flat_atomic_nand_i64_ret:
2027 ; GFX8: ; %bb.0:
2028 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2029 ; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v0
2030 ; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
2031 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
2032 ; GFX8-NEXT: flat_load_dword v5, v[5:6]
2033 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2034 ; GFX8-NEXT: .LBB52_1: ; %atomicrmw.start
2035 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2036 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2037 ; GFX8-NEXT: v_mov_b32_e32 v7, v5
2038 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
2039 ; GFX8-NEXT: v_and_b32_e32 v4, v7, v3
2040 ; GFX8-NEXT: v_and_b32_e32 v8, v6, v2
2041 ; GFX8-NEXT: v_not_b32_e32 v5, v4
2042 ; GFX8-NEXT: v_not_b32_e32 v4, v8
2043 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
2044 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2045 ; GFX8-NEXT: buffer_wbinvl1_vol
2046 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
2047 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2048 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2049 ; GFX8-NEXT: s_cbranch_execnz .LBB52_1
2050 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2051 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2052 ; GFX8-NEXT: v_mov_b32_e32 v0, v4
2053 ; GFX8-NEXT: v_mov_b32_e32 v1, v5
2054 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2055 ;
2056 ; GFX9-LABEL: flat_atomic_nand_i64_ret:
2057 ; GFX9: ; %bb.0:
2058 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2059 ; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
2060 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
2061 ; GFX9-NEXT: .LBB52_1: ; %atomicrmw.start
2062 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
2063 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2064 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
2065 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
2066 ; GFX9-NEXT: v_and_b32_e32 v4, v7, v3
2067 ; GFX9-NEXT: v_and_b32_e32 v8, v6, v2
2068 ; GFX9-NEXT: v_not_b32_e32 v5, v4
2069 ; GFX9-NEXT: v_not_b32_e32 v4, v8
2070 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
2071 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2072 ; GFX9-NEXT: buffer_wbinvl1_vol
2073 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
2074 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2075 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
2076 ; GFX9-NEXT: s_cbranch_execnz .LBB52_1
2077 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
2078 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
2079 ; GFX9-NEXT: v_mov_b32_e32 v0, v4
2080 ; GFX9-NEXT: v_mov_b32_e32 v1, v5
2081 ; GFX9-NEXT: s_setpc_b64 s[30:31]
2082 %result = atomicrmw nand ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
2083 ret i64 %result
2084 }
2086 define i64 @flat_atomic_nand_i64_ret_offset(ptr %out, i64 %in) {
2087 ; GFX7-LABEL: flat_atomic_nand_i64_ret_offset:
2088 ; GFX7: ; %bb.0:
2089 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2090 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 32, v0
2091 ; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
2092 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v0
2093 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2094 ; GFX7-NEXT: flat_load_dword v1, v[0:1]
2095 ; GFX7-NEXT: flat_load_dword v0, v[4:5]
2096 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
2097 ; GFX7-NEXT: .LBB53_1: ; %atomicrmw.start
2098 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
2099 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2100 ; GFX7-NEXT: v_mov_b32_e32 v9, v1
2101 ; GFX7-NEXT: v_mov_b32_e32 v8, v0
2102 ; GFX7-NEXT: v_and_b32_e32 v0, v9, v3
2103 ; GFX7-NEXT: v_and_b32_e32 v1, v8, v2
2104 ; GFX7-NEXT: v_not_b32_e32 v7, v0
2105 ; GFX7-NEXT: v_not_b32_e32 v6, v1
2106 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
2107 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2108 ; GFX7-NEXT: buffer_wbinvl1_vol
2109 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
2110 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2111 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
2112 ; GFX7-NEXT: s_cbranch_execnz .LBB53_1
2113 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
2114 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
2115 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2116 ;
2117 ; GFX8-LABEL: flat_atomic_nand_i64_ret_offset:
2118 ; GFX8: ; %bb.0:
2119 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2120 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 32, v0
2121 ; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
2122 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v0
2123 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2124 ; GFX8-NEXT: flat_load_dword v1, v[0:1]
2125 ; GFX8-NEXT: flat_load_dword v0, v[4:5]
2126 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2127 ; GFX8-NEXT: .LBB53_1: ; %atomicrmw.start
2128 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2129 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2130 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
2131 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
2132 ; GFX8-NEXT: v_and_b32_e32 v0, v9, v3
2133 ; GFX8-NEXT: v_and_b32_e32 v1, v8, v2
2134 ; GFX8-NEXT: v_not_b32_e32 v7, v0
2135 ; GFX8-NEXT: v_not_b32_e32 v6, v1
2136 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
2137 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2138 ; GFX8-NEXT: buffer_wbinvl1_vol
2139 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
2140 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2141 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2142 ; GFX8-NEXT: s_cbranch_execnz .LBB53_1
2143 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2144 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2145 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2146 ;
2147 ; GFX9-LABEL: flat_atomic_nand_i64_ret_offset:
2148 ; GFX9: ; %bb.0:
2149 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2150 ; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
2151 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
2152 ; GFX9-NEXT: .LBB53_1: ; %atomicrmw.start
2153 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
2154 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2155 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
2156 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
2157 ; GFX9-NEXT: v_and_b32_e32 v4, v7, v3
2158 ; GFX9-NEXT: v_and_b32_e32 v8, v6, v2
2159 ; GFX9-NEXT: v_not_b32_e32 v5, v4
2160 ; GFX9-NEXT: v_not_b32_e32 v4, v8
2161 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
2162 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2163 ; GFX9-NEXT: buffer_wbinvl1_vol
2164 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
2165 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2166 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
2167 ; GFX9-NEXT: s_cbranch_execnz .LBB53_1
2168 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
2169 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
2170 ; GFX9-NEXT: v_mov_b32_e32 v0, v4
2171 ; GFX9-NEXT: v_mov_b32_e32 v1, v5
2172 ; GFX9-NEXT: s_setpc_b64 s[30:31]
2173 %gep = getelementptr i64, ptr %out, i64 4
2174 %result = atomicrmw nand ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
2175 ret i64 %result
2176 }
2178 define amdgpu_gfx void @flat_atomic_nand_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
2179 ; GFX7-LABEL: flat_atomic_nand_i64_noret_scalar:
2180 ; GFX7: ; %bb.0:
2181 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2182 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
2183 ; GFX7-NEXT: s_add_u32 s34, s4, 4
2184 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
2185 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
2186 ; GFX7-NEXT: v_mov_b32_e32 v3, s34
2187 ; GFX7-NEXT: v_mov_b32_e32 v4, s35
2188 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
2189 ; GFX7-NEXT: flat_load_dword v3, v[3:4]
2190 ; GFX7-NEXT: v_mov_b32_e32 v4, s4
2191 ; GFX7-NEXT: s_mov_b64 s[34:35], 0
2192 ; GFX7-NEXT: v_mov_b32_e32 v5, s5
2193 ; GFX7-NEXT: .LBB54_1: ; %atomicrmw.start
2194 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
2195 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2196 ; GFX7-NEXT: v_and_b32_e32 v0, s7, v3
2197 ; GFX7-NEXT: v_and_b32_e32 v6, s6, v2
2198 ; GFX7-NEXT: v_not_b32_e32 v1, v0
2199 ; GFX7-NEXT: v_not_b32_e32 v0, v6
2200 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
2201 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2202 ; GFX7-NEXT: buffer_wbinvl1_vol
2203 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
2204 ; GFX7-NEXT: v_mov_b32_e32 v3, v1
2205 ; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
2206 ; GFX7-NEXT: v_mov_b32_e32 v2, v0
2207 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
2208 ; GFX7-NEXT: s_cbranch_execnz .LBB54_1
2209 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
2210 ; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
2211 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2212 ;
2213 ; GFX8-LABEL: flat_atomic_nand_i64_noret_scalar:
2214 ; GFX8: ; %bb.0:
2215 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2216 ; GFX8-NEXT: v_mov_b32_e32 v0, s4
2217 ; GFX8-NEXT: s_add_u32 s34, s4, 4
2218 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
2219 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
2220 ; GFX8-NEXT: v_mov_b32_e32 v3, s34
2221 ; GFX8-NEXT: v_mov_b32_e32 v4, s35
2222 ; GFX8-NEXT: flat_load_dword v2, v[0:1]
2223 ; GFX8-NEXT: flat_load_dword v3, v[3:4]
2224 ; GFX8-NEXT: v_mov_b32_e32 v4, s4
2225 ; GFX8-NEXT: s_mov_b64 s[34:35], 0
2226 ; GFX8-NEXT: v_mov_b32_e32 v5, s5
2227 ; GFX8-NEXT: .LBB54_1: ; %atomicrmw.start
2228 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2229 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2230 ; GFX8-NEXT: v_and_b32_e32 v0, s7, v3
2231 ; GFX8-NEXT: v_and_b32_e32 v6, s6, v2
2232 ; GFX8-NEXT: v_not_b32_e32 v1, v0
2233 ; GFX8-NEXT: v_not_b32_e32 v0, v6
2234 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
2235 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2236 ; GFX8-NEXT: buffer_wbinvl1_vol
2237 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
2238 ; GFX8-NEXT: v_mov_b32_e32 v3, v1
2239 ; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
2240 ; GFX8-NEXT: v_mov_b32_e32 v2, v0
2241 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
2242 ; GFX8-NEXT: s_cbranch_execnz .LBB54_1
2243 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2244 ; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
2245 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2246 ;
2247 ; GFX9-LABEL: flat_atomic_nand_i64_noret_scalar:
2248 ; GFX9: ; %bb.0:
2249 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2250 ; GFX9-NEXT: v_mov_b32_e32 v0, s4
2251 ; GFX9-NEXT: v_mov_b32_e32 v1, s5
2252 ; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
2253 ; GFX9-NEXT: v_mov_b32_e32 v4, s4
2254 ; GFX9-NEXT: s_mov_b64 s[34:35], 0
2255 ; GFX9-NEXT: v_mov_b32_e32 v5, s5
2256 ; GFX9-NEXT: .LBB54_1: ; %atomicrmw.start
2257 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
2258 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2259 ; GFX9-NEXT: v_and_b32_e32 v0, s7, v3
2260 ; GFX9-NEXT: v_and_b32_e32 v6, s6, v2
2261 ; GFX9-NEXT: v_not_b32_e32 v1, v0
2262 ; GFX9-NEXT: v_not_b32_e32 v0, v6
2263 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
2264 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2265 ; GFX9-NEXT: buffer_wbinvl1_vol
2266 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
2267 ; GFX9-NEXT: v_mov_b32_e32 v3, v1
2268 ; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
2269 ; GFX9-NEXT: v_mov_b32_e32 v2, v0
2270 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
2271 ; GFX9-NEXT: s_cbranch_execnz .LBB54_1
2272 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
2273 ; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
2274 ; GFX9-NEXT: s_setpc_b64 s[30:31]
2275 %tmp0 = atomicrmw nand ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
2276 ret void
2277 }
2279 define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
2280 ; GFX7-LABEL: flat_atomic_nand_i64_noret_offset_scalar:
2281 ; GFX7: ; %bb.0:
2282 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2283 ; GFX7-NEXT: s_add_u32 s34, s4, 32
2284 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
2285 ; GFX7-NEXT: s_add_u32 s36, s4, 36
2286 ; GFX7-NEXT: s_addc_u32 s37, s5, 0
2287 ; GFX7-NEXT: v_mov_b32_e32 v0, s36
2288 ; GFX7-NEXT: v_mov_b32_e32 v1, s37
2289 ; GFX7-NEXT: v_mov_b32_e32 v4, s34
2290 ; GFX7-NEXT: v_mov_b32_e32 v5, s35
2291 ; GFX7-NEXT: flat_load_dword v3, v[0:1]
2292 ; GFX7-NEXT: flat_load_dword v2, v[4:5]
2293 ; GFX7-NEXT: s_mov_b64 s[34:35], 0
2294 ; GFX7-NEXT: .LBB55_1: ; %atomicrmw.start
2295 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
2296 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2297 ; GFX7-NEXT: v_and_b32_e32 v0, s7, v3
2298 ; GFX7-NEXT: v_and_b32_e32 v6, s6, v2
2299 ; GFX7-NEXT: v_not_b32_e32 v1, v0
2300 ; GFX7-NEXT: v_not_b32_e32 v0, v6
2301 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
2302 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2303 ; GFX7-NEXT: buffer_wbinvl1_vol
2304 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
2305 ; GFX7-NEXT: v_mov_b32_e32 v3, v1
2306 ; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
2307 ; GFX7-NEXT: v_mov_b32_e32 v2, v0
2308 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
2309 ; GFX7-NEXT: s_cbranch_execnz .LBB55_1
2310 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
2311 ; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
2312 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2313 ;
2314 ; GFX8-LABEL: flat_atomic_nand_i64_noret_offset_scalar:
2315 ; GFX8: ; %bb.0:
2316 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2317 ; GFX8-NEXT: s_add_u32 s34, s4, 32
2318 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
2319 ; GFX8-NEXT: s_add_u32 s36, s4, 36
2320 ; GFX8-NEXT: s_addc_u32 s37, s5, 0
2321 ; GFX8-NEXT: v_mov_b32_e32 v0, s36
2322 ; GFX8-NEXT: v_mov_b32_e32 v1, s37
2323 ; GFX8-NEXT: v_mov_b32_e32 v4, s34
2324 ; GFX8-NEXT: v_mov_b32_e32 v5, s35
2325 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
2326 ; GFX8-NEXT: flat_load_dword v2, v[4:5]
2327 ; GFX8-NEXT: s_mov_b64 s[34:35], 0
2328 ; GFX8-NEXT: .LBB55_1: ; %atomicrmw.start
2329 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2330 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2331 ; GFX8-NEXT: v_and_b32_e32 v0, s7, v3
2332 ; GFX8-NEXT: v_and_b32_e32 v6, s6, v2
2333 ; GFX8-NEXT: v_not_b32_e32 v1, v0
2334 ; GFX8-NEXT: v_not_b32_e32 v0, v6
2335 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
2336 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2337 ; GFX8-NEXT: buffer_wbinvl1_vol
2338 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
2339 ; GFX8-NEXT: v_mov_b32_e32 v3, v1
2340 ; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
2341 ; GFX8-NEXT: v_mov_b32_e32 v2, v0
2342 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
2343 ; GFX8-NEXT: s_cbranch_execnz .LBB55_1
2344 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2345 ; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
2346 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2347 ;
2348 ; GFX9-LABEL: flat_atomic_nand_i64_noret_offset_scalar:
2349 ; GFX9: ; %bb.0:
2350 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2351 ; GFX9-NEXT: v_mov_b32_e32 v0, s4
2352 ; GFX9-NEXT: v_mov_b32_e32 v1, s5
2353 ; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
2354 ; GFX9-NEXT: v_mov_b32_e32 v4, s4
2355 ; GFX9-NEXT: s_mov_b64 s[34:35], 0
2356 ; GFX9-NEXT: v_mov_b32_e32 v5, s5
2357 ; GFX9-NEXT: .LBB55_1: ; %atomicrmw.start
2358 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
2359 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2360 ; GFX9-NEXT: v_and_b32_e32 v0, s7, v3
2361 ; GFX9-NEXT: v_and_b32_e32 v6, s6, v2
2362 ; GFX9-NEXT: v_not_b32_e32 v1, v0
2363 ; GFX9-NEXT: v_not_b32_e32 v0, v6
2364 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
2365 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2366 ; GFX9-NEXT: buffer_wbinvl1_vol
2367 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
2368 ; GFX9-NEXT: v_mov_b32_e32 v3, v1
2369 ; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
2370 ; GFX9-NEXT: v_mov_b32_e32 v2, v0
2371 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
2372 ; GFX9-NEXT: s_cbranch_execnz .LBB55_1
2373 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
2374 ; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
2375 ; GFX9-NEXT: s_setpc_b64 s[30:31]
2376 %gep = getelementptr i64, ptr %out, i64 4
2377 %tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
2378 ret void
2379 }
2381 define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
2382 ; GFX7-LABEL: flat_atomic_nand_i64_ret_scalar:
2383 ; GFX7: ; %bb.0:
2384 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2385 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
2386 ; GFX7-NEXT: s_add_u32 s34, s4, 4
2387 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
2388 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
2389 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
2390 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
2391 ; GFX7-NEXT: flat_load_dword v0, v[0:1]
2392 ; GFX7-NEXT: flat_load_dword v1, v[2:3]
2393 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
2394 ; GFX7-NEXT: s_mov_b64 s[34:35], 0
2395 ; GFX7-NEXT: v_mov_b32_e32 v3, s5
2396 ; GFX7-NEXT: .LBB56_1: ; %atomicrmw.start
2397 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
2398 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2399 ; GFX7-NEXT: v_mov_b32_e32 v7, v1
2400 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
2401 ; GFX7-NEXT: v_and_b32_e32 v0, s7, v7
2402 ; GFX7-NEXT: v_and_b32_e32 v1, s6, v6
2403 ; GFX7-NEXT: v_not_b32_e32 v5, v0
2404 ; GFX7-NEXT: v_not_b32_e32 v4, v1
2405 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
2406 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2407 ; GFX7-NEXT: buffer_wbinvl1_vol
2408 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
2409 ; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
2410 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
2411 ; GFX7-NEXT: s_cbranch_execnz .LBB56_1
2412 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
2413 ; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
2414 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2415 ;
2416 ; GFX8-LABEL: flat_atomic_nand_i64_ret_scalar:
2417 ; GFX8: ; %bb.0:
2418 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2419 ; GFX8-NEXT: v_mov_b32_e32 v0, s4
2420 ; GFX8-NEXT: s_add_u32 s34, s4, 4
2421 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
2422 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
2423 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
2424 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
2425 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
2426 ; GFX8-NEXT: flat_load_dword v1, v[2:3]
2427 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
2428 ; GFX8-NEXT: s_mov_b64 s[34:35], 0
2429 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
2430 ; GFX8-NEXT: .LBB56_1: ; %atomicrmw.start
2431 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2432 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2433 ; GFX8-NEXT: v_mov_b32_e32 v7, v1
2434 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
2435 ; GFX8-NEXT: v_and_b32_e32 v0, s7, v7
2436 ; GFX8-NEXT: v_and_b32_e32 v1, s6, v6
2437 ; GFX8-NEXT: v_not_b32_e32 v5, v0
2438 ; GFX8-NEXT: v_not_b32_e32 v4, v1
2439 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
2440 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2441 ; GFX8-NEXT: buffer_wbinvl1_vol
2442 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
2443 ; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
2444 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
2445 ; GFX8-NEXT: s_cbranch_execnz .LBB56_1
2446 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2447 ; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
2448 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2449 ;
2450 ; GFX9-LABEL: flat_atomic_nand_i64_ret_scalar:
2451 ; GFX9: ; %bb.0:
2452 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2453 ; GFX9-NEXT: v_mov_b32_e32 v0, s4
2454 ; GFX9-NEXT: v_mov_b32_e32 v1, s5
2455 ; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
2456 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
2457 ; GFX9-NEXT: s_mov_b64 s[34:35], 0
2458 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
2459 ; GFX9-NEXT: .LBB56_1: ; %atomicrmw.start
2460 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
2461 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2462 ; GFX9-NEXT: v_mov_b32_e32 v7, v1
2463 ; GFX9-NEXT: v_mov_b32_e32 v6, v0
2464 ; GFX9-NEXT: v_and_b32_e32 v0, s7, v7
2465 ; GFX9-NEXT: v_and_b32_e32 v1, s6, v6
2466 ; GFX9-NEXT: v_not_b32_e32 v5, v0
2467 ; GFX9-NEXT: v_not_b32_e32 v4, v1
2468 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
2469 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2470 ; GFX9-NEXT: buffer_wbinvl1_vol
2471 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
2472 ; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
2473 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
2474 ; GFX9-NEXT: s_cbranch_execnz .LBB56_1
2475 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
2476 ; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
2477 ; GFX9-NEXT: s_setpc_b64 s[30:31]
2478 %result = atomicrmw nand ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
2479 ret i64 %result
2480 }
2482 define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
2483 ; GFX7-LABEL: flat_atomic_nand_i64_ret_offset_scalar:
2484 ; GFX7: ; %bb.0:
2485 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2486 ; GFX7-NEXT: s_add_u32 s34, s4, 32
2487 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
2488 ; GFX7-NEXT: s_add_u32 s36, s4, 36
2489 ; GFX7-NEXT: s_addc_u32 s37, s5, 0
2490 ; GFX7-NEXT: v_mov_b32_e32 v0, s36
2491 ; GFX7-NEXT: v_mov_b32_e32 v1, s37
2492 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
2493 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
2494 ; GFX7-NEXT: flat_load_dword v1, v[0:1]
2495 ; GFX7-NEXT: flat_load_dword v0, v[2:3]
2496 ; GFX7-NEXT: s_mov_b64 s[34:35], 0
2497 ; GFX7-NEXT: .LBB57_1: ; %atomicrmw.start
2498 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
2499 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2500 ; GFX7-NEXT: v_mov_b32_e32 v7, v1
2501 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
2502 ; GFX7-NEXT: v_and_b32_e32 v0, s7, v7
2503 ; GFX7-NEXT: v_and_b32_e32 v1, s6, v6
2504 ; GFX7-NEXT: v_not_b32_e32 v5, v0
2505 ; GFX7-NEXT: v_not_b32_e32 v4, v1
2506 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
2507 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2508 ; GFX7-NEXT: buffer_wbinvl1_vol
2509 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
2510 ; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
2511 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
2512 ; GFX7-NEXT: s_cbranch_execnz .LBB57_1
2513 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
2514 ; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
2515 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2516 ;
2517 ; GFX8-LABEL: flat_atomic_nand_i64_ret_offset_scalar:
2518 ; GFX8: ; %bb.0:
2519 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2520 ; GFX8-NEXT: s_add_u32 s34, s4, 32
2521 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
2522 ; GFX8-NEXT: s_add_u32 s36, s4, 36
2523 ; GFX8-NEXT: s_addc_u32 s37, s5, 0
2524 ; GFX8-NEXT: v_mov_b32_e32 v0, s36
2525 ; GFX8-NEXT: v_mov_b32_e32 v1, s37
2526 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
2527 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
2528 ; GFX8-NEXT: flat_load_dword v1, v[0:1]
2529 ; GFX8-NEXT: flat_load_dword v0, v[2:3]
2530 ; GFX8-NEXT: s_mov_b64 s[34:35], 0
2531 ; GFX8-NEXT: .LBB57_1: ; %atomicrmw.start
2532 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2533 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2534 ; GFX8-NEXT: v_mov_b32_e32 v7, v1
2535 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
2536 ; GFX8-NEXT: v_and_b32_e32 v0, s7, v7
2537 ; GFX8-NEXT: v_and_b32_e32 v1, s6, v6
2538 ; GFX8-NEXT: v_not_b32_e32 v5, v0
2539 ; GFX8-NEXT: v_not_b32_e32 v4, v1
2540 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
2541 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2542 ; GFX8-NEXT: buffer_wbinvl1_vol
2543 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
2544 ; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
2545 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
2546 ; GFX8-NEXT: s_cbranch_execnz .LBB57_1
2547 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2548 ; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
2549 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2550 ;
2551 ; GFX9-LABEL: flat_atomic_nand_i64_ret_offset_scalar:
2552 ; GFX9: ; %bb.0:
2553 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2554 ; GFX9-NEXT: v_mov_b32_e32 v0, s4
2555 ; GFX9-NEXT: v_mov_b32_e32 v1, s5
2556 ; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
2557 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
2558 ; GFX9-NEXT: s_mov_b64 s[34:35], 0
2559 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
2560 ; GFX9-NEXT: .LBB57_1: ; %atomicrmw.start
2561 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
2562 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2563 ; GFX9-NEXT: v_mov_b32_e32 v7, v1
2564 ; GFX9-NEXT: v_mov_b32_e32 v6, v0
2565 ; GFX9-NEXT: v_and_b32_e32 v0, s7, v7
2566 ; GFX9-NEXT: v_and_b32_e32 v1, s6, v6
2567 ; GFX9-NEXT: v_not_b32_e32 v5, v0
2568 ; GFX9-NEXT: v_not_b32_e32 v4, v1
2569 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] offset:32 glc
2570 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2571 ; GFX9-NEXT: buffer_wbinvl1_vol
2572 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
2573 ; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
2574 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
2575 ; GFX9-NEXT: s_cbranch_execnz .LBB57_1
2576 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
2577 ; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
2578 ; GFX9-NEXT: s_setpc_b64 s[30:31]
2579 %gep = getelementptr i64, ptr %out, i64 4
2580 %result = atomicrmw nand ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
2581 ret i64 %result
2582 }
2584 define void @flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
2585 ; GFX7-LABEL: flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory:
2586 ; GFX7: ; %bb.0:
2587 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2588 ; GFX7-NEXT: v_add_i32_e32 v8, vcc, 32, v0
2589 ; GFX7-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
2590 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v0
2591 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2592 ; GFX7-NEXT: flat_load_dword v7, v[0:1]
2593 ; GFX7-NEXT: flat_load_dword v6, v[8:9]
2594 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
2595 ; GFX7-NEXT: .LBB58_1: ; %atomicrmw.start
2596 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
2597 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2598 ; GFX7-NEXT: v_and_b32_e32 v0, v7, v3
2599 ; GFX7-NEXT: v_and_b32_e32 v1, v6, v2
2600 ; GFX7-NEXT: v_not_b32_e32 v5, v0
2601 ; GFX7-NEXT: v_not_b32_e32 v4, v1
2602 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
2603 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2604 ; GFX7-NEXT: buffer_wbinvl1_vol
2605 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
2606 ; GFX7-NEXT: v_mov_b32_e32 v7, v1
2607 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2608 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
2609 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
2610 ; GFX7-NEXT: s_cbranch_execnz .LBB58_1
2611 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
2612 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
2613 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2614 ;
2615 ; GFX8-LABEL: flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory:
2616 ; GFX8: ; %bb.0:
2617 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2618 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 32, v0
2619 ; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
2620 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v0
2621 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2622 ; GFX8-NEXT: flat_load_dword v7, v[0:1]
2623 ; GFX8-NEXT: flat_load_dword v6, v[8:9]
2624 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2625 ; GFX8-NEXT: .LBB58_1: ; %atomicrmw.start
2626 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2627 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2628 ; GFX8-NEXT: v_and_b32_e32 v0, v7, v3
2629 ; GFX8-NEXT: v_and_b32_e32 v1, v6, v2
2630 ; GFX8-NEXT: v_not_b32_e32 v5, v0
2631 ; GFX8-NEXT: v_not_b32_e32 v4, v1
2632 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
2633 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2634 ; GFX8-NEXT: buffer_wbinvl1_vol
2635 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
2636 ; GFX8-NEXT: v_mov_b32_e32 v7, v1
2637 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2638 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
2639 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2640 ; GFX8-NEXT: s_cbranch_execnz .LBB58_1
2641 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2642 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2643 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2644 ;
2645 ; GFX9-LABEL: flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory:
2646 ; GFX9: ; %bb.0:
2647 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2648 ; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
2649 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
2650 ; GFX9-NEXT: .LBB58_1: ; %atomicrmw.start
2651 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
2652 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2653 ; GFX9-NEXT: v_and_b32_e32 v4, v7, v3
2654 ; GFX9-NEXT: v_and_b32_e32 v8, v6, v2
2655 ; GFX9-NEXT: v_not_b32_e32 v5, v4
2656 ; GFX9-NEXT: v_not_b32_e32 v4, v8
2657 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
2658 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2659 ; GFX9-NEXT: buffer_wbinvl1_vol
2660 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
2661 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
2662 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2663 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
2664 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
2665 ; GFX9-NEXT: s_cbranch_execnz .LBB58_1
2666 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
2667 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
2668 ; GFX9-NEXT: s_setpc_b64 s[30:31]
2669 %gep = getelementptr i64, ptr %out, i64 4
2670 %tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
2671 ret void
2672 }
2674 define i64 @flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
2675 ; GFX7-LABEL: flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory:
2676 ; GFX7: ; %bb.0:
2677 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2678 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 32, v0
2679 ; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
2680 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v0
2681 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2682 ; GFX7-NEXT: flat_load_dword v1, v[0:1]
2683 ; GFX7-NEXT: flat_load_dword v0, v[4:5]
2684 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
2685 ; GFX7-NEXT: .LBB59_1: ; %atomicrmw.start
2686 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
2687 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2688 ; GFX7-NEXT: v_mov_b32_e32 v9, v1
2689 ; GFX7-NEXT: v_mov_b32_e32 v8, v0
2690 ; GFX7-NEXT: v_and_b32_e32 v0, v9, v3
2691 ; GFX7-NEXT: v_and_b32_e32 v1, v8, v2
2692 ; GFX7-NEXT: v_not_b32_e32 v7, v0
2693 ; GFX7-NEXT: v_not_b32_e32 v6, v1
2694 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
2695 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2696 ; GFX7-NEXT: buffer_wbinvl1_vol
2697 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
2698 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2699 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
2700 ; GFX7-NEXT: s_cbranch_execnz .LBB59_1
2701 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
2702 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
2703 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2704 ;
2705 ; GFX8-LABEL: flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory:
2706 ; GFX8: ; %bb.0:
2707 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2708 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 32, v0
2709 ; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
2710 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v0
2711 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2712 ; GFX8-NEXT: flat_load_dword v1, v[0:1]
2713 ; GFX8-NEXT: flat_load_dword v0, v[4:5]
2714 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
2715 ; GFX8-NEXT: .LBB59_1: ; %atomicrmw.start
2716 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
2717 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2718 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
2719 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
2720 ; GFX8-NEXT: v_and_b32_e32 v0, v9, v3
2721 ; GFX8-NEXT: v_and_b32_e32 v1, v8, v2
2722 ; GFX8-NEXT: v_not_b32_e32 v7, v0
2723 ; GFX8-NEXT: v_not_b32_e32 v6, v1
2724 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
2725 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2726 ; GFX8-NEXT: buffer_wbinvl1_vol
2727 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
2728 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2729 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
2730 ; GFX8-NEXT: s_cbranch_execnz .LBB59_1
2731 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
2732 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
2733 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2734 ;
2735 ; GFX9-LABEL: flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory:
2736 ; GFX9: ; %bb.0:
2737 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2738 ; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
2739 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
2740 ; GFX9-NEXT: .LBB59_1: ; %atomicrmw.start
2741 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
2742 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2743 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
2744 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
2745 ; GFX9-NEXT: v_and_b32_e32 v4, v7, v3
2746 ; GFX9-NEXT: v_and_b32_e32 v8, v6, v2
2747 ; GFX9-NEXT: v_not_b32_e32 v5, v4
2748 ; GFX9-NEXT: v_not_b32_e32 v4, v8
2749 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
2750 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2751 ; GFX9-NEXT: buffer_wbinvl1_vol
2752 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
2753 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
2754 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
2755 ; GFX9-NEXT: s_cbranch_execnz .LBB59_1
2756 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
2757 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
2758 ; GFX9-NEXT: v_mov_b32_e32 v0, v4
2759 ; GFX9-NEXT: v_mov_b32_e32 v1, v5
2760 ; GFX9-NEXT: s_setpc_b64 s[30:31]
2761 %gep = getelementptr i64, ptr %out, i64 4
2762 %result = atomicrmw nand ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
2763 ret i64 %result
2764 }
2766 ; ---------------------------------------------------------------------
2767 ; atomicrmw or
2768 ; ---------------------------------------------------------------------
2770 define void @flat_atomic_or_i64_noret(ptr %ptr, i64 %in) {
2771 ; GFX7-LABEL: flat_atomic_or_i64_noret:
2772 ; GFX7: ; %bb.0:
2773 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2774 ; GFX7-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
2775 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2776 ; GFX7-NEXT: buffer_wbinvl1_vol
2777 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2778 ;
2779 ; GFX8-LABEL: flat_atomic_or_i64_noret:
2780 ; GFX8: ; %bb.0:
2781 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2782 ; GFX8-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
2783 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2784 ; GFX8-NEXT: buffer_wbinvl1_vol
2785 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2786 ;
2787 ; GFX9-LABEL: flat_atomic_or_i64_noret:
2788 ; GFX9: ; %bb.0:
2789 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2790 ; GFX9-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
2791 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2792 ; GFX9-NEXT: buffer_wbinvl1_vol
2793 ; GFX9-NEXT: s_setpc_b64 s[30:31]
2794 %tmp0 = atomicrmw or ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
2795 ret void
2796 }
2798 define void @flat_atomic_or_i64_noret_offset(ptr %out, i64 %in) {
2799 ; GFX7-LABEL: flat_atomic_or_i64_noret_offset:
2800 ; GFX7: ; %bb.0:
2801 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2802 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
2803 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2804 ; GFX7-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
2805 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2806 ; GFX7-NEXT: buffer_wbinvl1_vol
2807 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2808 ;
2809 ; GFX8-LABEL: flat_atomic_or_i64_noret_offset:
2810 ; GFX8: ; %bb.0:
2811 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2812 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
2813 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2814 ; GFX8-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
2815 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2816 ; GFX8-NEXT: buffer_wbinvl1_vol
2817 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2818 ;
2819 ; GFX9-LABEL: flat_atomic_or_i64_noret_offset:
2820 ; GFX9: ; %bb.0:
2821 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2822 ; GFX9-NEXT: flat_atomic_or_x2 v[0:1], v[2:3] offset:32
2823 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2824 ; GFX9-NEXT: buffer_wbinvl1_vol
2825 ; GFX9-NEXT: s_setpc_b64 s[30:31]
2826 %gep = getelementptr i64, ptr %out, i64 4
2827 %tmp0 = atomicrmw or ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
2828 ret void
2829 }
2831 define i64 @flat_atomic_or_i64_ret(ptr %ptr, i64 %in) {
2832 ; GFX7-LABEL: flat_atomic_or_i64_ret:
2833 ; GFX7: ; %bb.0:
2834 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2835 ; GFX7-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
2836 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2837 ; GFX7-NEXT: buffer_wbinvl1_vol
2838 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2839 ;
2840 ; GFX8-LABEL: flat_atomic_or_i64_ret:
2841 ; GFX8: ; %bb.0:
2842 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2843 ; GFX8-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
2844 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2845 ; GFX8-NEXT: buffer_wbinvl1_vol
2846 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2847 ;
2848 ; GFX9-LABEL: flat_atomic_or_i64_ret:
2849 ; GFX9: ; %bb.0:
2850 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2851 ; GFX9-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
2852 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2853 ; GFX9-NEXT: buffer_wbinvl1_vol
2854 ; GFX9-NEXT: s_setpc_b64 s[30:31]
2855 %result = atomicrmw or ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
2856 ret i64 %result
2857 }
2859 define i64 @flat_atomic_or_i64_ret_offset(ptr %out, i64 %in) {
2860 ; GFX7-LABEL: flat_atomic_or_i64_ret_offset:
2861 ; GFX7: ; %bb.0:
2862 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2863 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
2864 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2865 ; GFX7-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
2866 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2867 ; GFX7-NEXT: buffer_wbinvl1_vol
2868 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2869 ;
2870 ; GFX8-LABEL: flat_atomic_or_i64_ret_offset:
2871 ; GFX8: ; %bb.0:
2872 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2873 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
2874 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2875 ; GFX8-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
2876 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2877 ; GFX8-NEXT: buffer_wbinvl1_vol
2878 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2879 ;
2880 ; GFX9-LABEL: flat_atomic_or_i64_ret_offset:
2881 ; GFX9: ; %bb.0:
2882 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2883 ; GFX9-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
2884 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2885 ; GFX9-NEXT: buffer_wbinvl1_vol
2886 ; GFX9-NEXT: s_setpc_b64 s[30:31]
2887 %gep = getelementptr i64, ptr %out, i64 4
2888 %result = atomicrmw or ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
2889 ret i64 %result
2890 }
2892 define amdgpu_gfx void @flat_atomic_or_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
2893 ; GFX7-LABEL: flat_atomic_or_i64_noret_scalar:
2894 ; GFX7: ; %bb.0:
2895 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2896 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
2897 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
2898 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
2899 ; GFX7-NEXT: v_mov_b32_e32 v3, s5
2900 ; GFX7-NEXT: flat_atomic_or_x2 v[2:3], v[0:1]
2901 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2902 ; GFX7-NEXT: buffer_wbinvl1_vol
2903 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2904 ;
2905 ; GFX8-LABEL: flat_atomic_or_i64_noret_scalar:
2906 ; GFX8: ; %bb.0:
2907 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2908 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
2909 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
2910 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
2911 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
2912 ; GFX8-NEXT: flat_atomic_or_x2 v[2:3], v[0:1]
2913 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2914 ; GFX8-NEXT: buffer_wbinvl1_vol
2915 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2916 ;
2917 ; GFX9-LABEL: flat_atomic_or_i64_noret_scalar:
2918 ; GFX9: ; %bb.0:
2919 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2920 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
2921 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
2922 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
2923 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
2924 ; GFX9-NEXT: flat_atomic_or_x2 v[2:3], v[0:1]
2925 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2926 ; GFX9-NEXT: buffer_wbinvl1_vol
2927 ; GFX9-NEXT: s_setpc_b64 s[30:31]
2928 %tmp0 = atomicrmw or ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
2929 ret void
2930 }
2932 define amdgpu_gfx void @flat_atomic_or_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
2933 ; GFX7-LABEL: flat_atomic_or_i64_noret_offset_scalar:
2934 ; GFX7: ; %bb.0:
2935 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2936 ; GFX7-NEXT: s_add_u32 s34, s4, 32
2937 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
2938 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
2939 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
2940 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
2941 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
2942 ; GFX7-NEXT: flat_atomic_or_x2 v[2:3], v[0:1]
2943 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2944 ; GFX7-NEXT: buffer_wbinvl1_vol
2945 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2946 ;
2947 ; GFX8-LABEL: flat_atomic_or_i64_noret_offset_scalar:
2948 ; GFX8: ; %bb.0:
2949 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2950 ; GFX8-NEXT: s_add_u32 s34, s4, 32
2951 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
2952 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
2953 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
2954 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
2955 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
2956 ; GFX8-NEXT: flat_atomic_or_x2 v[2:3], v[0:1]
2957 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2958 ; GFX8-NEXT: buffer_wbinvl1_vol
2959 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2960 ;
2961 ; GFX9-LABEL: flat_atomic_or_i64_noret_offset_scalar:
2962 ; GFX9: ; %bb.0:
2963 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2964 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
2965 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
2966 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
2967 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
2968 ; GFX9-NEXT: flat_atomic_or_x2 v[2:3], v[0:1] offset:32
2969 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2970 ; GFX9-NEXT: buffer_wbinvl1_vol
2971 ; GFX9-NEXT: s_setpc_b64 s[30:31]
2972 %gep = getelementptr i64, ptr %out, i64 4
2973 %tmp0 = atomicrmw or ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
2974 ret void
2975 }
2977 define amdgpu_gfx i64 @flat_atomic_or_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
2978 ; GFX7-LABEL: flat_atomic_or_i64_ret_scalar:
2979 ; GFX7: ; %bb.0:
2980 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2981 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
2982 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
2983 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
2984 ; GFX7-NEXT: v_mov_b32_e32 v3, s5
2985 ; GFX7-NEXT: flat_atomic_or_x2 v[0:1], v[2:3], v[0:1] glc
2986 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2987 ; GFX7-NEXT: buffer_wbinvl1_vol
2988 ; GFX7-NEXT: s_setpc_b64 s[30:31]
2989 ;
2990 ; GFX8-LABEL: flat_atomic_or_i64_ret_scalar:
2991 ; GFX8: ; %bb.0:
2992 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2993 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
2994 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
2995 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
2996 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
2997 ; GFX8-NEXT: flat_atomic_or_x2 v[0:1], v[2:3], v[0:1] glc
2998 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2999 ; GFX8-NEXT: buffer_wbinvl1_vol
3000 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3001 ;
3002 ; GFX9-LABEL: flat_atomic_or_i64_ret_scalar:
3003 ; GFX9: ; %bb.0:
3004 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3005 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
3006 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
3007 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
3008 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
3009 ; GFX9-NEXT: flat_atomic_or_x2 v[0:1], v[2:3], v[0:1] glc
3010 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3011 ; GFX9-NEXT: buffer_wbinvl1_vol
3012 ; GFX9-NEXT: s_setpc_b64 s[30:31]
3013 %result = atomicrmw or ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
3014 ret i64 %result
3015 }
3017 define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
3018 ; GFX7-LABEL: flat_atomic_or_i64_ret_offset_scalar:
3019 ; GFX7: ; %bb.0:
3020 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3021 ; GFX7-NEXT: s_add_u32 s34, s4, 32
3022 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
3023 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
3024 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
3025 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
3026 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
3027 ; GFX7-NEXT: flat_atomic_or_x2 v[0:1], v[2:3], v[0:1] glc
3028 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3029 ; GFX7-NEXT: buffer_wbinvl1_vol
3030 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3031 ;
3032 ; GFX8-LABEL: flat_atomic_or_i64_ret_offset_scalar:
3033 ; GFX8: ; %bb.0:
3034 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3035 ; GFX8-NEXT: s_add_u32 s34, s4, 32
3036 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
3037 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
3038 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
3039 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
3040 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
3041 ; GFX8-NEXT: flat_atomic_or_x2 v[0:1], v[2:3], v[0:1] glc
3042 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3043 ; GFX8-NEXT: buffer_wbinvl1_vol
3044 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3045 ;
3046 ; GFX9-LABEL: flat_atomic_or_i64_ret_offset_scalar:
3047 ; GFX9: ; %bb.0:
3048 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3049 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
3050 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
3051 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
3052 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
3053 ; GFX9-NEXT: flat_atomic_or_x2 v[0:1], v[2:3], v[0:1] offset:32 glc
3054 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3055 ; GFX9-NEXT: buffer_wbinvl1_vol
3056 ; GFX9-NEXT: s_setpc_b64 s[30:31]
3057 %gep = getelementptr i64, ptr %out, i64 4
3058 %result = atomicrmw or ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
3059 ret i64 %result
3060 }
3062 define void @flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
3063 ; GFX7-LABEL: flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory:
3064 ; GFX7: ; %bb.0:
3065 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3066 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
3067 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3068 ; GFX7-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
3069 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3070 ; GFX7-NEXT: buffer_wbinvl1_vol
3071 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3072 ;
3073 ; GFX8-LABEL: flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory:
3074 ; GFX8: ; %bb.0:
3075 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3076 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
3077 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3078 ; GFX8-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
3079 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3080 ; GFX8-NEXT: buffer_wbinvl1_vol
3081 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3082 ;
3083 ; GFX9-LABEL: flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory:
3084 ; GFX9: ; %bb.0:
3085 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3086 ; GFX9-NEXT: flat_atomic_or_x2 v[0:1], v[2:3] offset:32
3087 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3088 ; GFX9-NEXT: buffer_wbinvl1_vol
3089 ; GFX9-NEXT: s_setpc_b64 s[30:31]
3090 %gep = getelementptr i64, ptr %out, i64 4
3091 %tmp0 = atomicrmw or ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
3092 ret void
3093 }
3095 define i64 @flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
3096 ; GFX7-LABEL: flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory:
3097 ; GFX7: ; %bb.0:
3098 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3099 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
3100 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3101 ; GFX7-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
3102 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3103 ; GFX7-NEXT: buffer_wbinvl1_vol
3104 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3105 ;
3106 ; GFX8-LABEL: flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory:
3107 ; GFX8: ; %bb.0:
3108 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3109 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
3110 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3111 ; GFX8-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
3112 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3113 ; GFX8-NEXT: buffer_wbinvl1_vol
3114 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3115 ;
3116 ; GFX9-LABEL: flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory:
3117 ; GFX9: ; %bb.0:
3118 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3119 ; GFX9-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
3120 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3121 ; GFX9-NEXT: buffer_wbinvl1_vol
3122 ; GFX9-NEXT: s_setpc_b64 s[30:31]
3123 %gep = getelementptr i64, ptr %out, i64 4
3124 %result = atomicrmw or ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
3125 ret i64 %result
3126 }
3128 ; ---------------------------------------------------------------------
3129 ; atomicrmw xor
3130 ; ---------------------------------------------------------------------
3132 define void @flat_atomic_xor_i64_noret(ptr %ptr, i64 %in) {
3133 ; GFX7-LABEL: flat_atomic_xor_i64_noret:
3134 ; GFX7: ; %bb.0:
3135 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3136 ; GFX7-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
3137 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3138 ; GFX7-NEXT: buffer_wbinvl1_vol
3139 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3140 ;
3141 ; GFX8-LABEL: flat_atomic_xor_i64_noret:
3142 ; GFX8: ; %bb.0:
3143 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3144 ; GFX8-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
3145 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3146 ; GFX8-NEXT: buffer_wbinvl1_vol
3147 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3148 ;
3149 ; GFX9-LABEL: flat_atomic_xor_i64_noret:
3150 ; GFX9: ; %bb.0:
3151 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3152 ; GFX9-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
3153 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3154 ; GFX9-NEXT: buffer_wbinvl1_vol
3155 ; GFX9-NEXT: s_setpc_b64 s[30:31]
3156 %tmp0 = atomicrmw xor ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
3157 ret void
3158 }
3160 define void @flat_atomic_xor_i64_noret_offset(ptr %out, i64 %in) {
3161 ; GFX7-LABEL: flat_atomic_xor_i64_noret_offset:
3162 ; GFX7: ; %bb.0:
3163 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3164 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
3165 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3166 ; GFX7-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
3167 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3168 ; GFX7-NEXT: buffer_wbinvl1_vol
3169 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3170 ;
3171 ; GFX8-LABEL: flat_atomic_xor_i64_noret_offset:
3172 ; GFX8: ; %bb.0:
3173 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3174 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
3175 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3176 ; GFX8-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
3177 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3178 ; GFX8-NEXT: buffer_wbinvl1_vol
3179 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3180 ;
3181 ; GFX9-LABEL: flat_atomic_xor_i64_noret_offset:
3182 ; GFX9: ; %bb.0:
3183 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3184 ; GFX9-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3] offset:32
3185 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3186 ; GFX9-NEXT: buffer_wbinvl1_vol
3187 ; GFX9-NEXT: s_setpc_b64 s[30:31]
3188 %gep = getelementptr i64, ptr %out, i64 4
3189 %tmp0 = atomicrmw xor ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
3190 ret void
3191 }
3193 define i64 @flat_atomic_xor_i64_ret(ptr %ptr, i64 %in) {
3194 ; GFX7-LABEL: flat_atomic_xor_i64_ret:
3195 ; GFX7: ; %bb.0:
3196 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3197 ; GFX7-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
3198 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3199 ; GFX7-NEXT: buffer_wbinvl1_vol
3200 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3201 ;
3202 ; GFX8-LABEL: flat_atomic_xor_i64_ret:
3203 ; GFX8: ; %bb.0:
3204 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3205 ; GFX8-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
3206 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3207 ; GFX8-NEXT: buffer_wbinvl1_vol
3208 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3209 ;
3210 ; GFX9-LABEL: flat_atomic_xor_i64_ret:
3211 ; GFX9: ; %bb.0:
3212 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3213 ; GFX9-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
3214 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3215 ; GFX9-NEXT: buffer_wbinvl1_vol
3216 ; GFX9-NEXT: s_setpc_b64 s[30:31]
3217 %result = atomicrmw xor ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
3218 ret i64 %result
3219 }
3221 define i64 @flat_atomic_xor_i64_ret_offset(ptr %out, i64 %in) {
3222 ; GFX7-LABEL: flat_atomic_xor_i64_ret_offset:
3223 ; GFX7: ; %bb.0:
3224 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3225 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
3226 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3227 ; GFX7-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
3228 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3229 ; GFX7-NEXT: buffer_wbinvl1_vol
3230 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3231 ;
3232 ; GFX8-LABEL: flat_atomic_xor_i64_ret_offset:
3233 ; GFX8: ; %bb.0:
3234 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3235 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
3236 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3237 ; GFX8-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
3238 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3239 ; GFX8-NEXT: buffer_wbinvl1_vol
3240 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3241 ;
3242 ; GFX9-LABEL: flat_atomic_xor_i64_ret_offset:
3243 ; GFX9: ; %bb.0:
3244 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3245 ; GFX9-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
3246 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3247 ; GFX9-NEXT: buffer_wbinvl1_vol
3248 ; GFX9-NEXT: s_setpc_b64 s[30:31]
3249 %gep = getelementptr i64, ptr %out, i64 4
3250 %result = atomicrmw xor ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
3251 ret i64 %result
3252 }
3254 define amdgpu_gfx void @flat_atomic_xor_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
3255 ; GFX7-LABEL: flat_atomic_xor_i64_noret_scalar:
3256 ; GFX7: ; %bb.0:
3257 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3258 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
3259 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
3260 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
3261 ; GFX7-NEXT: v_mov_b32_e32 v3, s5
3262 ; GFX7-NEXT: flat_atomic_xor_x2 v[2:3], v[0:1]
3263 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3264 ; GFX7-NEXT: buffer_wbinvl1_vol
3265 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3266 ;
3267 ; GFX8-LABEL: flat_atomic_xor_i64_noret_scalar:
3268 ; GFX8: ; %bb.0:
3269 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3270 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
3271 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
3272 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
3273 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
3274 ; GFX8-NEXT: flat_atomic_xor_x2 v[2:3], v[0:1]
3275 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3276 ; GFX8-NEXT: buffer_wbinvl1_vol
3277 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3278 ;
3279 ; GFX9-LABEL: flat_atomic_xor_i64_noret_scalar:
3280 ; GFX9: ; %bb.0:
3281 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3282 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
3283 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
3284 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
3285 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
3286 ; GFX9-NEXT: flat_atomic_xor_x2 v[2:3], v[0:1]
3287 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3288 ; GFX9-NEXT: buffer_wbinvl1_vol
3289 ; GFX9-NEXT: s_setpc_b64 s[30:31]
3290 %tmp0 = atomicrmw xor ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
3291 ret void
3292 }
3294 define amdgpu_gfx void @flat_atomic_xor_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
3295 ; GFX7-LABEL: flat_atomic_xor_i64_noret_offset_scalar:
3296 ; GFX7: ; %bb.0:
3297 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3298 ; GFX7-NEXT: s_add_u32 s34, s4, 32
3299 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
3300 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
3301 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
3302 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
3303 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
3304 ; GFX7-NEXT: flat_atomic_xor_x2 v[2:3], v[0:1]
3305 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3306 ; GFX7-NEXT: buffer_wbinvl1_vol
3307 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3308 ;
3309 ; GFX8-LABEL: flat_atomic_xor_i64_noret_offset_scalar:
3310 ; GFX8: ; %bb.0:
3311 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3312 ; GFX8-NEXT: s_add_u32 s34, s4, 32
3313 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
3314 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
3315 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
3316 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
3317 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
3318 ; GFX8-NEXT: flat_atomic_xor_x2 v[2:3], v[0:1]
3319 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3320 ; GFX8-NEXT: buffer_wbinvl1_vol
3321 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3322 ;
3323 ; GFX9-LABEL: flat_atomic_xor_i64_noret_offset_scalar:
3324 ; GFX9: ; %bb.0:
3325 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3326 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
3327 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
3328 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
3329 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
3330 ; GFX9-NEXT: flat_atomic_xor_x2 v[2:3], v[0:1] offset:32
3331 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3332 ; GFX9-NEXT: buffer_wbinvl1_vol
3333 ; GFX9-NEXT: s_setpc_b64 s[30:31]
3334 %gep = getelementptr i64, ptr %out, i64 4
3335 %tmp0 = atomicrmw xor ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
3336 ret void
3337 }
3339 define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
3340 ; GFX7-LABEL: flat_atomic_xor_i64_ret_scalar:
3341 ; GFX7: ; %bb.0:
3342 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3343 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
3344 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
3345 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
3346 ; GFX7-NEXT: v_mov_b32_e32 v3, s5
3347 ; GFX7-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3], v[0:1] glc
3348 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3349 ; GFX7-NEXT: buffer_wbinvl1_vol
3350 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3351 ;
3352 ; GFX8-LABEL: flat_atomic_xor_i64_ret_scalar:
3353 ; GFX8: ; %bb.0:
3354 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3355 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
3356 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
3357 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
3358 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
3359 ; GFX8-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3], v[0:1] glc
3360 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3361 ; GFX8-NEXT: buffer_wbinvl1_vol
3362 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3363 ;
3364 ; GFX9-LABEL: flat_atomic_xor_i64_ret_scalar:
3365 ; GFX9: ; %bb.0:
3366 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3367 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
3368 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
3369 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
3370 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
3371 ; GFX9-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3], v[0:1] glc
3372 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3373 ; GFX9-NEXT: buffer_wbinvl1_vol
3374 ; GFX9-NEXT: s_setpc_b64 s[30:31]
3375 %result = atomicrmw xor ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
3376 ret i64 %result
3377 }
3379 define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
3380 ; GFX7-LABEL: flat_atomic_xor_i64_ret_offset_scalar:
3381 ; GFX7: ; %bb.0:
3382 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3383 ; GFX7-NEXT: s_add_u32 s34, s4, 32
3384 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
3385 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
3386 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
3387 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
3388 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
3389 ; GFX7-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3], v[0:1] glc
3390 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3391 ; GFX7-NEXT: buffer_wbinvl1_vol
3392 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3393 ;
3394 ; GFX8-LABEL: flat_atomic_xor_i64_ret_offset_scalar:
3395 ; GFX8: ; %bb.0:
3396 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3397 ; GFX8-NEXT: s_add_u32 s34, s4, 32
3398 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
3399 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
3400 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
3401 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
3402 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
3403 ; GFX8-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3], v[0:1] glc
3404 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3405 ; GFX8-NEXT: buffer_wbinvl1_vol
3406 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3407 ;
3408 ; GFX9-LABEL: flat_atomic_xor_i64_ret_offset_scalar:
3409 ; GFX9: ; %bb.0:
3410 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3411 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
3412 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
3413 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
3414 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
3415 ; GFX9-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3], v[0:1] offset:32 glc
3416 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3417 ; GFX9-NEXT: buffer_wbinvl1_vol
3418 ; GFX9-NEXT: s_setpc_b64 s[30:31]
3419 %gep = getelementptr i64, ptr %out, i64 4
3420 %result = atomicrmw xor ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
3421 ret i64 %result
3422 }
3424 define void @flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
3425 ; GFX7-LABEL: flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory:
3426 ; GFX7: ; %bb.0:
3427 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3428 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
3429 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3430 ; GFX7-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
3431 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3432 ; GFX7-NEXT: buffer_wbinvl1_vol
3433 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3434 ;
3435 ; GFX8-LABEL: flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory:
3436 ; GFX8: ; %bb.0:
3437 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3438 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
3439 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3440 ; GFX8-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
3441 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3442 ; GFX8-NEXT: buffer_wbinvl1_vol
3443 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3444 ;
3445 ; GFX9-LABEL: flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory:
3446 ; GFX9: ; %bb.0:
3447 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3448 ; GFX9-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3] offset:32
3449 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3450 ; GFX9-NEXT: buffer_wbinvl1_vol
3451 ; GFX9-NEXT: s_setpc_b64 s[30:31]
3452 %gep = getelementptr i64, ptr %out, i64 4
3453 %tmp0 = atomicrmw xor ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
3454 ret void
3455 }
3457 define i64 @flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
3458 ; GFX7-LABEL: flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory:
3459 ; GFX7: ; %bb.0:
3460 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3461 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
3462 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3463 ; GFX7-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
3464 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3465 ; GFX7-NEXT: buffer_wbinvl1_vol
3466 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3467 ;
3468 ; GFX8-LABEL: flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory:
3469 ; GFX8: ; %bb.0:
3470 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3471 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
3472 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3473 ; GFX8-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
3474 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3475 ; GFX8-NEXT: buffer_wbinvl1_vol
3476 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3477 ;
3478 ; GFX9-LABEL: flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory:
3479 ; GFX9: ; %bb.0:
3480 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3481 ; GFX9-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
3482 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3483 ; GFX9-NEXT: buffer_wbinvl1_vol
3484 ; GFX9-NEXT: s_setpc_b64 s[30:31]
3485 %gep = getelementptr i64, ptr %out, i64 4
3486 %result = atomicrmw xor ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
3487 ret i64 %result
3488 }
3490 ; ---------------------------------------------------------------------
3491 ; atomicrmw max
3492 ; ---------------------------------------------------------------------
3494 define void @flat_atomic_max_i64_noret(ptr %ptr, i64 %in) {
3495 ; GFX7-LABEL: flat_atomic_max_i64_noret:
3496 ; GFX7: ; %bb.0:
3497 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3498 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 4, v0
3499 ; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
3500 ; GFX7-NEXT: flat_load_dword v6, v[0:1]
3501 ; GFX7-NEXT: flat_load_dword v7, v[4:5]
3502 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
3503 ; GFX7-NEXT: .LBB80_1: ; %atomicrmw.start
3504 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
3505 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3506 ; GFX7-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
3507 ; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
3508 ; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
3509 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
3510 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3511 ; GFX7-NEXT: buffer_wbinvl1_vol
3512 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
3513 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
3514 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3515 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
3516 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
3517 ; GFX7-NEXT: s_cbranch_execnz .LBB80_1
3518 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
3519 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
3520 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3521 ;
3522 ; GFX8-LABEL: flat_atomic_max_i64_noret:
3523 ; GFX8: ; %bb.0:
3524 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3525 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v0
3526 ; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
3527 ; GFX8-NEXT: flat_load_dword v6, v[0:1]
3528 ; GFX8-NEXT: flat_load_dword v7, v[4:5]
3529 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
3530 ; GFX8-NEXT: .LBB80_1: ; %atomicrmw.start
3531 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3532 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3533 ; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
3534 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
3535 ; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
3536 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
3537 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3538 ; GFX8-NEXT: buffer_wbinvl1_vol
3539 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
3540 ; GFX8-NEXT: v_mov_b32_e32 v7, v5
3541 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3542 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
3543 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
3544 ; GFX8-NEXT: s_cbranch_execnz .LBB80_1
3545 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
3546 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
3547 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3548 ;
3549 ; GFX9-LABEL: flat_atomic_max_i64_noret:
3550 ; GFX9: ; %bb.0:
3551 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3552 ; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
3553 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
3554 ; GFX9-NEXT: .LBB80_1: ; %atomicrmw.start
3555 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
3556 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3557 ; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
3558 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
3559 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
3560 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
3561 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3562 ; GFX9-NEXT: buffer_wbinvl1_vol
3563 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
3564 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
3565 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3566 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
3567 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
3568 ; GFX9-NEXT: s_cbranch_execnz .LBB80_1
3569 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
3570 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
3571 ; GFX9-NEXT: s_setpc_b64 s[30:31]
3572 %tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
3573 ret void
3574 }
3576 define void @flat_atomic_max_i64_noret_offset(ptr %out, i64 %in) {
3577 ; GFX7-LABEL: flat_atomic_max_i64_noret_offset:
3578 ; GFX7: ; %bb.0:
3579 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3580 ; GFX7-NEXT: v_add_i32_e32 v8, vcc, 32, v0
3581 ; GFX7-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
3582 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v0
3583 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3584 ; GFX7-NEXT: flat_load_dword v7, v[0:1]
3585 ; GFX7-NEXT: flat_load_dword v6, v[8:9]
3586 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
3587 ; GFX7-NEXT: .LBB81_1: ; %atomicrmw.start
3588 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
3589 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3590 ; GFX7-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
3591 ; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
3592 ; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
3593 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
3594 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3595 ; GFX7-NEXT: buffer_wbinvl1_vol
3596 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
3597 ; GFX7-NEXT: v_mov_b32_e32 v7, v1
3598 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3599 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
3600 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
3601 ; GFX7-NEXT: s_cbranch_execnz .LBB81_1
3602 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
3603 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
3604 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3605 ;
3606 ; GFX8-LABEL: flat_atomic_max_i64_noret_offset:
3607 ; GFX8: ; %bb.0:
3608 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3609 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 32, v0
3610 ; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
3611 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v0
3612 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3613 ; GFX8-NEXT: flat_load_dword v7, v[0:1]
3614 ; GFX8-NEXT: flat_load_dword v6, v[8:9]
3615 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
3616 ; GFX8-NEXT: .LBB81_1: ; %atomicrmw.start
3617 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3618 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3619 ; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
3620 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
3621 ; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
3622 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
3623 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3624 ; GFX8-NEXT: buffer_wbinvl1_vol
3625 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
3626 ; GFX8-NEXT: v_mov_b32_e32 v7, v1
3627 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3628 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
3629 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
3630 ; GFX8-NEXT: s_cbranch_execnz .LBB81_1
3631 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
3632 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
3633 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3634 ;
3635 ; GFX9-LABEL: flat_atomic_max_i64_noret_offset:
3636 ; GFX9: ; %bb.0:
3637 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3638 ; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
3639 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
3640 ; GFX9-NEXT: .LBB81_1: ; %atomicrmw.start
3641 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
3642 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3643 ; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
3644 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
3645 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
3646 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
3647 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3648 ; GFX9-NEXT: buffer_wbinvl1_vol
3649 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
3650 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
3651 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3652 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
3653 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
3654 ; GFX9-NEXT: s_cbranch_execnz .LBB81_1
3655 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
3656 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
3657 ; GFX9-NEXT: s_setpc_b64 s[30:31]
3658 %gep = getelementptr i64, ptr %out, i64 4
3659 %tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
3660 ret void
3661 }
3663 define i64 @flat_atomic_max_i64_ret(ptr %ptr, i64 %in) {
3664 ; GFX7-LABEL: flat_atomic_max_i64_ret:
3665 ; GFX7: ; %bb.0:
3666 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3667 ; GFX7-NEXT: v_add_i32_e32 v5, vcc, 4, v0
3668 ; GFX7-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
3669 ; GFX7-NEXT: flat_load_dword v4, v[0:1]
3670 ; GFX7-NEXT: flat_load_dword v5, v[5:6]
3671 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
3672 ; GFX7-NEXT: .LBB82_1: ; %atomicrmw.start
3673 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
3674 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3675 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
3676 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
3677 ; GFX7-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
3678 ; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
3679 ; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
3680 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
3681 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3682 ; GFX7-NEXT: buffer_wbinvl1_vol
3683 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
3684 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3685 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
3686 ; GFX7-NEXT: s_cbranch_execnz .LBB82_1
3687 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
3688 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
3689 ; GFX7-NEXT: v_mov_b32_e32 v0, v4
3690 ; GFX7-NEXT: v_mov_b32_e32 v1, v5
3691 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3692 ;
3693 ; GFX8-LABEL: flat_atomic_max_i64_ret:
3694 ; GFX8: ; %bb.0:
3695 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3696 ; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v0
3697 ; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
3698 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
3699 ; GFX8-NEXT: flat_load_dword v5, v[5:6]
3700 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
3701 ; GFX8-NEXT: .LBB82_1: ; %atomicrmw.start
3702 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3703 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3704 ; GFX8-NEXT: v_mov_b32_e32 v7, v5
3705 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
3706 ; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
3707 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
3708 ; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
3709 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
3710 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3711 ; GFX8-NEXT: buffer_wbinvl1_vol
3712 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
3713 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3714 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
3715 ; GFX8-NEXT: s_cbranch_execnz .LBB82_1
3716 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
3717 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
3718 ; GFX8-NEXT: v_mov_b32_e32 v0, v4
3719 ; GFX8-NEXT: v_mov_b32_e32 v1, v5
3720 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3721 ;
3722 ; GFX9-LABEL: flat_atomic_max_i64_ret:
3723 ; GFX9: ; %bb.0:
3724 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3725 ; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
3726 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
3727 ; GFX9-NEXT: .LBB82_1: ; %atomicrmw.start
3728 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
3729 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3730 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
3731 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
3732 ; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
3733 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
3734 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
3735 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
3736 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3737 ; GFX9-NEXT: buffer_wbinvl1_vol
3738 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
3739 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3740 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
3741 ; GFX9-NEXT: s_cbranch_execnz .LBB82_1
3742 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
3743 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
3744 ; GFX9-NEXT: v_mov_b32_e32 v0, v4
3745 ; GFX9-NEXT: v_mov_b32_e32 v1, v5
3746 ; GFX9-NEXT: s_setpc_b64 s[30:31]
3747 %result = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
3748 ret i64 %result
3749 }
3751 define i64 @flat_atomic_max_i64_ret_offset(ptr %out, i64 %in) {
3752 ; GFX7-LABEL: flat_atomic_max_i64_ret_offset:
3753 ; GFX7: ; %bb.0:
3754 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3755 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 32, v0
3756 ; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
3757 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v0
3758 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3759 ; GFX7-NEXT: flat_load_dword v1, v[0:1]
3760 ; GFX7-NEXT: flat_load_dword v0, v[4:5]
3761 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
3762 ; GFX7-NEXT: .LBB83_1: ; %atomicrmw.start
3763 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
3764 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3765 ; GFX7-NEXT: v_mov_b32_e32 v9, v1
3766 ; GFX7-NEXT: v_mov_b32_e32 v8, v0
3767 ; GFX7-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]
3768 ; GFX7-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
3769 ; GFX7-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
3770 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
3771 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3772 ; GFX7-NEXT: buffer_wbinvl1_vol
3773 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
3774 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3775 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
3776 ; GFX7-NEXT: s_cbranch_execnz .LBB83_1
3777 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
3778 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
3779 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3780 ;
3781 ; GFX8-LABEL: flat_atomic_max_i64_ret_offset:
3782 ; GFX8: ; %bb.0:
3783 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3784 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 32, v0
3785 ; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
3786 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v0
3787 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3788 ; GFX8-NEXT: flat_load_dword v1, v[0:1]
3789 ; GFX8-NEXT: flat_load_dword v0, v[4:5]
3790 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
3791 ; GFX8-NEXT: .LBB83_1: ; %atomicrmw.start
3792 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3793 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3794 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
3795 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
3796 ; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]
3797 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
3798 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
3799 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
3800 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3801 ; GFX8-NEXT: buffer_wbinvl1_vol
3802 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
3803 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3804 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
3805 ; GFX8-NEXT: s_cbranch_execnz .LBB83_1
3806 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
3807 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
3808 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3809 ;
3810 ; GFX9-LABEL: flat_atomic_max_i64_ret_offset:
3811 ; GFX9: ; %bb.0:
3812 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3813 ; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
3814 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
3815 ; GFX9-NEXT: .LBB83_1: ; %atomicrmw.start
3816 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
3817 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3818 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
3819 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
3820 ; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
3821 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
3822 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
3823 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
3824 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3825 ; GFX9-NEXT: buffer_wbinvl1_vol
3826 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
3827 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
3828 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
3829 ; GFX9-NEXT: s_cbranch_execnz .LBB83_1
3830 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
3831 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
3832 ; GFX9-NEXT: v_mov_b32_e32 v0, v4
3833 ; GFX9-NEXT: v_mov_b32_e32 v1, v5
3834 ; GFX9-NEXT: s_setpc_b64 s[30:31]
3835 %gep = getelementptr i64, ptr %out, i64 4
3836 %result = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
3837 ret i64 %result
3838 }
3840 define amdgpu_gfx void @flat_atomic_max_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
3841 ; GFX7-LABEL: flat_atomic_max_i64_noret_scalar:
3842 ; GFX7: ; %bb.0:
3843 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3844 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
3845 ; GFX7-NEXT: s_add_u32 s34, s4, 4
3846 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
3847 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
3848 ; GFX7-NEXT: v_mov_b32_e32 v3, s34
3849 ; GFX7-NEXT: v_mov_b32_e32 v4, s35
3850 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
3851 ; GFX7-NEXT: flat_load_dword v3, v[3:4]
3852 ; GFX7-NEXT: v_mov_b32_e32 v4, s4
3853 ; GFX7-NEXT: s_mov_b64 s[34:35], 0
3854 ; GFX7-NEXT: v_mov_b32_e32 v6, s7
3855 ; GFX7-NEXT: v_mov_b32_e32 v7, s6
3856 ; GFX7-NEXT: v_mov_b32_e32 v5, s5
3857 ; GFX7-NEXT: .LBB84_1: ; %atomicrmw.start
3858 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
3859 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3860 ; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
3861 ; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
3862 ; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
3863 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
3864 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3865 ; GFX7-NEXT: buffer_wbinvl1_vol
3866 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
3867 ; GFX7-NEXT: v_mov_b32_e32 v3, v1
3868 ; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
3869 ; GFX7-NEXT: v_mov_b32_e32 v2, v0
3870 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
3871 ; GFX7-NEXT: s_cbranch_execnz .LBB84_1
3872 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
3873 ; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
3874 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3875 ;
3876 ; GFX8-LABEL: flat_atomic_max_i64_noret_scalar:
3877 ; GFX8: ; %bb.0:
3878 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3879 ; GFX8-NEXT: v_mov_b32_e32 v0, s4
3880 ; GFX8-NEXT: s_add_u32 s34, s4, 4
3881 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
3882 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
3883 ; GFX8-NEXT: v_mov_b32_e32 v3, s34
3884 ; GFX8-NEXT: v_mov_b32_e32 v4, s35
3885 ; GFX8-NEXT: flat_load_dword v2, v[0:1]
3886 ; GFX8-NEXT: flat_load_dword v3, v[3:4]
3887 ; GFX8-NEXT: v_mov_b32_e32 v4, s4
3888 ; GFX8-NEXT: s_mov_b64 s[34:35], 0
3889 ; GFX8-NEXT: v_mov_b32_e32 v6, s7
3890 ; GFX8-NEXT: v_mov_b32_e32 v7, s6
3891 ; GFX8-NEXT: v_mov_b32_e32 v5, s5
3892 ; GFX8-NEXT: .LBB84_1: ; %atomicrmw.start
3893 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3894 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3895 ; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
3896 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
3897 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
3898 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
3899 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3900 ; GFX8-NEXT: buffer_wbinvl1_vol
3901 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
3902 ; GFX8-NEXT: v_mov_b32_e32 v3, v1
3903 ; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
3904 ; GFX8-NEXT: v_mov_b32_e32 v2, v0
3905 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
3906 ; GFX8-NEXT: s_cbranch_execnz .LBB84_1
3907 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
3908 ; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
3909 ; GFX8-NEXT: s_setpc_b64 s[30:31]
3910 ;
3911 ; GFX9-LABEL: flat_atomic_max_i64_noret_scalar:
3912 ; GFX9: ; %bb.0:
3913 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3914 ; GFX9-NEXT: v_mov_b32_e32 v0, s4
3915 ; GFX9-NEXT: v_mov_b32_e32 v1, s5
3916 ; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
3917 ; GFX9-NEXT: v_mov_b32_e32 v4, s4
3918 ; GFX9-NEXT: s_mov_b64 s[34:35], 0
3919 ; GFX9-NEXT: v_mov_b32_e32 v6, s7
3920 ; GFX9-NEXT: v_mov_b32_e32 v7, s6
3921 ; GFX9-NEXT: v_mov_b32_e32 v5, s5
3922 ; GFX9-NEXT: .LBB84_1: ; %atomicrmw.start
3923 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
3924 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3925 ; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
3926 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
3927 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
3928 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
3929 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3930 ; GFX9-NEXT: buffer_wbinvl1_vol
3931 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
3932 ; GFX9-NEXT: v_mov_b32_e32 v3, v1
3933 ; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
3934 ; GFX9-NEXT: v_mov_b32_e32 v2, v0
3935 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
3936 ; GFX9-NEXT: s_cbranch_execnz .LBB84_1
3937 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
3938 ; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
3939 ; GFX9-NEXT: s_setpc_b64 s[30:31]
3940 %tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
3941 ret void
3942 }
3944 define amdgpu_gfx void @flat_atomic_max_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
3945 ; GFX7-LABEL: flat_atomic_max_i64_noret_offset_scalar:
3946 ; GFX7: ; %bb.0:
3947 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3948 ; GFX7-NEXT: s_add_u32 s34, s4, 32
3949 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
3950 ; GFX7-NEXT: s_add_u32 s36, s4, 36
3951 ; GFX7-NEXT: s_addc_u32 s37, s5, 0
3952 ; GFX7-NEXT: v_mov_b32_e32 v0, s36
3953 ; GFX7-NEXT: v_mov_b32_e32 v1, s37
3954 ; GFX7-NEXT: v_mov_b32_e32 v4, s34
3955 ; GFX7-NEXT: v_mov_b32_e32 v5, s35
3956 ; GFX7-NEXT: flat_load_dword v3, v[0:1]
3957 ; GFX7-NEXT: flat_load_dword v2, v[4:5]
3958 ; GFX7-NEXT: s_mov_b64 s[34:35], 0
3959 ; GFX7-NEXT: v_mov_b32_e32 v6, s7
3960 ; GFX7-NEXT: v_mov_b32_e32 v7, s6
3961 ; GFX7-NEXT: .LBB85_1: ; %atomicrmw.start
3962 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
3963 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3964 ; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
3965 ; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
3966 ; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
3967 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
3968 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3969 ; GFX7-NEXT: buffer_wbinvl1_vol
3970 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
3971 ; GFX7-NEXT: v_mov_b32_e32 v3, v1
3972 ; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
3973 ; GFX7-NEXT: v_mov_b32_e32 v2, v0
3974 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
3975 ; GFX7-NEXT: s_cbranch_execnz .LBB85_1
3976 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
3977 ; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
3978 ; GFX7-NEXT: s_setpc_b64 s[30:31]
3979 ;
3980 ; GFX8-LABEL: flat_atomic_max_i64_noret_offset_scalar:
3981 ; GFX8: ; %bb.0:
3982 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
3983 ; GFX8-NEXT: s_add_u32 s34, s4, 32
3984 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
3985 ; GFX8-NEXT: s_add_u32 s36, s4, 36
3986 ; GFX8-NEXT: s_addc_u32 s37, s5, 0
3987 ; GFX8-NEXT: v_mov_b32_e32 v0, s36
3988 ; GFX8-NEXT: v_mov_b32_e32 v1, s37
3989 ; GFX8-NEXT: v_mov_b32_e32 v4, s34
3990 ; GFX8-NEXT: v_mov_b32_e32 v5, s35
3991 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
3992 ; GFX8-NEXT: flat_load_dword v2, v[4:5]
3993 ; GFX8-NEXT: s_mov_b64 s[34:35], 0
3994 ; GFX8-NEXT: v_mov_b32_e32 v6, s7
3995 ; GFX8-NEXT: v_mov_b32_e32 v7, s6
3996 ; GFX8-NEXT: .LBB85_1: ; %atomicrmw.start
3997 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
3998 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3999 ; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
4000 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
4001 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
4002 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
4003 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4004 ; GFX8-NEXT: buffer_wbinvl1_vol
4005 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
4006 ; GFX8-NEXT: v_mov_b32_e32 v3, v1
4007 ; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
4008 ; GFX8-NEXT: v_mov_b32_e32 v2, v0
4009 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
4010 ; GFX8-NEXT: s_cbranch_execnz .LBB85_1
4011 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4012 ; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
4013 ; GFX8-NEXT: s_setpc_b64 s[30:31]
4014 ;
4015 ; GFX9-LABEL: flat_atomic_max_i64_noret_offset_scalar:
4016 ; GFX9: ; %bb.0:
4017 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4018 ; GFX9-NEXT: v_mov_b32_e32 v0, s4
4019 ; GFX9-NEXT: v_mov_b32_e32 v1, s5
4020 ; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
4021 ; GFX9-NEXT: v_mov_b32_e32 v4, s4
4022 ; GFX9-NEXT: s_mov_b64 s[34:35], 0
4023 ; GFX9-NEXT: v_mov_b32_e32 v6, s7
4024 ; GFX9-NEXT: v_mov_b32_e32 v7, s6
4025 ; GFX9-NEXT: v_mov_b32_e32 v5, s5
4026 ; GFX9-NEXT: .LBB85_1: ; %atomicrmw.start
4027 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
4028 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4029 ; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
4030 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
4031 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
4032 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
4033 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4034 ; GFX9-NEXT: buffer_wbinvl1_vol
4035 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
4036 ; GFX9-NEXT: v_mov_b32_e32 v3, v1
4037 ; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
4038 ; GFX9-NEXT: v_mov_b32_e32 v2, v0
4039 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
4040 ; GFX9-NEXT: s_cbranch_execnz .LBB85_1
4041 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
4042 ; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
4043 ; GFX9-NEXT: s_setpc_b64 s[30:31]
4044 %gep = getelementptr i64, ptr %out, i64 4
4045 %tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
4046 ret void
4047 }
4049 define amdgpu_gfx i64 @flat_atomic_max_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
4050 ; GFX7-LABEL: flat_atomic_max_i64_ret_scalar:
4051 ; GFX7: ; %bb.0:
4052 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4053 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
4054 ; GFX7-NEXT: s_add_u32 s34, s4, 4
4055 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
4056 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
4057 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
4058 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
4059 ; GFX7-NEXT: flat_load_dword v0, v[0:1]
4060 ; GFX7-NEXT: flat_load_dword v1, v[2:3]
4061 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
4062 ; GFX7-NEXT: s_mov_b64 s[34:35], 0
4063 ; GFX7-NEXT: v_mov_b32_e32 v4, s7
4064 ; GFX7-NEXT: v_mov_b32_e32 v5, s6
4065 ; GFX7-NEXT: v_mov_b32_e32 v3, s5
4066 ; GFX7-NEXT: .LBB86_1: ; %atomicrmw.start
4067 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4068 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4069 ; GFX7-NEXT: v_mov_b32_e32 v9, v1
4070 ; GFX7-NEXT: v_mov_b32_e32 v8, v0
4071 ; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
4072 ; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
4073 ; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
4074 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
4075 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4076 ; GFX7-NEXT: buffer_wbinvl1_vol
4077 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
4078 ; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
4079 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
4080 ; GFX7-NEXT: s_cbranch_execnz .LBB86_1
4081 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4082 ; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
4083 ; GFX7-NEXT: s_setpc_b64 s[30:31]
4084 ;
4085 ; GFX8-LABEL: flat_atomic_max_i64_ret_scalar:
4086 ; GFX8: ; %bb.0:
4087 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4088 ; GFX8-NEXT: v_mov_b32_e32 v0, s4
4089 ; GFX8-NEXT: s_add_u32 s34, s4, 4
4090 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
4091 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
4092 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
4093 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
4094 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
4095 ; GFX8-NEXT: flat_load_dword v1, v[2:3]
4096 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
4097 ; GFX8-NEXT: s_mov_b64 s[34:35], 0
4098 ; GFX8-NEXT: v_mov_b32_e32 v4, s7
4099 ; GFX8-NEXT: v_mov_b32_e32 v5, s6
4100 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
4101 ; GFX8-NEXT: .LBB86_1: ; %atomicrmw.start
4102 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4103 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4104 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
4105 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
4106 ; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
4107 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
4108 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
4109 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
4110 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4111 ; GFX8-NEXT: buffer_wbinvl1_vol
4112 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
4113 ; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
4114 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
4115 ; GFX8-NEXT: s_cbranch_execnz .LBB86_1
4116 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4117 ; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
4118 ; GFX8-NEXT: s_setpc_b64 s[30:31]
4119 ;
4120 ; GFX9-LABEL: flat_atomic_max_i64_ret_scalar:
4121 ; GFX9: ; %bb.0:
4122 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4123 ; GFX9-NEXT: v_mov_b32_e32 v0, s4
4124 ; GFX9-NEXT: v_mov_b32_e32 v1, s5
4125 ; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
4126 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
4127 ; GFX9-NEXT: s_mov_b64 s[34:35], 0
4128 ; GFX9-NEXT: v_mov_b32_e32 v4, s7
4129 ; GFX9-NEXT: v_mov_b32_e32 v5, s6
4130 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
4131 ; GFX9-NEXT: .LBB86_1: ; %atomicrmw.start
4132 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
4133 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4134 ; GFX9-NEXT: v_mov_b32_e32 v9, v1
4135 ; GFX9-NEXT: v_mov_b32_e32 v8, v0
4136 ; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
4137 ; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
4138 ; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
4139 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
4140 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4141 ; GFX9-NEXT: buffer_wbinvl1_vol
4142 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
4143 ; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
4144 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
4145 ; GFX9-NEXT: s_cbranch_execnz .LBB86_1
4146 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
4147 ; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
4148 ; GFX9-NEXT: s_setpc_b64 s[30:31]
4149 %result = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
4150 ret i64 %result
4151 }
4153 define amdgpu_gfx i64 @flat_atomic_max_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
4154 ; GFX7-LABEL: flat_atomic_max_i64_ret_offset_scalar:
4155 ; GFX7: ; %bb.0:
4156 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4157 ; GFX7-NEXT: s_add_u32 s34, s4, 32
4158 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
4159 ; GFX7-NEXT: s_add_u32 s36, s4, 36
4160 ; GFX7-NEXT: s_addc_u32 s37, s5, 0
4161 ; GFX7-NEXT: v_mov_b32_e32 v0, s36
4162 ; GFX7-NEXT: v_mov_b32_e32 v1, s37
4163 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
4164 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
4165 ; GFX7-NEXT: flat_load_dword v1, v[0:1]
4166 ; GFX7-NEXT: flat_load_dword v0, v[2:3]
4167 ; GFX7-NEXT: s_mov_b64 s[34:35], 0
4168 ; GFX7-NEXT: v_mov_b32_e32 v4, s7
4169 ; GFX7-NEXT: v_mov_b32_e32 v5, s6
4170 ; GFX7-NEXT: .LBB87_1: ; %atomicrmw.start
4171 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4172 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4173 ; GFX7-NEXT: v_mov_b32_e32 v9, v1
4174 ; GFX7-NEXT: v_mov_b32_e32 v8, v0
4175 ; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
4176 ; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
4177 ; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
4178 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
4179 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4180 ; GFX7-NEXT: buffer_wbinvl1_vol
4181 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
4182 ; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
4183 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
4184 ; GFX7-NEXT: s_cbranch_execnz .LBB87_1
4185 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4186 ; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
4187 ; GFX7-NEXT: s_setpc_b64 s[30:31]
4188 ;
4189 ; GFX8-LABEL: flat_atomic_max_i64_ret_offset_scalar:
4190 ; GFX8: ; %bb.0:
4191 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4192 ; GFX8-NEXT: s_add_u32 s34, s4, 32
4193 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
4194 ; GFX8-NEXT: s_add_u32 s36, s4, 36
4195 ; GFX8-NEXT: s_addc_u32 s37, s5, 0
4196 ; GFX8-NEXT: v_mov_b32_e32 v0, s36
4197 ; GFX8-NEXT: v_mov_b32_e32 v1, s37
4198 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
4199 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
4200 ; GFX8-NEXT: flat_load_dword v1, v[0:1]
4201 ; GFX8-NEXT: flat_load_dword v0, v[2:3]
4202 ; GFX8-NEXT: s_mov_b64 s[34:35], 0
4203 ; GFX8-NEXT: v_mov_b32_e32 v4, s7
4204 ; GFX8-NEXT: v_mov_b32_e32 v5, s6
4205 ; GFX8-NEXT: .LBB87_1: ; %atomicrmw.start
4206 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4207 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4208 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
4209 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
4210 ; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
4211 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
4212 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
4213 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
4214 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4215 ; GFX8-NEXT: buffer_wbinvl1_vol
4216 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
4217 ; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
4218 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
4219 ; GFX8-NEXT: s_cbranch_execnz .LBB87_1
4220 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4221 ; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
4222 ; GFX8-NEXT: s_setpc_b64 s[30:31]
4223 ;
4224 ; GFX9-LABEL: flat_atomic_max_i64_ret_offset_scalar:
4225 ; GFX9: ; %bb.0:
4226 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4227 ; GFX9-NEXT: v_mov_b32_e32 v0, s4
4228 ; GFX9-NEXT: v_mov_b32_e32 v1, s5
4229 ; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
4230 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
4231 ; GFX9-NEXT: s_mov_b64 s[34:35], 0
4232 ; GFX9-NEXT: v_mov_b32_e32 v4, s7
4233 ; GFX9-NEXT: v_mov_b32_e32 v5, s6
4234 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
4235 ; GFX9-NEXT: .LBB87_1: ; %atomicrmw.start
4236 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
4237 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4238 ; GFX9-NEXT: v_mov_b32_e32 v9, v1
4239 ; GFX9-NEXT: v_mov_b32_e32 v8, v0
4240 ; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
4241 ; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
4242 ; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
4243 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc
4244 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4245 ; GFX9-NEXT: buffer_wbinvl1_vol
4246 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
4247 ; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
4248 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
4249 ; GFX9-NEXT: s_cbranch_execnz .LBB87_1
4250 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
4251 ; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
4252 ; GFX9-NEXT: s_setpc_b64 s[30:31]
4253 %gep = getelementptr i64, ptr %out, i64 4
4254 %result = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
4255 ret i64 %result
4256 }
4258 define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
4259 ; GFX7-LABEL: atomic_max_i64_addr64_offset:
4260 ; GFX7: ; %bb.0: ; %entry
4261 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
4262 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
4263 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4264 ; GFX7-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
4265 ; GFX7-NEXT: s_add_u32 s0, s0, s4
4266 ; GFX7-NEXT: s_addc_u32 s1, s1, s5
4267 ; GFX7-NEXT: s_add_u32 s0, s0, 32
4268 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
4269 ; GFX7-NEXT: v_mov_b32_e32 v5, s1
4270 ; GFX7-NEXT: v_mov_b32_e32 v4, s0
4271 ; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
4272 ; GFX7-NEXT: s_mov_b64 s[0:1], 0
4273 ; GFX7-NEXT: v_mov_b32_e32 v6, s3
4274 ; GFX7-NEXT: v_mov_b32_e32 v7, s2
4275 ; GFX7-NEXT: .LBB88_1: ; %atomicrmw.start
4276 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4277 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4278 ; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
4279 ; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
4280 ; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
4281 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
4282 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4283 ; GFX7-NEXT: buffer_wbinvl1_vol
4284 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
4285 ; GFX7-NEXT: v_mov_b32_e32 v3, v1
4286 ; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4287 ; GFX7-NEXT: v_mov_b32_e32 v2, v0
4288 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
4289 ; GFX7-NEXT: s_cbranch_execnz .LBB88_1
4290 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4291 ; GFX7-NEXT: s_endpgm
4292 ;
4293 ; GFX8-LABEL: atomic_max_i64_addr64_offset:
4294 ; GFX8: ; %bb.0: ; %entry
4295 ; GFX8-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
4296 ; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
4297 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
4298 ; GFX8-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
4299 ; GFX8-NEXT: s_add_u32 s0, s0, s4
4300 ; GFX8-NEXT: s_addc_u32 s1, s1, s5
4301 ; GFX8-NEXT: s_add_u32 s0, s0, 32
4302 ; GFX8-NEXT: s_addc_u32 s1, s1, 0
4303 ; GFX8-NEXT: v_mov_b32_e32 v5, s1
4304 ; GFX8-NEXT: v_mov_b32_e32 v4, s0
4305 ; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
4306 ; GFX8-NEXT: s_mov_b64 s[0:1], 0
4307 ; GFX8-NEXT: v_mov_b32_e32 v6, s3
4308 ; GFX8-NEXT: v_mov_b32_e32 v7, s2
4309 ; GFX8-NEXT: .LBB88_1: ; %atomicrmw.start
4310 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4311 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4312 ; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
4313 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
4314 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
4315 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
4316 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4317 ; GFX8-NEXT: buffer_wbinvl1_vol
4318 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
4319 ; GFX8-NEXT: v_mov_b32_e32 v3, v1
4320 ; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4321 ; GFX8-NEXT: v_mov_b32_e32 v2, v0
4322 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
4323 ; GFX8-NEXT: s_cbranch_execnz .LBB88_1
4324 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4325 ; GFX8-NEXT: s_endpgm
4326 ;
4327 ; GFX9-LABEL: atomic_max_i64_addr64_offset:
4328 ; GFX9: ; %bb.0: ; %entry
4329 ; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
4330 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
4331 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
4332 ; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
4333 ; GFX9-NEXT: s_add_u32 s0, s0, s4
4334 ; GFX9-NEXT: s_addc_u32 s1, s1, s5
4335 ; GFX9-NEXT: v_mov_b32_e32 v5, s1
4336 ; GFX9-NEXT: v_mov_b32_e32 v4, s0
4337 ; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[4:5] offset:32
4338 ; GFX9-NEXT: s_mov_b64 s[0:1], 0
4339 ; GFX9-NEXT: v_mov_b32_e32 v6, s3
4340 ; GFX9-NEXT: v_mov_b32_e32 v7, s2
4341 ; GFX9-NEXT: .LBB88_1: ; %atomicrmw.start
4342 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
4343 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4344 ; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
4345 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
4346 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
4347 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
4348 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4349 ; GFX9-NEXT: buffer_wbinvl1_vol
4350 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
4351 ; GFX9-NEXT: v_mov_b32_e32 v3, v1
4352 ; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4353 ; GFX9-NEXT: v_mov_b32_e32 v2, v0
4354 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
4355 ; GFX9-NEXT: s_cbranch_execnz .LBB88_1
4356 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
4357 ; GFX9-NEXT: s_endpgm
4358 entry:
4359 %ptr = getelementptr i64, ptr %out, i64 %index
4360 %gep = getelementptr i64, ptr %ptr, i64 4
4361 %tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
4362 ret void
4363 }
4365 define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
4366 ; GFX7-LABEL: atomic_max_i64_ret_addr64_offset:
4367 ; GFX7: ; %bb.0: ; %entry
4368 ; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
4369 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4370 ; GFX7-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
4371 ; GFX7-NEXT: s_add_u32 s0, s0, s6
4372 ; GFX7-NEXT: s_addc_u32 s1, s1, s7
4373 ; GFX7-NEXT: s_add_u32 s0, s0, 32
4374 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
4375 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
4376 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
4377 ; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
4378 ; GFX7-NEXT: s_mov_b64 s[0:1], 0
4379 ; GFX7-NEXT: v_mov_b32_e32 v4, s5
4380 ; GFX7-NEXT: v_mov_b32_e32 v5, s4
4381 ; GFX7-NEXT: .LBB89_1: ; %atomicrmw.start
4382 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4383 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4384 ; GFX7-NEXT: v_mov_b32_e32 v9, v3
4385 ; GFX7-NEXT: v_mov_b32_e32 v8, v2
4386 ; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
4387 ; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
4388 ; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
4389 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
4390 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4391 ; GFX7-NEXT: buffer_wbinvl1_vol
4392 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
4393 ; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4394 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
4395 ; GFX7-NEXT: s_cbranch_execnz .LBB89_1
4396 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4397 ; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
4398 ; GFX7-NEXT: v_mov_b32_e32 v0, s2
4399 ; GFX7-NEXT: v_mov_b32_e32 v1, s3
4400 ; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
4401 ; GFX7-NEXT: s_endpgm
4402 ;
4403 ; GFX8-LABEL: atomic_max_i64_ret_addr64_offset:
4404 ; GFX8: ; %bb.0: ; %entry
4405 ; GFX8-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
4406 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
4407 ; GFX8-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
4408 ; GFX8-NEXT: s_add_u32 s0, s0, s6
4409 ; GFX8-NEXT: s_addc_u32 s1, s1, s7
4410 ; GFX8-NEXT: s_add_u32 s0, s0, 32
4411 ; GFX8-NEXT: s_addc_u32 s1, s1, 0
4412 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
4413 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
4414 ; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
4415 ; GFX8-NEXT: s_mov_b64 s[0:1], 0
4416 ; GFX8-NEXT: v_mov_b32_e32 v4, s5
4417 ; GFX8-NEXT: v_mov_b32_e32 v5, s4
4418 ; GFX8-NEXT: .LBB89_1: ; %atomicrmw.start
4419 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4420 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4421 ; GFX8-NEXT: v_mov_b32_e32 v9, v3
4422 ; GFX8-NEXT: v_mov_b32_e32 v8, v2
4423 ; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
4424 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
4425 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
4426 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
4427 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4428 ; GFX8-NEXT: buffer_wbinvl1_vol
4429 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
4430 ; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4431 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
4432 ; GFX8-NEXT: s_cbranch_execnz .LBB89_1
4433 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4434 ; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
4435 ; GFX8-NEXT: v_mov_b32_e32 v0, s2
4436 ; GFX8-NEXT: v_mov_b32_e32 v1, s3
4437 ; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
4438 ; GFX8-NEXT: s_endpgm
4439 ;
4440 ; GFX9-LABEL: atomic_max_i64_ret_addr64_offset:
4441 ; GFX9: ; %bb.0: ; %entry
4442 ; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
4443 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
4444 ; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
4445 ; GFX9-NEXT: s_add_u32 s0, s8, s0
4446 ; GFX9-NEXT: s_addc_u32 s1, s9, s1
4447 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
4448 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
4449 ; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
4450 ; GFX9-NEXT: s_mov_b64 s[0:1], 0
4451 ; GFX9-NEXT: v_mov_b32_e32 v4, s13
4452 ; GFX9-NEXT: v_mov_b32_e32 v5, s12
4453 ; GFX9-NEXT: .LBB89_1: ; %atomicrmw.start
4454 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
4455 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4456 ; GFX9-NEXT: v_mov_b32_e32 v9, v3
4457 ; GFX9-NEXT: v_mov_b32_e32 v8, v2
4458 ; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9]
4459 ; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
4460 ; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
4461 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] offset:32 glc
4462 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4463 ; GFX9-NEXT: buffer_wbinvl1_vol
4464 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
4465 ; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4466 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
4467 ; GFX9-NEXT: s_cbranch_execnz .LBB89_1
4468 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
4469 ; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
4470 ; GFX9-NEXT: v_mov_b32_e32 v0, s10
4471 ; GFX9-NEXT: v_mov_b32_e32 v1, s11
4472 ; GFX9-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
4473 ; GFX9-NEXT: s_endpgm
4474 entry:
4475 %ptr = getelementptr i64, ptr %out, i64 %index
4476 %gep = getelementptr i64, ptr %ptr, i64 4
4477 %tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
4478 store i64 %tmp0, ptr %out2
4479 ret void
4480 }
4482 define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index) {
4483 ; GFX7-LABEL: atomic_max_i64_addr64:
4484 ; GFX7: ; %bb.0: ; %entry
4485 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
4486 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
4487 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4488 ; GFX7-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
4489 ; GFX7-NEXT: s_add_u32 s0, s0, s4
4490 ; GFX7-NEXT: s_addc_u32 s1, s1, s5
4491 ; GFX7-NEXT: v_mov_b32_e32 v5, s1
4492 ; GFX7-NEXT: v_mov_b32_e32 v4, s0
4493 ; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
4494 ; GFX7-NEXT: s_mov_b64 s[0:1], 0
4495 ; GFX7-NEXT: v_mov_b32_e32 v6, s3
4496 ; GFX7-NEXT: v_mov_b32_e32 v7, s2
4497 ; GFX7-NEXT: .LBB90_1: ; %atomicrmw.start
4498 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4499 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4500 ; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
4501 ; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
4502 ; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
4503 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
4504 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4505 ; GFX7-NEXT: buffer_wbinvl1_vol
4506 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
4507 ; GFX7-NEXT: v_mov_b32_e32 v3, v1
4508 ; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4509 ; GFX7-NEXT: v_mov_b32_e32 v2, v0
4510 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
4511 ; GFX7-NEXT: s_cbranch_execnz .LBB90_1
4512 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4513 ; GFX7-NEXT: s_endpgm
4514 ;
4515 ; GFX8-LABEL: atomic_max_i64_addr64:
4516 ; GFX8: ; %bb.0: ; %entry
4517 ; GFX8-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
4518 ; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
4519 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
4520 ; GFX8-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
4521 ; GFX8-NEXT: s_add_u32 s0, s0, s4
4522 ; GFX8-NEXT: s_addc_u32 s1, s1, s5
4523 ; GFX8-NEXT: v_mov_b32_e32 v5, s1
4524 ; GFX8-NEXT: v_mov_b32_e32 v4, s0
4525 ; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
4526 ; GFX8-NEXT: s_mov_b64 s[0:1], 0
4527 ; GFX8-NEXT: v_mov_b32_e32 v6, s3
4528 ; GFX8-NEXT: v_mov_b32_e32 v7, s2
4529 ; GFX8-NEXT: .LBB90_1: ; %atomicrmw.start
4530 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4531 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4532 ; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
4533 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
4534 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
4535 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
4536 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4537 ; GFX8-NEXT: buffer_wbinvl1_vol
4538 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
4539 ; GFX8-NEXT: v_mov_b32_e32 v3, v1
4540 ; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4541 ; GFX8-NEXT: v_mov_b32_e32 v2, v0
4542 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
4543 ; GFX8-NEXT: s_cbranch_execnz .LBB90_1
4544 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4545 ; GFX8-NEXT: s_endpgm
4546 ;
4547 ; GFX9-LABEL: atomic_max_i64_addr64:
4548 ; GFX9: ; %bb.0: ; %entry
4549 ; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
4550 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
4551 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
4552 ; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
4553 ; GFX9-NEXT: s_add_u32 s0, s0, s4
4554 ; GFX9-NEXT: s_addc_u32 s1, s1, s5
4555 ; GFX9-NEXT: v_mov_b32_e32 v5, s1
4556 ; GFX9-NEXT: v_mov_b32_e32 v4, s0
4557 ; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
4558 ; GFX9-NEXT: s_mov_b64 s[0:1], 0
4559 ; GFX9-NEXT: v_mov_b32_e32 v6, s3
4560 ; GFX9-NEXT: v_mov_b32_e32 v7, s2
4561 ; GFX9-NEXT: .LBB90_1: ; %atomicrmw.start
4562 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
4563 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4564 ; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
4565 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
4566 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
4567 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
4568 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4569 ; GFX9-NEXT: buffer_wbinvl1_vol
4570 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
4571 ; GFX9-NEXT: v_mov_b32_e32 v3, v1
4572 ; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4573 ; GFX9-NEXT: v_mov_b32_e32 v2, v0
4574 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
4575 ; GFX9-NEXT: s_cbranch_execnz .LBB90_1
4576 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
4577 ; GFX9-NEXT: s_endpgm
4578 entry:
4579 %ptr = getelementptr i64, ptr %out, i64 %index
4580 %tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
4581 ret void
4582 }
4584 define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
4585 ; GFX7-LABEL: atomic_max_i64_ret_addr64:
4586 ; GFX7: ; %bb.0: ; %entry
4587 ; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
4588 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4589 ; GFX7-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
4590 ; GFX7-NEXT: s_add_u32 s0, s0, s6
4591 ; GFX7-NEXT: s_addc_u32 s1, s1, s7
4592 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
4593 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
4594 ; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
4595 ; GFX7-NEXT: s_mov_b64 s[0:1], 0
4596 ; GFX7-NEXT: v_mov_b32_e32 v4, s5
4597 ; GFX7-NEXT: v_mov_b32_e32 v5, s4
4598 ; GFX7-NEXT: .LBB91_1: ; %atomicrmw.start
4599 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4600 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4601 ; GFX7-NEXT: v_mov_b32_e32 v9, v3
4602 ; GFX7-NEXT: v_mov_b32_e32 v8, v2
4603 ; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
4604 ; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
4605 ; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
4606 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
4607 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4608 ; GFX7-NEXT: buffer_wbinvl1_vol
4609 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
4610 ; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4611 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
4612 ; GFX7-NEXT: s_cbranch_execnz .LBB91_1
4613 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4614 ; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
4615 ; GFX7-NEXT: v_mov_b32_e32 v0, s2
4616 ; GFX7-NEXT: v_mov_b32_e32 v1, s3
4617 ; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
4618 ; GFX7-NEXT: s_endpgm
4619 ;
4620 ; GFX8-LABEL: atomic_max_i64_ret_addr64:
4621 ; GFX8: ; %bb.0: ; %entry
4622 ; GFX8-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
4623 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
4624 ; GFX8-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
4625 ; GFX8-NEXT: s_add_u32 s0, s0, s6
4626 ; GFX8-NEXT: s_addc_u32 s1, s1, s7
4627 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
4628 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
4629 ; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
4630 ; GFX8-NEXT: s_mov_b64 s[0:1], 0
4631 ; GFX8-NEXT: v_mov_b32_e32 v4, s5
4632 ; GFX8-NEXT: v_mov_b32_e32 v5, s4
4633 ; GFX8-NEXT: .LBB91_1: ; %atomicrmw.start
4634 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4635 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4636 ; GFX8-NEXT: v_mov_b32_e32 v9, v3
4637 ; GFX8-NEXT: v_mov_b32_e32 v8, v2
4638 ; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
4639 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
4640 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
4641 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
4642 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4643 ; GFX8-NEXT: buffer_wbinvl1_vol
4644 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
4645 ; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4646 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
4647 ; GFX8-NEXT: s_cbranch_execnz .LBB91_1
4648 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4649 ; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
4650 ; GFX8-NEXT: v_mov_b32_e32 v0, s2
4651 ; GFX8-NEXT: v_mov_b32_e32 v1, s3
4652 ; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
4653 ; GFX8-NEXT: s_endpgm
4654 ;
4655 ; GFX9-LABEL: atomic_max_i64_ret_addr64:
4656 ; GFX9: ; %bb.0: ; %entry
4657 ; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
4658 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
4659 ; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
4660 ; GFX9-NEXT: s_add_u32 s0, s8, s0
4661 ; GFX9-NEXT: s_addc_u32 s1, s9, s1
4662 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
4663 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
4664 ; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
4665 ; GFX9-NEXT: s_mov_b64 s[0:1], 0
4666 ; GFX9-NEXT: v_mov_b32_e32 v4, s13
4667 ; GFX9-NEXT: v_mov_b32_e32 v5, s12
4668 ; GFX9-NEXT: .LBB91_1: ; %atomicrmw.start
4669 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
4670 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4671 ; GFX9-NEXT: v_mov_b32_e32 v9, v3
4672 ; GFX9-NEXT: v_mov_b32_e32 v8, v2
4673 ; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9]
4674 ; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
4675 ; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
4676 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
4677 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4678 ; GFX9-NEXT: buffer_wbinvl1_vol
4679 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
4680 ; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
4681 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
4682 ; GFX9-NEXT: s_cbranch_execnz .LBB91_1
4683 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
4684 ; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
4685 ; GFX9-NEXT: v_mov_b32_e32 v0, s10
4686 ; GFX9-NEXT: v_mov_b32_e32 v1, s11
4687 ; GFX9-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
4688 ; GFX9-NEXT: s_endpgm
4689 entry:
4690 %ptr = getelementptr i64, ptr %out, i64 %index
4691 %tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
4692 store i64 %tmp0, ptr %out2
4693 ret void
4694 }
4696 define void @flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
4697 ; GFX7-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory:
4698 ; GFX7: ; %bb.0:
4699 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4700 ; GFX7-NEXT: v_add_i32_e32 v8, vcc, 32, v0
4701 ; GFX7-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
4702 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v0
4703 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
4704 ; GFX7-NEXT: flat_load_dword v7, v[0:1]
4705 ; GFX7-NEXT: flat_load_dword v6, v[8:9]
4706 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
4707 ; GFX7-NEXT: .LBB92_1: ; %atomicrmw.start
4708 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4709 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4710 ; GFX7-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
4711 ; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
4712 ; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
4713 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
4714 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4715 ; GFX7-NEXT: buffer_wbinvl1_vol
4716 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
4717 ; GFX7-NEXT: v_mov_b32_e32 v7, v1
4718 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4719 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
4720 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
4721 ; GFX7-NEXT: s_cbranch_execnz .LBB92_1
4722 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4723 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
4724 ; GFX7-NEXT: s_setpc_b64 s[30:31]
4725 ;
4726 ; GFX8-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory:
4727 ; GFX8: ; %bb.0:
4728 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4729 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 32, v0
4730 ; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
4731 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v0
4732 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
4733 ; GFX8-NEXT: flat_load_dword v7, v[0:1]
4734 ; GFX8-NEXT: flat_load_dword v6, v[8:9]
4735 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
4736 ; GFX8-NEXT: .LBB92_1: ; %atomicrmw.start
4737 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4738 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4739 ; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
4740 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
4741 ; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
4742 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
4743 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4744 ; GFX8-NEXT: buffer_wbinvl1_vol
4745 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
4746 ; GFX8-NEXT: v_mov_b32_e32 v7, v1
4747 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4748 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
4749 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
4750 ; GFX8-NEXT: s_cbranch_execnz .LBB92_1
4751 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4752 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
4753 ; GFX8-NEXT: s_setpc_b64 s[30:31]
4754 ;
4755 ; GFX9-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory:
4756 ; GFX9: ; %bb.0:
4757 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4758 ; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
4759 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
4760 ; GFX9-NEXT: .LBB92_1: ; %atomicrmw.start
4761 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
4762 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4763 ; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
4764 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
4765 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
4766 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
4767 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4768 ; GFX9-NEXT: buffer_wbinvl1_vol
4769 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4770 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
4771 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4772 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
4773 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
4774 ; GFX9-NEXT: s_cbranch_execnz .LBB92_1
4775 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
4776 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
4777 ; GFX9-NEXT: s_setpc_b64 s[30:31]
4778 %gep = getelementptr i64, ptr %out, i64 4
4779 %tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
4780 ret void
4781 }
4783 define i64 @flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
4784 ; GFX7-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:
4785 ; GFX7: ; %bb.0:
4786 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4787 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 32, v0
4788 ; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
4789 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v0
4790 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
4791 ; GFX7-NEXT: flat_load_dword v1, v[0:1]
4792 ; GFX7-NEXT: flat_load_dword v0, v[4:5]
4793 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
4794 ; GFX7-NEXT: .LBB93_1: ; %atomicrmw.start
4795 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4796 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4797 ; GFX7-NEXT: v_mov_b32_e32 v9, v1
4798 ; GFX7-NEXT: v_mov_b32_e32 v8, v0
4799 ; GFX7-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]
4800 ; GFX7-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
4801 ; GFX7-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
4802 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
4803 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4804 ; GFX7-NEXT: buffer_wbinvl1_vol
4805 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
4806 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4807 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
4808 ; GFX7-NEXT: s_cbranch_execnz .LBB93_1
4809 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4810 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
4811 ; GFX7-NEXT: s_setpc_b64 s[30:31]
4812 ;
4813 ; GFX8-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:
4814 ; GFX8: ; %bb.0:
4815 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4816 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 32, v0
4817 ; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
4818 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v0
4819 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
4820 ; GFX8-NEXT: flat_load_dword v1, v[0:1]
4821 ; GFX8-NEXT: flat_load_dword v0, v[4:5]
4822 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
4823 ; GFX8-NEXT: .LBB93_1: ; %atomicrmw.start
4824 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4825 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4826 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
4827 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
4828 ; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]
4829 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
4830 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
4831 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
4832 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4833 ; GFX8-NEXT: buffer_wbinvl1_vol
4834 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
4835 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4836 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
4837 ; GFX8-NEXT: s_cbranch_execnz .LBB93_1
4838 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4839 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
4840 ; GFX8-NEXT: s_setpc_b64 s[30:31]
4841 ;
4842 ; GFX9-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:
4843 ; GFX9: ; %bb.0:
4844 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4845 ; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
4846 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
4847 ; GFX9-NEXT: .LBB93_1: ; %atomicrmw.start
4848 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
4849 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4850 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
4851 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
4852 ; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
4853 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
4854 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
4855 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
4856 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4857 ; GFX9-NEXT: buffer_wbinvl1_vol
4858 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4859 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4860 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
4861 ; GFX9-NEXT: s_cbranch_execnz .LBB93_1
4862 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
4863 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
4864 ; GFX9-NEXT: v_mov_b32_e32 v0, v4
4865 ; GFX9-NEXT: v_mov_b32_e32 v1, v5
4866 ; GFX9-NEXT: s_setpc_b64 s[30:31]
4867 %gep = getelementptr i64, ptr %out, i64 4
4868 %result = atomicrmw max ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
4869 ret i64 %result
4870 }
4872 ; ---------------------------------------------------------------------
4873 ; atomicrmw umax
4874 ; ---------------------------------------------------------------------
4876 define void @flat_atomic_umax_i64_noret(ptr %ptr, i64 %in) {
4877 ; GFX7-LABEL: flat_atomic_umax_i64_noret:
4878 ; GFX7: ; %bb.0:
4879 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4880 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 4, v0
4881 ; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
4882 ; GFX7-NEXT: flat_load_dword v6, v[0:1]
4883 ; GFX7-NEXT: flat_load_dword v7, v[4:5]
4884 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
4885 ; GFX7-NEXT: .LBB94_1: ; %atomicrmw.start
4886 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4887 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4888 ; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
4889 ; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
4890 ; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
4891 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
4892 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4893 ; GFX7-NEXT: buffer_wbinvl1_vol
4894 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4895 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
4896 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4897 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
4898 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
4899 ; GFX7-NEXT: s_cbranch_execnz .LBB94_1
4900 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4901 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
4902 ; GFX7-NEXT: s_setpc_b64 s[30:31]
4903 ;
4904 ; GFX8-LABEL: flat_atomic_umax_i64_noret:
4905 ; GFX8: ; %bb.0:
4906 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4907 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v0
4908 ; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
4909 ; GFX8-NEXT: flat_load_dword v6, v[0:1]
4910 ; GFX8-NEXT: flat_load_dword v7, v[4:5]
4911 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
4912 ; GFX8-NEXT: .LBB94_1: ; %atomicrmw.start
4913 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
4914 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4915 ; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
4916 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
4917 ; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
4918 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
4919 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4920 ; GFX8-NEXT: buffer_wbinvl1_vol
4921 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4922 ; GFX8-NEXT: v_mov_b32_e32 v7, v5
4923 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4924 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
4925 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
4926 ; GFX8-NEXT: s_cbranch_execnz .LBB94_1
4927 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
4928 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
4929 ; GFX8-NEXT: s_setpc_b64 s[30:31]
4930 ;
4931 ; GFX9-LABEL: flat_atomic_umax_i64_noret:
4932 ; GFX9: ; %bb.0:
4933 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4934 ; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
4935 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
4936 ; GFX9-NEXT: .LBB94_1: ; %atomicrmw.start
4937 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
4938 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4939 ; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
4940 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
4941 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
4942 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
4943 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4944 ; GFX9-NEXT: buffer_wbinvl1_vol
4945 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
4946 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
4947 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4948 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
4949 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
4950 ; GFX9-NEXT: s_cbranch_execnz .LBB94_1
4951 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
4952 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
4953 ; GFX9-NEXT: s_setpc_b64 s[30:31]
4954 %tmp0 = atomicrmw umax ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
4955 ret void
4956 }
4958 define void @flat_atomic_umax_i64_noret_offset(ptr %out, i64 %in) {
4959 ; GFX7-LABEL: flat_atomic_umax_i64_noret_offset:
4960 ; GFX7: ; %bb.0:
4961 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4962 ; GFX7-NEXT: v_add_i32_e32 v8, vcc, 32, v0
4963 ; GFX7-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
4964 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v0
4965 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
4966 ; GFX7-NEXT: flat_load_dword v7, v[0:1]
4967 ; GFX7-NEXT: flat_load_dword v6, v[8:9]
4968 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
4969 ; GFX7-NEXT: .LBB95_1: ; %atomicrmw.start
4970 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
4971 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4972 ; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
4973 ; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
4974 ; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
4975 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
4976 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4977 ; GFX7-NEXT: buffer_wbinvl1_vol
4978 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
4979 ; GFX7-NEXT: v_mov_b32_e32 v7, v1
4980 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
4981 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
4982 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
4983 ; GFX7-NEXT: s_cbranch_execnz .LBB95_1
4984 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
4985 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
4986 ; GFX7-NEXT: s_setpc_b64 s[30:31]
4987 ;
4988 ; GFX8-LABEL: flat_atomic_umax_i64_noret_offset:
4989 ; GFX8: ; %bb.0:
4990 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
4991 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 32, v0
4992 ; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
4993 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v0
4994 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
4995 ; GFX8-NEXT: flat_load_dword v7, v[0:1]
4996 ; GFX8-NEXT: flat_load_dword v6, v[8:9]
4997 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
4998 ; GFX8-NEXT: .LBB95_1: ; %atomicrmw.start
4999 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5000 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5001 ; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
5002 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
5003 ; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
5004 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
5005 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5006 ; GFX8-NEXT: buffer_wbinvl1_vol
5007 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
5008 ; GFX8-NEXT: v_mov_b32_e32 v7, v1
5009 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5010 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
5011 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
5012 ; GFX8-NEXT: s_cbranch_execnz .LBB95_1
5013 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5014 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
5015 ; GFX8-NEXT: s_setpc_b64 s[30:31]
5016 ;
5017 ; GFX9-LABEL: flat_atomic_umax_i64_noret_offset:
5018 ; GFX9: ; %bb.0:
5019 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5020 ; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
5021 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
5022 ; GFX9-NEXT: .LBB95_1: ; %atomicrmw.start
5023 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
5024 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5025 ; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
5026 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
5027 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
5028 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
5029 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5030 ; GFX9-NEXT: buffer_wbinvl1_vol
5031 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
5032 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
5033 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5034 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
5035 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
5036 ; GFX9-NEXT: s_cbranch_execnz .LBB95_1
5037 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
5038 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
5039 ; GFX9-NEXT: s_setpc_b64 s[30:31]
5040 %gep = getelementptr i64, ptr %out, i64 4
5041 %tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
5042 ret void
5043 }
5045 define i64 @flat_atomic_umax_i64_ret(ptr %ptr, i64 %in) {
5046 ; GFX7-LABEL: flat_atomic_umax_i64_ret:
5047 ; GFX7: ; %bb.0:
5048 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5049 ; GFX7-NEXT: v_add_i32_e32 v5, vcc, 4, v0
5050 ; GFX7-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
5051 ; GFX7-NEXT: flat_load_dword v4, v[0:1]
5052 ; GFX7-NEXT: flat_load_dword v5, v[5:6]
5053 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
5054 ; GFX7-NEXT: .LBB96_1: ; %atomicrmw.start
5055 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5056 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5057 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
5058 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
5059 ; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
5060 ; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
5061 ; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
5062 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
5063 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5064 ; GFX7-NEXT: buffer_wbinvl1_vol
5065 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
5066 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5067 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
5068 ; GFX7-NEXT: s_cbranch_execnz .LBB96_1
5069 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5070 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
5071 ; GFX7-NEXT: v_mov_b32_e32 v0, v4
5072 ; GFX7-NEXT: v_mov_b32_e32 v1, v5
5073 ; GFX7-NEXT: s_setpc_b64 s[30:31]
5074 ;
5075 ; GFX8-LABEL: flat_atomic_umax_i64_ret:
5076 ; GFX8: ; %bb.0:
5077 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5078 ; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v0
5079 ; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
5080 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
5081 ; GFX8-NEXT: flat_load_dword v5, v[5:6]
5082 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
5083 ; GFX8-NEXT: .LBB96_1: ; %atomicrmw.start
5084 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5085 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5086 ; GFX8-NEXT: v_mov_b32_e32 v7, v5
5087 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
5088 ; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
5089 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
5090 ; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
5091 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
5092 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5093 ; GFX8-NEXT: buffer_wbinvl1_vol
5094 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
5095 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5096 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
5097 ; GFX8-NEXT: s_cbranch_execnz .LBB96_1
5098 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5099 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
5100 ; GFX8-NEXT: v_mov_b32_e32 v0, v4
5101 ; GFX8-NEXT: v_mov_b32_e32 v1, v5
5102 ; GFX8-NEXT: s_setpc_b64 s[30:31]
5103 ;
5104 ; GFX9-LABEL: flat_atomic_umax_i64_ret:
5105 ; GFX9: ; %bb.0:
5106 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5107 ; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
5108 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
5109 ; GFX9-NEXT: .LBB96_1: ; %atomicrmw.start
5110 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
5111 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5112 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
5113 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
5114 ; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
5115 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
5116 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
5117 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
5118 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5119 ; GFX9-NEXT: buffer_wbinvl1_vol
5120 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
5121 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5122 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
5123 ; GFX9-NEXT: s_cbranch_execnz .LBB96_1
5124 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
5125 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
5126 ; GFX9-NEXT: v_mov_b32_e32 v0, v4
5127 ; GFX9-NEXT: v_mov_b32_e32 v1, v5
5128 ; GFX9-NEXT: s_setpc_b64 s[30:31]
5129 %result = atomicrmw umax ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
5130 ret i64 %result
5131 }
5133 define i64 @flat_atomic_umax_i64_ret_offset(ptr %out, i64 %in) {
5134 ; GFX7-LABEL: flat_atomic_umax_i64_ret_offset:
5135 ; GFX7: ; %bb.0:
5136 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5137 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 32, v0
5138 ; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
5139 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v0
5140 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
5141 ; GFX7-NEXT: flat_load_dword v1, v[0:1]
5142 ; GFX7-NEXT: flat_load_dword v0, v[4:5]
5143 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
5144 ; GFX7-NEXT: .LBB97_1: ; %atomicrmw.start
5145 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5146 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5147 ; GFX7-NEXT: v_mov_b32_e32 v9, v1
5148 ; GFX7-NEXT: v_mov_b32_e32 v8, v0
5149 ; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]
5150 ; GFX7-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
5151 ; GFX7-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
5152 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
5153 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5154 ; GFX7-NEXT: buffer_wbinvl1_vol
5155 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
5156 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5157 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
5158 ; GFX7-NEXT: s_cbranch_execnz .LBB97_1
5159 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5160 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
5161 ; GFX7-NEXT: s_setpc_b64 s[30:31]
5162 ;
5163 ; GFX8-LABEL: flat_atomic_umax_i64_ret_offset:
5164 ; GFX8: ; %bb.0:
5165 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5166 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 32, v0
5167 ; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
5168 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v0
5169 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
5170 ; GFX8-NEXT: flat_load_dword v1, v[0:1]
5171 ; GFX8-NEXT: flat_load_dword v0, v[4:5]
5172 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
5173 ; GFX8-NEXT: .LBB97_1: ; %atomicrmw.start
5174 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5175 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5176 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
5177 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
5178 ; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]
5179 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
5180 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
5181 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
5182 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5183 ; GFX8-NEXT: buffer_wbinvl1_vol
5184 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
5185 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5186 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
5187 ; GFX8-NEXT: s_cbranch_execnz .LBB97_1
5188 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5189 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
5190 ; GFX8-NEXT: s_setpc_b64 s[30:31]
5191 ;
5192 ; GFX9-LABEL: flat_atomic_umax_i64_ret_offset:
5193 ; GFX9: ; %bb.0:
5194 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5195 ; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
5196 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
5197 ; GFX9-NEXT: .LBB97_1: ; %atomicrmw.start
5198 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
5199 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5200 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
5201 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
5202 ; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
5203 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
5204 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
5205 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
5206 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5207 ; GFX9-NEXT: buffer_wbinvl1_vol
5208 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
5209 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5210 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
5211 ; GFX9-NEXT: s_cbranch_execnz .LBB97_1
5212 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
5213 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
5214 ; GFX9-NEXT: v_mov_b32_e32 v0, v4
5215 ; GFX9-NEXT: v_mov_b32_e32 v1, v5
5216 ; GFX9-NEXT: s_setpc_b64 s[30:31]
5217 %gep = getelementptr i64, ptr %out, i64 4
5218 %result = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
5219 ret i64 %result
5220 }
5222 define amdgpu_gfx void @flat_atomic_umax_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
5223 ; GFX7-LABEL: flat_atomic_umax_i64_noret_scalar:
5224 ; GFX7: ; %bb.0:
5225 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5226 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
5227 ; GFX7-NEXT: s_add_u32 s34, s4, 4
5228 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
5229 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
5230 ; GFX7-NEXT: v_mov_b32_e32 v3, s34
5231 ; GFX7-NEXT: v_mov_b32_e32 v4, s35
5232 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
5233 ; GFX7-NEXT: flat_load_dword v3, v[3:4]
5234 ; GFX7-NEXT: v_mov_b32_e32 v4, s4
5235 ; GFX7-NEXT: s_mov_b64 s[34:35], 0
5236 ; GFX7-NEXT: v_mov_b32_e32 v6, s7
5237 ; GFX7-NEXT: v_mov_b32_e32 v7, s6
5238 ; GFX7-NEXT: v_mov_b32_e32 v5, s5
5239 ; GFX7-NEXT: .LBB98_1: ; %atomicrmw.start
5240 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5241 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5242 ; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
5243 ; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
5244 ; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
5245 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
5246 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5247 ; GFX7-NEXT: buffer_wbinvl1_vol
5248 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
5249 ; GFX7-NEXT: v_mov_b32_e32 v3, v1
5250 ; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
5251 ; GFX7-NEXT: v_mov_b32_e32 v2, v0
5252 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
5253 ; GFX7-NEXT: s_cbranch_execnz .LBB98_1
5254 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5255 ; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
5256 ; GFX7-NEXT: s_setpc_b64 s[30:31]
5257 ;
5258 ; GFX8-LABEL: flat_atomic_umax_i64_noret_scalar:
5259 ; GFX8: ; %bb.0:
5260 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5261 ; GFX8-NEXT: v_mov_b32_e32 v0, s4
5262 ; GFX8-NEXT: s_add_u32 s34, s4, 4
5263 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
5264 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
5265 ; GFX8-NEXT: v_mov_b32_e32 v3, s34
5266 ; GFX8-NEXT: v_mov_b32_e32 v4, s35
5267 ; GFX8-NEXT: flat_load_dword v2, v[0:1]
5268 ; GFX8-NEXT: flat_load_dword v3, v[3:4]
5269 ; GFX8-NEXT: v_mov_b32_e32 v4, s4
5270 ; GFX8-NEXT: s_mov_b64 s[34:35], 0
5271 ; GFX8-NEXT: v_mov_b32_e32 v6, s7
5272 ; GFX8-NEXT: v_mov_b32_e32 v7, s6
5273 ; GFX8-NEXT: v_mov_b32_e32 v5, s5
5274 ; GFX8-NEXT: .LBB98_1: ; %atomicrmw.start
5275 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5276 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5277 ; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
5278 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
5279 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
5280 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
5281 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5282 ; GFX8-NEXT: buffer_wbinvl1_vol
5283 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
5284 ; GFX8-NEXT: v_mov_b32_e32 v3, v1
5285 ; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
5286 ; GFX8-NEXT: v_mov_b32_e32 v2, v0
5287 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
5288 ; GFX8-NEXT: s_cbranch_execnz .LBB98_1
5289 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5290 ; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
5291 ; GFX8-NEXT: s_setpc_b64 s[30:31]
5292 ;
5293 ; GFX9-LABEL: flat_atomic_umax_i64_noret_scalar:
5294 ; GFX9: ; %bb.0:
5295 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5296 ; GFX9-NEXT: v_mov_b32_e32 v0, s4
5297 ; GFX9-NEXT: v_mov_b32_e32 v1, s5
5298 ; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
5299 ; GFX9-NEXT: v_mov_b32_e32 v4, s4
5300 ; GFX9-NEXT: s_mov_b64 s[34:35], 0
5301 ; GFX9-NEXT: v_mov_b32_e32 v6, s7
5302 ; GFX9-NEXT: v_mov_b32_e32 v7, s6
5303 ; GFX9-NEXT: v_mov_b32_e32 v5, s5
5304 ; GFX9-NEXT: .LBB98_1: ; %atomicrmw.start
5305 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
5306 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5307 ; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
5308 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
5309 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
5310 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
5311 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5312 ; GFX9-NEXT: buffer_wbinvl1_vol
5313 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
5314 ; GFX9-NEXT: v_mov_b32_e32 v3, v1
5315 ; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
5316 ; GFX9-NEXT: v_mov_b32_e32 v2, v0
5317 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
5318 ; GFX9-NEXT: s_cbranch_execnz .LBB98_1
5319 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
5320 ; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
5321 ; GFX9-NEXT: s_setpc_b64 s[30:31]
5322 %tmp0 = atomicrmw umax ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
5323 ret void
5324 }
5326 define amdgpu_gfx void @flat_atomic_umax_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
5327 ; GFX7-LABEL: flat_atomic_umax_i64_noret_offset_scalar:
5328 ; GFX7: ; %bb.0:
5329 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5330 ; GFX7-NEXT: s_add_u32 s34, s4, 32
5331 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
5332 ; GFX7-NEXT: s_add_u32 s36, s4, 36
5333 ; GFX7-NEXT: s_addc_u32 s37, s5, 0
5334 ; GFX7-NEXT: v_mov_b32_e32 v0, s36
5335 ; GFX7-NEXT: v_mov_b32_e32 v1, s37
5336 ; GFX7-NEXT: v_mov_b32_e32 v4, s34
5337 ; GFX7-NEXT: v_mov_b32_e32 v5, s35
5338 ; GFX7-NEXT: flat_load_dword v3, v[0:1]
5339 ; GFX7-NEXT: flat_load_dword v2, v[4:5]
5340 ; GFX7-NEXT: s_mov_b64 s[34:35], 0
5341 ; GFX7-NEXT: v_mov_b32_e32 v6, s7
5342 ; GFX7-NEXT: v_mov_b32_e32 v7, s6
5343 ; GFX7-NEXT: .LBB99_1: ; %atomicrmw.start
5344 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5345 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5346 ; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
5347 ; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
5348 ; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
5349 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
5350 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5351 ; GFX7-NEXT: buffer_wbinvl1_vol
5352 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
5353 ; GFX7-NEXT: v_mov_b32_e32 v3, v1
5354 ; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
5355 ; GFX7-NEXT: v_mov_b32_e32 v2, v0
5356 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
5357 ; GFX7-NEXT: s_cbranch_execnz .LBB99_1
5358 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5359 ; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
5360 ; GFX7-NEXT: s_setpc_b64 s[30:31]
5361 ;
5362 ; GFX8-LABEL: flat_atomic_umax_i64_noret_offset_scalar:
5363 ; GFX8: ; %bb.0:
5364 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5365 ; GFX8-NEXT: s_add_u32 s34, s4, 32
5366 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
5367 ; GFX8-NEXT: s_add_u32 s36, s4, 36
5368 ; GFX8-NEXT: s_addc_u32 s37, s5, 0
5369 ; GFX8-NEXT: v_mov_b32_e32 v0, s36
5370 ; GFX8-NEXT: v_mov_b32_e32 v1, s37
5371 ; GFX8-NEXT: v_mov_b32_e32 v4, s34
5372 ; GFX8-NEXT: v_mov_b32_e32 v5, s35
5373 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
5374 ; GFX8-NEXT: flat_load_dword v2, v[4:5]
5375 ; GFX8-NEXT: s_mov_b64 s[34:35], 0
5376 ; GFX8-NEXT: v_mov_b32_e32 v6, s7
5377 ; GFX8-NEXT: v_mov_b32_e32 v7, s6
5378 ; GFX8-NEXT: .LBB99_1: ; %atomicrmw.start
5379 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5380 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5381 ; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
5382 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
5383 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
5384 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
5385 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5386 ; GFX8-NEXT: buffer_wbinvl1_vol
5387 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
5388 ; GFX8-NEXT: v_mov_b32_e32 v3, v1
5389 ; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
5390 ; GFX8-NEXT: v_mov_b32_e32 v2, v0
5391 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
5392 ; GFX8-NEXT: s_cbranch_execnz .LBB99_1
5393 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5394 ; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
5395 ; GFX8-NEXT: s_setpc_b64 s[30:31]
5396 ;
5397 ; GFX9-LABEL: flat_atomic_umax_i64_noret_offset_scalar:
5398 ; GFX9: ; %bb.0:
5399 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5400 ; GFX9-NEXT: v_mov_b32_e32 v0, s4
5401 ; GFX9-NEXT: v_mov_b32_e32 v1, s5
5402 ; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
5403 ; GFX9-NEXT: v_mov_b32_e32 v4, s4
5404 ; GFX9-NEXT: s_mov_b64 s[34:35], 0
5405 ; GFX9-NEXT: v_mov_b32_e32 v6, s7
5406 ; GFX9-NEXT: v_mov_b32_e32 v7, s6
5407 ; GFX9-NEXT: v_mov_b32_e32 v5, s5
5408 ; GFX9-NEXT: .LBB99_1: ; %atomicrmw.start
5409 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
5410 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5411 ; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
5412 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
5413 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
5414 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
5415 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5416 ; GFX9-NEXT: buffer_wbinvl1_vol
5417 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
5418 ; GFX9-NEXT: v_mov_b32_e32 v3, v1
5419 ; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
5420 ; GFX9-NEXT: v_mov_b32_e32 v2, v0
5421 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
5422 ; GFX9-NEXT: s_cbranch_execnz .LBB99_1
5423 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
5424 ; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
5425 ; GFX9-NEXT: s_setpc_b64 s[30:31]
5426 %gep = getelementptr i64, ptr %out, i64 4
5427 %tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
5428 ret void
5429 }
5431 define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
5432 ; GFX7-LABEL: flat_atomic_umax_i64_ret_scalar:
5433 ; GFX7: ; %bb.0:
5434 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5435 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
5436 ; GFX7-NEXT: s_add_u32 s34, s4, 4
5437 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
5438 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
5439 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
5440 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
5441 ; GFX7-NEXT: flat_load_dword v0, v[0:1]
5442 ; GFX7-NEXT: flat_load_dword v1, v[2:3]
5443 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
5444 ; GFX7-NEXT: s_mov_b64 s[34:35], 0
5445 ; GFX7-NEXT: v_mov_b32_e32 v4, s7
5446 ; GFX7-NEXT: v_mov_b32_e32 v5, s6
5447 ; GFX7-NEXT: v_mov_b32_e32 v3, s5
5448 ; GFX7-NEXT: .LBB100_1: ; %atomicrmw.start
5449 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5450 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5451 ; GFX7-NEXT: v_mov_b32_e32 v9, v1
5452 ; GFX7-NEXT: v_mov_b32_e32 v8, v0
5453 ; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
5454 ; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
5455 ; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
5456 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
5457 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5458 ; GFX7-NEXT: buffer_wbinvl1_vol
5459 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
5460 ; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
5461 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
5462 ; GFX7-NEXT: s_cbranch_execnz .LBB100_1
5463 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5464 ; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
5465 ; GFX7-NEXT: s_setpc_b64 s[30:31]
5466 ;
5467 ; GFX8-LABEL: flat_atomic_umax_i64_ret_scalar:
5468 ; GFX8: ; %bb.0:
5469 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5470 ; GFX8-NEXT: v_mov_b32_e32 v0, s4
5471 ; GFX8-NEXT: s_add_u32 s34, s4, 4
5472 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
5473 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
5474 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
5475 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
5476 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
5477 ; GFX8-NEXT: flat_load_dword v1, v[2:3]
5478 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
5479 ; GFX8-NEXT: s_mov_b64 s[34:35], 0
5480 ; GFX8-NEXT: v_mov_b32_e32 v4, s7
5481 ; GFX8-NEXT: v_mov_b32_e32 v5, s6
5482 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
5483 ; GFX8-NEXT: .LBB100_1: ; %atomicrmw.start
5484 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5485 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5486 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
5487 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
5488 ; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
5489 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
5490 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
5491 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
5492 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5493 ; GFX8-NEXT: buffer_wbinvl1_vol
5494 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
5495 ; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
5496 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
5497 ; GFX8-NEXT: s_cbranch_execnz .LBB100_1
5498 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5499 ; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
5500 ; GFX8-NEXT: s_setpc_b64 s[30:31]
5501 ;
5502 ; GFX9-LABEL: flat_atomic_umax_i64_ret_scalar:
5503 ; GFX9: ; %bb.0:
5504 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5505 ; GFX9-NEXT: v_mov_b32_e32 v0, s4
5506 ; GFX9-NEXT: v_mov_b32_e32 v1, s5
5507 ; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
5508 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
5509 ; GFX9-NEXT: s_mov_b64 s[34:35], 0
5510 ; GFX9-NEXT: v_mov_b32_e32 v4, s7
5511 ; GFX9-NEXT: v_mov_b32_e32 v5, s6
5512 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
5513 ; GFX9-NEXT: .LBB100_1: ; %atomicrmw.start
5514 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
5515 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5516 ; GFX9-NEXT: v_mov_b32_e32 v9, v1
5517 ; GFX9-NEXT: v_mov_b32_e32 v8, v0
5518 ; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
5519 ; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
5520 ; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
5521 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
5522 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5523 ; GFX9-NEXT: buffer_wbinvl1_vol
5524 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
5525 ; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
5526 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
5527 ; GFX9-NEXT: s_cbranch_execnz .LBB100_1
5528 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
5529 ; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
5530 ; GFX9-NEXT: s_setpc_b64 s[30:31]
5531 %result = atomicrmw umax ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
5532 ret i64 %result
5533 }
5535 define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
5536 ; GFX7-LABEL: flat_atomic_umax_i64_ret_offset_scalar:
5537 ; GFX7: ; %bb.0:
5538 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5539 ; GFX7-NEXT: s_add_u32 s34, s4, 32
5540 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
5541 ; GFX7-NEXT: s_add_u32 s36, s4, 36
5542 ; GFX7-NEXT: s_addc_u32 s37, s5, 0
5543 ; GFX7-NEXT: v_mov_b32_e32 v0, s36
5544 ; GFX7-NEXT: v_mov_b32_e32 v1, s37
5545 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
5546 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
5547 ; GFX7-NEXT: flat_load_dword v1, v[0:1]
5548 ; GFX7-NEXT: flat_load_dword v0, v[2:3]
5549 ; GFX7-NEXT: s_mov_b64 s[34:35], 0
5550 ; GFX7-NEXT: v_mov_b32_e32 v4, s7
5551 ; GFX7-NEXT: v_mov_b32_e32 v5, s6
5552 ; GFX7-NEXT: .LBB101_1: ; %atomicrmw.start
5553 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5554 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5555 ; GFX7-NEXT: v_mov_b32_e32 v9, v1
5556 ; GFX7-NEXT: v_mov_b32_e32 v8, v0
5557 ; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
5558 ; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
5559 ; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
5560 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
5561 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5562 ; GFX7-NEXT: buffer_wbinvl1_vol
5563 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
5564 ; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
5565 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
5566 ; GFX7-NEXT: s_cbranch_execnz .LBB101_1
5567 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5568 ; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
5569 ; GFX7-NEXT: s_setpc_b64 s[30:31]
5570 ;
5571 ; GFX8-LABEL: flat_atomic_umax_i64_ret_offset_scalar:
5572 ; GFX8: ; %bb.0:
5573 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5574 ; GFX8-NEXT: s_add_u32 s34, s4, 32
5575 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
5576 ; GFX8-NEXT: s_add_u32 s36, s4, 36
5577 ; GFX8-NEXT: s_addc_u32 s37, s5, 0
5578 ; GFX8-NEXT: v_mov_b32_e32 v0, s36
5579 ; GFX8-NEXT: v_mov_b32_e32 v1, s37
5580 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
5581 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
5582 ; GFX8-NEXT: flat_load_dword v1, v[0:1]
5583 ; GFX8-NEXT: flat_load_dword v0, v[2:3]
5584 ; GFX8-NEXT: s_mov_b64 s[34:35], 0
5585 ; GFX8-NEXT: v_mov_b32_e32 v4, s7
5586 ; GFX8-NEXT: v_mov_b32_e32 v5, s6
5587 ; GFX8-NEXT: .LBB101_1: ; %atomicrmw.start
5588 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5589 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5590 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
5591 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
5592 ; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
5593 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
5594 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
5595 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
5596 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5597 ; GFX8-NEXT: buffer_wbinvl1_vol
5598 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
5599 ; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
5600 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
5601 ; GFX8-NEXT: s_cbranch_execnz .LBB101_1
5602 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5603 ; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
5604 ; GFX8-NEXT: s_setpc_b64 s[30:31]
5605 ;
5606 ; GFX9-LABEL: flat_atomic_umax_i64_ret_offset_scalar:
5607 ; GFX9: ; %bb.0:
5608 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5609 ; GFX9-NEXT: v_mov_b32_e32 v0, s4
5610 ; GFX9-NEXT: v_mov_b32_e32 v1, s5
5611 ; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
5612 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
5613 ; GFX9-NEXT: s_mov_b64 s[34:35], 0
5614 ; GFX9-NEXT: v_mov_b32_e32 v4, s7
5615 ; GFX9-NEXT: v_mov_b32_e32 v5, s6
5616 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
5617 ; GFX9-NEXT: .LBB101_1: ; %atomicrmw.start
5618 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
5619 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5620 ; GFX9-NEXT: v_mov_b32_e32 v9, v1
5621 ; GFX9-NEXT: v_mov_b32_e32 v8, v0
5622 ; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
5623 ; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
5624 ; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
5625 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc
5626 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5627 ; GFX9-NEXT: buffer_wbinvl1_vol
5628 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
5629 ; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
5630 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
5631 ; GFX9-NEXT: s_cbranch_execnz .LBB101_1
5632 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
5633 ; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
5634 ; GFX9-NEXT: s_setpc_b64 s[30:31]
5635 %gep = getelementptr i64, ptr %out, i64 4
5636 %result = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
5637 ret i64 %result
5638 }
5640 define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
5641 ; GFX7-LABEL: atomic_umax_i64_addr64_offset:
5642 ; GFX7: ; %bb.0: ; %entry
5643 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
5644 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
5645 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5646 ; GFX7-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
5647 ; GFX7-NEXT: s_add_u32 s0, s0, s4
5648 ; GFX7-NEXT: s_addc_u32 s1, s1, s5
5649 ; GFX7-NEXT: s_add_u32 s0, s0, 32
5650 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
5651 ; GFX7-NEXT: v_mov_b32_e32 v5, s1
5652 ; GFX7-NEXT: v_mov_b32_e32 v4, s0
5653 ; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
5654 ; GFX7-NEXT: s_mov_b64 s[0:1], 0
5655 ; GFX7-NEXT: v_mov_b32_e32 v6, s3
5656 ; GFX7-NEXT: v_mov_b32_e32 v7, s2
5657 ; GFX7-NEXT: .LBB102_1: ; %atomicrmw.start
5658 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5659 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5660 ; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
5661 ; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
5662 ; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
5663 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
5664 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5665 ; GFX7-NEXT: buffer_wbinvl1_vol
5666 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
5667 ; GFX7-NEXT: v_mov_b32_e32 v3, v1
5668 ; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
5669 ; GFX7-NEXT: v_mov_b32_e32 v2, v0
5670 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
5671 ; GFX7-NEXT: s_cbranch_execnz .LBB102_1
5672 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5673 ; GFX7-NEXT: s_endpgm
5674 ;
5675 ; GFX8-LABEL: atomic_umax_i64_addr64_offset:
5676 ; GFX8: ; %bb.0: ; %entry
5677 ; GFX8-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
5678 ; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
5679 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
5680 ; GFX8-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
5681 ; GFX8-NEXT: s_add_u32 s0, s0, s4
5682 ; GFX8-NEXT: s_addc_u32 s1, s1, s5
5683 ; GFX8-NEXT: s_add_u32 s0, s0, 32
5684 ; GFX8-NEXT: s_addc_u32 s1, s1, 0
5685 ; GFX8-NEXT: v_mov_b32_e32 v5, s1
5686 ; GFX8-NEXT: v_mov_b32_e32 v4, s0
5687 ; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
5688 ; GFX8-NEXT: s_mov_b64 s[0:1], 0
5689 ; GFX8-NEXT: v_mov_b32_e32 v6, s3
5690 ; GFX8-NEXT: v_mov_b32_e32 v7, s2
5691 ; GFX8-NEXT: .LBB102_1: ; %atomicrmw.start
5692 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5693 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5694 ; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
5695 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
5696 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
5697 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
5698 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5699 ; GFX8-NEXT: buffer_wbinvl1_vol
5700 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
5701 ; GFX8-NEXT: v_mov_b32_e32 v3, v1
5702 ; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
5703 ; GFX8-NEXT: v_mov_b32_e32 v2, v0
5704 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
5705 ; GFX8-NEXT: s_cbranch_execnz .LBB102_1
5706 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5707 ; GFX8-NEXT: s_endpgm
5708 ;
5709 ; GFX9-LABEL: atomic_umax_i64_addr64_offset:
5710 ; GFX9: ; %bb.0: ; %entry
5711 ; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
5712 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
5713 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
5714 ; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
5715 ; GFX9-NEXT: s_add_u32 s0, s0, s4
5716 ; GFX9-NEXT: s_addc_u32 s1, s1, s5
5717 ; GFX9-NEXT: v_mov_b32_e32 v5, s1
5718 ; GFX9-NEXT: v_mov_b32_e32 v4, s0
5719 ; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[4:5] offset:32
5720 ; GFX9-NEXT: s_mov_b64 s[0:1], 0
5721 ; GFX9-NEXT: v_mov_b32_e32 v6, s3
5722 ; GFX9-NEXT: v_mov_b32_e32 v7, s2
5723 ; GFX9-NEXT: .LBB102_1: ; %atomicrmw.start
5724 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
5725 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5726 ; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
5727 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
5728 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
5729 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
5730 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5731 ; GFX9-NEXT: buffer_wbinvl1_vol
5732 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
5733 ; GFX9-NEXT: v_mov_b32_e32 v3, v1
5734 ; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
5735 ; GFX9-NEXT: v_mov_b32_e32 v2, v0
5736 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
5737 ; GFX9-NEXT: s_cbranch_execnz .LBB102_1
5738 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
5739 ; GFX9-NEXT: s_endpgm
5740 entry:
5741 %ptr = getelementptr i64, ptr %out, i64 %index
5742 %gep = getelementptr i64, ptr %ptr, i64 4
5743 %tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
5744 ret void
5745 }
5747 define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
5748 ; GFX7-LABEL: atomic_umax_i64_ret_addr64_offset:
5749 ; GFX7: ; %bb.0: ; %entry
5750 ; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
5751 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5752 ; GFX7-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
5753 ; GFX7-NEXT: s_add_u32 s0, s0, s6
5754 ; GFX7-NEXT: s_addc_u32 s1, s1, s7
5755 ; GFX7-NEXT: s_add_u32 s0, s0, 32
5756 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
5757 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
5758 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
5759 ; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
5760 ; GFX7-NEXT: s_mov_b64 s[0:1], 0
5761 ; GFX7-NEXT: v_mov_b32_e32 v4, s5
5762 ; GFX7-NEXT: v_mov_b32_e32 v5, s4
5763 ; GFX7-NEXT: .LBB103_1: ; %atomicrmw.start
5764 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5765 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5766 ; GFX7-NEXT: v_mov_b32_e32 v9, v3
5767 ; GFX7-NEXT: v_mov_b32_e32 v8, v2
5768 ; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
5769 ; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
5770 ; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
5771 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
5772 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5773 ; GFX7-NEXT: buffer_wbinvl1_vol
5774 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
5775 ; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
5776 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
5777 ; GFX7-NEXT: s_cbranch_execnz .LBB103_1
5778 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5779 ; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
5780 ; GFX7-NEXT: v_mov_b32_e32 v0, s2
5781 ; GFX7-NEXT: v_mov_b32_e32 v1, s3
5782 ; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
5783 ; GFX7-NEXT: s_endpgm
5784 ;
5785 ; GFX8-LABEL: atomic_umax_i64_ret_addr64_offset:
5786 ; GFX8: ; %bb.0: ; %entry
5787 ; GFX8-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
5788 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
5789 ; GFX8-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
5790 ; GFX8-NEXT: s_add_u32 s0, s0, s6
5791 ; GFX8-NEXT: s_addc_u32 s1, s1, s7
5792 ; GFX8-NEXT: s_add_u32 s0, s0, 32
5793 ; GFX8-NEXT: s_addc_u32 s1, s1, 0
5794 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
5795 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
5796 ; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
5797 ; GFX8-NEXT: s_mov_b64 s[0:1], 0
5798 ; GFX8-NEXT: v_mov_b32_e32 v4, s5
5799 ; GFX8-NEXT: v_mov_b32_e32 v5, s4
5800 ; GFX8-NEXT: .LBB103_1: ; %atomicrmw.start
5801 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5802 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5803 ; GFX8-NEXT: v_mov_b32_e32 v9, v3
5804 ; GFX8-NEXT: v_mov_b32_e32 v8, v2
5805 ; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
5806 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
5807 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
5808 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
5809 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5810 ; GFX8-NEXT: buffer_wbinvl1_vol
5811 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
5812 ; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
5813 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
5814 ; GFX8-NEXT: s_cbranch_execnz .LBB103_1
5815 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5816 ; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
5817 ; GFX8-NEXT: v_mov_b32_e32 v0, s2
5818 ; GFX8-NEXT: v_mov_b32_e32 v1, s3
5819 ; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
5820 ; GFX8-NEXT: s_endpgm
5821 ;
5822 ; GFX9-LABEL: atomic_umax_i64_ret_addr64_offset:
5823 ; GFX9: ; %bb.0: ; %entry
5824 ; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
5825 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
5826 ; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
5827 ; GFX9-NEXT: s_add_u32 s0, s8, s0
5828 ; GFX9-NEXT: s_addc_u32 s1, s9, s1
5829 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
5830 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
5831 ; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
5832 ; GFX9-NEXT: s_mov_b64 s[0:1], 0
5833 ; GFX9-NEXT: v_mov_b32_e32 v4, s13
5834 ; GFX9-NEXT: v_mov_b32_e32 v5, s12
5835 ; GFX9-NEXT: .LBB103_1: ; %atomicrmw.start
5836 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
5837 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5838 ; GFX9-NEXT: v_mov_b32_e32 v9, v3
5839 ; GFX9-NEXT: v_mov_b32_e32 v8, v2
5840 ; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9]
5841 ; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
5842 ; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
5843 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] offset:32 glc
5844 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5845 ; GFX9-NEXT: buffer_wbinvl1_vol
5846 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
5847 ; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
5848 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
5849 ; GFX9-NEXT: s_cbranch_execnz .LBB103_1
5850 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
5851 ; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
5852 ; GFX9-NEXT: v_mov_b32_e32 v0, s10
5853 ; GFX9-NEXT: v_mov_b32_e32 v1, s11
5854 ; GFX9-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
5855 ; GFX9-NEXT: s_endpgm
5856 entry:
5857 %ptr = getelementptr i64, ptr %out, i64 %index
5858 %gep = getelementptr i64, ptr %ptr, i64 4
5859 %tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
5860 store i64 %tmp0, ptr %out2
5861 ret void
5862 }
5864 define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
5865 ; GFX7-LABEL: atomic_umax_i64_ret_addr64:
5866 ; GFX7: ; %bb.0: ; %entry
5867 ; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
5868 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5869 ; GFX7-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
5870 ; GFX7-NEXT: s_add_u32 s0, s0, s6
5871 ; GFX7-NEXT: s_addc_u32 s1, s1, s7
5872 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
5873 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
5874 ; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
5875 ; GFX7-NEXT: s_mov_b64 s[0:1], 0
5876 ; GFX7-NEXT: v_mov_b32_e32 v4, s5
5877 ; GFX7-NEXT: v_mov_b32_e32 v5, s4
5878 ; GFX7-NEXT: .LBB104_1: ; %atomicrmw.start
5879 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5880 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5881 ; GFX7-NEXT: v_mov_b32_e32 v9, v3
5882 ; GFX7-NEXT: v_mov_b32_e32 v8, v2
5883 ; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
5884 ; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
5885 ; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
5886 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
5887 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5888 ; GFX7-NEXT: buffer_wbinvl1_vol
5889 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
5890 ; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
5891 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
5892 ; GFX7-NEXT: s_cbranch_execnz .LBB104_1
5893 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
5894 ; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
5895 ; GFX7-NEXT: v_mov_b32_e32 v0, s2
5896 ; GFX7-NEXT: v_mov_b32_e32 v1, s3
5897 ; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
5898 ; GFX7-NEXT: s_endpgm
5899 ;
5900 ; GFX8-LABEL: atomic_umax_i64_ret_addr64:
5901 ; GFX8: ; %bb.0: ; %entry
5902 ; GFX8-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
5903 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
5904 ; GFX8-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
5905 ; GFX8-NEXT: s_add_u32 s0, s0, s6
5906 ; GFX8-NEXT: s_addc_u32 s1, s1, s7
5907 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
5908 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
5909 ; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
5910 ; GFX8-NEXT: s_mov_b64 s[0:1], 0
5911 ; GFX8-NEXT: v_mov_b32_e32 v4, s5
5912 ; GFX8-NEXT: v_mov_b32_e32 v5, s4
5913 ; GFX8-NEXT: .LBB104_1: ; %atomicrmw.start
5914 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
5915 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5916 ; GFX8-NEXT: v_mov_b32_e32 v9, v3
5917 ; GFX8-NEXT: v_mov_b32_e32 v8, v2
5918 ; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
5919 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
5920 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
5921 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
5922 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5923 ; GFX8-NEXT: buffer_wbinvl1_vol
5924 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
5925 ; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
5926 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
5927 ; GFX8-NEXT: s_cbranch_execnz .LBB104_1
5928 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
5929 ; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
5930 ; GFX8-NEXT: v_mov_b32_e32 v0, s2
5931 ; GFX8-NEXT: v_mov_b32_e32 v1, s3
5932 ; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
5933 ; GFX8-NEXT: s_endpgm
5934 ;
5935 ; GFX9-LABEL: atomic_umax_i64_ret_addr64:
5936 ; GFX9: ; %bb.0: ; %entry
5937 ; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
5938 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
5939 ; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
5940 ; GFX9-NEXT: s_add_u32 s0, s8, s0
5941 ; GFX9-NEXT: s_addc_u32 s1, s9, s1
5942 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
5943 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
5944 ; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
5945 ; GFX9-NEXT: s_mov_b64 s[0:1], 0
5946 ; GFX9-NEXT: v_mov_b32_e32 v4, s13
5947 ; GFX9-NEXT: v_mov_b32_e32 v5, s12
5948 ; GFX9-NEXT: .LBB104_1: ; %atomicrmw.start
5949 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
5950 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5951 ; GFX9-NEXT: v_mov_b32_e32 v9, v3
5952 ; GFX9-NEXT: v_mov_b32_e32 v8, v2
5953 ; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9]
5954 ; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
5955 ; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
5956 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
5957 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5958 ; GFX9-NEXT: buffer_wbinvl1_vol
5959 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
5960 ; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
5961 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
5962 ; GFX9-NEXT: s_cbranch_execnz .LBB104_1
5963 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
5964 ; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
5965 ; GFX9-NEXT: v_mov_b32_e32 v0, s10
5966 ; GFX9-NEXT: v_mov_b32_e32 v1, s11
5967 ; GFX9-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
5968 ; GFX9-NEXT: s_endpgm
5969 entry:
5970 %ptr = getelementptr i64, ptr %out, i64 %index
5971 %tmp0 = atomicrmw umax ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
5972 store i64 %tmp0, ptr %out2
5973 ret void
5974 }
5976 define void @flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
5977 ; GFX7-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory:
5978 ; GFX7: ; %bb.0:
5979 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
5980 ; GFX7-NEXT: v_add_i32_e32 v8, vcc, 32, v0
5981 ; GFX7-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
5982 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v0
5983 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
5984 ; GFX7-NEXT: flat_load_dword v7, v[0:1]
5985 ; GFX7-NEXT: flat_load_dword v6, v[8:9]
5986 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
5987 ; GFX7-NEXT: .LBB105_1: ; %atomicrmw.start
5988 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
5989 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5990 ; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
5991 ; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
5992 ; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
5993 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
5994 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5995 ; GFX7-NEXT: buffer_wbinvl1_vol
5996 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
5997 ; GFX7-NEXT: v_mov_b32_e32 v7, v1
5998 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
5999 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
6000 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
6001 ; GFX7-NEXT: s_cbranch_execnz .LBB105_1
6002 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6003 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
6004 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6005 ;
6006 ; GFX8-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory:
6007 ; GFX8: ; %bb.0:
6008 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6009 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 32, v0
6010 ; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
6011 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v0
6012 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
6013 ; GFX8-NEXT: flat_load_dword v7, v[0:1]
6014 ; GFX8-NEXT: flat_load_dword v6, v[8:9]
6015 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
6016 ; GFX8-NEXT: .LBB105_1: ; %atomicrmw.start
6017 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6018 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6019 ; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
6020 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
6021 ; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
6022 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
6023 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6024 ; GFX8-NEXT: buffer_wbinvl1_vol
6025 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
6026 ; GFX8-NEXT: v_mov_b32_e32 v7, v1
6027 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6028 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
6029 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
6030 ; GFX8-NEXT: s_cbranch_execnz .LBB105_1
6031 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6032 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
6033 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6034 ;
6035 ; GFX9-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory:
6036 ; GFX9: ; %bb.0:
6037 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6038 ; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
6039 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
6040 ; GFX9-NEXT: .LBB105_1: ; %atomicrmw.start
6041 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
6042 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6043 ; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
6044 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
6045 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
6046 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
6047 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6048 ; GFX9-NEXT: buffer_wbinvl1_vol
6049 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
6050 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
6051 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6052 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
6053 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
6054 ; GFX9-NEXT: s_cbranch_execnz .LBB105_1
6055 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
6056 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
6057 ; GFX9-NEXT: s_setpc_b64 s[30:31]
6058 %gep = getelementptr i64, ptr %out, i64 4
6059 %tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
6060 ret void
6061 }
6063 define i64 @flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
6064 ; GFX7-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:
6065 ; GFX7: ; %bb.0:
6066 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6067 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 32, v0
6068 ; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
6069 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v0
6070 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
6071 ; GFX7-NEXT: flat_load_dword v1, v[0:1]
6072 ; GFX7-NEXT: flat_load_dword v0, v[4:5]
6073 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
6074 ; GFX7-NEXT: .LBB106_1: ; %atomicrmw.start
6075 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
6076 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6077 ; GFX7-NEXT: v_mov_b32_e32 v9, v1
6078 ; GFX7-NEXT: v_mov_b32_e32 v8, v0
6079 ; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]
6080 ; GFX7-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
6081 ; GFX7-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
6082 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
6083 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6084 ; GFX7-NEXT: buffer_wbinvl1_vol
6085 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
6086 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6087 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
6088 ; GFX7-NEXT: s_cbranch_execnz .LBB106_1
6089 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6090 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
6091 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6092 ;
6093 ; GFX8-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:
6094 ; GFX8: ; %bb.0:
6095 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6096 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 32, v0
6097 ; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
6098 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v0
6099 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
6100 ; GFX8-NEXT: flat_load_dword v1, v[0:1]
6101 ; GFX8-NEXT: flat_load_dword v0, v[4:5]
6102 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
6103 ; GFX8-NEXT: .LBB106_1: ; %atomicrmw.start
6104 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6105 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6106 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
6107 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
6108 ; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]
6109 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
6110 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
6111 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
6112 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6113 ; GFX8-NEXT: buffer_wbinvl1_vol
6114 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
6115 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6116 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
6117 ; GFX8-NEXT: s_cbranch_execnz .LBB106_1
6118 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6119 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
6120 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6121 ;
6122 ; GFX9-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:
6123 ; GFX9: ; %bb.0:
6124 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6125 ; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
6126 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
6127 ; GFX9-NEXT: .LBB106_1: ; %atomicrmw.start
6128 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
6129 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6130 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
6131 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
6132 ; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
6133 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
6134 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
6135 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
6136 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6137 ; GFX9-NEXT: buffer_wbinvl1_vol
6138 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
6139 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6140 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
6141 ; GFX9-NEXT: s_cbranch_execnz .LBB106_1
6142 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
6143 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
6144 ; GFX9-NEXT: v_mov_b32_e32 v0, v4
6145 ; GFX9-NEXT: v_mov_b32_e32 v1, v5
6146 ; GFX9-NEXT: s_setpc_b64 s[30:31]
6147 %gep = getelementptr i64, ptr %out, i64 4
6148 %result = atomicrmw umax ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
6149 ret i64 %result
6150 }
6152 ; ---------------------------------------------------------------------
6153 ; atomicrmw umin
6154 ; ---------------------------------------------------------------------
6156 define void @flat_atomic_umin_i64_noret(ptr %ptr, i64 %in) {
6157 ; GFX7-LABEL: flat_atomic_umin_i64_noret:
6158 ; GFX7: ; %bb.0:
6159 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6160 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 4, v0
6161 ; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
6162 ; GFX7-NEXT: flat_load_dword v6, v[0:1]
6163 ; GFX7-NEXT: flat_load_dword v7, v[4:5]
6164 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
6165 ; GFX7-NEXT: .LBB107_1: ; %atomicrmw.start
6166 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
6167 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6168 ; GFX7-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
6169 ; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
6170 ; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
6171 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
6172 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6173 ; GFX7-NEXT: buffer_wbinvl1_vol
6174 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
6175 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
6176 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6177 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
6178 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
6179 ; GFX7-NEXT: s_cbranch_execnz .LBB107_1
6180 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6181 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
6182 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6183 ;
6184 ; GFX8-LABEL: flat_atomic_umin_i64_noret:
6185 ; GFX8: ; %bb.0:
6186 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6187 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v0
6188 ; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
6189 ; GFX8-NEXT: flat_load_dword v6, v[0:1]
6190 ; GFX8-NEXT: flat_load_dword v7, v[4:5]
6191 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
6192 ; GFX8-NEXT: .LBB107_1: ; %atomicrmw.start
6193 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6194 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6195 ; GFX8-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
6196 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
6197 ; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
6198 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
6199 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6200 ; GFX8-NEXT: buffer_wbinvl1_vol
6201 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
6202 ; GFX8-NEXT: v_mov_b32_e32 v7, v5
6203 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6204 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
6205 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
6206 ; GFX8-NEXT: s_cbranch_execnz .LBB107_1
6207 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6208 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
6209 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6210 ;
6211 ; GFX9-LABEL: flat_atomic_umin_i64_noret:
6212 ; GFX9: ; %bb.0:
6213 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6214 ; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
6215 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
6216 ; GFX9-NEXT: .LBB107_1: ; %atomicrmw.start
6217 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
6218 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6219 ; GFX9-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
6220 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
6221 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
6222 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
6223 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6224 ; GFX9-NEXT: buffer_wbinvl1_vol
6225 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
6226 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
6227 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6228 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
6229 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
6230 ; GFX9-NEXT: s_cbranch_execnz .LBB107_1
6231 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
6232 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
6233 ; GFX9-NEXT: s_setpc_b64 s[30:31]
6234 %tmp0 = atomicrmw umin ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
6235 ret void
6236 }
6238 define void @flat_atomic_umin_i64_noret_offset(ptr %out, i64 %in) {
6239 ; GFX7-LABEL: flat_atomic_umin_i64_noret_offset:
6240 ; GFX7: ; %bb.0:
6241 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6242 ; GFX7-NEXT: v_add_i32_e32 v8, vcc, 32, v0
6243 ; GFX7-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
6244 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v0
6245 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
6246 ; GFX7-NEXT: flat_load_dword v7, v[0:1]
6247 ; GFX7-NEXT: flat_load_dword v6, v[8:9]
6248 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
6249 ; GFX7-NEXT: .LBB108_1: ; %atomicrmw.start
6250 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
6251 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6252 ; GFX7-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
6253 ; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
6254 ; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
6255 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
6256 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6257 ; GFX7-NEXT: buffer_wbinvl1_vol
6258 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
6259 ; GFX7-NEXT: v_mov_b32_e32 v7, v1
6260 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6261 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
6262 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
6263 ; GFX7-NEXT: s_cbranch_execnz .LBB108_1
6264 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6265 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
6266 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6267 ;
6268 ; GFX8-LABEL: flat_atomic_umin_i64_noret_offset:
6269 ; GFX8: ; %bb.0:
6270 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6271 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 32, v0
6272 ; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
6273 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v0
6274 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
6275 ; GFX8-NEXT: flat_load_dword v7, v[0:1]
6276 ; GFX8-NEXT: flat_load_dword v6, v[8:9]
6277 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
6278 ; GFX8-NEXT: .LBB108_1: ; %atomicrmw.start
6279 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6280 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6281 ; GFX8-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
6282 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
6283 ; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
6284 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
6285 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6286 ; GFX8-NEXT: buffer_wbinvl1_vol
6287 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
6288 ; GFX8-NEXT: v_mov_b32_e32 v7, v1
6289 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6290 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
6291 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
6292 ; GFX8-NEXT: s_cbranch_execnz .LBB108_1
6293 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6294 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
6295 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6296 ;
6297 ; GFX9-LABEL: flat_atomic_umin_i64_noret_offset:
6298 ; GFX9: ; %bb.0:
6299 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6300 ; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
6301 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
6302 ; GFX9-NEXT: .LBB108_1: ; %atomicrmw.start
6303 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
6304 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6305 ; GFX9-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
6306 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
6307 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
6308 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
6309 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6310 ; GFX9-NEXT: buffer_wbinvl1_vol
6311 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
6312 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
6313 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6314 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
6315 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
6316 ; GFX9-NEXT: s_cbranch_execnz .LBB108_1
6317 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
6318 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
6319 ; GFX9-NEXT: s_setpc_b64 s[30:31]
6320 %gep = getelementptr i64, ptr %out, i64 4
6321 %tmp0 = atomicrmw umin ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
6322 ret void
6323 }
6325 define i64 @flat_atomic_umin_i64_ret(ptr %ptr, i64 %in) {
6326 ; GFX7-LABEL: flat_atomic_umin_i64_ret:
6327 ; GFX7: ; %bb.0:
6328 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6329 ; GFX7-NEXT: v_add_i32_e32 v5, vcc, 4, v0
6330 ; GFX7-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
6331 ; GFX7-NEXT: flat_load_dword v4, v[0:1]
6332 ; GFX7-NEXT: flat_load_dword v5, v[5:6]
6333 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
6334 ; GFX7-NEXT: .LBB109_1: ; %atomicrmw.start
6335 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
6336 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6337 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
6338 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
6339 ; GFX7-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
6340 ; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
6341 ; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
6342 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
6343 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6344 ; GFX7-NEXT: buffer_wbinvl1_vol
6345 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
6346 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6347 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
6348 ; GFX7-NEXT: s_cbranch_execnz .LBB109_1
6349 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6350 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
6351 ; GFX7-NEXT: v_mov_b32_e32 v0, v4
6352 ; GFX7-NEXT: v_mov_b32_e32 v1, v5
6353 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6354 ;
6355 ; GFX8-LABEL: flat_atomic_umin_i64_ret:
6356 ; GFX8: ; %bb.0:
6357 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6358 ; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v0
6359 ; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
6360 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
6361 ; GFX8-NEXT: flat_load_dword v5, v[5:6]
6362 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
6363 ; GFX8-NEXT: .LBB109_1: ; %atomicrmw.start
6364 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6365 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6366 ; GFX8-NEXT: v_mov_b32_e32 v7, v5
6367 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
6368 ; GFX8-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
6369 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
6370 ; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
6371 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
6372 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6373 ; GFX8-NEXT: buffer_wbinvl1_vol
6374 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
6375 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6376 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
6377 ; GFX8-NEXT: s_cbranch_execnz .LBB109_1
6378 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6379 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
6380 ; GFX8-NEXT: v_mov_b32_e32 v0, v4
6381 ; GFX8-NEXT: v_mov_b32_e32 v1, v5
6382 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6383 ;
6384 ; GFX9-LABEL: flat_atomic_umin_i64_ret:
6385 ; GFX9: ; %bb.0:
6386 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6387 ; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
6388 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
6389 ; GFX9-NEXT: .LBB109_1: ; %atomicrmw.start
6390 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
6391 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6392 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
6393 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
6394 ; GFX9-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
6395 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
6396 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
6397 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
6398 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6399 ; GFX9-NEXT: buffer_wbinvl1_vol
6400 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
6401 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6402 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
6403 ; GFX9-NEXT: s_cbranch_execnz .LBB109_1
6404 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
6405 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
6406 ; GFX9-NEXT: v_mov_b32_e32 v0, v4
6407 ; GFX9-NEXT: v_mov_b32_e32 v1, v5
6408 ; GFX9-NEXT: s_setpc_b64 s[30:31]
6409 %result = atomicrmw umin ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
6410 ret i64 %result
6411 }
6413 define i64 @flat_atomic_umin_i64_ret_offset(ptr %out, i64 %in) {
6414 ; GFX7-LABEL: flat_atomic_umin_i64_ret_offset:
6415 ; GFX7: ; %bb.0:
6416 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6417 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 32, v0
6418 ; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
6419 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v0
6420 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
6421 ; GFX7-NEXT: flat_load_dword v1, v[0:1]
6422 ; GFX7-NEXT: flat_load_dword v0, v[4:5]
6423 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
6424 ; GFX7-NEXT: .LBB110_1: ; %atomicrmw.start
6425 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
6426 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6427 ; GFX7-NEXT: v_mov_b32_e32 v9, v1
6428 ; GFX7-NEXT: v_mov_b32_e32 v8, v0
6429 ; GFX7-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]
6430 ; GFX7-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
6431 ; GFX7-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
6432 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
6433 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6434 ; GFX7-NEXT: buffer_wbinvl1_vol
6435 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
6436 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6437 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
6438 ; GFX7-NEXT: s_cbranch_execnz .LBB110_1
6439 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6440 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
6441 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6442 ;
6443 ; GFX8-LABEL: flat_atomic_umin_i64_ret_offset:
6444 ; GFX8: ; %bb.0:
6445 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6446 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 32, v0
6447 ; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
6448 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v0
6449 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
6450 ; GFX8-NEXT: flat_load_dword v1, v[0:1]
6451 ; GFX8-NEXT: flat_load_dword v0, v[4:5]
6452 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
6453 ; GFX8-NEXT: .LBB110_1: ; %atomicrmw.start
6454 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6455 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6456 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
6457 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
6458 ; GFX8-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]
6459 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
6460 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
6461 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
6462 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6463 ; GFX8-NEXT: buffer_wbinvl1_vol
6464 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
6465 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6466 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
6467 ; GFX8-NEXT: s_cbranch_execnz .LBB110_1
6468 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6469 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
6470 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6471 ;
6472 ; GFX9-LABEL: flat_atomic_umin_i64_ret_offset:
6473 ; GFX9: ; %bb.0:
6474 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6475 ; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
6476 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
6477 ; GFX9-NEXT: .LBB110_1: ; %atomicrmw.start
6478 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
6479 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6480 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
6481 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
6482 ; GFX9-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
6483 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
6484 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
6485 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
6486 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6487 ; GFX9-NEXT: buffer_wbinvl1_vol
6488 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
6489 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6490 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
6491 ; GFX9-NEXT: s_cbranch_execnz .LBB110_1
6492 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
6493 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
6494 ; GFX9-NEXT: v_mov_b32_e32 v0, v4
6495 ; GFX9-NEXT: v_mov_b32_e32 v1, v5
6496 ; GFX9-NEXT: s_setpc_b64 s[30:31]
6497 %gep = getelementptr i64, ptr %out, i64 4
6498 %result = atomicrmw umin ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
6499 ret i64 %result
6500 }
6502 define amdgpu_gfx void @flat_atomic_umin_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
6503 ; GFX7-LABEL: flat_atomic_umin_i64_noret_scalar:
6504 ; GFX7: ; %bb.0:
6505 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6506 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
6507 ; GFX7-NEXT: s_add_u32 s34, s4, 4
6508 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
6509 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
6510 ; GFX7-NEXT: v_mov_b32_e32 v3, s34
6511 ; GFX7-NEXT: v_mov_b32_e32 v4, s35
6512 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
6513 ; GFX7-NEXT: flat_load_dword v3, v[3:4]
6514 ; GFX7-NEXT: v_mov_b32_e32 v4, s4
6515 ; GFX7-NEXT: s_mov_b64 s[34:35], 0
6516 ; GFX7-NEXT: v_mov_b32_e32 v6, s7
6517 ; GFX7-NEXT: v_mov_b32_e32 v7, s6
6518 ; GFX7-NEXT: v_mov_b32_e32 v5, s5
6519 ; GFX7-NEXT: .LBB111_1: ; %atomicrmw.start
6520 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
6521 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6522 ; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
6523 ; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
6524 ; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
6525 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
6526 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6527 ; GFX7-NEXT: buffer_wbinvl1_vol
6528 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
6529 ; GFX7-NEXT: v_mov_b32_e32 v3, v1
6530 ; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
6531 ; GFX7-NEXT: v_mov_b32_e32 v2, v0
6532 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
6533 ; GFX7-NEXT: s_cbranch_execnz .LBB111_1
6534 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6535 ; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
6536 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6537 ;
6538 ; GFX8-LABEL: flat_atomic_umin_i64_noret_scalar:
6539 ; GFX8: ; %bb.0:
6540 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6541 ; GFX8-NEXT: v_mov_b32_e32 v0, s4
6542 ; GFX8-NEXT: s_add_u32 s34, s4, 4
6543 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
6544 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
6545 ; GFX8-NEXT: v_mov_b32_e32 v3, s34
6546 ; GFX8-NEXT: v_mov_b32_e32 v4, s35
6547 ; GFX8-NEXT: flat_load_dword v2, v[0:1]
6548 ; GFX8-NEXT: flat_load_dword v3, v[3:4]
6549 ; GFX8-NEXT: v_mov_b32_e32 v4, s4
6550 ; GFX8-NEXT: s_mov_b64 s[34:35], 0
6551 ; GFX8-NEXT: v_mov_b32_e32 v6, s7
6552 ; GFX8-NEXT: v_mov_b32_e32 v7, s6
6553 ; GFX8-NEXT: v_mov_b32_e32 v5, s5
6554 ; GFX8-NEXT: .LBB111_1: ; %atomicrmw.start
6555 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6556 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6557 ; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
6558 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
6559 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
6560 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
6561 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6562 ; GFX8-NEXT: buffer_wbinvl1_vol
6563 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
6564 ; GFX8-NEXT: v_mov_b32_e32 v3, v1
6565 ; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
6566 ; GFX8-NEXT: v_mov_b32_e32 v2, v0
6567 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
6568 ; GFX8-NEXT: s_cbranch_execnz .LBB111_1
6569 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6570 ; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
6571 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6572 ;
6573 ; GFX9-LABEL: flat_atomic_umin_i64_noret_scalar:
6574 ; GFX9: ; %bb.0:
6575 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6576 ; GFX9-NEXT: v_mov_b32_e32 v0, s4
6577 ; GFX9-NEXT: v_mov_b32_e32 v1, s5
6578 ; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
6579 ; GFX9-NEXT: v_mov_b32_e32 v4, s4
6580 ; GFX9-NEXT: s_mov_b64 s[34:35], 0
6581 ; GFX9-NEXT: v_mov_b32_e32 v6, s7
6582 ; GFX9-NEXT: v_mov_b32_e32 v7, s6
6583 ; GFX9-NEXT: v_mov_b32_e32 v5, s5
6584 ; GFX9-NEXT: .LBB111_1: ; %atomicrmw.start
6585 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
6586 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6587 ; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
6588 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
6589 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
6590 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
6591 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6592 ; GFX9-NEXT: buffer_wbinvl1_vol
6593 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
6594 ; GFX9-NEXT: v_mov_b32_e32 v3, v1
6595 ; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
6596 ; GFX9-NEXT: v_mov_b32_e32 v2, v0
6597 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
6598 ; GFX9-NEXT: s_cbranch_execnz .LBB111_1
6599 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
6600 ; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
6601 ; GFX9-NEXT: s_setpc_b64 s[30:31]
6602 %tmp0 = atomicrmw umin ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
6603 ret void
6604 }
6606 define amdgpu_gfx void @flat_atomic_umin_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
6607 ; GFX7-LABEL: flat_atomic_umin_i64_noret_offset_scalar:
6608 ; GFX7: ; %bb.0:
6609 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6610 ; GFX7-NEXT: s_add_u32 s34, s4, 32
6611 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
6612 ; GFX7-NEXT: s_add_u32 s36, s4, 36
6613 ; GFX7-NEXT: s_addc_u32 s37, s5, 0
6614 ; GFX7-NEXT: v_mov_b32_e32 v0, s36
6615 ; GFX7-NEXT: v_mov_b32_e32 v1, s37
6616 ; GFX7-NEXT: v_mov_b32_e32 v4, s34
6617 ; GFX7-NEXT: v_mov_b32_e32 v5, s35
6618 ; GFX7-NEXT: flat_load_dword v3, v[0:1]
6619 ; GFX7-NEXT: flat_load_dword v2, v[4:5]
6620 ; GFX7-NEXT: s_mov_b64 s[34:35], 0
6621 ; GFX7-NEXT: v_mov_b32_e32 v6, s7
6622 ; GFX7-NEXT: v_mov_b32_e32 v7, s6
6623 ; GFX7-NEXT: .LBB112_1: ; %atomicrmw.start
6624 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
6625 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6626 ; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
6627 ; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
6628 ; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
6629 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
6630 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6631 ; GFX7-NEXT: buffer_wbinvl1_vol
6632 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
6633 ; GFX7-NEXT: v_mov_b32_e32 v3, v1
6634 ; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
6635 ; GFX7-NEXT: v_mov_b32_e32 v2, v0
6636 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
6637 ; GFX7-NEXT: s_cbranch_execnz .LBB112_1
6638 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6639 ; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
6640 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6641 ;
6642 ; GFX8-LABEL: flat_atomic_umin_i64_noret_offset_scalar:
6643 ; GFX8: ; %bb.0:
6644 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6645 ; GFX8-NEXT: s_add_u32 s34, s4, 32
6646 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
6647 ; GFX8-NEXT: s_add_u32 s36, s4, 36
6648 ; GFX8-NEXT: s_addc_u32 s37, s5, 0
6649 ; GFX8-NEXT: v_mov_b32_e32 v0, s36
6650 ; GFX8-NEXT: v_mov_b32_e32 v1, s37
6651 ; GFX8-NEXT: v_mov_b32_e32 v4, s34
6652 ; GFX8-NEXT: v_mov_b32_e32 v5, s35
6653 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
6654 ; GFX8-NEXT: flat_load_dword v2, v[4:5]
6655 ; GFX8-NEXT: s_mov_b64 s[34:35], 0
6656 ; GFX8-NEXT: v_mov_b32_e32 v6, s7
6657 ; GFX8-NEXT: v_mov_b32_e32 v7, s6
6658 ; GFX8-NEXT: .LBB112_1: ; %atomicrmw.start
6659 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6660 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6661 ; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
6662 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
6663 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
6664 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
6665 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6666 ; GFX8-NEXT: buffer_wbinvl1_vol
6667 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
6668 ; GFX8-NEXT: v_mov_b32_e32 v3, v1
6669 ; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
6670 ; GFX8-NEXT: v_mov_b32_e32 v2, v0
6671 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
6672 ; GFX8-NEXT: s_cbranch_execnz .LBB112_1
6673 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6674 ; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
6675 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6676 ;
6677 ; GFX9-LABEL: flat_atomic_umin_i64_noret_offset_scalar:
6678 ; GFX9: ; %bb.0:
6679 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6680 ; GFX9-NEXT: v_mov_b32_e32 v0, s4
6681 ; GFX9-NEXT: v_mov_b32_e32 v1, s5
6682 ; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
6683 ; GFX9-NEXT: v_mov_b32_e32 v4, s4
6684 ; GFX9-NEXT: s_mov_b64 s[34:35], 0
6685 ; GFX9-NEXT: v_mov_b32_e32 v6, s7
6686 ; GFX9-NEXT: v_mov_b32_e32 v7, s6
6687 ; GFX9-NEXT: v_mov_b32_e32 v5, s5
6688 ; GFX9-NEXT: .LBB112_1: ; %atomicrmw.start
6689 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
6690 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6691 ; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
6692 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
6693 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
6694 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
6695 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6696 ; GFX9-NEXT: buffer_wbinvl1_vol
6697 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
6698 ; GFX9-NEXT: v_mov_b32_e32 v3, v1
6699 ; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
6700 ; GFX9-NEXT: v_mov_b32_e32 v2, v0
6701 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
6702 ; GFX9-NEXT: s_cbranch_execnz .LBB112_1
6703 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
6704 ; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
6705 ; GFX9-NEXT: s_setpc_b64 s[30:31]
6706 %gep = getelementptr i64, ptr %out, i64 4
6707 %tmp0 = atomicrmw umin ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
6708 ret void
6709 }
6711 define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
6712 ; GFX7-LABEL: flat_atomic_umin_i64_ret_scalar:
6713 ; GFX7: ; %bb.0:
6714 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6715 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
6716 ; GFX7-NEXT: s_add_u32 s34, s4, 4
6717 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
6718 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
6719 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
6720 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
6721 ; GFX7-NEXT: flat_load_dword v0, v[0:1]
6722 ; GFX7-NEXT: flat_load_dword v1, v[2:3]
6723 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
6724 ; GFX7-NEXT: s_mov_b64 s[34:35], 0
6725 ; GFX7-NEXT: v_mov_b32_e32 v4, s7
6726 ; GFX7-NEXT: v_mov_b32_e32 v5, s6
6727 ; GFX7-NEXT: v_mov_b32_e32 v3, s5
6728 ; GFX7-NEXT: .LBB113_1: ; %atomicrmw.start
6729 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
6730 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6731 ; GFX7-NEXT: v_mov_b32_e32 v9, v1
6732 ; GFX7-NEXT: v_mov_b32_e32 v8, v0
6733 ; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
6734 ; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
6735 ; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
6736 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
6737 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6738 ; GFX7-NEXT: buffer_wbinvl1_vol
6739 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
6740 ; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
6741 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
6742 ; GFX7-NEXT: s_cbranch_execnz .LBB113_1
6743 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6744 ; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
6745 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6746 ;
6747 ; GFX8-LABEL: flat_atomic_umin_i64_ret_scalar:
6748 ; GFX8: ; %bb.0:
6749 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6750 ; GFX8-NEXT: v_mov_b32_e32 v0, s4
6751 ; GFX8-NEXT: s_add_u32 s34, s4, 4
6752 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
6753 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
6754 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
6755 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
6756 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
6757 ; GFX8-NEXT: flat_load_dword v1, v[2:3]
6758 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
6759 ; GFX8-NEXT: s_mov_b64 s[34:35], 0
6760 ; GFX8-NEXT: v_mov_b32_e32 v4, s7
6761 ; GFX8-NEXT: v_mov_b32_e32 v5, s6
6762 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
6763 ; GFX8-NEXT: .LBB113_1: ; %atomicrmw.start
6764 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6765 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6766 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
6767 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
6768 ; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
6769 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
6770 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
6771 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
6772 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6773 ; GFX8-NEXT: buffer_wbinvl1_vol
6774 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
6775 ; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
6776 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
6777 ; GFX8-NEXT: s_cbranch_execnz .LBB113_1
6778 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6779 ; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
6780 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6781 ;
6782 ; GFX9-LABEL: flat_atomic_umin_i64_ret_scalar:
6783 ; GFX9: ; %bb.0:
6784 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6785 ; GFX9-NEXT: v_mov_b32_e32 v0, s4
6786 ; GFX9-NEXT: v_mov_b32_e32 v1, s5
6787 ; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
6788 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
6789 ; GFX9-NEXT: s_mov_b64 s[34:35], 0
6790 ; GFX9-NEXT: v_mov_b32_e32 v4, s7
6791 ; GFX9-NEXT: v_mov_b32_e32 v5, s6
6792 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
6793 ; GFX9-NEXT: .LBB113_1: ; %atomicrmw.start
6794 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
6795 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6796 ; GFX9-NEXT: v_mov_b32_e32 v9, v1
6797 ; GFX9-NEXT: v_mov_b32_e32 v8, v0
6798 ; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
6799 ; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
6800 ; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
6801 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
6802 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6803 ; GFX9-NEXT: buffer_wbinvl1_vol
6804 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
6805 ; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
6806 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
6807 ; GFX9-NEXT: s_cbranch_execnz .LBB113_1
6808 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
6809 ; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
6810 ; GFX9-NEXT: s_setpc_b64 s[30:31]
6811 %result = atomicrmw umin ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
6812 ret i64 %result
6813 }
6815 define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
6816 ; GFX7-LABEL: flat_atomic_umin_i64_ret_offset_scalar:
6817 ; GFX7: ; %bb.0:
6818 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6819 ; GFX7-NEXT: s_add_u32 s34, s4, 32
6820 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
6821 ; GFX7-NEXT: s_add_u32 s36, s4, 36
6822 ; GFX7-NEXT: s_addc_u32 s37, s5, 0
6823 ; GFX7-NEXT: v_mov_b32_e32 v0, s36
6824 ; GFX7-NEXT: v_mov_b32_e32 v1, s37
6825 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
6826 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
6827 ; GFX7-NEXT: flat_load_dword v1, v[0:1]
6828 ; GFX7-NEXT: flat_load_dword v0, v[2:3]
6829 ; GFX7-NEXT: s_mov_b64 s[34:35], 0
6830 ; GFX7-NEXT: v_mov_b32_e32 v4, s7
6831 ; GFX7-NEXT: v_mov_b32_e32 v5, s6
6832 ; GFX7-NEXT: .LBB114_1: ; %atomicrmw.start
6833 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
6834 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6835 ; GFX7-NEXT: v_mov_b32_e32 v9, v1
6836 ; GFX7-NEXT: v_mov_b32_e32 v8, v0
6837 ; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
6838 ; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
6839 ; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
6840 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
6841 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6842 ; GFX7-NEXT: buffer_wbinvl1_vol
6843 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
6844 ; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
6845 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
6846 ; GFX7-NEXT: s_cbranch_execnz .LBB114_1
6847 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6848 ; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
6849 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6850 ;
6851 ; GFX8-LABEL: flat_atomic_umin_i64_ret_offset_scalar:
6852 ; GFX8: ; %bb.0:
6853 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6854 ; GFX8-NEXT: s_add_u32 s34, s4, 32
6855 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
6856 ; GFX8-NEXT: s_add_u32 s36, s4, 36
6857 ; GFX8-NEXT: s_addc_u32 s37, s5, 0
6858 ; GFX8-NEXT: v_mov_b32_e32 v0, s36
6859 ; GFX8-NEXT: v_mov_b32_e32 v1, s37
6860 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
6861 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
6862 ; GFX8-NEXT: flat_load_dword v1, v[0:1]
6863 ; GFX8-NEXT: flat_load_dword v0, v[2:3]
6864 ; GFX8-NEXT: s_mov_b64 s[34:35], 0
6865 ; GFX8-NEXT: v_mov_b32_e32 v4, s7
6866 ; GFX8-NEXT: v_mov_b32_e32 v5, s6
6867 ; GFX8-NEXT: .LBB114_1: ; %atomicrmw.start
6868 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6869 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6870 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
6871 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
6872 ; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
6873 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
6874 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
6875 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
6876 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6877 ; GFX8-NEXT: buffer_wbinvl1_vol
6878 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
6879 ; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
6880 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
6881 ; GFX8-NEXT: s_cbranch_execnz .LBB114_1
6882 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6883 ; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
6884 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6885 ;
6886 ; GFX9-LABEL: flat_atomic_umin_i64_ret_offset_scalar:
6887 ; GFX9: ; %bb.0:
6888 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6889 ; GFX9-NEXT: v_mov_b32_e32 v0, s4
6890 ; GFX9-NEXT: v_mov_b32_e32 v1, s5
6891 ; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
6892 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
6893 ; GFX9-NEXT: s_mov_b64 s[34:35], 0
6894 ; GFX9-NEXT: v_mov_b32_e32 v4, s7
6895 ; GFX9-NEXT: v_mov_b32_e32 v5, s6
6896 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
6897 ; GFX9-NEXT: .LBB114_1: ; %atomicrmw.start
6898 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
6899 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6900 ; GFX9-NEXT: v_mov_b32_e32 v9, v1
6901 ; GFX9-NEXT: v_mov_b32_e32 v8, v0
6902 ; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
6903 ; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
6904 ; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
6905 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc
6906 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6907 ; GFX9-NEXT: buffer_wbinvl1_vol
6908 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
6909 ; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
6910 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
6911 ; GFX9-NEXT: s_cbranch_execnz .LBB114_1
6912 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
6913 ; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
6914 ; GFX9-NEXT: s_setpc_b64 s[30:31]
6915 %gep = getelementptr i64, ptr %out, i64 4
6916 %result = atomicrmw umin ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
6917 ret i64 %result
6918 }
6920 define void @flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
6921 ; GFX7-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory:
6922 ; GFX7: ; %bb.0:
6923 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6924 ; GFX7-NEXT: v_add_i32_e32 v8, vcc, 32, v0
6925 ; GFX7-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
6926 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v0
6927 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
6928 ; GFX7-NEXT: flat_load_dword v7, v[0:1]
6929 ; GFX7-NEXT: flat_load_dword v6, v[8:9]
6930 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
6931 ; GFX7-NEXT: .LBB115_1: ; %atomicrmw.start
6932 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
6933 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6934 ; GFX7-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
6935 ; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
6936 ; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
6937 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
6938 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6939 ; GFX7-NEXT: buffer_wbinvl1_vol
6940 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
6941 ; GFX7-NEXT: v_mov_b32_e32 v7, v1
6942 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6943 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
6944 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
6945 ; GFX7-NEXT: s_cbranch_execnz .LBB115_1
6946 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
6947 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
6948 ; GFX7-NEXT: s_setpc_b64 s[30:31]
6949 ;
6950 ; GFX8-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory:
6951 ; GFX8: ; %bb.0:
6952 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6953 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 32, v0
6954 ; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
6955 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v0
6956 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
6957 ; GFX8-NEXT: flat_load_dword v7, v[0:1]
6958 ; GFX8-NEXT: flat_load_dword v6, v[8:9]
6959 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
6960 ; GFX8-NEXT: .LBB115_1: ; %atomicrmw.start
6961 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
6962 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6963 ; GFX8-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
6964 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
6965 ; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
6966 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
6967 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6968 ; GFX8-NEXT: buffer_wbinvl1_vol
6969 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
6970 ; GFX8-NEXT: v_mov_b32_e32 v7, v1
6971 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6972 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
6973 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
6974 ; GFX8-NEXT: s_cbranch_execnz .LBB115_1
6975 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
6976 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
6977 ; GFX8-NEXT: s_setpc_b64 s[30:31]
6978 ;
6979 ; GFX9-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory:
6980 ; GFX9: ; %bb.0:
6981 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
6982 ; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
6983 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
6984 ; GFX9-NEXT: .LBB115_1: ; %atomicrmw.start
6985 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
6986 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6987 ; GFX9-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
6988 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
6989 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
6990 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
6991 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6992 ; GFX9-NEXT: buffer_wbinvl1_vol
6993 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
6994 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
6995 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
6996 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
6997 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
6998 ; GFX9-NEXT: s_cbranch_execnz .LBB115_1
6999 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
7000 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
7001 ; GFX9-NEXT: s_setpc_b64 s[30:31]
7002 %gep = getelementptr i64, ptr %out, i64 4
7003 %tmp0 = atomicrmw umin ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
7004 ret void
7005 }
7007 define i64 @flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
7008 ; GFX7-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:
7009 ; GFX7: ; %bb.0:
7010 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7011 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 32, v0
7012 ; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
7013 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v0
7014 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
7015 ; GFX7-NEXT: flat_load_dword v1, v[0:1]
7016 ; GFX7-NEXT: flat_load_dword v0, v[4:5]
7017 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
7018 ; GFX7-NEXT: .LBB116_1: ; %atomicrmw.start
7019 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7020 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7021 ; GFX7-NEXT: v_mov_b32_e32 v9, v1
7022 ; GFX7-NEXT: v_mov_b32_e32 v8, v0
7023 ; GFX7-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]
7024 ; GFX7-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
7025 ; GFX7-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
7026 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
7027 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7028 ; GFX7-NEXT: buffer_wbinvl1_vol
7029 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
7030 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7031 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
7032 ; GFX7-NEXT: s_cbranch_execnz .LBB116_1
7033 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
7034 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
7035 ; GFX7-NEXT: s_setpc_b64 s[30:31]
7036 ;
7037 ; GFX8-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:
7038 ; GFX8: ; %bb.0:
7039 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7040 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 32, v0
7041 ; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
7042 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v0
7043 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
7044 ; GFX8-NEXT: flat_load_dword v1, v[0:1]
7045 ; GFX8-NEXT: flat_load_dword v0, v[4:5]
7046 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
7047 ; GFX8-NEXT: .LBB116_1: ; %atomicrmw.start
7048 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7049 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7050 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
7051 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
7052 ; GFX8-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]
7053 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
7054 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
7055 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
7056 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7057 ; GFX8-NEXT: buffer_wbinvl1_vol
7058 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
7059 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7060 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
7061 ; GFX8-NEXT: s_cbranch_execnz .LBB116_1
7062 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7063 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
7064 ; GFX8-NEXT: s_setpc_b64 s[30:31]
7065 ;
7066 ; GFX9-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:
7067 ; GFX9: ; %bb.0:
7068 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7069 ; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
7070 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
7071 ; GFX9-NEXT: .LBB116_1: ; %atomicrmw.start
7072 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
7073 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7074 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
7075 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
7076 ; GFX9-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
7077 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
7078 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
7079 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
7080 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7081 ; GFX9-NEXT: buffer_wbinvl1_vol
7082 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
7083 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7084 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
7085 ; GFX9-NEXT: s_cbranch_execnz .LBB116_1
7086 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
7087 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
7088 ; GFX9-NEXT: v_mov_b32_e32 v0, v4
7089 ; GFX9-NEXT: v_mov_b32_e32 v1, v5
7090 ; GFX9-NEXT: s_setpc_b64 s[30:31]
7091 %gep = getelementptr i64, ptr %out, i64 4
7092 %result = atomicrmw umin ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
7093 ret i64 %result
7094 }
7096 ; ---------------------------------------------------------------------
7097 ; atomicrmw min
7098 ; ---------------------------------------------------------------------
7100 define void @flat_atomic_min_i64_noret(ptr %ptr, i64 %in) {
7101 ; GFX7-LABEL: flat_atomic_min_i64_noret:
7102 ; GFX7: ; %bb.0:
7103 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7104 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 4, v0
7105 ; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
7106 ; GFX7-NEXT: flat_load_dword v6, v[0:1]
7107 ; GFX7-NEXT: flat_load_dword v7, v[4:5]
7108 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
7109 ; GFX7-NEXT: .LBB117_1: ; %atomicrmw.start
7110 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7111 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7112 ; GFX7-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
7113 ; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
7114 ; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
7115 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
7116 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7117 ; GFX7-NEXT: buffer_wbinvl1_vol
7118 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
7119 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
7120 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7121 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
7122 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
7123 ; GFX7-NEXT: s_cbranch_execnz .LBB117_1
7124 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
7125 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
7126 ; GFX7-NEXT: s_setpc_b64 s[30:31]
7127 ;
7128 ; GFX8-LABEL: flat_atomic_min_i64_noret:
7129 ; GFX8: ; %bb.0:
7130 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7131 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v0
7132 ; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
7133 ; GFX8-NEXT: flat_load_dword v6, v[0:1]
7134 ; GFX8-NEXT: flat_load_dword v7, v[4:5]
7135 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
7136 ; GFX8-NEXT: .LBB117_1: ; %atomicrmw.start
7137 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7138 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7139 ; GFX8-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
7140 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
7141 ; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
7142 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
7143 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7144 ; GFX8-NEXT: buffer_wbinvl1_vol
7145 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
7146 ; GFX8-NEXT: v_mov_b32_e32 v7, v5
7147 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7148 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
7149 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
7150 ; GFX8-NEXT: s_cbranch_execnz .LBB117_1
7151 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7152 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
7153 ; GFX8-NEXT: s_setpc_b64 s[30:31]
7154 ;
7155 ; GFX9-LABEL: flat_atomic_min_i64_noret:
7156 ; GFX9: ; %bb.0:
7157 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7158 ; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
7159 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
7160 ; GFX9-NEXT: .LBB117_1: ; %atomicrmw.start
7161 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
7162 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7163 ; GFX9-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
7164 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
7165 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
7166 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
7167 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7168 ; GFX9-NEXT: buffer_wbinvl1_vol
7169 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
7170 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
7171 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7172 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
7173 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
7174 ; GFX9-NEXT: s_cbranch_execnz .LBB117_1
7175 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
7176 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
7177 ; GFX9-NEXT: s_setpc_b64 s[30:31]
7178 %tmp0 = atomicrmw min ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
7179 ret void
7180 }
7182 define void @flat_atomic_min_i64_noret_offset(ptr %out, i64 %in) {
7183 ; GFX7-LABEL: flat_atomic_min_i64_noret_offset:
7184 ; GFX7: ; %bb.0:
7185 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7186 ; GFX7-NEXT: v_add_i32_e32 v8, vcc, 32, v0
7187 ; GFX7-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
7188 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v0
7189 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
7190 ; GFX7-NEXT: flat_load_dword v7, v[0:1]
7191 ; GFX7-NEXT: flat_load_dword v6, v[8:9]
7192 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
7193 ; GFX7-NEXT: .LBB118_1: ; %atomicrmw.start
7194 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7195 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7196 ; GFX7-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
7197 ; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
7198 ; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
7199 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
7200 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7201 ; GFX7-NEXT: buffer_wbinvl1_vol
7202 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
7203 ; GFX7-NEXT: v_mov_b32_e32 v7, v1
7204 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7205 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
7206 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
7207 ; GFX7-NEXT: s_cbranch_execnz .LBB118_1
7208 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
7209 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
7210 ; GFX7-NEXT: s_setpc_b64 s[30:31]
7211 ;
7212 ; GFX8-LABEL: flat_atomic_min_i64_noret_offset:
7213 ; GFX8: ; %bb.0:
7214 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7215 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 32, v0
7216 ; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
7217 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v0
7218 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
7219 ; GFX8-NEXT: flat_load_dword v7, v[0:1]
7220 ; GFX8-NEXT: flat_load_dword v6, v[8:9]
7221 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
7222 ; GFX8-NEXT: .LBB118_1: ; %atomicrmw.start
7223 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7224 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7225 ; GFX8-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
7226 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
7227 ; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
7228 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
7229 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7230 ; GFX8-NEXT: buffer_wbinvl1_vol
7231 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
7232 ; GFX8-NEXT: v_mov_b32_e32 v7, v1
7233 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7234 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
7235 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
7236 ; GFX8-NEXT: s_cbranch_execnz .LBB118_1
7237 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7238 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
7239 ; GFX8-NEXT: s_setpc_b64 s[30:31]
7240 ;
7241 ; GFX9-LABEL: flat_atomic_min_i64_noret_offset:
7242 ; GFX9: ; %bb.0:
7243 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7244 ; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
7245 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
7246 ; GFX9-NEXT: .LBB118_1: ; %atomicrmw.start
7247 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
7248 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7249 ; GFX9-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
7250 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
7251 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
7252 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
7253 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7254 ; GFX9-NEXT: buffer_wbinvl1_vol
7255 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
7256 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
7257 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7258 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
7259 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
7260 ; GFX9-NEXT: s_cbranch_execnz .LBB118_1
7261 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
7262 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
7263 ; GFX9-NEXT: s_setpc_b64 s[30:31]
7264 %gep = getelementptr i64, ptr %out, i64 4
7265 %tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
7266 ret void
7267 }
7269 define i64 @flat_atomic_min_i64_ret(ptr %ptr, i64 %in) {
7270 ; GFX7-LABEL: flat_atomic_min_i64_ret:
7271 ; GFX7: ; %bb.0:
7272 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7273 ; GFX7-NEXT: v_add_i32_e32 v5, vcc, 4, v0
7274 ; GFX7-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
7275 ; GFX7-NEXT: flat_load_dword v4, v[0:1]
7276 ; GFX7-NEXT: flat_load_dword v5, v[5:6]
7277 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
7278 ; GFX7-NEXT: .LBB119_1: ; %atomicrmw.start
7279 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7280 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7281 ; GFX7-NEXT: v_mov_b32_e32 v7, v5
7282 ; GFX7-NEXT: v_mov_b32_e32 v6, v4
7283 ; GFX7-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
7284 ; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
7285 ; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
7286 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
7287 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7288 ; GFX7-NEXT: buffer_wbinvl1_vol
7289 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
7290 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7291 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
7292 ; GFX7-NEXT: s_cbranch_execnz .LBB119_1
7293 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
7294 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
7295 ; GFX7-NEXT: v_mov_b32_e32 v0, v4
7296 ; GFX7-NEXT: v_mov_b32_e32 v1, v5
7297 ; GFX7-NEXT: s_setpc_b64 s[30:31]
7298 ;
7299 ; GFX8-LABEL: flat_atomic_min_i64_ret:
7300 ; GFX8: ; %bb.0:
7301 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7302 ; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v0
7303 ; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
7304 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
7305 ; GFX8-NEXT: flat_load_dword v5, v[5:6]
7306 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
7307 ; GFX8-NEXT: .LBB119_1: ; %atomicrmw.start
7308 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7309 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7310 ; GFX8-NEXT: v_mov_b32_e32 v7, v5
7311 ; GFX8-NEXT: v_mov_b32_e32 v6, v4
7312 ; GFX8-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
7313 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
7314 ; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
7315 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
7316 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7317 ; GFX8-NEXT: buffer_wbinvl1_vol
7318 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
7319 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7320 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
7321 ; GFX8-NEXT: s_cbranch_execnz .LBB119_1
7322 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7323 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
7324 ; GFX8-NEXT: v_mov_b32_e32 v0, v4
7325 ; GFX8-NEXT: v_mov_b32_e32 v1, v5
7326 ; GFX8-NEXT: s_setpc_b64 s[30:31]
7327 ;
7328 ; GFX9-LABEL: flat_atomic_min_i64_ret:
7329 ; GFX9: ; %bb.0:
7330 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7331 ; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
7332 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
7333 ; GFX9-NEXT: .LBB119_1: ; %atomicrmw.start
7334 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
7335 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7336 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
7337 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
7338 ; GFX9-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
7339 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
7340 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
7341 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
7342 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7343 ; GFX9-NEXT: buffer_wbinvl1_vol
7344 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
7345 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7346 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
7347 ; GFX9-NEXT: s_cbranch_execnz .LBB119_1
7348 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
7349 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
7350 ; GFX9-NEXT: v_mov_b32_e32 v0, v4
7351 ; GFX9-NEXT: v_mov_b32_e32 v1, v5
7352 ; GFX9-NEXT: s_setpc_b64 s[30:31]
7353 %result = atomicrmw min ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
7354 ret i64 %result
7355 }
7357 define i64 @flat_atomic_min_i64_ret_offset(ptr %out, i64 %in) {
7358 ; GFX7-LABEL: flat_atomic_min_i64_ret_offset:
7359 ; GFX7: ; %bb.0:
7360 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7361 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 32, v0
7362 ; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
7363 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v0
7364 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
7365 ; GFX7-NEXT: flat_load_dword v1, v[0:1]
7366 ; GFX7-NEXT: flat_load_dword v0, v[4:5]
7367 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
7368 ; GFX7-NEXT: .LBB120_1: ; %atomicrmw.start
7369 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7370 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7371 ; GFX7-NEXT: v_mov_b32_e32 v9, v1
7372 ; GFX7-NEXT: v_mov_b32_e32 v8, v0
7373 ; GFX7-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]
7374 ; GFX7-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
7375 ; GFX7-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
7376 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
7377 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7378 ; GFX7-NEXT: buffer_wbinvl1_vol
7379 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
7380 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7381 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
7382 ; GFX7-NEXT: s_cbranch_execnz .LBB120_1
7383 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
7384 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
7385 ; GFX7-NEXT: s_setpc_b64 s[30:31]
7386 ;
7387 ; GFX8-LABEL: flat_atomic_min_i64_ret_offset:
7388 ; GFX8: ; %bb.0:
7389 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7390 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 32, v0
7391 ; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
7392 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v0
7393 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
7394 ; GFX8-NEXT: flat_load_dword v1, v[0:1]
7395 ; GFX8-NEXT: flat_load_dword v0, v[4:5]
7396 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
7397 ; GFX8-NEXT: .LBB120_1: ; %atomicrmw.start
7398 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7399 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7400 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
7401 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
7402 ; GFX8-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]
7403 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
7404 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
7405 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
7406 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7407 ; GFX8-NEXT: buffer_wbinvl1_vol
7408 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
7409 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7410 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
7411 ; GFX8-NEXT: s_cbranch_execnz .LBB120_1
7412 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7413 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
7414 ; GFX8-NEXT: s_setpc_b64 s[30:31]
7415 ;
7416 ; GFX9-LABEL: flat_atomic_min_i64_ret_offset:
7417 ; GFX9: ; %bb.0:
7418 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7419 ; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
7420 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
7421 ; GFX9-NEXT: .LBB120_1: ; %atomicrmw.start
7422 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
7423 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7424 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
7425 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
7426 ; GFX9-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
7427 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
7428 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
7429 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
7430 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7431 ; GFX9-NEXT: buffer_wbinvl1_vol
7432 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
7433 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
7434 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
7435 ; GFX9-NEXT: s_cbranch_execnz .LBB120_1
7436 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
7437 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
7438 ; GFX9-NEXT: v_mov_b32_e32 v0, v4
7439 ; GFX9-NEXT: v_mov_b32_e32 v1, v5
7440 ; GFX9-NEXT: s_setpc_b64 s[30:31]
7441 %gep = getelementptr i64, ptr %out, i64 4
7442 %result = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
7443 ret i64 %result
7444 }
7446 define amdgpu_gfx void @flat_atomic_min_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
7447 ; GFX7-LABEL: flat_atomic_min_i64_noret_scalar:
7448 ; GFX7: ; %bb.0:
7449 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7450 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
7451 ; GFX7-NEXT: s_add_u32 s34, s4, 4
7452 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
7453 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
7454 ; GFX7-NEXT: v_mov_b32_e32 v3, s34
7455 ; GFX7-NEXT: v_mov_b32_e32 v4, s35
7456 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
7457 ; GFX7-NEXT: flat_load_dword v3, v[3:4]
7458 ; GFX7-NEXT: v_mov_b32_e32 v4, s4
7459 ; GFX7-NEXT: s_mov_b64 s[34:35], 0
7460 ; GFX7-NEXT: v_mov_b32_e32 v6, s7
7461 ; GFX7-NEXT: v_mov_b32_e32 v7, s6
7462 ; GFX7-NEXT: v_mov_b32_e32 v5, s5
7463 ; GFX7-NEXT: .LBB121_1: ; %atomicrmw.start
7464 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7465 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7466 ; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
7467 ; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
7468 ; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
7469 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
7470 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7471 ; GFX7-NEXT: buffer_wbinvl1_vol
7472 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
7473 ; GFX7-NEXT: v_mov_b32_e32 v3, v1
7474 ; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
7475 ; GFX7-NEXT: v_mov_b32_e32 v2, v0
7476 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
7477 ; GFX7-NEXT: s_cbranch_execnz .LBB121_1
7478 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
7479 ; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
7480 ; GFX7-NEXT: s_setpc_b64 s[30:31]
7481 ;
7482 ; GFX8-LABEL: flat_atomic_min_i64_noret_scalar:
7483 ; GFX8: ; %bb.0:
7484 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7485 ; GFX8-NEXT: v_mov_b32_e32 v0, s4
7486 ; GFX8-NEXT: s_add_u32 s34, s4, 4
7487 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
7488 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
7489 ; GFX8-NEXT: v_mov_b32_e32 v3, s34
7490 ; GFX8-NEXT: v_mov_b32_e32 v4, s35
7491 ; GFX8-NEXT: flat_load_dword v2, v[0:1]
7492 ; GFX8-NEXT: flat_load_dword v3, v[3:4]
7493 ; GFX8-NEXT: v_mov_b32_e32 v4, s4
7494 ; GFX8-NEXT: s_mov_b64 s[34:35], 0
7495 ; GFX8-NEXT: v_mov_b32_e32 v6, s7
7496 ; GFX8-NEXT: v_mov_b32_e32 v7, s6
7497 ; GFX8-NEXT: v_mov_b32_e32 v5, s5
7498 ; GFX8-NEXT: .LBB121_1: ; %atomicrmw.start
7499 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7500 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7501 ; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
7502 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
7503 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
7504 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
7505 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7506 ; GFX8-NEXT: buffer_wbinvl1_vol
7507 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
7508 ; GFX8-NEXT: v_mov_b32_e32 v3, v1
7509 ; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
7510 ; GFX8-NEXT: v_mov_b32_e32 v2, v0
7511 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
7512 ; GFX8-NEXT: s_cbranch_execnz .LBB121_1
7513 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7514 ; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
7515 ; GFX8-NEXT: s_setpc_b64 s[30:31]
7516 ;
7517 ; GFX9-LABEL: flat_atomic_min_i64_noret_scalar:
7518 ; GFX9: ; %bb.0:
7519 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7520 ; GFX9-NEXT: v_mov_b32_e32 v0, s4
7521 ; GFX9-NEXT: v_mov_b32_e32 v1, s5
7522 ; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
7523 ; GFX9-NEXT: v_mov_b32_e32 v4, s4
7524 ; GFX9-NEXT: s_mov_b64 s[34:35], 0
7525 ; GFX9-NEXT: v_mov_b32_e32 v6, s7
7526 ; GFX9-NEXT: v_mov_b32_e32 v7, s6
7527 ; GFX9-NEXT: v_mov_b32_e32 v5, s5
7528 ; GFX9-NEXT: .LBB121_1: ; %atomicrmw.start
7529 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
7530 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7531 ; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
7532 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
7533 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
7534 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
7535 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7536 ; GFX9-NEXT: buffer_wbinvl1_vol
7537 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
7538 ; GFX9-NEXT: v_mov_b32_e32 v3, v1
7539 ; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
7540 ; GFX9-NEXT: v_mov_b32_e32 v2, v0
7541 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
7542 ; GFX9-NEXT: s_cbranch_execnz .LBB121_1
7543 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
7544 ; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
7545 ; GFX9-NEXT: s_setpc_b64 s[30:31]
7546 %tmp0 = atomicrmw min ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
7547 ret void
7548 }
7550 define amdgpu_gfx void @flat_atomic_min_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
7551 ; GFX7-LABEL: flat_atomic_min_i64_noret_offset_scalar:
7552 ; GFX7: ; %bb.0:
7553 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7554 ; GFX7-NEXT: s_add_u32 s34, s4, 32
7555 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
7556 ; GFX7-NEXT: s_add_u32 s36, s4, 36
7557 ; GFX7-NEXT: s_addc_u32 s37, s5, 0
7558 ; GFX7-NEXT: v_mov_b32_e32 v0, s36
7559 ; GFX7-NEXT: v_mov_b32_e32 v1, s37
7560 ; GFX7-NEXT: v_mov_b32_e32 v4, s34
7561 ; GFX7-NEXT: v_mov_b32_e32 v5, s35
7562 ; GFX7-NEXT: flat_load_dword v3, v[0:1]
7563 ; GFX7-NEXT: flat_load_dword v2, v[4:5]
7564 ; GFX7-NEXT: s_mov_b64 s[34:35], 0
7565 ; GFX7-NEXT: v_mov_b32_e32 v6, s7
7566 ; GFX7-NEXT: v_mov_b32_e32 v7, s6
7567 ; GFX7-NEXT: .LBB122_1: ; %atomicrmw.start
7568 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7569 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7570 ; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
7571 ; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
7572 ; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
7573 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
7574 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7575 ; GFX7-NEXT: buffer_wbinvl1_vol
7576 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
7577 ; GFX7-NEXT: v_mov_b32_e32 v3, v1
7578 ; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
7579 ; GFX7-NEXT: v_mov_b32_e32 v2, v0
7580 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
7581 ; GFX7-NEXT: s_cbranch_execnz .LBB122_1
7582 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
7583 ; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
7584 ; GFX7-NEXT: s_setpc_b64 s[30:31]
7585 ;
7586 ; GFX8-LABEL: flat_atomic_min_i64_noret_offset_scalar:
7587 ; GFX8: ; %bb.0:
7588 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7589 ; GFX8-NEXT: s_add_u32 s34, s4, 32
7590 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
7591 ; GFX8-NEXT: s_add_u32 s36, s4, 36
7592 ; GFX8-NEXT: s_addc_u32 s37, s5, 0
7593 ; GFX8-NEXT: v_mov_b32_e32 v0, s36
7594 ; GFX8-NEXT: v_mov_b32_e32 v1, s37
7595 ; GFX8-NEXT: v_mov_b32_e32 v4, s34
7596 ; GFX8-NEXT: v_mov_b32_e32 v5, s35
7597 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
7598 ; GFX8-NEXT: flat_load_dword v2, v[4:5]
7599 ; GFX8-NEXT: s_mov_b64 s[34:35], 0
7600 ; GFX8-NEXT: v_mov_b32_e32 v6, s7
7601 ; GFX8-NEXT: v_mov_b32_e32 v7, s6
7602 ; GFX8-NEXT: .LBB122_1: ; %atomicrmw.start
7603 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7604 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7605 ; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
7606 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
7607 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
7608 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
7609 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7610 ; GFX8-NEXT: buffer_wbinvl1_vol
7611 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
7612 ; GFX8-NEXT: v_mov_b32_e32 v3, v1
7613 ; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
7614 ; GFX8-NEXT: v_mov_b32_e32 v2, v0
7615 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
7616 ; GFX8-NEXT: s_cbranch_execnz .LBB122_1
7617 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7618 ; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
7619 ; GFX8-NEXT: s_setpc_b64 s[30:31]
7620 ;
7621 ; GFX9-LABEL: flat_atomic_min_i64_noret_offset_scalar:
7622 ; GFX9: ; %bb.0:
7623 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7624 ; GFX9-NEXT: v_mov_b32_e32 v0, s4
7625 ; GFX9-NEXT: v_mov_b32_e32 v1, s5
7626 ; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
7627 ; GFX9-NEXT: v_mov_b32_e32 v4, s4
7628 ; GFX9-NEXT: s_mov_b64 s[34:35], 0
7629 ; GFX9-NEXT: v_mov_b32_e32 v6, s7
7630 ; GFX9-NEXT: v_mov_b32_e32 v7, s6
7631 ; GFX9-NEXT: v_mov_b32_e32 v5, s5
7632 ; GFX9-NEXT: .LBB122_1: ; %atomicrmw.start
7633 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
7634 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7635 ; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
7636 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
7637 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
7638 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
7639 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7640 ; GFX9-NEXT: buffer_wbinvl1_vol
7641 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
7642 ; GFX9-NEXT: v_mov_b32_e32 v3, v1
7643 ; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
7644 ; GFX9-NEXT: v_mov_b32_e32 v2, v0
7645 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
7646 ; GFX9-NEXT: s_cbranch_execnz .LBB122_1
7647 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
7648 ; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
7649 ; GFX9-NEXT: s_setpc_b64 s[30:31]
7650 %gep = getelementptr i64, ptr %out, i64 4
7651 %tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
7652 ret void
7653 }
7655 define amdgpu_gfx i64 @flat_atomic_min_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
7656 ; GFX7-LABEL: flat_atomic_min_i64_ret_scalar:
7657 ; GFX7: ; %bb.0:
7658 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7659 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
7660 ; GFX7-NEXT: s_add_u32 s34, s4, 4
7661 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
7662 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
7663 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
7664 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
7665 ; GFX7-NEXT: flat_load_dword v0, v[0:1]
7666 ; GFX7-NEXT: flat_load_dword v1, v[2:3]
7667 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
7668 ; GFX7-NEXT: s_mov_b64 s[34:35], 0
7669 ; GFX7-NEXT: v_mov_b32_e32 v4, s7
7670 ; GFX7-NEXT: v_mov_b32_e32 v5, s6
7671 ; GFX7-NEXT: v_mov_b32_e32 v3, s5
7672 ; GFX7-NEXT: .LBB123_1: ; %atomicrmw.start
7673 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7674 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7675 ; GFX7-NEXT: v_mov_b32_e32 v9, v1
7676 ; GFX7-NEXT: v_mov_b32_e32 v8, v0
7677 ; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
7678 ; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
7679 ; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
7680 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
7681 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7682 ; GFX7-NEXT: buffer_wbinvl1_vol
7683 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
7684 ; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
7685 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
7686 ; GFX7-NEXT: s_cbranch_execnz .LBB123_1
7687 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
7688 ; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
7689 ; GFX7-NEXT: s_setpc_b64 s[30:31]
7690 ;
7691 ; GFX8-LABEL: flat_atomic_min_i64_ret_scalar:
7692 ; GFX8: ; %bb.0:
7693 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7694 ; GFX8-NEXT: v_mov_b32_e32 v0, s4
7695 ; GFX8-NEXT: s_add_u32 s34, s4, 4
7696 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
7697 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
7698 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
7699 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
7700 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
7701 ; GFX8-NEXT: flat_load_dword v1, v[2:3]
7702 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
7703 ; GFX8-NEXT: s_mov_b64 s[34:35], 0
7704 ; GFX8-NEXT: v_mov_b32_e32 v4, s7
7705 ; GFX8-NEXT: v_mov_b32_e32 v5, s6
7706 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
7707 ; GFX8-NEXT: .LBB123_1: ; %atomicrmw.start
7708 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7709 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7710 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
7711 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
7712 ; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
7713 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
7714 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
7715 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
7716 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7717 ; GFX8-NEXT: buffer_wbinvl1_vol
7718 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
7719 ; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
7720 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
7721 ; GFX8-NEXT: s_cbranch_execnz .LBB123_1
7722 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7723 ; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
7724 ; GFX8-NEXT: s_setpc_b64 s[30:31]
7725 ;
7726 ; GFX9-LABEL: flat_atomic_min_i64_ret_scalar:
7727 ; GFX9: ; %bb.0:
7728 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7729 ; GFX9-NEXT: v_mov_b32_e32 v0, s4
7730 ; GFX9-NEXT: v_mov_b32_e32 v1, s5
7731 ; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
7732 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
7733 ; GFX9-NEXT: s_mov_b64 s[34:35], 0
7734 ; GFX9-NEXT: v_mov_b32_e32 v4, s7
7735 ; GFX9-NEXT: v_mov_b32_e32 v5, s6
7736 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
7737 ; GFX9-NEXT: .LBB123_1: ; %atomicrmw.start
7738 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
7739 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7740 ; GFX9-NEXT: v_mov_b32_e32 v9, v1
7741 ; GFX9-NEXT: v_mov_b32_e32 v8, v0
7742 ; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
7743 ; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
7744 ; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
7745 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
7746 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7747 ; GFX9-NEXT: buffer_wbinvl1_vol
7748 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
7749 ; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
7750 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
7751 ; GFX9-NEXT: s_cbranch_execnz .LBB123_1
7752 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
7753 ; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
7754 ; GFX9-NEXT: s_setpc_b64 s[30:31]
7755 %result = atomicrmw min ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
7756 ret i64 %result
7757 }
7759 define amdgpu_gfx i64 @flat_atomic_min_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
7760 ; GFX7-LABEL: flat_atomic_min_i64_ret_offset_scalar:
7761 ; GFX7: ; %bb.0:
7762 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7763 ; GFX7-NEXT: s_add_u32 s34, s4, 32
7764 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
7765 ; GFX7-NEXT: s_add_u32 s36, s4, 36
7766 ; GFX7-NEXT: s_addc_u32 s37, s5, 0
7767 ; GFX7-NEXT: v_mov_b32_e32 v0, s36
7768 ; GFX7-NEXT: v_mov_b32_e32 v1, s37
7769 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
7770 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
7771 ; GFX7-NEXT: flat_load_dword v1, v[0:1]
7772 ; GFX7-NEXT: flat_load_dword v0, v[2:3]
7773 ; GFX7-NEXT: s_mov_b64 s[34:35], 0
7774 ; GFX7-NEXT: v_mov_b32_e32 v4, s7
7775 ; GFX7-NEXT: v_mov_b32_e32 v5, s6
7776 ; GFX7-NEXT: .LBB124_1: ; %atomicrmw.start
7777 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7778 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7779 ; GFX7-NEXT: v_mov_b32_e32 v9, v1
7780 ; GFX7-NEXT: v_mov_b32_e32 v8, v0
7781 ; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
7782 ; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
7783 ; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
7784 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
7785 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7786 ; GFX7-NEXT: buffer_wbinvl1_vol
7787 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
7788 ; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
7789 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
7790 ; GFX7-NEXT: s_cbranch_execnz .LBB124_1
7791 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
7792 ; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
7793 ; GFX7-NEXT: s_setpc_b64 s[30:31]
7794 ;
7795 ; GFX8-LABEL: flat_atomic_min_i64_ret_offset_scalar:
7796 ; GFX8: ; %bb.0:
7797 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7798 ; GFX8-NEXT: s_add_u32 s34, s4, 32
7799 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
7800 ; GFX8-NEXT: s_add_u32 s36, s4, 36
7801 ; GFX8-NEXT: s_addc_u32 s37, s5, 0
7802 ; GFX8-NEXT: v_mov_b32_e32 v0, s36
7803 ; GFX8-NEXT: v_mov_b32_e32 v1, s37
7804 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
7805 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
7806 ; GFX8-NEXT: flat_load_dword v1, v[0:1]
7807 ; GFX8-NEXT: flat_load_dword v0, v[2:3]
7808 ; GFX8-NEXT: s_mov_b64 s[34:35], 0
7809 ; GFX8-NEXT: v_mov_b32_e32 v4, s7
7810 ; GFX8-NEXT: v_mov_b32_e32 v5, s6
7811 ; GFX8-NEXT: .LBB124_1: ; %atomicrmw.start
7812 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7813 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7814 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
7815 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
7816 ; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
7817 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
7818 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
7819 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
7820 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7821 ; GFX8-NEXT: buffer_wbinvl1_vol
7822 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
7823 ; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
7824 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
7825 ; GFX8-NEXT: s_cbranch_execnz .LBB124_1
7826 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7827 ; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
7828 ; GFX8-NEXT: s_setpc_b64 s[30:31]
7829 ;
7830 ; GFX9-LABEL: flat_atomic_min_i64_ret_offset_scalar:
7831 ; GFX9: ; %bb.0:
7832 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
7833 ; GFX9-NEXT: v_mov_b32_e32 v0, s4
7834 ; GFX9-NEXT: v_mov_b32_e32 v1, s5
7835 ; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
7836 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
7837 ; GFX9-NEXT: s_mov_b64 s[34:35], 0
7838 ; GFX9-NEXT: v_mov_b32_e32 v4, s7
7839 ; GFX9-NEXT: v_mov_b32_e32 v5, s6
7840 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
7841 ; GFX9-NEXT: .LBB124_1: ; %atomicrmw.start
7842 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
7843 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7844 ; GFX9-NEXT: v_mov_b32_e32 v9, v1
7845 ; GFX9-NEXT: v_mov_b32_e32 v8, v0
7846 ; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
7847 ; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
7848 ; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
7849 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc
7850 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7851 ; GFX9-NEXT: buffer_wbinvl1_vol
7852 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
7853 ; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
7854 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
7855 ; GFX9-NEXT: s_cbranch_execnz .LBB124_1
7856 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
7857 ; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]
7858 ; GFX9-NEXT: s_setpc_b64 s[30:31]
7859 %gep = getelementptr i64, ptr %out, i64 4
7860 %result = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
7861 ret i64 %result
7862 }
7864 define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
7865 ; GFX7-LABEL: atomic_min_i64_addr64_offset:
7866 ; GFX7: ; %bb.0: ; %entry
7867 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
7868 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
7869 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7870 ; GFX7-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
7871 ; GFX7-NEXT: s_add_u32 s0, s0, s4
7872 ; GFX7-NEXT: s_addc_u32 s1, s1, s5
7873 ; GFX7-NEXT: s_add_u32 s0, s0, 32
7874 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
7875 ; GFX7-NEXT: v_mov_b32_e32 v5, s1
7876 ; GFX7-NEXT: v_mov_b32_e32 v4, s0
7877 ; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
7878 ; GFX7-NEXT: s_mov_b64 s[0:1], 0
7879 ; GFX7-NEXT: v_mov_b32_e32 v6, s3
7880 ; GFX7-NEXT: v_mov_b32_e32 v7, s2
7881 ; GFX7-NEXT: .LBB125_1: ; %atomicrmw.start
7882 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7883 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7884 ; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
7885 ; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
7886 ; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
7887 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
7888 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7889 ; GFX7-NEXT: buffer_wbinvl1_vol
7890 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
7891 ; GFX7-NEXT: v_mov_b32_e32 v3, v1
7892 ; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
7893 ; GFX7-NEXT: v_mov_b32_e32 v2, v0
7894 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
7895 ; GFX7-NEXT: s_cbranch_execnz .LBB125_1
7896 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
7897 ; GFX7-NEXT: s_endpgm
7898 ;
7899 ; GFX8-LABEL: atomic_min_i64_addr64_offset:
7900 ; GFX8: ; %bb.0: ; %entry
7901 ; GFX8-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
7902 ; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
7903 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
7904 ; GFX8-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
7905 ; GFX8-NEXT: s_add_u32 s0, s0, s4
7906 ; GFX8-NEXT: s_addc_u32 s1, s1, s5
7907 ; GFX8-NEXT: s_add_u32 s0, s0, 32
7908 ; GFX8-NEXT: s_addc_u32 s1, s1, 0
7909 ; GFX8-NEXT: v_mov_b32_e32 v5, s1
7910 ; GFX8-NEXT: v_mov_b32_e32 v4, s0
7911 ; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
7912 ; GFX8-NEXT: s_mov_b64 s[0:1], 0
7913 ; GFX8-NEXT: v_mov_b32_e32 v6, s3
7914 ; GFX8-NEXT: v_mov_b32_e32 v7, s2
7915 ; GFX8-NEXT: .LBB125_1: ; %atomicrmw.start
7916 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
7917 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7918 ; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
7919 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
7920 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
7921 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
7922 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7923 ; GFX8-NEXT: buffer_wbinvl1_vol
7924 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
7925 ; GFX8-NEXT: v_mov_b32_e32 v3, v1
7926 ; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
7927 ; GFX8-NEXT: v_mov_b32_e32 v2, v0
7928 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
7929 ; GFX8-NEXT: s_cbranch_execnz .LBB125_1
7930 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
7931 ; GFX8-NEXT: s_endpgm
7932 ;
7933 ; GFX9-LABEL: atomic_min_i64_addr64_offset:
7934 ; GFX9: ; %bb.0: ; %entry
7935 ; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
7936 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
7937 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
7938 ; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
7939 ; GFX9-NEXT: s_add_u32 s0, s0, s4
7940 ; GFX9-NEXT: s_addc_u32 s1, s1, s5
7941 ; GFX9-NEXT: v_mov_b32_e32 v5, s1
7942 ; GFX9-NEXT: v_mov_b32_e32 v4, s0
7943 ; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[4:5] offset:32
7944 ; GFX9-NEXT: s_mov_b64 s[0:1], 0
7945 ; GFX9-NEXT: v_mov_b32_e32 v6, s3
7946 ; GFX9-NEXT: v_mov_b32_e32 v7, s2
7947 ; GFX9-NEXT: .LBB125_1: ; %atomicrmw.start
7948 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
7949 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7950 ; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
7951 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
7952 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
7953 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
7954 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7955 ; GFX9-NEXT: buffer_wbinvl1_vol
7956 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
7957 ; GFX9-NEXT: v_mov_b32_e32 v3, v1
7958 ; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
7959 ; GFX9-NEXT: v_mov_b32_e32 v2, v0
7960 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
7961 ; GFX9-NEXT: s_cbranch_execnz .LBB125_1
7962 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
7963 ; GFX9-NEXT: s_endpgm
7964 entry:
7965 %ptr = getelementptr i64, ptr %out, i64 %index
7966 %gep = getelementptr i64, ptr %ptr, i64 4
7967 %tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
7968 ret void
7969 }
7971 define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
7972 ; GFX7-LABEL: atomic_min_i64_ret_addr64_offset:
7973 ; GFX7: ; %bb.0: ; %entry
7974 ; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
7975 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7976 ; GFX7-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
7977 ; GFX7-NEXT: s_add_u32 s0, s0, s6
7978 ; GFX7-NEXT: s_addc_u32 s1, s1, s7
7979 ; GFX7-NEXT: s_add_u32 s0, s0, 32
7980 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
7981 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
7982 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
7983 ; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
7984 ; GFX7-NEXT: s_mov_b64 s[0:1], 0
7985 ; GFX7-NEXT: v_mov_b32_e32 v4, s5
7986 ; GFX7-NEXT: v_mov_b32_e32 v5, s4
7987 ; GFX7-NEXT: .LBB126_1: ; %atomicrmw.start
7988 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
7989 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7990 ; GFX7-NEXT: v_mov_b32_e32 v9, v3
7991 ; GFX7-NEXT: v_mov_b32_e32 v8, v2
7992 ; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
7993 ; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
7994 ; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
7995 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
7996 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7997 ; GFX7-NEXT: buffer_wbinvl1_vol
7998 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
7999 ; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
8000 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
8001 ; GFX7-NEXT: s_cbranch_execnz .LBB126_1
8002 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
8003 ; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
8004 ; GFX7-NEXT: v_mov_b32_e32 v0, s2
8005 ; GFX7-NEXT: v_mov_b32_e32 v1, s3
8006 ; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
8007 ; GFX7-NEXT: s_endpgm
8008 ;
8009 ; GFX8-LABEL: atomic_min_i64_ret_addr64_offset:
8010 ; GFX8: ; %bb.0: ; %entry
8011 ; GFX8-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
8012 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
8013 ; GFX8-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
8014 ; GFX8-NEXT: s_add_u32 s0, s0, s6
8015 ; GFX8-NEXT: s_addc_u32 s1, s1, s7
8016 ; GFX8-NEXT: s_add_u32 s0, s0, 32
8017 ; GFX8-NEXT: s_addc_u32 s1, s1, 0
8018 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
8019 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
8020 ; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
8021 ; GFX8-NEXT: s_mov_b64 s[0:1], 0
8022 ; GFX8-NEXT: v_mov_b32_e32 v4, s5
8023 ; GFX8-NEXT: v_mov_b32_e32 v5, s4
8024 ; GFX8-NEXT: .LBB126_1: ; %atomicrmw.start
8025 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
8026 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8027 ; GFX8-NEXT: v_mov_b32_e32 v9, v3
8028 ; GFX8-NEXT: v_mov_b32_e32 v8, v2
8029 ; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
8030 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
8031 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
8032 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
8033 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8034 ; GFX8-NEXT: buffer_wbinvl1_vol
8035 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
8036 ; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
8037 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
8038 ; GFX8-NEXT: s_cbranch_execnz .LBB126_1
8039 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
8040 ; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
8041 ; GFX8-NEXT: v_mov_b32_e32 v0, s2
8042 ; GFX8-NEXT: v_mov_b32_e32 v1, s3
8043 ; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
8044 ; GFX8-NEXT: s_endpgm
8045 ;
8046 ; GFX9-LABEL: atomic_min_i64_ret_addr64_offset:
8047 ; GFX9: ; %bb.0: ; %entry
8048 ; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
8049 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
8050 ; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
8051 ; GFX9-NEXT: s_add_u32 s0, s8, s0
8052 ; GFX9-NEXT: s_addc_u32 s1, s9, s1
8053 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
8054 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
8055 ; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
8056 ; GFX9-NEXT: s_mov_b64 s[0:1], 0
8057 ; GFX9-NEXT: v_mov_b32_e32 v4, s13
8058 ; GFX9-NEXT: v_mov_b32_e32 v5, s12
8059 ; GFX9-NEXT: .LBB126_1: ; %atomicrmw.start
8060 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
8061 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8062 ; GFX9-NEXT: v_mov_b32_e32 v9, v3
8063 ; GFX9-NEXT: v_mov_b32_e32 v8, v2
8064 ; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9]
8065 ; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
8066 ; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
8067 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] offset:32 glc
8068 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8069 ; GFX9-NEXT: buffer_wbinvl1_vol
8070 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
8071 ; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
8072 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
8073 ; GFX9-NEXT: s_cbranch_execnz .LBB126_1
8074 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
8075 ; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
8076 ; GFX9-NEXT: v_mov_b32_e32 v0, s10
8077 ; GFX9-NEXT: v_mov_b32_e32 v1, s11
8078 ; GFX9-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
8079 ; GFX9-NEXT: s_endpgm
8080 entry:
8081 %ptr = getelementptr i64, ptr %out, i64 %index
8082 %gep = getelementptr i64, ptr %ptr, i64 4
8083 %tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
8084 store i64 %tmp0, ptr %out2
8085 ret void
8086 }
8088 define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
8089 ; GFX7-LABEL: atomic_min_i64:
8090 ; GFX7: ; %bb.0: ; %entry
8091 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
8092 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
8093 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8094 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
8095 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
8096 ; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
8097 ; GFX7-NEXT: v_mov_b32_e32 v5, s1
8098 ; GFX7-NEXT: v_mov_b32_e32 v6, s3
8099 ; GFX7-NEXT: v_mov_b32_e32 v7, s2
8100 ; GFX7-NEXT: v_mov_b32_e32 v4, s0
8101 ; GFX7-NEXT: .LBB127_1: ; %atomicrmw.start
8102 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
8103 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8104 ; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
8105 ; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
8106 ; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
8107 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
8108 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8109 ; GFX7-NEXT: buffer_wbinvl1_vol
8110 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
8111 ; GFX7-NEXT: v_mov_b32_e32 v3, v1
8112 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8113 ; GFX7-NEXT: v_mov_b32_e32 v2, v0
8114 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
8115 ; GFX7-NEXT: s_cbranch_execnz .LBB127_1
8116 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
8117 ; GFX7-NEXT: s_endpgm
8118 ;
8119 ; GFX8-LABEL: atomic_min_i64:
8120 ; GFX8: ; %bb.0: ; %entry
8121 ; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
8122 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
8123 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
8124 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
8125 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
8126 ; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
8127 ; GFX8-NEXT: v_mov_b32_e32 v5, s1
8128 ; GFX8-NEXT: v_mov_b32_e32 v6, s3
8129 ; GFX8-NEXT: v_mov_b32_e32 v7, s2
8130 ; GFX8-NEXT: v_mov_b32_e32 v4, s0
8131 ; GFX8-NEXT: .LBB127_1: ; %atomicrmw.start
8132 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
8133 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8134 ; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
8135 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
8136 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
8137 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
8138 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8139 ; GFX8-NEXT: buffer_wbinvl1_vol
8140 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
8141 ; GFX8-NEXT: v_mov_b32_e32 v3, v1
8142 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8143 ; GFX8-NEXT: v_mov_b32_e32 v2, v0
8144 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
8145 ; GFX8-NEXT: s_cbranch_execnz .LBB127_1
8146 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
8147 ; GFX8-NEXT: s_endpgm
8148 ;
8149 ; GFX9-LABEL: atomic_min_i64:
8150 ; GFX9: ; %bb.0: ; %entry
8151 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
8152 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
8153 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
8154 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
8155 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
8156 ; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
8157 ; GFX9-NEXT: v_mov_b32_e32 v5, s1
8158 ; GFX9-NEXT: v_mov_b32_e32 v6, s3
8159 ; GFX9-NEXT: v_mov_b32_e32 v7, s2
8160 ; GFX9-NEXT: v_mov_b32_e32 v4, s0
8161 ; GFX9-NEXT: .LBB127_1: ; %atomicrmw.start
8162 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
8163 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8164 ; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
8165 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
8166 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
8167 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
8168 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8169 ; GFX9-NEXT: buffer_wbinvl1_vol
8170 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
8171 ; GFX9-NEXT: v_mov_b32_e32 v3, v1
8172 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8173 ; GFX9-NEXT: v_mov_b32_e32 v2, v0
8174 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
8175 ; GFX9-NEXT: s_cbranch_execnz .LBB127_1
8176 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
8177 ; GFX9-NEXT: s_endpgm
8178 entry:
8179 %tmp0 = atomicrmw min ptr %out, i64 %in seq_cst, !noalias.addrspace !1
8180 ret void
8181 }
8183 define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
8184 ; GFX7-LABEL: atomic_min_i64_ret_addr64:
8185 ; GFX7: ; %bb.0: ; %entry
8186 ; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
8187 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8188 ; GFX7-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
8189 ; GFX7-NEXT: s_add_u32 s0, s0, s6
8190 ; GFX7-NEXT: s_addc_u32 s1, s1, s7
8191 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
8192 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
8193 ; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
8194 ; GFX7-NEXT: s_mov_b64 s[0:1], 0
8195 ; GFX7-NEXT: v_mov_b32_e32 v4, s5
8196 ; GFX7-NEXT: v_mov_b32_e32 v5, s4
8197 ; GFX7-NEXT: .LBB128_1: ; %atomicrmw.start
8198 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
8199 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8200 ; GFX7-NEXT: v_mov_b32_e32 v9, v3
8201 ; GFX7-NEXT: v_mov_b32_e32 v8, v2
8202 ; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
8203 ; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
8204 ; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
8205 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
8206 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8207 ; GFX7-NEXT: buffer_wbinvl1_vol
8208 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
8209 ; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
8210 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
8211 ; GFX7-NEXT: s_cbranch_execnz .LBB128_1
8212 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
8213 ; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
8214 ; GFX7-NEXT: v_mov_b32_e32 v0, s2
8215 ; GFX7-NEXT: v_mov_b32_e32 v1, s3
8216 ; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
8217 ; GFX7-NEXT: s_endpgm
8218 ;
8219 ; GFX8-LABEL: atomic_min_i64_ret_addr64:
8220 ; GFX8: ; %bb.0: ; %entry
8221 ; GFX8-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
8222 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
8223 ; GFX8-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
8224 ; GFX8-NEXT: s_add_u32 s0, s0, s6
8225 ; GFX8-NEXT: s_addc_u32 s1, s1, s7
8226 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
8227 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
8228 ; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
8229 ; GFX8-NEXT: s_mov_b64 s[0:1], 0
8230 ; GFX8-NEXT: v_mov_b32_e32 v4, s5
8231 ; GFX8-NEXT: v_mov_b32_e32 v5, s4
8232 ; GFX8-NEXT: .LBB128_1: ; %atomicrmw.start
8233 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
8234 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8235 ; GFX8-NEXT: v_mov_b32_e32 v9, v3
8236 ; GFX8-NEXT: v_mov_b32_e32 v8, v2
8237 ; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
8238 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
8239 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
8240 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
8241 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8242 ; GFX8-NEXT: buffer_wbinvl1_vol
8243 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
8244 ; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
8245 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
8246 ; GFX8-NEXT: s_cbranch_execnz .LBB128_1
8247 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
8248 ; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
8249 ; GFX8-NEXT: v_mov_b32_e32 v0, s2
8250 ; GFX8-NEXT: v_mov_b32_e32 v1, s3
8251 ; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
8252 ; GFX8-NEXT: s_endpgm
8253 ;
8254 ; GFX9-LABEL: atomic_min_i64_ret_addr64:
8255 ; GFX9: ; %bb.0: ; %entry
8256 ; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
8257 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
8258 ; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
8259 ; GFX9-NEXT: s_add_u32 s0, s8, s0
8260 ; GFX9-NEXT: s_addc_u32 s1, s9, s1
8261 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
8262 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
8263 ; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
8264 ; GFX9-NEXT: s_mov_b64 s[0:1], 0
8265 ; GFX9-NEXT: v_mov_b32_e32 v4, s13
8266 ; GFX9-NEXT: v_mov_b32_e32 v5, s12
8267 ; GFX9-NEXT: .LBB128_1: ; %atomicrmw.start
8268 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
8269 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8270 ; GFX9-NEXT: v_mov_b32_e32 v9, v3
8271 ; GFX9-NEXT: v_mov_b32_e32 v8, v2
8272 ; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9]
8273 ; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
8274 ; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
8275 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
8276 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8277 ; GFX9-NEXT: buffer_wbinvl1_vol
8278 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
8279 ; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
8280 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
8281 ; GFX9-NEXT: s_cbranch_execnz .LBB128_1
8282 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
8283 ; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
8284 ; GFX9-NEXT: v_mov_b32_e32 v0, s10
8285 ; GFX9-NEXT: v_mov_b32_e32 v1, s11
8286 ; GFX9-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
8287 ; GFX9-NEXT: s_endpgm
8288 entry:
8289 %ptr = getelementptr i64, ptr %out, i64 %index
8290 %tmp0 = atomicrmw min ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1, !noalias.addrspace !1
8291 store i64 %tmp0, ptr %out2
8292 ret void
8293 }
8295 define void @flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
8296 ; GFX7-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory:
8297 ; GFX7: ; %bb.0:
8298 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8299 ; GFX7-NEXT: v_add_i32_e32 v8, vcc, 32, v0
8300 ; GFX7-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
8301 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v0
8302 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8303 ; GFX7-NEXT: flat_load_dword v7, v[0:1]
8304 ; GFX7-NEXT: flat_load_dword v6, v[8:9]
8305 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
8306 ; GFX7-NEXT: .LBB129_1: ; %atomicrmw.start
8307 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
8308 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8309 ; GFX7-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
8310 ; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
8311 ; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
8312 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
8313 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8314 ; GFX7-NEXT: buffer_wbinvl1_vol
8315 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
8316 ; GFX7-NEXT: v_mov_b32_e32 v7, v1
8317 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8318 ; GFX7-NEXT: v_mov_b32_e32 v6, v0
8319 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
8320 ; GFX7-NEXT: s_cbranch_execnz .LBB129_1
8321 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
8322 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
8323 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8324 ;
8325 ; GFX8-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory:
8326 ; GFX8: ; %bb.0:
8327 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8328 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 32, v0
8329 ; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
8330 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v0
8331 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8332 ; GFX8-NEXT: flat_load_dword v7, v[0:1]
8333 ; GFX8-NEXT: flat_load_dword v6, v[8:9]
8334 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
8335 ; GFX8-NEXT: .LBB129_1: ; %atomicrmw.start
8336 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
8337 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8338 ; GFX8-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
8339 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
8340 ; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
8341 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
8342 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8343 ; GFX8-NEXT: buffer_wbinvl1_vol
8344 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
8345 ; GFX8-NEXT: v_mov_b32_e32 v7, v1
8346 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8347 ; GFX8-NEXT: v_mov_b32_e32 v6, v0
8348 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
8349 ; GFX8-NEXT: s_cbranch_execnz .LBB129_1
8350 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
8351 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
8352 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8353 ;
8354 ; GFX9-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory:
8355 ; GFX9: ; %bb.0:
8356 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8357 ; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
8358 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
8359 ; GFX9-NEXT: .LBB129_1: ; %atomicrmw.start
8360 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
8361 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8362 ; GFX9-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
8363 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
8364 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
8365 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
8366 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8367 ; GFX9-NEXT: buffer_wbinvl1_vol
8368 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
8369 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
8370 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8371 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
8372 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
8373 ; GFX9-NEXT: s_cbranch_execnz .LBB129_1
8374 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
8375 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
8376 ; GFX9-NEXT: s_setpc_b64 s[30:31]
8377 %gep = getelementptr i64, ptr %out, i64 4
8378 %tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
8379 ret void
8380 }
8382 define i64 @flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
8383 ; GFX7-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:
8384 ; GFX7: ; %bb.0:
8385 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8386 ; GFX7-NEXT: v_add_i32_e32 v4, vcc, 32, v0
8387 ; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
8388 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v0
8389 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8390 ; GFX7-NEXT: flat_load_dword v1, v[0:1]
8391 ; GFX7-NEXT: flat_load_dword v0, v[4:5]
8392 ; GFX7-NEXT: s_mov_b64 s[4:5], 0
8393 ; GFX7-NEXT: .LBB130_1: ; %atomicrmw.start
8394 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
8395 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8396 ; GFX7-NEXT: v_mov_b32_e32 v9, v1
8397 ; GFX7-NEXT: v_mov_b32_e32 v8, v0
8398 ; GFX7-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]
8399 ; GFX7-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
8400 ; GFX7-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
8401 ; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
8402 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8403 ; GFX7-NEXT: buffer_wbinvl1_vol
8404 ; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
8405 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8406 ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
8407 ; GFX7-NEXT: s_cbranch_execnz .LBB130_1
8408 ; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
8409 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
8410 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8411 ;
8412 ; GFX8-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:
8413 ; GFX8: ; %bb.0:
8414 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8415 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 32, v0
8416 ; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
8417 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v0
8418 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8419 ; GFX8-NEXT: flat_load_dword v1, v[0:1]
8420 ; GFX8-NEXT: flat_load_dword v0, v[4:5]
8421 ; GFX8-NEXT: s_mov_b64 s[4:5], 0
8422 ; GFX8-NEXT: .LBB130_1: ; %atomicrmw.start
8423 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
8424 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8425 ; GFX8-NEXT: v_mov_b32_e32 v9, v1
8426 ; GFX8-NEXT: v_mov_b32_e32 v8, v0
8427 ; GFX8-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]
8428 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
8429 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
8430 ; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
8431 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8432 ; GFX8-NEXT: buffer_wbinvl1_vol
8433 ; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
8434 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8435 ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
8436 ; GFX8-NEXT: s_cbranch_execnz .LBB130_1
8437 ; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
8438 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
8439 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8440 ;
8441 ; GFX9-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:
8442 ; GFX9: ; %bb.0:
8443 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8444 ; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
8445 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
8446 ; GFX9-NEXT: .LBB130_1: ; %atomicrmw.start
8447 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
8448 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8449 ; GFX9-NEXT: v_mov_b32_e32 v7, v5
8450 ; GFX9-NEXT: v_mov_b32_e32 v6, v4
8451 ; GFX9-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
8452 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
8453 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
8454 ; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
8455 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8456 ; GFX9-NEXT: buffer_wbinvl1_vol
8457 ; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
8458 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
8459 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
8460 ; GFX9-NEXT: s_cbranch_execnz .LBB130_1
8461 ; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
8462 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
8463 ; GFX9-NEXT: v_mov_b32_e32 v0, v4
8464 ; GFX9-NEXT: v_mov_b32_e32 v1, v5
8465 ; GFX9-NEXT: s_setpc_b64 s[30:31]
8466 %gep = getelementptr i64, ptr %out, i64 4
8467 %result = atomicrmw min ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
8468 ret i64 %result
8469 }
8471 ; ---------------------------------------------------------------------
8472 ; atomicrmw uinc_wrap
8473 ; ---------------------------------------------------------------------
8475 define void @flat_atomic_uinc_wrap_i64_noret(ptr %ptr, i64 %in) {
8476 ; GFX7-LABEL: flat_atomic_uinc_wrap_i64_noret:
8477 ; GFX7: ; %bb.0:
8478 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8479 ; GFX7-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
8480 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8481 ; GFX7-NEXT: buffer_wbinvl1_vol
8482 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8483 ;
8484 ; GFX8-LABEL: flat_atomic_uinc_wrap_i64_noret:
8485 ; GFX8: ; %bb.0:
8486 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8487 ; GFX8-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
8488 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8489 ; GFX8-NEXT: buffer_wbinvl1_vol
8490 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8491 ;
8492 ; GFX9-LABEL: flat_atomic_uinc_wrap_i64_noret:
8493 ; GFX9: ; %bb.0:
8494 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8495 ; GFX9-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
8496 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8497 ; GFX9-NEXT: buffer_wbinvl1_vol
8498 ; GFX9-NEXT: s_setpc_b64 s[30:31]
8499 %tmp0 = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
8500 ret void
8501 }
8503 define void @flat_atomic_uinc_wrap_i64_noret_offset(ptr %out, i64 %in) {
8504 ; GFX7-LABEL: flat_atomic_uinc_wrap_i64_noret_offset:
8505 ; GFX7: ; %bb.0:
8506 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8507 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
8508 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8509 ; GFX7-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
8510 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8511 ; GFX7-NEXT: buffer_wbinvl1_vol
8512 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8513 ;
8514 ; GFX8-LABEL: flat_atomic_uinc_wrap_i64_noret_offset:
8515 ; GFX8: ; %bb.0:
8516 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8517 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
8518 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8519 ; GFX8-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
8520 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8521 ; GFX8-NEXT: buffer_wbinvl1_vol
8522 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8523 ;
8524 ; GFX9-LABEL: flat_atomic_uinc_wrap_i64_noret_offset:
8525 ; GFX9: ; %bb.0:
8526 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8527 ; GFX9-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] offset:32
8528 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8529 ; GFX9-NEXT: buffer_wbinvl1_vol
8530 ; GFX9-NEXT: s_setpc_b64 s[30:31]
8531 %gep = getelementptr i64, ptr %out, i64 4
8532 %tmp0 = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
8533 ret void
8534 }
8536 define i64 @flat_atomic_uinc_wrap_i64_ret(ptr %ptr, i64 %in) {
8537 ; GFX7-LABEL: flat_atomic_uinc_wrap_i64_ret:
8538 ; GFX7: ; %bb.0:
8539 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8540 ; GFX7-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
8541 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8542 ; GFX7-NEXT: buffer_wbinvl1_vol
8543 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8544 ;
8545 ; GFX8-LABEL: flat_atomic_uinc_wrap_i64_ret:
8546 ; GFX8: ; %bb.0:
8547 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8548 ; GFX8-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
8549 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8550 ; GFX8-NEXT: buffer_wbinvl1_vol
8551 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8552 ;
8553 ; GFX9-LABEL: flat_atomic_uinc_wrap_i64_ret:
8554 ; GFX9: ; %bb.0:
8555 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8556 ; GFX9-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
8557 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8558 ; GFX9-NEXT: buffer_wbinvl1_vol
8559 ; GFX9-NEXT: s_setpc_b64 s[30:31]
8560 %result = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
8561 ret i64 %result
8562 }
8564 define i64 @flat_atomic_uinc_wrap_i64_ret_offset(ptr %out, i64 %in) {
8565 ; GFX7-LABEL: flat_atomic_uinc_wrap_i64_ret_offset:
8566 ; GFX7: ; %bb.0:
8567 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8568 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
8569 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8570 ; GFX7-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
8571 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8572 ; GFX7-NEXT: buffer_wbinvl1_vol
8573 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8574 ;
8575 ; GFX8-LABEL: flat_atomic_uinc_wrap_i64_ret_offset:
8576 ; GFX8: ; %bb.0:
8577 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8578 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
8579 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8580 ; GFX8-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
8581 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8582 ; GFX8-NEXT: buffer_wbinvl1_vol
8583 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8584 ;
8585 ; GFX9-LABEL: flat_atomic_uinc_wrap_i64_ret_offset:
8586 ; GFX9: ; %bb.0:
8587 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8588 ; GFX9-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
8589 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8590 ; GFX9-NEXT: buffer_wbinvl1_vol
8591 ; GFX9-NEXT: s_setpc_b64 s[30:31]
8592 %gep = getelementptr i64, ptr %out, i64 4
8593 %result = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
8594 ret i64 %result
8595 }
8597 define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
8598 ; GFX7-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar:
8599 ; GFX7: ; %bb.0:
8600 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8601 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
8602 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
8603 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
8604 ; GFX7-NEXT: v_mov_b32_e32 v3, s5
8605 ; GFX7-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
8606 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8607 ; GFX7-NEXT: buffer_wbinvl1_vol
8608 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8609 ;
8610 ; GFX8-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar:
8611 ; GFX8: ; %bb.0:
8612 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8613 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
8614 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
8615 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
8616 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
8617 ; GFX8-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
8618 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8619 ; GFX8-NEXT: buffer_wbinvl1_vol
8620 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8621 ;
8622 ; GFX9-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar:
8623 ; GFX9: ; %bb.0:
8624 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8625 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
8626 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
8627 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
8628 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
8629 ; GFX9-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
8630 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8631 ; GFX9-NEXT: buffer_wbinvl1_vol
8632 ; GFX9-NEXT: s_setpc_b64 s[30:31]
8633 %tmp0 = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
8634 ret void
8635 }
8637 define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
8638 ; GFX7-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar:
8639 ; GFX7: ; %bb.0:
8640 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8641 ; GFX7-NEXT: s_add_u32 s34, s4, 32
8642 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
8643 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
8644 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
8645 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
8646 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
8647 ; GFX7-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
8648 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8649 ; GFX7-NEXT: buffer_wbinvl1_vol
8650 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8651 ;
8652 ; GFX8-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar:
8653 ; GFX8: ; %bb.0:
8654 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8655 ; GFX8-NEXT: s_add_u32 s34, s4, 32
8656 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
8657 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
8658 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
8659 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
8660 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
8661 ; GFX8-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
8662 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8663 ; GFX8-NEXT: buffer_wbinvl1_vol
8664 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8665 ;
8666 ; GFX9-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar:
8667 ; GFX9: ; %bb.0:
8668 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8669 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
8670 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
8671 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
8672 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
8673 ; GFX9-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] offset:32
8674 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8675 ; GFX9-NEXT: buffer_wbinvl1_vol
8676 ; GFX9-NEXT: s_setpc_b64 s[30:31]
8677 %gep = getelementptr i64, ptr %out, i64 4
8678 %tmp0 = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
8679 ret void
8680 }
8682 define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
8683 ; GFX7-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar:
8684 ; GFX7: ; %bb.0:
8685 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8686 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
8687 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
8688 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
8689 ; GFX7-NEXT: v_mov_b32_e32 v3, s5
8690 ; GFX7-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
8691 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8692 ; GFX7-NEXT: buffer_wbinvl1_vol
8693 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8694 ;
8695 ; GFX8-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar:
8696 ; GFX8: ; %bb.0:
8697 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8698 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
8699 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
8700 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
8701 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
8702 ; GFX8-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
8703 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8704 ; GFX8-NEXT: buffer_wbinvl1_vol
8705 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8706 ;
8707 ; GFX9-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar:
8708 ; GFX9: ; %bb.0:
8709 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8710 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
8711 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
8712 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
8713 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
8714 ; GFX9-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
8715 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8716 ; GFX9-NEXT: buffer_wbinvl1_vol
8717 ; GFX9-NEXT: s_setpc_b64 s[30:31]
8718 %result = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
8719 ret i64 %result
8720 }
8722 define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
8723 ; GFX7-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar:
8724 ; GFX7: ; %bb.0:
8725 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8726 ; GFX7-NEXT: s_add_u32 s34, s4, 32
8727 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
8728 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
8729 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
8730 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
8731 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
8732 ; GFX7-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
8733 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8734 ; GFX7-NEXT: buffer_wbinvl1_vol
8735 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8736 ;
8737 ; GFX8-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar:
8738 ; GFX8: ; %bb.0:
8739 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8740 ; GFX8-NEXT: s_add_u32 s34, s4, 32
8741 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
8742 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
8743 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
8744 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
8745 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
8746 ; GFX8-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
8747 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8748 ; GFX8-NEXT: buffer_wbinvl1_vol
8749 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8750 ;
8751 ; GFX9-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar:
8752 ; GFX9: ; %bb.0:
8753 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8754 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
8755 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
8756 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
8757 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
8758 ; GFX9-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] offset:32 glc
8759 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8760 ; GFX9-NEXT: buffer_wbinvl1_vol
8761 ; GFX9-NEXT: s_setpc_b64 s[30:31]
8762 %gep = getelementptr i64, ptr %out, i64 4
8763 %result = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
8764 ret i64 %result
8765 }
8767 define void @flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
8768 ; GFX7-LABEL: flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory:
8769 ; GFX7: ; %bb.0:
8770 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8771 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
8772 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8773 ; GFX7-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
8774 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8775 ; GFX7-NEXT: buffer_wbinvl1_vol
8776 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8777 ;
8778 ; GFX8-LABEL: flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory:
8779 ; GFX8: ; %bb.0:
8780 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8781 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
8782 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8783 ; GFX8-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
8784 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8785 ; GFX8-NEXT: buffer_wbinvl1_vol
8786 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8787 ;
8788 ; GFX9-LABEL: flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory:
8789 ; GFX9: ; %bb.0:
8790 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8791 ; GFX9-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] offset:32
8792 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8793 ; GFX9-NEXT: buffer_wbinvl1_vol
8794 ; GFX9-NEXT: s_setpc_b64 s[30:31]
8795 %gep = getelementptr i64, ptr %out, i64 4
8796 %tmp0 = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
8797 ret void
8798 }
8800 define i64 @flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
8801 ; GFX7-LABEL: flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory:
8802 ; GFX7: ; %bb.0:
8803 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8804 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
8805 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8806 ; GFX7-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
8807 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8808 ; GFX7-NEXT: buffer_wbinvl1_vol
8809 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8810 ;
8811 ; GFX8-LABEL: flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory:
8812 ; GFX8: ; %bb.0:
8813 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8814 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
8815 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8816 ; GFX8-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
8817 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8818 ; GFX8-NEXT: buffer_wbinvl1_vol
8819 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8820 ;
8821 ; GFX9-LABEL: flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory:
8822 ; GFX9: ; %bb.0:
8823 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8824 ; GFX9-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
8825 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8826 ; GFX9-NEXT: buffer_wbinvl1_vol
8827 ; GFX9-NEXT: s_setpc_b64 s[30:31]
8828 %gep = getelementptr i64, ptr %out, i64 4
8829 %result = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
8830 ret i64 %result
8831 }
8833 ; ---------------------------------------------------------------------
8834 ; atomicrmw udec_wrap
8835 ; ---------------------------------------------------------------------
8837 define void @flat_atomic_udec_wrap_i64_noret(ptr %ptr, i64 %in) {
8838 ; GFX7-LABEL: flat_atomic_udec_wrap_i64_noret:
8839 ; GFX7: ; %bb.0:
8840 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8841 ; GFX7-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
8842 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8843 ; GFX7-NEXT: buffer_wbinvl1_vol
8844 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8845 ;
8846 ; GFX8-LABEL: flat_atomic_udec_wrap_i64_noret:
8847 ; GFX8: ; %bb.0:
8848 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8849 ; GFX8-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
8850 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8851 ; GFX8-NEXT: buffer_wbinvl1_vol
8852 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8853 ;
8854 ; GFX9-LABEL: flat_atomic_udec_wrap_i64_noret:
8855 ; GFX9: ; %bb.0:
8856 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8857 ; GFX9-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
8858 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8859 ; GFX9-NEXT: buffer_wbinvl1_vol
8860 ; GFX9-NEXT: s_setpc_b64 s[30:31]
8861 %tmp0 = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
8862 ret void
8863 }
8865 define void @flat_atomic_udec_wrap_i64_noret_offset(ptr %out, i64 %in) {
8866 ; GFX7-LABEL: flat_atomic_udec_wrap_i64_noret_offset:
8867 ; GFX7: ; %bb.0:
8868 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8869 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
8870 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8871 ; GFX7-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
8872 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8873 ; GFX7-NEXT: buffer_wbinvl1_vol
8874 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8875 ;
8876 ; GFX8-LABEL: flat_atomic_udec_wrap_i64_noret_offset:
8877 ; GFX8: ; %bb.0:
8878 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8879 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
8880 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8881 ; GFX8-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
8882 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8883 ; GFX8-NEXT: buffer_wbinvl1_vol
8884 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8885 ;
8886 ; GFX9-LABEL: flat_atomic_udec_wrap_i64_noret_offset:
8887 ; GFX9: ; %bb.0:
8888 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8889 ; GFX9-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] offset:32
8890 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8891 ; GFX9-NEXT: buffer_wbinvl1_vol
8892 ; GFX9-NEXT: s_setpc_b64 s[30:31]
8893 %gep = getelementptr i64, ptr %out, i64 4
8894 %tmp0 = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
8895 ret void
8896 }
8898 define i64 @flat_atomic_udec_wrap_i64_ret(ptr %ptr, i64 %in) {
8899 ; GFX7-LABEL: flat_atomic_udec_wrap_i64_ret:
8900 ; GFX7: ; %bb.0:
8901 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8902 ; GFX7-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
8903 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8904 ; GFX7-NEXT: buffer_wbinvl1_vol
8905 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8906 ;
8907 ; GFX8-LABEL: flat_atomic_udec_wrap_i64_ret:
8908 ; GFX8: ; %bb.0:
8909 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8910 ; GFX8-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
8911 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8912 ; GFX8-NEXT: buffer_wbinvl1_vol
8913 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8914 ;
8915 ; GFX9-LABEL: flat_atomic_udec_wrap_i64_ret:
8916 ; GFX9: ; %bb.0:
8917 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8918 ; GFX9-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
8919 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8920 ; GFX9-NEXT: buffer_wbinvl1_vol
8921 ; GFX9-NEXT: s_setpc_b64 s[30:31]
8922 %result = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
8923 ret i64 %result
8924 }
8926 define i64 @flat_atomic_udec_wrap_i64_ret_offset(ptr %out, i64 %in) {
8927 ; GFX7-LABEL: flat_atomic_udec_wrap_i64_ret_offset:
8928 ; GFX7: ; %bb.0:
8929 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8930 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
8931 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8932 ; GFX7-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
8933 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8934 ; GFX7-NEXT: buffer_wbinvl1_vol
8935 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8936 ;
8937 ; GFX8-LABEL: flat_atomic_udec_wrap_i64_ret_offset:
8938 ; GFX8: ; %bb.0:
8939 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8940 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
8941 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
8942 ; GFX8-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
8943 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8944 ; GFX8-NEXT: buffer_wbinvl1_vol
8945 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8946 ;
8947 ; GFX9-LABEL: flat_atomic_udec_wrap_i64_ret_offset:
8948 ; GFX9: ; %bb.0:
8949 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8950 ; GFX9-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
8951 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8952 ; GFX9-NEXT: buffer_wbinvl1_vol
8953 ; GFX9-NEXT: s_setpc_b64 s[30:31]
8954 %gep = getelementptr i64, ptr %out, i64 4
8955 %result = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
8956 ret i64 %result
8957 }
8959 define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
8960 ; GFX7-LABEL: flat_atomic_udec_wrap_i64_noret_scalar:
8961 ; GFX7: ; %bb.0:
8962 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8963 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
8964 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
8965 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
8966 ; GFX7-NEXT: v_mov_b32_e32 v3, s5
8967 ; GFX7-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
8968 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8969 ; GFX7-NEXT: buffer_wbinvl1_vol
8970 ; GFX7-NEXT: s_setpc_b64 s[30:31]
8971 ;
8972 ; GFX8-LABEL: flat_atomic_udec_wrap_i64_noret_scalar:
8973 ; GFX8: ; %bb.0:
8974 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8975 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
8976 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
8977 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
8978 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
8979 ; GFX8-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
8980 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8981 ; GFX8-NEXT: buffer_wbinvl1_vol
8982 ; GFX8-NEXT: s_setpc_b64 s[30:31]
8983 ;
8984 ; GFX9-LABEL: flat_atomic_udec_wrap_i64_noret_scalar:
8985 ; GFX9: ; %bb.0:
8986 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
8987 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
8988 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
8989 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
8990 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
8991 ; GFX9-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
8992 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8993 ; GFX9-NEXT: buffer_wbinvl1_vol
8994 ; GFX9-NEXT: s_setpc_b64 s[30:31]
8995 %tmp0 = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
8996 ret void
8997 }
8999 define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
9000 ; GFX7-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar:
9001 ; GFX7: ; %bb.0:
9002 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9003 ; GFX7-NEXT: s_add_u32 s34, s4, 32
9004 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
9005 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
9006 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
9007 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
9008 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
9009 ; GFX7-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
9010 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9011 ; GFX7-NEXT: buffer_wbinvl1_vol
9012 ; GFX7-NEXT: s_setpc_b64 s[30:31]
9013 ;
9014 ; GFX8-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar:
9015 ; GFX8: ; %bb.0:
9016 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9017 ; GFX8-NEXT: s_add_u32 s34, s4, 32
9018 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
9019 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
9020 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
9021 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
9022 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
9023 ; GFX8-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
9024 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9025 ; GFX8-NEXT: buffer_wbinvl1_vol
9026 ; GFX8-NEXT: s_setpc_b64 s[30:31]
9027 ;
9028 ; GFX9-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar:
9029 ; GFX9: ; %bb.0:
9030 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9031 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
9032 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
9033 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
9034 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
9035 ; GFX9-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1] offset:32
9036 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9037 ; GFX9-NEXT: buffer_wbinvl1_vol
9038 ; GFX9-NEXT: s_setpc_b64 s[30:31]
9039 %gep = getelementptr i64, ptr %out, i64 4
9040 %tmp0 = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
9041 ret void
9042 }
9044 define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
9045 ; GFX7-LABEL: flat_atomic_udec_wrap_i64_ret_scalar:
9046 ; GFX7: ; %bb.0:
9047 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9048 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
9049 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
9050 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
9051 ; GFX7-NEXT: v_mov_b32_e32 v3, s5
9052 ; GFX7-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
9053 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9054 ; GFX7-NEXT: buffer_wbinvl1_vol
9055 ; GFX7-NEXT: s_setpc_b64 s[30:31]
9056 ;
9057 ; GFX8-LABEL: flat_atomic_udec_wrap_i64_ret_scalar:
9058 ; GFX8: ; %bb.0:
9059 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9060 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
9061 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
9062 ; GFX8-NEXT: v_mov_b32_e32 v2, s4
9063 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
9064 ; GFX8-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
9065 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9066 ; GFX8-NEXT: buffer_wbinvl1_vol
9067 ; GFX8-NEXT: s_setpc_b64 s[30:31]
9068 ;
9069 ; GFX9-LABEL: flat_atomic_udec_wrap_i64_ret_scalar:
9070 ; GFX9: ; %bb.0:
9071 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9072 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
9073 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
9074 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
9075 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
9076 ; GFX9-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
9077 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9078 ; GFX9-NEXT: buffer_wbinvl1_vol
9079 ; GFX9-NEXT: s_setpc_b64 s[30:31]
9080 %result = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
9081 ret i64 %result
9082 }
9084 define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
9085 ; GFX7-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar:
9086 ; GFX7: ; %bb.0:
9087 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9088 ; GFX7-NEXT: s_add_u32 s34, s4, 32
9089 ; GFX7-NEXT: s_addc_u32 s35, s5, 0
9090 ; GFX7-NEXT: v_mov_b32_e32 v2, s34
9091 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
9092 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
9093 ; GFX7-NEXT: v_mov_b32_e32 v3, s35
9094 ; GFX7-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
9095 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9096 ; GFX7-NEXT: buffer_wbinvl1_vol
9097 ; GFX7-NEXT: s_setpc_b64 s[30:31]
9098 ;
9099 ; GFX8-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar:
9100 ; GFX8: ; %bb.0:
9101 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9102 ; GFX8-NEXT: s_add_u32 s34, s4, 32
9103 ; GFX8-NEXT: s_addc_u32 s35, s5, 0
9104 ; GFX8-NEXT: v_mov_b32_e32 v2, s34
9105 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
9106 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
9107 ; GFX8-NEXT: v_mov_b32_e32 v3, s35
9108 ; GFX8-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
9109 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9110 ; GFX8-NEXT: buffer_wbinvl1_vol
9111 ; GFX8-NEXT: s_setpc_b64 s[30:31]
9112 ;
9113 ; GFX9-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar:
9114 ; GFX9: ; %bb.0:
9115 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9116 ; GFX9-NEXT: v_mov_b32_e32 v0, s6
9117 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
9118 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
9119 ; GFX9-NEXT: v_mov_b32_e32 v3, s5
9120 ; GFX9-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] offset:32 glc
9121 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9122 ; GFX9-NEXT: buffer_wbinvl1_vol
9123 ; GFX9-NEXT: s_setpc_b64 s[30:31]
9124 %gep = getelementptr i64, ptr %out, i64 4
9125 %result = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
9126 ret i64 %result
9127 }
9129 define void @flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
9130 ; GFX7-LABEL: flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory:
9131 ; GFX7: ; %bb.0:
9132 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9133 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
9134 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
9135 ; GFX7-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
9136 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9137 ; GFX7-NEXT: buffer_wbinvl1_vol
9138 ; GFX7-NEXT: s_setpc_b64 s[30:31]
9139 ;
9140 ; GFX8-LABEL: flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory:
9141 ; GFX8: ; %bb.0:
9142 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9143 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
9144 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
9145 ; GFX8-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
9146 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9147 ; GFX8-NEXT: buffer_wbinvl1_vol
9148 ; GFX8-NEXT: s_setpc_b64 s[30:31]
9149 ;
9150 ; GFX9-LABEL: flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory:
9151 ; GFX9: ; %bb.0:
9152 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9153 ; GFX9-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] offset:32
9154 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9155 ; GFX9-NEXT: buffer_wbinvl1_vol
9156 ; GFX9-NEXT: s_setpc_b64 s[30:31]
9157 %gep = getelementptr i64, ptr %out, i64 4
9158 %tmp0 = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
9159 ret void
9160 }
9162 define i64 @flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
9163 ; GFX7-LABEL: flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory:
9164 ; GFX7: ; %bb.0:
9165 ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9166 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0
9167 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
9168 ; GFX7-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
9169 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9170 ; GFX7-NEXT: buffer_wbinvl1_vol
9171 ; GFX7-NEXT: s_setpc_b64 s[30:31]
9172 ;
9173 ; GFX8-LABEL: flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory:
9174 ; GFX8: ; %bb.0:
9175 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9176 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0
9177 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
9178 ; GFX8-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
9179 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9180 ; GFX8-NEXT: buffer_wbinvl1_vol
9181 ; GFX8-NEXT: s_setpc_b64 s[30:31]
9182 ;
9183 ; GFX9-LABEL: flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory:
9184 ; GFX9: ; %bb.0:
9185 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
9186 ; GFX9-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
9187 ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9188 ; GFX9-NEXT: buffer_wbinvl1_vol
9189 ; GFX9-NEXT: s_setpc_b64 s[30:31]
9190 %gep = getelementptr i64, ptr %out, i64 4
9191 %result = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
9192 ret i64 %result
9193 }
9195 !0 = !{}
9196 !1 = !{i32 5, i32 6}