| blob | 321f572d57cb21b2aea7809c4b661c58a52e98c4 |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx700 < %s | FileCheck --check-prefixes=GFX7 %s
3 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1010 < %s | FileCheck --check-prefixes=GFX10-WGP %s
4 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1010 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX10-CU %s
5 ; RUN: llc -mtriple=amdgcn-amd-amdpal -O0 -mcpu=gfx700 -amdgcn-skip-cache-invalidations < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
6 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx90a < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
7 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx90a -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
8 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx940 < %s | FileCheck -check-prefixes=GFX940-NOTTGSPLIT %s
9 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx940 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
10 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
11 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
12 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
13 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
15 define amdgpu_kernel void @flat_agent_unordered_load(
16 ; GFX7-LABEL: flat_agent_unordered_load:
17 ; GFX7: ; %bb.0: ; %entry
18 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
19 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x2
20 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
21 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
22 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
23 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
24 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
25 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
26 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
27 ; GFX7-NEXT: flat_store_dword v[0:1], v2
28 ; GFX7-NEXT: s_endpgm
29 ;
30 ; GFX10-WGP-LABEL: flat_agent_unordered_load:
31 ; GFX10-WGP: ; %bb.0: ; %entry
32 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
33 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
34 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
35 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
36 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
37 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1]
38 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
39 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
40 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
41 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
42 ; GFX10-WGP-NEXT: s_endpgm
43 ;
44 ; GFX10-CU-LABEL: flat_agent_unordered_load:
45 ; GFX10-CU: ; %bb.0: ; %entry
46 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
47 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
48 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
49 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
50 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
51 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]
52 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
53 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
54 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
55 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
56 ; GFX10-CU-NEXT: s_endpgm
57 ;
58 ; SKIP-CACHE-INV-LABEL: flat_agent_unordered_load:
59 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
60 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
61 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
62 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
63 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
64 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
65 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]
66 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
67 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
68 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
69 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
70 ; SKIP-CACHE-INV-NEXT: s_endpgm
71 ;
72 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_unordered_load:
73 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
74 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
75 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
76 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
77 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
78 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
79 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
80 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
81 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
82 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
83 ;
84 ; GFX90A-TGSPLIT-LABEL: flat_agent_unordered_load:
85 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
86 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
87 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
88 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
89 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
90 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
91 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
92 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
93 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
94 ; GFX90A-TGSPLIT-NEXT: s_endpgm
95 ;
96 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_unordered_load:
97 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
98 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
99 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
100 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
101 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
102 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
103 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
104 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
105 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
106 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
107 ;
108 ; GFX940-TGSPLIT-LABEL: flat_agent_unordered_load:
109 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
110 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
111 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
112 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
113 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
114 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
115 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
116 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
117 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
118 ; GFX940-TGSPLIT-NEXT: s_endpgm
119 ;
120 ; GFX11-WGP-LABEL: flat_agent_unordered_load:
121 ; GFX11-WGP: ; %bb.0: ; %entry
122 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
123 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
124 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
125 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
126 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
127 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1]
128 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
129 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
130 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
131 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
132 ; GFX11-WGP-NEXT: s_endpgm
133 ;
134 ; GFX11-CU-LABEL: flat_agent_unordered_load:
135 ; GFX11-CU: ; %bb.0: ; %entry
136 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
137 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
138 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
139 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
140 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
141 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]
142 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
143 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
144 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
145 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
146 ; GFX11-CU-NEXT: s_endpgm
147 ;
148 ; GFX12-WGP-LABEL: flat_agent_unordered_load:
149 ; GFX12-WGP: ; %bb.0: ; %entry
150 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
151 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
152 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
153 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
154 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
155 ; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
156 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
157 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
158 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
159 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
160 ; GFX12-WGP-NEXT: s_endpgm
161 ;
162 ; GFX12-CU-LABEL: flat_agent_unordered_load:
163 ; GFX12-CU: ; %bb.0: ; %entry
164 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
165 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
166 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
167 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
168 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
169 ; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
170 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
171 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
172 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
173 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
174 ; GFX12-CU-NEXT: s_endpgm
175 ptr %in, ptr %out) {
176 entry:
177 %val = load atomic i32, ptr %in syncscope("agent") unordered, align 4
178 store i32 %val, ptr %out
179 ret void
180 }
182 define amdgpu_kernel void @flat_agent_monotonic_load(
183 ; GFX7-LABEL: flat_agent_monotonic_load:
184 ; GFX7: ; %bb.0: ; %entry
185 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
186 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x2
187 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
188 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
189 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
190 ; GFX7-NEXT: flat_load_dword v2, v[0:1] glc
191 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
192 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
193 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
194 ; GFX7-NEXT: flat_store_dword v[0:1], v2
195 ; GFX7-NEXT: s_endpgm
196 ;
197 ; GFX10-WGP-LABEL: flat_agent_monotonic_load:
198 ; GFX10-WGP: ; %bb.0: ; %entry
199 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
200 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
201 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
202 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
203 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
204 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc dlc
205 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
206 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
207 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
208 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
209 ; GFX10-WGP-NEXT: s_endpgm
210 ;
211 ; GFX10-CU-LABEL: flat_agent_monotonic_load:
212 ; GFX10-CU: ; %bb.0: ; %entry
213 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
214 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
215 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
216 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
217 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
218 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] glc dlc
219 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
220 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
221 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
222 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
223 ; GFX10-CU-NEXT: s_endpgm
224 ;
225 ; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_load:
226 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
227 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
228 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
229 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
230 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
231 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
232 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1] glc
233 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
234 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
235 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
236 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
237 ; SKIP-CACHE-INV-NEXT: s_endpgm
238 ;
239 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_load:
240 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
241 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
242 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
243 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
244 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
245 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc
246 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
247 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
248 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
249 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
250 ;
251 ; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_load:
252 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
253 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
254 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
255 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
256 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
257 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc
258 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
259 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
260 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
261 ; GFX90A-TGSPLIT-NEXT: s_endpgm
262 ;
263 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_load:
264 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
265 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
266 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
267 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
268 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
269 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc1
270 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
271 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
272 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
273 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
274 ;
275 ; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_load:
276 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
277 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
278 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
279 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
280 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
281 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc1
282 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
283 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
284 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
285 ; GFX940-TGSPLIT-NEXT: s_endpgm
286 ;
287 ; GFX11-WGP-LABEL: flat_agent_monotonic_load:
288 ; GFX11-WGP: ; %bb.0: ; %entry
289 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
290 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
291 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
292 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
293 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
294 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc
295 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
296 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
297 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
298 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
299 ; GFX11-WGP-NEXT: s_endpgm
300 ;
301 ; GFX11-CU-LABEL: flat_agent_monotonic_load:
302 ; GFX11-CU: ; %bb.0: ; %entry
303 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
304 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
305 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
306 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
307 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
308 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1] glc
309 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
310 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
311 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
312 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
313 ; GFX11-CU-NEXT: s_endpgm
314 ;
315 ; GFX12-WGP-LABEL: flat_agent_monotonic_load:
316 ; GFX12-WGP: ; %bb.0: ; %entry
317 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
318 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
319 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
320 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
321 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
322 ; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_DEV
323 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
324 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
325 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
326 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
327 ; GFX12-WGP-NEXT: s_endpgm
328 ;
329 ; GFX12-CU-LABEL: flat_agent_monotonic_load:
330 ; GFX12-CU: ; %bb.0: ; %entry
331 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
332 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
333 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
334 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
335 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
336 ; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_DEV
337 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
338 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
339 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
340 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
341 ; GFX12-CU-NEXT: s_endpgm
342 ptr %in, ptr %out) {
343 entry:
344 %val = load atomic i32, ptr %in syncscope("agent") monotonic, align 4
345 store i32 %val, ptr %out
346 ret void
347 }
349 define amdgpu_kernel void @flat_agent_acquire_load(
350 ; GFX7-LABEL: flat_agent_acquire_load:
351 ; GFX7: ; %bb.0: ; %entry
352 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
353 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x2
354 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
355 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
356 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
357 ; GFX7-NEXT: flat_load_dword v2, v[0:1] glc
358 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
359 ; GFX7-NEXT: buffer_wbinvl1_vol
360 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
361 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
362 ; GFX7-NEXT: flat_store_dword v[0:1], v2
363 ; GFX7-NEXT: s_endpgm
364 ;
365 ; GFX10-WGP-LABEL: flat_agent_acquire_load:
366 ; GFX10-WGP: ; %bb.0: ; %entry
367 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
368 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
369 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
370 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
371 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
372 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc dlc
373 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
374 ; GFX10-WGP-NEXT: buffer_gl1_inv
375 ; GFX10-WGP-NEXT: buffer_gl0_inv
376 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
377 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
378 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
379 ; GFX10-WGP-NEXT: s_endpgm
380 ;
381 ; GFX10-CU-LABEL: flat_agent_acquire_load:
382 ; GFX10-CU: ; %bb.0: ; %entry
383 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
384 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
385 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
386 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
387 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
388 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] glc dlc
389 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
390 ; GFX10-CU-NEXT: buffer_gl1_inv
391 ; GFX10-CU-NEXT: buffer_gl0_inv
392 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
393 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
394 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
395 ; GFX10-CU-NEXT: s_endpgm
396 ;
397 ; SKIP-CACHE-INV-LABEL: flat_agent_acquire_load:
398 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
399 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
400 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
401 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
402 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
403 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
404 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1] glc
405 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
406 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
407 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
408 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
409 ; SKIP-CACHE-INV-NEXT: s_endpgm
410 ;
411 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_load:
412 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
413 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
414 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
415 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
416 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
417 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc
418 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
419 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
420 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
421 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
422 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
423 ;
424 ; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_load:
425 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
426 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
427 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
428 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
429 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
430 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc
431 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
432 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
433 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
434 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
435 ; GFX90A-TGSPLIT-NEXT: s_endpgm
436 ;
437 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_load:
438 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
439 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
440 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
441 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
442 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
443 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc1
444 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
445 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
446 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
447 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
448 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
449 ;
450 ; GFX940-TGSPLIT-LABEL: flat_agent_acquire_load:
451 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
452 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
453 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
454 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
455 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
456 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc1
457 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
458 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
459 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
460 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
461 ; GFX940-TGSPLIT-NEXT: s_endpgm
462 ;
463 ; GFX11-WGP-LABEL: flat_agent_acquire_load:
464 ; GFX11-WGP: ; %bb.0: ; %entry
465 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
466 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
467 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
468 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
469 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
470 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc
471 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
472 ; GFX11-WGP-NEXT: buffer_gl1_inv
473 ; GFX11-WGP-NEXT: buffer_gl0_inv
474 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
475 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
476 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
477 ; GFX11-WGP-NEXT: s_endpgm
478 ;
479 ; GFX11-CU-LABEL: flat_agent_acquire_load:
480 ; GFX11-CU: ; %bb.0: ; %entry
481 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
482 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
483 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
484 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
485 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
486 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1] glc
487 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
488 ; GFX11-CU-NEXT: buffer_gl1_inv
489 ; GFX11-CU-NEXT: buffer_gl0_inv
490 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
491 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
492 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
493 ; GFX11-CU-NEXT: s_endpgm
494 ;
495 ; GFX12-WGP-LABEL: flat_agent_acquire_load:
496 ; GFX12-WGP: ; %bb.0: ; %entry
497 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
498 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
499 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
500 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
501 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
502 ; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_DEV
503 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
504 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
505 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
506 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
507 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
508 ; GFX12-WGP-NEXT: s_endpgm
509 ;
510 ; GFX12-CU-LABEL: flat_agent_acquire_load:
511 ; GFX12-CU: ; %bb.0: ; %entry
512 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
513 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
514 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
515 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
516 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
517 ; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_DEV
518 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
519 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
520 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
521 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
522 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
523 ; GFX12-CU-NEXT: s_endpgm
524 ptr %in, ptr %out) {
525 entry:
526 %val = load atomic i32, ptr %in syncscope("agent") acquire, align 4
527 store i32 %val, ptr %out
528 ret void
529 }
531 define amdgpu_kernel void @flat_agent_seq_cst_load(
532 ; GFX7-LABEL: flat_agent_seq_cst_load:
533 ; GFX7: ; %bb.0: ; %entry
534 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
535 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x2
536 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
537 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
538 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
539 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
540 ; GFX7-NEXT: flat_load_dword v2, v[0:1] glc
541 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
542 ; GFX7-NEXT: buffer_wbinvl1_vol
543 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
544 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
545 ; GFX7-NEXT: flat_store_dword v[0:1], v2
546 ; GFX7-NEXT: s_endpgm
547 ;
548 ; GFX10-WGP-LABEL: flat_agent_seq_cst_load:
549 ; GFX10-WGP: ; %bb.0: ; %entry
550 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
551 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
552 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
553 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
554 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
555 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
556 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
557 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc dlc
558 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
559 ; GFX10-WGP-NEXT: buffer_gl1_inv
560 ; GFX10-WGP-NEXT: buffer_gl0_inv
561 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
562 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
563 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
564 ; GFX10-WGP-NEXT: s_endpgm
565 ;
566 ; GFX10-CU-LABEL: flat_agent_seq_cst_load:
567 ; GFX10-CU: ; %bb.0: ; %entry
568 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
569 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
570 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
571 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
572 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
573 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
574 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
575 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] glc dlc
576 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
577 ; GFX10-CU-NEXT: buffer_gl1_inv
578 ; GFX10-CU-NEXT: buffer_gl0_inv
579 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
580 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
581 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
582 ; GFX10-CU-NEXT: s_endpgm
583 ;
584 ; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_load:
585 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
586 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
587 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
588 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
589 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
590 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
591 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
592 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1] glc
593 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
594 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
595 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
596 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
597 ; SKIP-CACHE-INV-NEXT: s_endpgm
598 ;
599 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_load:
600 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
601 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
602 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
603 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
604 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
605 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
606 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc
607 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
608 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
609 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
610 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
611 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
612 ;
613 ; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_load:
614 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
615 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
616 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
617 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
618 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
619 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
620 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc
621 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
622 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
623 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
624 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
625 ; GFX90A-TGSPLIT-NEXT: s_endpgm
626 ;
627 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_load:
628 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
629 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
630 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
631 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
632 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
633 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
634 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc1
635 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
636 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
637 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
638 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
639 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
640 ;
641 ; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_load:
642 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
643 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
644 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
645 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
646 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
647 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
648 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc1
649 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
650 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
651 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
652 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
653 ; GFX940-TGSPLIT-NEXT: s_endpgm
654 ;
655 ; GFX11-WGP-LABEL: flat_agent_seq_cst_load:
656 ; GFX11-WGP: ; %bb.0: ; %entry
657 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
658 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
659 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
660 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
661 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
662 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
663 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
664 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc
665 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
666 ; GFX11-WGP-NEXT: buffer_gl1_inv
667 ; GFX11-WGP-NEXT: buffer_gl0_inv
668 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
669 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
670 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
671 ; GFX11-WGP-NEXT: s_endpgm
672 ;
673 ; GFX11-CU-LABEL: flat_agent_seq_cst_load:
674 ; GFX11-CU: ; %bb.0: ; %entry
675 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
676 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
677 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
678 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
679 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
680 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
681 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
682 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1] glc
683 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
684 ; GFX11-CU-NEXT: buffer_gl1_inv
685 ; GFX11-CU-NEXT: buffer_gl0_inv
686 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
687 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
688 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
689 ; GFX11-CU-NEXT: s_endpgm
690 ;
691 ; GFX12-WGP-LABEL: flat_agent_seq_cst_load:
692 ; GFX12-WGP: ; %bb.0: ; %entry
693 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
694 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
695 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
696 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
697 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
698 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
699 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
700 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
701 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
702 ; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_DEV
703 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
704 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
705 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
706 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
707 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
708 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
709 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
710 ; GFX12-WGP-NEXT: s_endpgm
711 ;
712 ; GFX12-CU-LABEL: flat_agent_seq_cst_load:
713 ; GFX12-CU: ; %bb.0: ; %entry
714 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
715 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
716 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
717 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
718 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
719 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
720 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
721 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
722 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
723 ; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_DEV
724 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
725 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
726 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
727 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
728 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
729 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
730 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
731 ; GFX12-CU-NEXT: s_endpgm
732 ptr %in, ptr %out) {
733 entry:
734 %val = load atomic i32, ptr %in syncscope("agent") seq_cst, align 4
735 store i32 %val, ptr %out
736 ret void
737 }
739 define amdgpu_kernel void @flat_agent_unordered_store(
740 ; GFX7-LABEL: flat_agent_unordered_store:
741 ; GFX7: ; %bb.0: ; %entry
742 ; GFX7-NEXT: s_load_dword s4, s[8:9], 0x0
743 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x2
744 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
745 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
746 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
747 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
748 ; GFX7-NEXT: flat_store_dword v[0:1], v2
749 ; GFX7-NEXT: s_endpgm
750 ;
751 ; GFX10-WGP-LABEL: flat_agent_unordered_store:
752 ; GFX10-WGP: ; %bb.0: ; %entry
753 ; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x0
754 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
755 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
756 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
757 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
758 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
759 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
760 ; GFX10-WGP-NEXT: s_endpgm
761 ;
762 ; GFX10-CU-LABEL: flat_agent_unordered_store:
763 ; GFX10-CU: ; %bb.0: ; %entry
764 ; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x0
765 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
766 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
767 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
768 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
769 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
770 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
771 ; GFX10-CU-NEXT: s_endpgm
772 ;
773 ; SKIP-CACHE-INV-LABEL: flat_agent_unordered_store:
774 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
775 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x0
776 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
777 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
778 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
779 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
780 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
781 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
782 ; SKIP-CACHE-INV-NEXT: s_endpgm
783 ;
784 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_unordered_store:
785 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
786 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x0
787 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
788 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
789 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
790 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
791 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
792 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
793 ;
794 ; GFX90A-TGSPLIT-LABEL: flat_agent_unordered_store:
795 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
796 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x0
797 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
798 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
799 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
800 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
801 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
802 ; GFX90A-TGSPLIT-NEXT: s_endpgm
803 ;
804 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_unordered_store:
805 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
806 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x0
807 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
808 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
809 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
810 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
811 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
812 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
813 ;
814 ; GFX940-TGSPLIT-LABEL: flat_agent_unordered_store:
815 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
816 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x0
817 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
818 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
819 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
820 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
821 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
822 ; GFX940-TGSPLIT-NEXT: s_endpgm
823 ;
824 ; GFX11-WGP-LABEL: flat_agent_unordered_store:
825 ; GFX11-WGP: ; %bb.0: ; %entry
826 ; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
827 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
828 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
829 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
830 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
831 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
832 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
833 ; GFX11-WGP-NEXT: s_endpgm
834 ;
835 ; GFX11-CU-LABEL: flat_agent_unordered_store:
836 ; GFX11-CU: ; %bb.0: ; %entry
837 ; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
838 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
839 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
840 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
841 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
842 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
843 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
844 ; GFX11-CU-NEXT: s_endpgm
845 ;
846 ; GFX12-WGP-LABEL: flat_agent_unordered_store:
847 ; GFX12-WGP: ; %bb.0: ; %entry
848 ; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
849 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
850 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
851 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
852 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
853 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
854 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
855 ; GFX12-WGP-NEXT: s_endpgm
856 ;
857 ; GFX12-CU-LABEL: flat_agent_unordered_store:
858 ; GFX12-CU: ; %bb.0: ; %entry
859 ; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
860 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
861 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
862 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
863 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
864 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
865 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
866 ; GFX12-CU-NEXT: s_endpgm
867 i32 %in, ptr %out) {
868 entry:
869 store atomic i32 %in, ptr %out syncscope("agent") unordered, align 4
870 ret void
871 }
873 define amdgpu_kernel void @flat_agent_monotonic_store(
874 ; GFX7-LABEL: flat_agent_monotonic_store:
875 ; GFX7: ; %bb.0: ; %entry
876 ; GFX7-NEXT: s_load_dword s4, s[8:9], 0x0
877 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x2
878 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
879 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
880 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
881 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
882 ; GFX7-NEXT: flat_store_dword v[0:1], v2
883 ; GFX7-NEXT: s_endpgm
884 ;
885 ; GFX10-WGP-LABEL: flat_agent_monotonic_store:
886 ; GFX10-WGP: ; %bb.0: ; %entry
887 ; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x0
888 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
889 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
890 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
891 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
892 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
893 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
894 ; GFX10-WGP-NEXT: s_endpgm
895 ;
896 ; GFX10-CU-LABEL: flat_agent_monotonic_store:
897 ; GFX10-CU: ; %bb.0: ; %entry
898 ; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x0
899 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
900 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
901 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
902 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
903 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
904 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
905 ; GFX10-CU-NEXT: s_endpgm
906 ;
907 ; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_store:
908 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
909 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x0
910 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
911 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
912 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
913 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
914 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
915 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
916 ; SKIP-CACHE-INV-NEXT: s_endpgm
917 ;
918 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_store:
919 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
920 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x0
921 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
922 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
923 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
924 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
925 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
926 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
927 ;
928 ; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_store:
929 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
930 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x0
931 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
932 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
933 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
934 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
935 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
936 ; GFX90A-TGSPLIT-NEXT: s_endpgm
937 ;
938 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_store:
939 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
940 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x0
941 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
942 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
943 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
944 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
945 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
946 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
947 ;
948 ; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_store:
949 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
950 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x0
951 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
952 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
953 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
954 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
955 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
956 ; GFX940-TGSPLIT-NEXT: s_endpgm
957 ;
958 ; GFX11-WGP-LABEL: flat_agent_monotonic_store:
959 ; GFX11-WGP: ; %bb.0: ; %entry
960 ; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
961 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
962 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
963 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
964 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
965 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
966 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
967 ; GFX11-WGP-NEXT: s_endpgm
968 ;
969 ; GFX11-CU-LABEL: flat_agent_monotonic_store:
970 ; GFX11-CU: ; %bb.0: ; %entry
971 ; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
972 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
973 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
974 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
975 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
976 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
977 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
978 ; GFX11-CU-NEXT: s_endpgm
979 ;
980 ; GFX12-WGP-LABEL: flat_agent_monotonic_store:
981 ; GFX12-WGP: ; %bb.0: ; %entry
982 ; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
983 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
984 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
985 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
986 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
987 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
988 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_DEV
989 ; GFX12-WGP-NEXT: s_endpgm
990 ;
991 ; GFX12-CU-LABEL: flat_agent_monotonic_store:
992 ; GFX12-CU: ; %bb.0: ; %entry
993 ; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
994 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
995 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
996 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
997 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
998 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
999 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_DEV
1000 ; GFX12-CU-NEXT: s_endpgm
1001 i32 %in, ptr %out) {
1002 entry:
1003 store atomic i32 %in, ptr %out syncscope("agent") monotonic, align 4
1004 ret void
1005 }
1007 define amdgpu_kernel void @flat_agent_release_store(
1008 ; GFX7-LABEL: flat_agent_release_store:
1009 ; GFX7: ; %bb.0: ; %entry
1010 ; GFX7-NEXT: s_load_dword s4, s[8:9], 0x0
1011 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x2
1012 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1013 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1014 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1015 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
1016 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1017 ; GFX7-NEXT: flat_store_dword v[0:1], v2
1018 ; GFX7-NEXT: s_endpgm
1019 ;
1020 ; GFX10-WGP-LABEL: flat_agent_release_store:
1021 ; GFX10-WGP: ; %bb.0: ; %entry
1022 ; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x0
1023 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
1024 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1025 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
1026 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
1027 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
1028 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1029 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1030 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
1031 ; GFX10-WGP-NEXT: s_endpgm
1032 ;
1033 ; GFX10-CU-LABEL: flat_agent_release_store:
1034 ; GFX10-CU: ; %bb.0: ; %entry
1035 ; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x0
1036 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
1037 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1038 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
1039 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
1040 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
1041 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1042 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
1043 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
1044 ; GFX10-CU-NEXT: s_endpgm
1045 ;
1046 ; SKIP-CACHE-INV-LABEL: flat_agent_release_store:
1047 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1048 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x0
1049 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
1050 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1051 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
1052 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
1053 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
1054 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1055 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
1056 ; SKIP-CACHE-INV-NEXT: s_endpgm
1057 ;
1058 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_store:
1059 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1060 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x0
1061 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
1062 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1063 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1064 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1065 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1066 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
1067 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1068 ;
1069 ; GFX90A-TGSPLIT-LABEL: flat_agent_release_store:
1070 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1071 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x0
1072 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
1073 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1074 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1075 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1076 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1077 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
1078 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1079 ;
1080 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_store:
1081 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1082 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x0
1083 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
1084 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1085 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1086 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1087 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
1088 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1089 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
1090 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1091 ;
1092 ; GFX940-TGSPLIT-LABEL: flat_agent_release_store:
1093 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1094 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x0
1095 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
1096 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1097 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1098 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1099 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
1100 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1101 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
1102 ; GFX940-TGSPLIT-NEXT: s_endpgm
1103 ;
1104 ; GFX11-WGP-LABEL: flat_agent_release_store:
1105 ; GFX11-WGP: ; %bb.0: ; %entry
1106 ; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
1107 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
1108 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1109 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
1110 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
1111 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
1112 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1113 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1114 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
1115 ; GFX11-WGP-NEXT: s_endpgm
1116 ;
1117 ; GFX11-CU-LABEL: flat_agent_release_store:
1118 ; GFX11-CU: ; %bb.0: ; %entry
1119 ; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
1120 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
1121 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1122 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
1123 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
1124 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
1125 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1126 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
1127 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
1128 ; GFX11-CU-NEXT: s_endpgm
1129 ;
1130 ; GFX12-WGP-LABEL: flat_agent_release_store:
1131 ; GFX12-WGP: ; %bb.0: ; %entry
1132 ; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
1133 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
1134 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
1135 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
1136 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
1137 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
1138 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
1139 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
1140 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
1141 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
1142 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_DEV
1143 ; GFX12-WGP-NEXT: s_endpgm
1144 ;
1145 ; GFX12-CU-LABEL: flat_agent_release_store:
1146 ; GFX12-CU: ; %bb.0: ; %entry
1147 ; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
1148 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
1149 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
1150 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
1151 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
1152 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
1153 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
1154 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
1155 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
1156 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
1157 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_DEV
1158 ; GFX12-CU-NEXT: s_endpgm
1159 i32 %in, ptr %out) {
1160 entry:
1161 store atomic i32 %in, ptr %out syncscope("agent") release, align 4
1162 ret void
1163 }
1165 define amdgpu_kernel void @flat_agent_seq_cst_store(
1166 ; GFX7-LABEL: flat_agent_seq_cst_store:
1167 ; GFX7: ; %bb.0: ; %entry
1168 ; GFX7-NEXT: s_load_dword s4, s[8:9], 0x0
1169 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x2
1170 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1171 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1172 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1173 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
1174 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1175 ; GFX7-NEXT: flat_store_dword v[0:1], v2
1176 ; GFX7-NEXT: s_endpgm
1177 ;
1178 ; GFX10-WGP-LABEL: flat_agent_seq_cst_store:
1179 ; GFX10-WGP: ; %bb.0: ; %entry
1180 ; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x0
1181 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
1182 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1183 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
1184 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
1185 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
1186 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1187 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1188 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
1189 ; GFX10-WGP-NEXT: s_endpgm
1190 ;
1191 ; GFX10-CU-LABEL: flat_agent_seq_cst_store:
1192 ; GFX10-CU: ; %bb.0: ; %entry
1193 ; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x0
1194 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
1195 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1196 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
1197 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
1198 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
1199 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1200 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
1201 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
1202 ; GFX10-CU-NEXT: s_endpgm
1203 ;
1204 ; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_store:
1205 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1206 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x0
1207 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
1208 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1209 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
1210 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
1211 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
1212 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1213 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
1214 ; SKIP-CACHE-INV-NEXT: s_endpgm
1215 ;
1216 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_store:
1217 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1218 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x0
1219 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
1220 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1221 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1222 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1223 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1224 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
1225 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1226 ;
1227 ; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_store:
1228 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1229 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x0
1230 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
1231 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1232 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1233 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1234 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1235 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
1236 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1237 ;
1238 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_store:
1239 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1240 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x0
1241 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
1242 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1243 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1244 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1245 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
1246 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1247 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
1248 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1249 ;
1250 ; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_store:
1251 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1252 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x0
1253 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
1254 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1255 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1256 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1257 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
1258 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1259 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
1260 ; GFX940-TGSPLIT-NEXT: s_endpgm
1261 ;
1262 ; GFX11-WGP-LABEL: flat_agent_seq_cst_store:
1263 ; GFX11-WGP: ; %bb.0: ; %entry
1264 ; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
1265 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
1266 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1267 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
1268 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
1269 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
1270 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1271 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1272 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
1273 ; GFX11-WGP-NEXT: s_endpgm
1274 ;
1275 ; GFX11-CU-LABEL: flat_agent_seq_cst_store:
1276 ; GFX11-CU: ; %bb.0: ; %entry
1277 ; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
1278 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
1279 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1280 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
1281 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
1282 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
1283 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1284 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
1285 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
1286 ; GFX11-CU-NEXT: s_endpgm
1287 ;
1288 ; GFX12-WGP-LABEL: flat_agent_seq_cst_store:
1289 ; GFX12-WGP: ; %bb.0: ; %entry
1290 ; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
1291 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
1292 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
1293 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
1294 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
1295 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
1296 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
1297 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
1298 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
1299 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
1300 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_DEV
1301 ; GFX12-WGP-NEXT: s_endpgm
1302 ;
1303 ; GFX12-CU-LABEL: flat_agent_seq_cst_store:
1304 ; GFX12-CU: ; %bb.0: ; %entry
1305 ; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
1306 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
1307 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
1308 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
1309 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
1310 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
1311 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
1312 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
1313 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
1314 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
1315 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_DEV
1316 ; GFX12-CU-NEXT: s_endpgm
1317 i32 %in, ptr %out) {
1318 entry:
1319 store atomic i32 %in, ptr %out syncscope("agent") seq_cst, align 4
1320 ret void
1321 }
1323 define amdgpu_kernel void @flat_agent_monotonic_atomicrmw(
1324 ; GFX7-LABEL: flat_agent_monotonic_atomicrmw:
1325 ; GFX7: ; %bb.0: ; %entry
1326 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
1327 ; GFX7-NEXT: s_load_dword s4, s[8:9], 0x2
1328 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1329 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1330 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1331 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
1332 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
1333 ; GFX7-NEXT: s_endpgm
1334 ;
1335 ; GFX10-WGP-LABEL: flat_agent_monotonic_atomicrmw:
1336 ; GFX10-WGP: ; %bb.0: ; %entry
1337 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
1338 ; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x8
1339 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1340 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
1341 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
1342 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
1343 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
1344 ; GFX10-WGP-NEXT: s_endpgm
1345 ;
1346 ; GFX10-CU-LABEL: flat_agent_monotonic_atomicrmw:
1347 ; GFX10-CU: ; %bb.0: ; %entry
1348 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
1349 ; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x8
1350 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1351 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
1352 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
1353 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
1354 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
1355 ; GFX10-CU-NEXT: s_endpgm
1356 ;
1357 ; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_atomicrmw:
1358 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1359 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
1360 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x2
1361 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1362 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
1363 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
1364 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
1365 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
1366 ; SKIP-CACHE-INV-NEXT: s_endpgm
1367 ;
1368 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_atomicrmw:
1369 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1370 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
1371 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
1372 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1373 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1374 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1375 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1376 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1377 ;
1378 ; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_atomicrmw:
1379 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1380 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
1381 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
1382 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1383 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1384 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1385 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1386 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1387 ;
1388 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_atomicrmw:
1389 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1390 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
1391 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
1392 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1393 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1394 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1395 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1396 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1397 ;
1398 ; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_atomicrmw:
1399 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1400 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
1401 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
1402 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1403 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1404 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1405 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1406 ; GFX940-TGSPLIT-NEXT: s_endpgm
1407 ;
1408 ; GFX11-WGP-LABEL: flat_agent_monotonic_atomicrmw:
1409 ; GFX11-WGP: ; %bb.0: ; %entry
1410 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
1411 ; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
1412 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1413 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
1414 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
1415 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
1416 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
1417 ; GFX11-WGP-NEXT: s_endpgm
1418 ;
1419 ; GFX11-CU-LABEL: flat_agent_monotonic_atomicrmw:
1420 ; GFX11-CU: ; %bb.0: ; %entry
1421 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
1422 ; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
1423 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1424 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
1425 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
1426 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
1427 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
1428 ; GFX11-CU-NEXT: s_endpgm
1429 ;
1430 ; GFX12-WGP-LABEL: flat_agent_monotonic_atomicrmw:
1431 ; GFX12-WGP: ; %bb.0: ; %entry
1432 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
1433 ; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
1434 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
1435 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
1436 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
1437 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
1438 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
1439 ; GFX12-WGP-NEXT: s_endpgm
1440 ;
1441 ; GFX12-CU-LABEL: flat_agent_monotonic_atomicrmw:
1442 ; GFX12-CU: ; %bb.0: ; %entry
1443 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
1444 ; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
1445 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
1446 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
1447 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
1448 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
1449 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
1450 ; GFX12-CU-NEXT: s_endpgm
1451 ptr %out, i32 %in) {
1452 entry:
1453 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") monotonic
1454 ret void
1455 }
1457 define amdgpu_kernel void @flat_agent_acquire_atomicrmw(
1458 ; GFX7-LABEL: flat_agent_acquire_atomicrmw:
1459 ; GFX7: ; %bb.0: ; %entry
1460 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
1461 ; GFX7-NEXT: s_load_dword s4, s[8:9], 0x2
1462 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1463 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1464 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1465 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
1466 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
1467 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1468 ; GFX7-NEXT: buffer_wbinvl1_vol
1469 ; GFX7-NEXT: s_endpgm
1470 ;
1471 ; GFX10-WGP-LABEL: flat_agent_acquire_atomicrmw:
1472 ; GFX10-WGP: ; %bb.0: ; %entry
1473 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
1474 ; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x8
1475 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1476 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
1477 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
1478 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
1479 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
1480 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1481 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1482 ; GFX10-WGP-NEXT: buffer_gl1_inv
1483 ; GFX10-WGP-NEXT: buffer_gl0_inv
1484 ; GFX10-WGP-NEXT: s_endpgm
1485 ;
1486 ; GFX10-CU-LABEL: flat_agent_acquire_atomicrmw:
1487 ; GFX10-CU: ; %bb.0: ; %entry
1488 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
1489 ; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x8
1490 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1491 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
1492 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
1493 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
1494 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
1495 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1496 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
1497 ; GFX10-CU-NEXT: buffer_gl1_inv
1498 ; GFX10-CU-NEXT: buffer_gl0_inv
1499 ; GFX10-CU-NEXT: s_endpgm
1500 ;
1501 ; SKIP-CACHE-INV-LABEL: flat_agent_acquire_atomicrmw:
1502 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1503 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
1504 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x2
1505 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1506 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
1507 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
1508 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
1509 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
1510 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1511 ; SKIP-CACHE-INV-NEXT: s_endpgm
1512 ;
1513 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_atomicrmw:
1514 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1515 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
1516 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
1517 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1518 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1519 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1520 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1521 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1522 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
1523 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1524 ;
1525 ; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_atomicrmw:
1526 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1527 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
1528 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
1529 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1530 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1531 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1532 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1533 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
1534 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
1535 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1536 ;
1537 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_atomicrmw:
1538 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1539 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
1540 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
1541 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1542 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1543 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1544 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1545 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1546 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
1547 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1548 ;
1549 ; GFX940-TGSPLIT-LABEL: flat_agent_acquire_atomicrmw:
1550 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1551 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
1552 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
1553 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1554 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1555 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1556 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1557 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
1558 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
1559 ; GFX940-TGSPLIT-NEXT: s_endpgm
1560 ;
1561 ; GFX11-WGP-LABEL: flat_agent_acquire_atomicrmw:
1562 ; GFX11-WGP: ; %bb.0: ; %entry
1563 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
1564 ; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
1565 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1566 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
1567 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
1568 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
1569 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
1570 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1571 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1572 ; GFX11-WGP-NEXT: buffer_gl1_inv
1573 ; GFX11-WGP-NEXT: buffer_gl0_inv
1574 ; GFX11-WGP-NEXT: s_endpgm
1575 ;
1576 ; GFX11-CU-LABEL: flat_agent_acquire_atomicrmw:
1577 ; GFX11-CU: ; %bb.0: ; %entry
1578 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
1579 ; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
1580 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1581 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
1582 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
1583 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
1584 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
1585 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1586 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
1587 ; GFX11-CU-NEXT: buffer_gl1_inv
1588 ; GFX11-CU-NEXT: buffer_gl0_inv
1589 ; GFX11-CU-NEXT: s_endpgm
1590 ;
1591 ; GFX12-WGP-LABEL: flat_agent_acquire_atomicrmw:
1592 ; GFX12-WGP: ; %bb.0: ; %entry
1593 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
1594 ; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
1595 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
1596 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
1597 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
1598 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
1599 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
1600 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
1601 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
1602 ; GFX12-WGP-NEXT: s_endpgm
1603 ;
1604 ; GFX12-CU-LABEL: flat_agent_acquire_atomicrmw:
1605 ; GFX12-CU: ; %bb.0: ; %entry
1606 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
1607 ; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
1608 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
1609 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
1610 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
1611 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
1612 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
1613 ; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
1614 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
1615 ; GFX12-CU-NEXT: s_endpgm
1616 ptr %out, i32 %in) {
1617 entry:
1618 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") acquire
1619 ret void
1620 }
1622 define amdgpu_kernel void @flat_agent_release_atomicrmw(
1623 ; GFX7-LABEL: flat_agent_release_atomicrmw:
1624 ; GFX7: ; %bb.0: ; %entry
1625 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
1626 ; GFX7-NEXT: s_load_dword s4, s[8:9], 0x2
1627 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1628 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1629 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1630 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
1631 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1632 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
1633 ; GFX7-NEXT: s_endpgm
1634 ;
1635 ; GFX10-WGP-LABEL: flat_agent_release_atomicrmw:
1636 ; GFX10-WGP: ; %bb.0: ; %entry
1637 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
1638 ; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x8
1639 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1640 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
1641 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
1642 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
1643 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1644 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1645 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
1646 ; GFX10-WGP-NEXT: s_endpgm
1647 ;
1648 ; GFX10-CU-LABEL: flat_agent_release_atomicrmw:
1649 ; GFX10-CU: ; %bb.0: ; %entry
1650 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
1651 ; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x8
1652 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1653 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
1654 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
1655 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
1656 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1657 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
1658 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
1659 ; GFX10-CU-NEXT: s_endpgm
1660 ;
1661 ; SKIP-CACHE-INV-LABEL: flat_agent_release_atomicrmw:
1662 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1663 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
1664 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x2
1665 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1666 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
1667 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
1668 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
1669 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1670 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
1671 ; SKIP-CACHE-INV-NEXT: s_endpgm
1672 ;
1673 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_atomicrmw:
1674 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1675 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
1676 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
1677 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1678 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1679 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1680 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1681 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1682 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1683 ;
1684 ; GFX90A-TGSPLIT-LABEL: flat_agent_release_atomicrmw:
1685 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1686 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
1687 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
1688 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1689 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1690 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1691 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1692 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1693 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1694 ;
1695 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_atomicrmw:
1696 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1697 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
1698 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
1699 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1700 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1701 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1702 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
1703 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1704 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1705 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1706 ;
1707 ; GFX940-TGSPLIT-LABEL: flat_agent_release_atomicrmw:
1708 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1709 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
1710 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
1711 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1712 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1713 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1714 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
1715 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1716 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1717 ; GFX940-TGSPLIT-NEXT: s_endpgm
1718 ;
1719 ; GFX11-WGP-LABEL: flat_agent_release_atomicrmw:
1720 ; GFX11-WGP: ; %bb.0: ; %entry
1721 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
1722 ; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
1723 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1724 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
1725 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
1726 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
1727 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1728 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1729 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
1730 ; GFX11-WGP-NEXT: s_endpgm
1731 ;
1732 ; GFX11-CU-LABEL: flat_agent_release_atomicrmw:
1733 ; GFX11-CU: ; %bb.0: ; %entry
1734 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
1735 ; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
1736 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1737 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
1738 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
1739 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
1740 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1741 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
1742 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
1743 ; GFX11-CU-NEXT: s_endpgm
1744 ;
1745 ; GFX12-WGP-LABEL: flat_agent_release_atomicrmw:
1746 ; GFX12-WGP: ; %bb.0: ; %entry
1747 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
1748 ; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
1749 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
1750 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
1751 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
1752 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
1753 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
1754 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
1755 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
1756 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
1757 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
1758 ; GFX12-WGP-NEXT: s_endpgm
1759 ;
1760 ; GFX12-CU-LABEL: flat_agent_release_atomicrmw:
1761 ; GFX12-CU: ; %bb.0: ; %entry
1762 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
1763 ; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
1764 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
1765 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
1766 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
1767 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
1768 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
1769 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
1770 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
1771 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
1772 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
1773 ; GFX12-CU-NEXT: s_endpgm
1774 ptr %out, i32 %in) {
1775 entry:
1776 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") release
1777 ret void
1778 }
1780 define amdgpu_kernel void @flat_agent_acq_rel_atomicrmw(
1781 ; GFX7-LABEL: flat_agent_acq_rel_atomicrmw:
1782 ; GFX7: ; %bb.0: ; %entry
1783 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
1784 ; GFX7-NEXT: s_load_dword s4, s[8:9], 0x2
1785 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1786 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1787 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1788 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
1789 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1790 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
1791 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1792 ; GFX7-NEXT: buffer_wbinvl1_vol
1793 ; GFX7-NEXT: s_endpgm
1794 ;
1795 ; GFX10-WGP-LABEL: flat_agent_acq_rel_atomicrmw:
1796 ; GFX10-WGP: ; %bb.0: ; %entry
1797 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
1798 ; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x8
1799 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1800 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
1801 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
1802 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
1803 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1804 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1805 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
1806 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1807 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1808 ; GFX10-WGP-NEXT: buffer_gl1_inv
1809 ; GFX10-WGP-NEXT: buffer_gl0_inv
1810 ; GFX10-WGP-NEXT: s_endpgm
1811 ;
1812 ; GFX10-CU-LABEL: flat_agent_acq_rel_atomicrmw:
1813 ; GFX10-CU: ; %bb.0: ; %entry
1814 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
1815 ; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x8
1816 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1817 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
1818 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
1819 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
1820 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1821 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
1822 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
1823 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1824 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
1825 ; GFX10-CU-NEXT: buffer_gl1_inv
1826 ; GFX10-CU-NEXT: buffer_gl0_inv
1827 ; GFX10-CU-NEXT: s_endpgm
1828 ;
1829 ; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_atomicrmw:
1830 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1831 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
1832 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x2
1833 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1834 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
1835 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
1836 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
1837 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1838 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
1839 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1840 ; SKIP-CACHE-INV-NEXT: s_endpgm
1841 ;
1842 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_atomicrmw:
1843 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1844 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
1845 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
1846 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1847 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1848 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1849 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1850 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1851 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1852 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
1853 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1854 ;
1855 ; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_atomicrmw:
1856 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1857 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
1858 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
1859 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1860 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1861 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1862 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1863 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1864 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
1865 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
1866 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1867 ;
1868 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_atomicrmw:
1869 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1870 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
1871 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
1872 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1873 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1874 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1875 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
1876 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1877 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1878 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1879 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
1880 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1881 ;
1882 ; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_atomicrmw:
1883 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1884 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
1885 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
1886 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1887 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1888 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1889 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
1890 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1891 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1892 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
1893 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
1894 ; GFX940-TGSPLIT-NEXT: s_endpgm
1895 ;
1896 ; GFX11-WGP-LABEL: flat_agent_acq_rel_atomicrmw:
1897 ; GFX11-WGP: ; %bb.0: ; %entry
1898 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
1899 ; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
1900 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1901 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
1902 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
1903 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
1904 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1905 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1906 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
1907 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1908 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1909 ; GFX11-WGP-NEXT: buffer_gl1_inv
1910 ; GFX11-WGP-NEXT: buffer_gl0_inv
1911 ; GFX11-WGP-NEXT: s_endpgm
1912 ;
1913 ; GFX11-CU-LABEL: flat_agent_acq_rel_atomicrmw:
1914 ; GFX11-CU: ; %bb.0: ; %entry
1915 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
1916 ; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
1917 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1918 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
1919 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
1920 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
1921 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1922 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
1923 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
1924 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1925 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
1926 ; GFX11-CU-NEXT: buffer_gl1_inv
1927 ; GFX11-CU-NEXT: buffer_gl0_inv
1928 ; GFX11-CU-NEXT: s_endpgm
1929 ;
1930 ; GFX12-WGP-LABEL: flat_agent_acq_rel_atomicrmw:
1931 ; GFX12-WGP: ; %bb.0: ; %entry
1932 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
1933 ; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
1934 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
1935 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
1936 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
1937 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
1938 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
1939 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
1940 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
1941 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
1942 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
1943 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
1944 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
1945 ; GFX12-WGP-NEXT: s_endpgm
1946 ;
1947 ; GFX12-CU-LABEL: flat_agent_acq_rel_atomicrmw:
1948 ; GFX12-CU: ; %bb.0: ; %entry
1949 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
1950 ; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
1951 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
1952 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
1953 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
1954 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
1955 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
1956 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
1957 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
1958 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
1959 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
1960 ; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
1961 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
1962 ; GFX12-CU-NEXT: s_endpgm
1963 ptr %out, i32 %in) {
1964 entry:
1965 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") acq_rel
1966 ret void
1967 }
1969 define amdgpu_kernel void @flat_agent_seq_cst_atomicrmw(
1970 ; GFX7-LABEL: flat_agent_seq_cst_atomicrmw:
1971 ; GFX7: ; %bb.0: ; %entry
1972 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
1973 ; GFX7-NEXT: s_load_dword s4, s[8:9], 0x2
1974 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1975 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1976 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1977 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
1978 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1979 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
1980 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1981 ; GFX7-NEXT: buffer_wbinvl1_vol
1982 ; GFX7-NEXT: s_endpgm
1983 ;
1984 ; GFX10-WGP-LABEL: flat_agent_seq_cst_atomicrmw:
1985 ; GFX10-WGP: ; %bb.0: ; %entry
1986 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
1987 ; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x8
1988 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1989 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
1990 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
1991 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
1992 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1993 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1994 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
1995 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1996 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1997 ; GFX10-WGP-NEXT: buffer_gl1_inv
1998 ; GFX10-WGP-NEXT: buffer_gl0_inv
1999 ; GFX10-WGP-NEXT: s_endpgm
2000 ;
2001 ; GFX10-CU-LABEL: flat_agent_seq_cst_atomicrmw:
2002 ; GFX10-CU: ; %bb.0: ; %entry
2003 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
2004 ; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x8
2005 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2006 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
2007 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
2008 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
2009 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2010 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
2011 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
2012 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2013 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
2014 ; GFX10-CU-NEXT: buffer_gl1_inv
2015 ; GFX10-CU-NEXT: buffer_gl0_inv
2016 ; GFX10-CU-NEXT: s_endpgm
2017 ;
2018 ; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_atomicrmw:
2019 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2020 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
2021 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x2
2022 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2023 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
2024 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
2025 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
2026 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2027 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
2028 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2029 ; SKIP-CACHE-INV-NEXT: s_endpgm
2030 ;
2031 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_atomicrmw:
2032 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
2033 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
2034 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
2035 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2036 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
2037 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
2038 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2039 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
2040 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2041 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
2042 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
2043 ;
2044 ; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_atomicrmw:
2045 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
2046 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
2047 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
2048 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2049 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
2050 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
2051 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2052 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
2053 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2054 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
2055 ; GFX90A-TGSPLIT-NEXT: s_endpgm
2056 ;
2057 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_atomicrmw:
2058 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
2059 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
2060 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
2061 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2062 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
2063 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
2064 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
2065 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2066 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
2067 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2068 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
2069 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
2070 ;
2071 ; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_atomicrmw:
2072 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
2073 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
2074 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
2075 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2076 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
2077 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
2078 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
2079 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2080 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
2081 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2082 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
2083 ; GFX940-TGSPLIT-NEXT: s_endpgm
2084 ;
2085 ; GFX11-WGP-LABEL: flat_agent_seq_cst_atomicrmw:
2086 ; GFX11-WGP: ; %bb.0: ; %entry
2087 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
2088 ; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
2089 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2090 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
2091 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
2092 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
2093 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2094 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2095 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
2096 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2097 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2098 ; GFX11-WGP-NEXT: buffer_gl1_inv
2099 ; GFX11-WGP-NEXT: buffer_gl0_inv
2100 ; GFX11-WGP-NEXT: s_endpgm
2101 ;
2102 ; GFX11-CU-LABEL: flat_agent_seq_cst_atomicrmw:
2103 ; GFX11-CU: ; %bb.0: ; %entry
2104 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
2105 ; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
2106 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2107 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
2108 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
2109 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
2110 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2111 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
2112 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
2113 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2114 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
2115 ; GFX11-CU-NEXT: buffer_gl1_inv
2116 ; GFX11-CU-NEXT: buffer_gl0_inv
2117 ; GFX11-CU-NEXT: s_endpgm
2118 ;
2119 ; GFX12-WGP-LABEL: flat_agent_seq_cst_atomicrmw:
2120 ; GFX12-WGP: ; %bb.0: ; %entry
2121 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
2122 ; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
2123 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
2124 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
2125 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
2126 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
2127 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
2128 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
2129 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
2130 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
2131 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
2132 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
2133 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
2134 ; GFX12-WGP-NEXT: s_endpgm
2135 ;
2136 ; GFX12-CU-LABEL: flat_agent_seq_cst_atomicrmw:
2137 ; GFX12-CU: ; %bb.0: ; %entry
2138 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
2139 ; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
2140 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
2141 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
2142 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
2143 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
2144 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
2145 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
2146 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
2147 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
2148 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
2149 ; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
2150 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
2151 ; GFX12-CU-NEXT: s_endpgm
2152 ptr %out, i32 %in) {
2153 entry:
2154 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") seq_cst
2155 ret void
2156 }
2158 define amdgpu_kernel void @flat_agent_acquire_ret_atomicrmw(
2159 ; GFX7-LABEL: flat_agent_acquire_ret_atomicrmw:
2160 ; GFX7: ; %bb.0: ; %entry
2161 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
2162 ; GFX7-NEXT: s_load_dword s6, s[8:9], 0x2
2163 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2164 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
2165 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
2166 ; GFX7-NEXT: v_mov_b32_e32 v2, s6
2167 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2168 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2169 ; GFX7-NEXT: buffer_wbinvl1_vol
2170 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
2171 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
2172 ; GFX7-NEXT: flat_store_dword v[0:1], v2
2173 ; GFX7-NEXT: s_endpgm
2174 ;
2175 ; GFX10-WGP-LABEL: flat_agent_acquire_ret_atomicrmw:
2176 ; GFX10-WGP: ; %bb.0: ; %entry
2177 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
2178 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x8
2179 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
2180 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
2181 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
2182 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s6
2183 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2184 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2185 ; GFX10-WGP-NEXT: buffer_gl1_inv
2186 ; GFX10-WGP-NEXT: buffer_gl0_inv
2187 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
2188 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
2189 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
2190 ; GFX10-WGP-NEXT: s_endpgm
2191 ;
2192 ; GFX10-CU-LABEL: flat_agent_acquire_ret_atomicrmw:
2193 ; GFX10-CU: ; %bb.0: ; %entry
2194 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
2195 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x8
2196 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2197 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
2198 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
2199 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s6
2200 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2201 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2202 ; GFX10-CU-NEXT: buffer_gl1_inv
2203 ; GFX10-CU-NEXT: buffer_gl0_inv
2204 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
2205 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
2206 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
2207 ; GFX10-CU-NEXT: s_endpgm
2208 ;
2209 ; SKIP-CACHE-INV-LABEL: flat_agent_acquire_ret_atomicrmw:
2210 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2211 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
2212 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x2
2213 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2214 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
2215 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
2216 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
2217 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2218 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2219 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
2220 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
2221 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
2222 ; SKIP-CACHE-INV-NEXT: s_endpgm
2223 ;
2224 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_ret_atomicrmw:
2225 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
2226 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
2227 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x8
2228 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2229 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2230 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s6
2231 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2232 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2233 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
2234 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2235 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
2236 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
2237 ;
2238 ; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_ret_atomicrmw:
2239 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
2240 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
2241 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x8
2242 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2243 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2244 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s6
2245 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2246 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2247 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
2248 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2249 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
2250 ; GFX90A-TGSPLIT-NEXT: s_endpgm
2251 ;
2252 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_ret_atomicrmw:
2253 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
2254 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
2255 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
2256 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2257 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2258 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
2259 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
2260 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2261 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
2262 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2263 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
2264 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
2265 ;
2266 ; GFX940-TGSPLIT-LABEL: flat_agent_acquire_ret_atomicrmw:
2267 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
2268 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
2269 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
2270 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2271 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2272 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
2273 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
2274 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2275 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
2276 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2277 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
2278 ; GFX940-TGSPLIT-NEXT: s_endpgm
2279 ;
2280 ; GFX11-WGP-LABEL: flat_agent_acquire_ret_atomicrmw:
2281 ; GFX11-WGP: ; %bb.0: ; %entry
2282 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
2283 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x8
2284 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2285 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
2286 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
2287 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s2
2288 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
2289 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2290 ; GFX11-WGP-NEXT: buffer_gl1_inv
2291 ; GFX11-WGP-NEXT: buffer_gl0_inv
2292 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
2293 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
2294 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
2295 ; GFX11-WGP-NEXT: s_endpgm
2296 ;
2297 ; GFX11-CU-LABEL: flat_agent_acquire_ret_atomicrmw:
2298 ; GFX11-CU: ; %bb.0: ; %entry
2299 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
2300 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x8
2301 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2302 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
2303 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
2304 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
2305 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
2306 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2307 ; GFX11-CU-NEXT: buffer_gl1_inv
2308 ; GFX11-CU-NEXT: buffer_gl0_inv
2309 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
2310 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
2311 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
2312 ; GFX11-CU-NEXT: s_endpgm
2313 ;
2314 ; GFX12-WGP-LABEL: flat_agent_acquire_ret_atomicrmw:
2315 ; GFX12-WGP: ; %bb.0: ; %entry
2316 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
2317 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x8
2318 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
2319 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
2320 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
2321 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
2322 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
2323 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
2324 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
2325 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
2326 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
2327 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
2328 ; GFX12-WGP-NEXT: s_endpgm
2329 ;
2330 ; GFX12-CU-LABEL: flat_agent_acquire_ret_atomicrmw:
2331 ; GFX12-CU: ; %bb.0: ; %entry
2332 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
2333 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x8
2334 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
2335 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
2336 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
2337 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
2338 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
2339 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
2340 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
2341 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
2342 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
2343 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
2344 ; GFX12-CU-NEXT: s_endpgm
2345 ptr %out, i32 %in) {
2346 entry:
2347 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") acquire
2348 store i32 %val, ptr %out, align 4
2349 ret void
2350 }
2352 define amdgpu_kernel void @flat_agent_acq_rel_ret_atomicrmw(
2353 ; GFX7-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2354 ; GFX7: ; %bb.0: ; %entry
2355 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
2356 ; GFX7-NEXT: s_load_dword s6, s[8:9], 0x2
2357 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2358 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
2359 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
2360 ; GFX7-NEXT: v_mov_b32_e32 v2, s6
2361 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2362 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2363 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2364 ; GFX7-NEXT: buffer_wbinvl1_vol
2365 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
2366 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
2367 ; GFX7-NEXT: flat_store_dword v[0:1], v2
2368 ; GFX7-NEXT: s_endpgm
2369 ;
2370 ; GFX10-WGP-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2371 ; GFX10-WGP: ; %bb.0: ; %entry
2372 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
2373 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x8
2374 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
2375 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
2376 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
2377 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s6
2378 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2379 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2380 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2381 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2382 ; GFX10-WGP-NEXT: buffer_gl1_inv
2383 ; GFX10-WGP-NEXT: buffer_gl0_inv
2384 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
2385 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
2386 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
2387 ; GFX10-WGP-NEXT: s_endpgm
2388 ;
2389 ; GFX10-CU-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2390 ; GFX10-CU: ; %bb.0: ; %entry
2391 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
2392 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x8
2393 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2394 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
2395 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
2396 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s6
2397 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2398 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
2399 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2400 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2401 ; GFX10-CU-NEXT: buffer_gl1_inv
2402 ; GFX10-CU-NEXT: buffer_gl0_inv
2403 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
2404 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
2405 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
2406 ; GFX10-CU-NEXT: s_endpgm
2407 ;
2408 ; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2409 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2410 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
2411 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x2
2412 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2413 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
2414 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
2415 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
2416 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2417 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2418 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2419 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
2420 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
2421 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
2422 ; SKIP-CACHE-INV-NEXT: s_endpgm
2423 ;
2424 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2425 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
2426 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
2427 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x8
2428 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2429 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2430 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s6
2431 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2432 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2433 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2434 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
2435 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2436 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
2437 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
2438 ;
2439 ; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2440 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
2441 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
2442 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x8
2443 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2444 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2445 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s6
2446 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2447 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2448 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2449 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
2450 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2451 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
2452 ; GFX90A-TGSPLIT-NEXT: s_endpgm
2453 ;
2454 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2455 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
2456 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
2457 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
2458 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2459 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2460 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
2461 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
2462 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2463 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
2464 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2465 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
2466 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2467 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
2468 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
2469 ;
2470 ; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2471 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
2472 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
2473 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
2474 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2475 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2476 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
2477 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
2478 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2479 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
2480 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2481 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
2482 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2483 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
2484 ; GFX940-TGSPLIT-NEXT: s_endpgm
2485 ;
2486 ; GFX11-WGP-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2487 ; GFX11-WGP: ; %bb.0: ; %entry
2488 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
2489 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x8
2490 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2491 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
2492 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
2493 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s2
2494 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2495 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2496 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
2497 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2498 ; GFX11-WGP-NEXT: buffer_gl1_inv
2499 ; GFX11-WGP-NEXT: buffer_gl0_inv
2500 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
2501 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
2502 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
2503 ; GFX11-WGP-NEXT: s_endpgm
2504 ;
2505 ; GFX11-CU-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2506 ; GFX11-CU: ; %bb.0: ; %entry
2507 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
2508 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x8
2509 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2510 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
2511 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
2512 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
2513 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2514 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
2515 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
2516 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2517 ; GFX11-CU-NEXT: buffer_gl1_inv
2518 ; GFX11-CU-NEXT: buffer_gl0_inv
2519 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
2520 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
2521 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
2522 ; GFX11-CU-NEXT: s_endpgm
2523 ;
2524 ; GFX12-WGP-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2525 ; GFX12-WGP: ; %bb.0: ; %entry
2526 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
2527 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x8
2528 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
2529 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
2530 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
2531 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
2532 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
2533 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
2534 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
2535 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
2536 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
2537 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
2538 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
2539 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
2540 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
2541 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
2542 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
2543 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
2544 ; GFX12-WGP-NEXT: s_endpgm
2545 ;
2546 ; GFX12-CU-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2547 ; GFX12-CU: ; %bb.0: ; %entry
2548 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
2549 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x8
2550 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
2551 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
2552 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
2553 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
2554 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
2555 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
2556 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
2557 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
2558 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
2559 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
2560 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
2561 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
2562 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
2563 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
2564 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
2565 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
2566 ; GFX12-CU-NEXT: s_endpgm
2567 ptr %out, i32 %in) {
2568 entry:
2569 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") acq_rel
2570 store i32 %val, ptr %out, align 4
2571 ret void
2572 }
2574 define amdgpu_kernel void @flat_agent_seq_cst_ret_atomicrmw(
2575 ; GFX7-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2576 ; GFX7: ; %bb.0: ; %entry
2577 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
2578 ; GFX7-NEXT: s_load_dword s6, s[8:9], 0x2
2579 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2580 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
2581 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
2582 ; GFX7-NEXT: v_mov_b32_e32 v2, s6
2583 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2584 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2585 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2586 ; GFX7-NEXT: buffer_wbinvl1_vol
2587 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
2588 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
2589 ; GFX7-NEXT: flat_store_dword v[0:1], v2
2590 ; GFX7-NEXT: s_endpgm
2591 ;
2592 ; GFX10-WGP-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2593 ; GFX10-WGP: ; %bb.0: ; %entry
2594 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
2595 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x8
2596 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
2597 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
2598 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
2599 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s6
2600 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2601 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2602 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2603 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2604 ; GFX10-WGP-NEXT: buffer_gl1_inv
2605 ; GFX10-WGP-NEXT: buffer_gl0_inv
2606 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
2607 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
2608 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
2609 ; GFX10-WGP-NEXT: s_endpgm
2610 ;
2611 ; GFX10-CU-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2612 ; GFX10-CU: ; %bb.0: ; %entry
2613 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
2614 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x8
2615 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2616 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
2617 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
2618 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s6
2619 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2620 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
2621 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2622 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2623 ; GFX10-CU-NEXT: buffer_gl1_inv
2624 ; GFX10-CU-NEXT: buffer_gl0_inv
2625 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
2626 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
2627 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
2628 ; GFX10-CU-NEXT: s_endpgm
2629 ;
2630 ; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2631 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2632 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
2633 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x2
2634 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2635 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
2636 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
2637 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
2638 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2639 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2640 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2641 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
2642 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
2643 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
2644 ; SKIP-CACHE-INV-NEXT: s_endpgm
2645 ;
2646 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2647 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
2648 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
2649 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x8
2650 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2651 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2652 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s6
2653 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2654 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2655 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2656 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
2657 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2658 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
2659 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
2660 ;
2661 ; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2662 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
2663 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
2664 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x8
2665 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2666 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2667 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s6
2668 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2669 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2670 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2671 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
2672 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2673 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
2674 ; GFX90A-TGSPLIT-NEXT: s_endpgm
2675 ;
2676 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2677 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
2678 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
2679 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
2680 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2681 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2682 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
2683 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
2684 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2685 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
2686 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2687 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
2688 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2689 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
2690 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
2691 ;
2692 ; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2693 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
2694 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
2695 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
2696 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2697 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2698 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
2699 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
2700 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2701 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
2702 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2703 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
2704 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2705 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
2706 ; GFX940-TGSPLIT-NEXT: s_endpgm
2707 ;
2708 ; GFX11-WGP-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2709 ; GFX11-WGP: ; %bb.0: ; %entry
2710 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
2711 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x8
2712 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2713 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
2714 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
2715 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s2
2716 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2717 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2718 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
2719 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2720 ; GFX11-WGP-NEXT: buffer_gl1_inv
2721 ; GFX11-WGP-NEXT: buffer_gl0_inv
2722 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
2723 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
2724 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
2725 ; GFX11-WGP-NEXT: s_endpgm
2726 ;
2727 ; GFX11-CU-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2728 ; GFX11-CU: ; %bb.0: ; %entry
2729 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
2730 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x8
2731 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2732 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
2733 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
2734 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
2735 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2736 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
2737 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
2738 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2739 ; GFX11-CU-NEXT: buffer_gl1_inv
2740 ; GFX11-CU-NEXT: buffer_gl0_inv
2741 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
2742 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
2743 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
2744 ; GFX11-CU-NEXT: s_endpgm
2745 ;
2746 ; GFX12-WGP-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2747 ; GFX12-WGP: ; %bb.0: ; %entry
2748 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
2749 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x8
2750 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
2751 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
2752 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
2753 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
2754 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
2755 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
2756 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
2757 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
2758 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
2759 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
2760 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
2761 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
2762 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
2763 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
2764 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
2765 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
2766 ; GFX12-WGP-NEXT: s_endpgm
2767 ;
2768 ; GFX12-CU-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2769 ; GFX12-CU: ; %bb.0: ; %entry
2770 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
2771 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x8
2772 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
2773 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
2774 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
2775 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
2776 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
2777 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
2778 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
2779 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
2780 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
2781 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
2782 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
2783 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
2784 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
2785 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
2786 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
2787 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
2788 ; GFX12-CU-NEXT: s_endpgm
2789 ptr %out, i32 %in) {
2790 entry:
2791 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") seq_cst
2792 store i32 %val, ptr %out, align 4
2793 ret void
2794 }
2796 define amdgpu_kernel void @flat_agent_monotonic_monotonic_cmpxchg(
2797 ; GFX7-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2798 ; GFX7: ; %bb.0: ; %entry
2799 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
2800 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
2801 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
2802 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
2803 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
2804 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2805 ; GFX7-NEXT: s_mov_b32 s4, s8
2806 ; GFX7-NEXT: s_mov_b32 s5, s9
2807 ; GFX7-NEXT: s_mov_b32 s9, s10
2808 ; GFX7-NEXT: s_mov_b32 s8, s11
2809 ; GFX7-NEXT: s_add_u32 s4, s4, s9
2810 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
2811 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
2812 ; GFX7-NEXT: s_mov_b32 s5, s8
2813 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
2814 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
2815 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
2816 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
2817 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
2818 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
2819 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2820 ; GFX7-NEXT: s_endpgm
2821 ;
2822 ; GFX10-WGP-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2823 ; GFX10-WGP: ; %bb.0: ; %entry
2824 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
2825 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
2826 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
2827 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
2828 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
2829 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
2830 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
2831 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
2832 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
2833 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
2834 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
2835 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
2836 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
2837 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
2838 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
2839 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
2840 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
2841 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
2842 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
2843 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
2844 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2845 ; GFX10-WGP-NEXT: s_endpgm
2846 ;
2847 ; GFX10-CU-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2848 ; GFX10-CU: ; %bb.0: ; %entry
2849 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
2850 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
2851 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
2852 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
2853 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
2854 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2855 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
2856 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
2857 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
2858 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
2859 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
2860 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
2861 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
2862 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
2863 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
2864 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
2865 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
2866 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
2867 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
2868 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
2869 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2870 ; GFX10-CU-NEXT: s_endpgm
2871 ;
2872 ; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2873 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2874 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
2875 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
2876 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
2877 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
2878 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
2879 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2880 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
2881 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
2882 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
2883 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
2884 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
2885 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
2886 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
2887 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
2888 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
2889 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
2890 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
2891 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
2892 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
2893 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
2894 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2895 ; SKIP-CACHE-INV-NEXT: s_endpgm
2896 ;
2897 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2898 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
2899 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
2900 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
2901 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
2902 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2903 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
2904 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
2905 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
2906 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
2907 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2908 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2909 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
2910 ;
2911 ; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2912 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
2913 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
2914 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
2915 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
2916 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2917 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
2918 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
2919 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
2920 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
2921 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2922 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2923 ; GFX90A-TGSPLIT-NEXT: s_endpgm
2924 ;
2925 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2926 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
2927 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
2928 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
2929 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
2930 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2931 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
2932 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
2933 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
2934 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
2935 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2936 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2937 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
2938 ;
2939 ; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2940 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
2941 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
2942 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
2943 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
2944 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2945 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
2946 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
2947 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
2948 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
2949 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2950 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2951 ; GFX940-TGSPLIT-NEXT: s_endpgm
2952 ;
2953 ; GFX11-WGP-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2954 ; GFX11-WGP: ; %bb.0: ; %entry
2955 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
2956 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
2957 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
2958 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2959 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
2960 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
2961 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
2962 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
2963 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
2964 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
2965 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2966 ; GFX11-WGP-NEXT: s_endpgm
2967 ;
2968 ; GFX11-CU-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2969 ; GFX11-CU: ; %bb.0: ; %entry
2970 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
2971 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
2972 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
2973 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2974 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
2975 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
2976 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
2977 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
2978 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
2979 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
2980 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2981 ; GFX11-CU-NEXT: s_endpgm
2982 ;
2983 ; GFX12-WGP-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2984 ; GFX12-WGP: ; %bb.0: ; %entry
2985 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
2986 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
2987 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
2988 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
2989 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
2990 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
2991 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
2992 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
2993 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
2994 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
2995 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
2996 ; GFX12-WGP-NEXT: s_endpgm
2997 ;
2998 ; GFX12-CU-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2999 ; GFX12-CU: ; %bb.0: ; %entry
3000 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3001 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
3002 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
3003 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
3004 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
3005 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
3006 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3007 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
3008 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
3009 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
3010 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
3011 ; GFX12-CU-NEXT: s_endpgm
3012 ptr %out, i32 %in, i32 %old) {
3013 entry:
3014 %gep = getelementptr i32, ptr %out, i32 4
3015 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") monotonic monotonic
3016 ret void
3017 }
3019 define amdgpu_kernel void @flat_agent_acquire_monotonic_cmpxchg(
3020 ; GFX7-LABEL: flat_agent_acquire_monotonic_cmpxchg:
3021 ; GFX7: ; %bb.0: ; %entry
3022 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
3023 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3024 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
3025 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
3026 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
3027 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3028 ; GFX7-NEXT: s_mov_b32 s4, s8
3029 ; GFX7-NEXT: s_mov_b32 s5, s9
3030 ; GFX7-NEXT: s_mov_b32 s9, s10
3031 ; GFX7-NEXT: s_mov_b32 s8, s11
3032 ; GFX7-NEXT: s_add_u32 s4, s4, s9
3033 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
3034 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3035 ; GFX7-NEXT: s_mov_b32 s5, s8
3036 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
3037 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
3038 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3039 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
3040 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
3041 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
3042 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3043 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3044 ; GFX7-NEXT: buffer_wbinvl1_vol
3045 ; GFX7-NEXT: s_endpgm
3046 ;
3047 ; GFX10-WGP-LABEL: flat_agent_acquire_monotonic_cmpxchg:
3048 ; GFX10-WGP: ; %bb.0: ; %entry
3049 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
3050 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3051 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
3052 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
3053 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
3054 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3055 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
3056 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
3057 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
3058 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
3059 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
3060 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
3061 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3062 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
3063 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
3064 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
3065 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3066 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
3067 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
3068 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
3069 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3070 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3071 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3072 ; GFX10-WGP-NEXT: buffer_gl1_inv
3073 ; GFX10-WGP-NEXT: buffer_gl0_inv
3074 ; GFX10-WGP-NEXT: s_endpgm
3075 ;
3076 ; GFX10-CU-LABEL: flat_agent_acquire_monotonic_cmpxchg:
3077 ; GFX10-CU: ; %bb.0: ; %entry
3078 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
3079 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3080 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
3081 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
3082 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
3083 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3084 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
3085 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
3086 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
3087 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
3088 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
3089 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
3090 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3091 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
3092 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
3093 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
3094 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3095 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
3096 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
3097 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
3098 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3099 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3100 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
3101 ; GFX10-CU-NEXT: buffer_gl1_inv
3102 ; GFX10-CU-NEXT: buffer_gl0_inv
3103 ; GFX10-CU-NEXT: s_endpgm
3104 ;
3105 ; SKIP-CACHE-INV-LABEL: flat_agent_acquire_monotonic_cmpxchg:
3106 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
3107 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
3108 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
3109 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
3110 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
3111 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
3112 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3113 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
3114 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
3115 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
3116 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
3117 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
3118 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
3119 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
3120 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
3121 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
3122 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
3123 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3124 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
3125 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
3126 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
3127 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3128 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3129 ; SKIP-CACHE-INV-NEXT: s_endpgm
3130 ;
3131 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_monotonic_cmpxchg:
3132 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
3133 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3134 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3135 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3136 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3137 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3138 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
3139 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3140 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3141 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
3142 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3143 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3144 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
3145 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
3146 ;
3147 ; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_monotonic_cmpxchg:
3148 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
3149 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3150 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3151 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3152 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3153 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3154 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
3155 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3156 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3157 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
3158 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3159 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
3160 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
3161 ; GFX90A-TGSPLIT-NEXT: s_endpgm
3162 ;
3163 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_monotonic_cmpxchg:
3164 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
3165 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3166 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3167 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3168 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3169 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3170 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
3171 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3172 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3173 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3174 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3175 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3176 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
3177 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
3178 ;
3179 ; GFX940-TGSPLIT-LABEL: flat_agent_acquire_monotonic_cmpxchg:
3180 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
3181 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3182 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3183 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3184 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3185 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3186 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
3187 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3188 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3189 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3190 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3191 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
3192 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
3193 ; GFX940-TGSPLIT-NEXT: s_endpgm
3194 ;
3195 ; GFX11-WGP-LABEL: flat_agent_acquire_monotonic_cmpxchg:
3196 ; GFX11-WGP: ; %bb.0: ; %entry
3197 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3198 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
3199 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
3200 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3201 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
3202 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
3203 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3204 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
3205 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
3206 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
3207 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3208 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3209 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3210 ; GFX11-WGP-NEXT: buffer_gl1_inv
3211 ; GFX11-WGP-NEXT: buffer_gl0_inv
3212 ; GFX11-WGP-NEXT: s_endpgm
3213 ;
3214 ; GFX11-CU-LABEL: flat_agent_acquire_monotonic_cmpxchg:
3215 ; GFX11-CU: ; %bb.0: ; %entry
3216 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3217 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
3218 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
3219 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3220 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
3221 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
3222 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3223 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
3224 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
3225 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
3226 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3227 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3228 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
3229 ; GFX11-CU-NEXT: buffer_gl1_inv
3230 ; GFX11-CU-NEXT: buffer_gl0_inv
3231 ; GFX11-CU-NEXT: s_endpgm
3232 ;
3233 ; GFX12-WGP-LABEL: flat_agent_acquire_monotonic_cmpxchg:
3234 ; GFX12-WGP: ; %bb.0: ; %entry
3235 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3236 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
3237 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
3238 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
3239 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
3240 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
3241 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3242 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
3243 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
3244 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
3245 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
3246 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
3247 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
3248 ; GFX12-WGP-NEXT: s_endpgm
3249 ;
3250 ; GFX12-CU-LABEL: flat_agent_acquire_monotonic_cmpxchg:
3251 ; GFX12-CU: ; %bb.0: ; %entry
3252 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3253 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
3254 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
3255 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
3256 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
3257 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
3258 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3259 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
3260 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
3261 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
3262 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
3263 ; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
3264 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
3265 ; GFX12-CU-NEXT: s_endpgm
3266 ptr %out, i32 %in, i32 %old) {
3267 entry:
3268 %gep = getelementptr i32, ptr %out, i32 4
3269 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
3270 ret void
3271 }
3273 define amdgpu_kernel void @flat_agent_release_monotonic_cmpxchg(
3274 ; GFX7-LABEL: flat_agent_release_monotonic_cmpxchg:
3275 ; GFX7: ; %bb.0: ; %entry
3276 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
3277 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3278 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
3279 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
3280 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
3281 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3282 ; GFX7-NEXT: s_mov_b32 s4, s8
3283 ; GFX7-NEXT: s_mov_b32 s5, s9
3284 ; GFX7-NEXT: s_mov_b32 s9, s10
3285 ; GFX7-NEXT: s_mov_b32 s8, s11
3286 ; GFX7-NEXT: s_add_u32 s4, s4, s9
3287 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
3288 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3289 ; GFX7-NEXT: s_mov_b32 s5, s8
3290 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
3291 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
3292 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3293 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
3294 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
3295 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
3296 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3297 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3298 ; GFX7-NEXT: s_endpgm
3299 ;
3300 ; GFX10-WGP-LABEL: flat_agent_release_monotonic_cmpxchg:
3301 ; GFX10-WGP: ; %bb.0: ; %entry
3302 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
3303 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3304 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
3305 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
3306 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
3307 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3308 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
3309 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
3310 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
3311 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
3312 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
3313 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
3314 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3315 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
3316 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
3317 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
3318 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3319 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
3320 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
3321 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
3322 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3323 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3324 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3325 ; GFX10-WGP-NEXT: s_endpgm
3326 ;
3327 ; GFX10-CU-LABEL: flat_agent_release_monotonic_cmpxchg:
3328 ; GFX10-CU: ; %bb.0: ; %entry
3329 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
3330 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3331 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
3332 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
3333 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
3334 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3335 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
3336 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
3337 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
3338 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
3339 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
3340 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
3341 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3342 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
3343 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
3344 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
3345 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3346 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
3347 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
3348 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
3349 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3350 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
3351 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3352 ; GFX10-CU-NEXT: s_endpgm
3353 ;
3354 ; SKIP-CACHE-INV-LABEL: flat_agent_release_monotonic_cmpxchg:
3355 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
3356 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
3357 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
3358 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
3359 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
3360 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
3361 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3362 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
3363 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
3364 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
3365 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
3366 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
3367 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
3368 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
3369 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
3370 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
3371 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
3372 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3373 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
3374 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
3375 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
3376 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3377 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3378 ; SKIP-CACHE-INV-NEXT: s_endpgm
3379 ;
3380 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_monotonic_cmpxchg:
3381 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
3382 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3383 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3384 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3385 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3386 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3387 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
3388 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3389 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3390 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
3391 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3392 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3393 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
3394 ;
3395 ; GFX90A-TGSPLIT-LABEL: flat_agent_release_monotonic_cmpxchg:
3396 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
3397 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3398 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3399 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3400 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3401 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3402 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
3403 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3404 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3405 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
3406 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3407 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3408 ; GFX90A-TGSPLIT-NEXT: s_endpgm
3409 ;
3410 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_monotonic_cmpxchg:
3411 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
3412 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3413 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3414 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3415 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3416 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3417 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
3418 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3419 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3420 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3421 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
3422 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3423 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3424 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
3425 ;
3426 ; GFX940-TGSPLIT-LABEL: flat_agent_release_monotonic_cmpxchg:
3427 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
3428 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3429 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3430 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3431 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3432 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3433 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
3434 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3435 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3436 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3437 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
3438 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3439 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3440 ; GFX940-TGSPLIT-NEXT: s_endpgm
3441 ;
3442 ; GFX11-WGP-LABEL: flat_agent_release_monotonic_cmpxchg:
3443 ; GFX11-WGP: ; %bb.0: ; %entry
3444 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3445 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
3446 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
3447 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3448 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
3449 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
3450 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3451 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
3452 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
3453 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
3454 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3455 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3456 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3457 ; GFX11-WGP-NEXT: s_endpgm
3458 ;
3459 ; GFX11-CU-LABEL: flat_agent_release_monotonic_cmpxchg:
3460 ; GFX11-CU: ; %bb.0: ; %entry
3461 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3462 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
3463 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
3464 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3465 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
3466 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
3467 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3468 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
3469 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
3470 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
3471 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3472 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
3473 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3474 ; GFX11-CU-NEXT: s_endpgm
3475 ;
3476 ; GFX12-WGP-LABEL: flat_agent_release_monotonic_cmpxchg:
3477 ; GFX12-WGP: ; %bb.0: ; %entry
3478 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3479 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
3480 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
3481 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
3482 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
3483 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
3484 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3485 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
3486 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
3487 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
3488 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
3489 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
3490 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
3491 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
3492 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
3493 ; GFX12-WGP-NEXT: s_endpgm
3494 ;
3495 ; GFX12-CU-LABEL: flat_agent_release_monotonic_cmpxchg:
3496 ; GFX12-CU: ; %bb.0: ; %entry
3497 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3498 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
3499 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
3500 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
3501 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
3502 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
3503 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3504 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
3505 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
3506 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
3507 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
3508 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
3509 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
3510 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
3511 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
3512 ; GFX12-CU-NEXT: s_endpgm
3513 ptr %out, i32 %in, i32 %old) {
3514 entry:
3515 %gep = getelementptr i32, ptr %out, i32 4
3516 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") release monotonic
3517 ret void
3518 }
3520 define amdgpu_kernel void @flat_agent_acq_rel_monotonic_cmpxchg(
3521 ; GFX7-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
3522 ; GFX7: ; %bb.0: ; %entry
3523 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
3524 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3525 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
3526 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
3527 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
3528 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3529 ; GFX7-NEXT: s_mov_b32 s4, s8
3530 ; GFX7-NEXT: s_mov_b32 s5, s9
3531 ; GFX7-NEXT: s_mov_b32 s9, s10
3532 ; GFX7-NEXT: s_mov_b32 s8, s11
3533 ; GFX7-NEXT: s_add_u32 s4, s4, s9
3534 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
3535 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3536 ; GFX7-NEXT: s_mov_b32 s5, s8
3537 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
3538 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
3539 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3540 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
3541 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
3542 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
3543 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3544 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3545 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3546 ; GFX7-NEXT: buffer_wbinvl1_vol
3547 ; GFX7-NEXT: s_endpgm
3548 ;
3549 ; GFX10-WGP-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
3550 ; GFX10-WGP: ; %bb.0: ; %entry
3551 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
3552 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3553 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
3554 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
3555 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
3556 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3557 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
3558 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
3559 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
3560 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
3561 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
3562 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
3563 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3564 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
3565 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
3566 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
3567 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3568 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
3569 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
3570 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
3571 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3572 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3573 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3574 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3575 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3576 ; GFX10-WGP-NEXT: buffer_gl1_inv
3577 ; GFX10-WGP-NEXT: buffer_gl0_inv
3578 ; GFX10-WGP-NEXT: s_endpgm
3579 ;
3580 ; GFX10-CU-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
3581 ; GFX10-CU: ; %bb.0: ; %entry
3582 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
3583 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3584 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
3585 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
3586 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
3587 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3588 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
3589 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
3590 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
3591 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
3592 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
3593 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
3594 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3595 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
3596 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
3597 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
3598 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3599 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
3600 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
3601 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
3602 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3603 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
3604 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3605 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3606 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
3607 ; GFX10-CU-NEXT: buffer_gl1_inv
3608 ; GFX10-CU-NEXT: buffer_gl0_inv
3609 ; GFX10-CU-NEXT: s_endpgm
3610 ;
3611 ; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
3612 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
3613 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
3614 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
3615 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
3616 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
3617 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
3618 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3619 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
3620 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
3621 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
3622 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
3623 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
3624 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
3625 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
3626 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
3627 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
3628 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
3629 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3630 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
3631 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
3632 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
3633 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3634 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3635 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3636 ; SKIP-CACHE-INV-NEXT: s_endpgm
3637 ;
3638 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
3639 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
3640 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3641 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3642 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3643 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3644 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3645 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
3646 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3647 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3648 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
3649 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3650 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3651 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3652 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
3653 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
3654 ;
3655 ; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
3656 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
3657 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3658 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3659 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3660 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3661 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3662 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
3663 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3664 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3665 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
3666 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3667 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3668 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
3669 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
3670 ; GFX90A-TGSPLIT-NEXT: s_endpgm
3671 ;
3672 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
3673 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
3674 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3675 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3676 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3677 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3678 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3679 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
3680 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3681 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3682 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3683 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
3684 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3685 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3686 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3687 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
3688 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
3689 ;
3690 ; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
3691 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
3692 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3693 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3694 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3695 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3696 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3697 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
3698 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3699 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3700 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3701 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
3702 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3703 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3704 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
3705 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
3706 ; GFX940-TGSPLIT-NEXT: s_endpgm
3707 ;
3708 ; GFX11-WGP-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
3709 ; GFX11-WGP: ; %bb.0: ; %entry
3710 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3711 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
3712 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
3713 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3714 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
3715 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
3716 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3717 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
3718 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
3719 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
3720 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3721 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3722 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3723 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3724 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3725 ; GFX11-WGP-NEXT: buffer_gl1_inv
3726 ; GFX11-WGP-NEXT: buffer_gl0_inv
3727 ; GFX11-WGP-NEXT: s_endpgm
3728 ;
3729 ; GFX11-CU-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
3730 ; GFX11-CU: ; %bb.0: ; %entry
3731 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3732 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
3733 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
3734 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3735 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
3736 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
3737 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3738 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
3739 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
3740 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
3741 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3742 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
3743 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3744 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3745 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
3746 ; GFX11-CU-NEXT: buffer_gl1_inv
3747 ; GFX11-CU-NEXT: buffer_gl0_inv
3748 ; GFX11-CU-NEXT: s_endpgm
3749 ;
3750 ; GFX12-WGP-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
3751 ; GFX12-WGP: ; %bb.0: ; %entry
3752 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3753 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
3754 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
3755 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
3756 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
3757 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
3758 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3759 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
3760 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
3761 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
3762 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
3763 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
3764 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
3765 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
3766 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
3767 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
3768 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
3769 ; GFX12-WGP-NEXT: s_endpgm
3770 ;
3771 ; GFX12-CU-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
3772 ; GFX12-CU: ; %bb.0: ; %entry
3773 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3774 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
3775 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
3776 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
3777 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
3778 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
3779 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3780 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
3781 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
3782 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
3783 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
3784 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
3785 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
3786 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
3787 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
3788 ; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
3789 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
3790 ; GFX12-CU-NEXT: s_endpgm
3791 ptr %out, i32 %in, i32 %old) {
3792 entry:
3793 %gep = getelementptr i32, ptr %out, i32 4
3794 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
3795 ret void
3796 }
3798 define amdgpu_kernel void @flat_agent_seq_cst_monotonic_cmpxchg(
3799 ; GFX7-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
3800 ; GFX7: ; %bb.0: ; %entry
3801 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
3802 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3803 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
3804 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
3805 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
3806 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3807 ; GFX7-NEXT: s_mov_b32 s4, s8
3808 ; GFX7-NEXT: s_mov_b32 s5, s9
3809 ; GFX7-NEXT: s_mov_b32 s9, s10
3810 ; GFX7-NEXT: s_mov_b32 s8, s11
3811 ; GFX7-NEXT: s_add_u32 s4, s4, s9
3812 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
3813 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3814 ; GFX7-NEXT: s_mov_b32 s5, s8
3815 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
3816 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
3817 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3818 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
3819 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
3820 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
3821 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3822 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3823 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3824 ; GFX7-NEXT: buffer_wbinvl1_vol
3825 ; GFX7-NEXT: s_endpgm
3826 ;
3827 ; GFX10-WGP-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
3828 ; GFX10-WGP: ; %bb.0: ; %entry
3829 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
3830 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3831 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
3832 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
3833 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
3834 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3835 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
3836 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
3837 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
3838 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
3839 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
3840 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
3841 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3842 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
3843 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
3844 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
3845 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3846 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
3847 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
3848 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
3849 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3850 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3851 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3852 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3853 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3854 ; GFX10-WGP-NEXT: buffer_gl1_inv
3855 ; GFX10-WGP-NEXT: buffer_gl0_inv
3856 ; GFX10-WGP-NEXT: s_endpgm
3857 ;
3858 ; GFX10-CU-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
3859 ; GFX10-CU: ; %bb.0: ; %entry
3860 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
3861 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3862 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
3863 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
3864 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
3865 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3866 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
3867 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
3868 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
3869 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
3870 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
3871 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
3872 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3873 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
3874 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
3875 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
3876 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3877 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
3878 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
3879 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
3880 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3881 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
3882 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3883 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3884 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
3885 ; GFX10-CU-NEXT: buffer_gl1_inv
3886 ; GFX10-CU-NEXT: buffer_gl0_inv
3887 ; GFX10-CU-NEXT: s_endpgm
3888 ;
3889 ; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
3890 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
3891 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
3892 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
3893 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
3894 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
3895 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
3896 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3897 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
3898 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
3899 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
3900 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
3901 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
3902 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
3903 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
3904 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
3905 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
3906 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
3907 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3908 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
3909 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
3910 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
3911 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3912 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3913 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3914 ; SKIP-CACHE-INV-NEXT: s_endpgm
3915 ;
3916 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
3917 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
3918 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3919 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3920 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3921 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3922 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3923 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
3924 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3925 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3926 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
3927 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3928 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3929 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3930 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
3931 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
3932 ;
3933 ; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
3934 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
3935 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3936 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3937 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3938 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3939 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3940 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
3941 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3942 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3943 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
3944 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3945 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3946 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
3947 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
3948 ; GFX90A-TGSPLIT-NEXT: s_endpgm
3949 ;
3950 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
3951 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
3952 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3953 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3954 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3955 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3956 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3957 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
3958 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3959 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3960 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3961 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
3962 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3963 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3964 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3965 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
3966 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
3967 ;
3968 ; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
3969 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
3970 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3971 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3972 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3973 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3974 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3975 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
3976 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3977 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3978 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3979 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
3980 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3981 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3982 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
3983 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
3984 ; GFX940-TGSPLIT-NEXT: s_endpgm
3985 ;
3986 ; GFX11-WGP-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
3987 ; GFX11-WGP: ; %bb.0: ; %entry
3988 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3989 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
3990 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
3991 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3992 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
3993 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
3994 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3995 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
3996 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
3997 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
3998 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3999 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4000 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4001 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4002 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4003 ; GFX11-WGP-NEXT: buffer_gl1_inv
4004 ; GFX11-WGP-NEXT: buffer_gl0_inv
4005 ; GFX11-WGP-NEXT: s_endpgm
4006 ;
4007 ; GFX11-CU-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
4008 ; GFX11-CU: ; %bb.0: ; %entry
4009 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4010 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4011 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4012 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4013 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
4014 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
4015 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4016 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
4017 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
4018 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
4019 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4020 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
4021 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4022 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4023 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
4024 ; GFX11-CU-NEXT: buffer_gl1_inv
4025 ; GFX11-CU-NEXT: buffer_gl0_inv
4026 ; GFX11-CU-NEXT: s_endpgm
4027 ;
4028 ; GFX12-WGP-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
4029 ; GFX12-WGP: ; %bb.0: ; %entry
4030 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4031 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
4032 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
4033 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
4034 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
4035 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
4036 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4037 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
4038 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
4039 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
4040 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
4041 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
4042 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
4043 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
4044 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
4045 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
4046 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
4047 ; GFX12-WGP-NEXT: s_endpgm
4048 ;
4049 ; GFX12-CU-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
4050 ; GFX12-CU: ; %bb.0: ; %entry
4051 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4052 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4053 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4054 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
4055 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
4056 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
4057 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4058 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
4059 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
4060 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
4061 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
4062 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
4063 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
4064 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
4065 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
4066 ; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
4067 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
4068 ; GFX12-CU-NEXT: s_endpgm
4069 ptr %out, i32 %in, i32 %old) {
4070 entry:
4071 %gep = getelementptr i32, ptr %out, i32 4
4072 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
4073 ret void
4074 }
4076 define amdgpu_kernel void @flat_agent_monotonic_acquire_cmpxchg(
4077 ; GFX7-LABEL: flat_agent_monotonic_acquire_cmpxchg:
4078 ; GFX7: ; %bb.0: ; %entry
4079 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
4080 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4081 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
4082 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
4083 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
4084 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4085 ; GFX7-NEXT: s_mov_b32 s4, s8
4086 ; GFX7-NEXT: s_mov_b32 s5, s9
4087 ; GFX7-NEXT: s_mov_b32 s9, s10
4088 ; GFX7-NEXT: s_mov_b32 s8, s11
4089 ; GFX7-NEXT: s_add_u32 s4, s4, s9
4090 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
4091 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4092 ; GFX7-NEXT: s_mov_b32 s5, s8
4093 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
4094 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
4095 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4096 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
4097 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
4098 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
4099 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4100 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4101 ; GFX7-NEXT: buffer_wbinvl1_vol
4102 ; GFX7-NEXT: s_endpgm
4103 ;
4104 ; GFX10-WGP-LABEL: flat_agent_monotonic_acquire_cmpxchg:
4105 ; GFX10-WGP: ; %bb.0: ; %entry
4106 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
4107 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4108 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
4109 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
4110 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
4111 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4112 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
4113 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
4114 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
4115 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
4116 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
4117 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
4118 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4119 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
4120 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
4121 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
4122 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4123 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
4124 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
4125 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
4126 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4127 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4128 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4129 ; GFX10-WGP-NEXT: buffer_gl1_inv
4130 ; GFX10-WGP-NEXT: buffer_gl0_inv
4131 ; GFX10-WGP-NEXT: s_endpgm
4132 ;
4133 ; GFX10-CU-LABEL: flat_agent_monotonic_acquire_cmpxchg:
4134 ; GFX10-CU: ; %bb.0: ; %entry
4135 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
4136 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4137 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
4138 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
4139 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
4140 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4141 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
4142 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
4143 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
4144 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
4145 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
4146 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
4147 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4148 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
4149 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
4150 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
4151 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4152 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
4153 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
4154 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
4155 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4156 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4157 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
4158 ; GFX10-CU-NEXT: buffer_gl1_inv
4159 ; GFX10-CU-NEXT: buffer_gl0_inv
4160 ; GFX10-CU-NEXT: s_endpgm
4161 ;
4162 ; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_acquire_cmpxchg:
4163 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
4164 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
4165 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
4166 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
4167 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
4168 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
4169 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4170 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
4171 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
4172 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
4173 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
4174 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
4175 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
4176 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
4177 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
4178 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
4179 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
4180 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4181 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
4182 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
4183 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
4184 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4185 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4186 ; SKIP-CACHE-INV-NEXT: s_endpgm
4187 ;
4188 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_acquire_cmpxchg:
4189 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
4190 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4191 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
4192 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
4193 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4194 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
4195 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
4196 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4197 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4198 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
4199 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4200 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4201 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
4202 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
4203 ;
4204 ; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_acquire_cmpxchg:
4205 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
4206 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4207 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
4208 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
4209 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4210 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
4211 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
4212 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4213 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4214 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
4215 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4216 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4217 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
4218 ; GFX90A-TGSPLIT-NEXT: s_endpgm
4219 ;
4220 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_acquire_cmpxchg:
4221 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
4222 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
4223 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
4224 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
4225 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4226 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
4227 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
4228 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4229 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4230 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4231 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4232 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4233 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
4234 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
4235 ;
4236 ; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_acquire_cmpxchg:
4237 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
4238 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
4239 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
4240 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
4241 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4242 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
4243 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
4244 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4245 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4246 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4247 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4248 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4249 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
4250 ; GFX940-TGSPLIT-NEXT: s_endpgm
4251 ;
4252 ; GFX11-WGP-LABEL: flat_agent_monotonic_acquire_cmpxchg:
4253 ; GFX11-WGP: ; %bb.0: ; %entry
4254 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4255 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
4256 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
4257 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4258 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
4259 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
4260 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4261 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
4262 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
4263 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
4264 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4265 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4266 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4267 ; GFX11-WGP-NEXT: buffer_gl1_inv
4268 ; GFX11-WGP-NEXT: buffer_gl0_inv
4269 ; GFX11-WGP-NEXT: s_endpgm
4270 ;
4271 ; GFX11-CU-LABEL: flat_agent_monotonic_acquire_cmpxchg:
4272 ; GFX11-CU: ; %bb.0: ; %entry
4273 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4274 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4275 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4276 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4277 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
4278 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
4279 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4280 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
4281 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
4282 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
4283 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4284 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4285 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
4286 ; GFX11-CU-NEXT: buffer_gl1_inv
4287 ; GFX11-CU-NEXT: buffer_gl0_inv
4288 ; GFX11-CU-NEXT: s_endpgm
4289 ;
4290 ; GFX12-WGP-LABEL: flat_agent_monotonic_acquire_cmpxchg:
4291 ; GFX12-WGP: ; %bb.0: ; %entry
4292 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4293 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
4294 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
4295 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
4296 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
4297 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
4298 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4299 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
4300 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
4301 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
4302 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
4303 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
4304 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
4305 ; GFX12-WGP-NEXT: s_endpgm
4306 ;
4307 ; GFX12-CU-LABEL: flat_agent_monotonic_acquire_cmpxchg:
4308 ; GFX12-CU: ; %bb.0: ; %entry
4309 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4310 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4311 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4312 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
4313 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
4314 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
4315 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4316 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
4317 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
4318 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
4319 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
4320 ; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
4321 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
4322 ; GFX12-CU-NEXT: s_endpgm
4323 ptr %out, i32 %in, i32 %old) {
4324 entry:
4325 %gep = getelementptr i32, ptr %out, i32 4
4326 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") monotonic acquire
4327 ret void
4328 }
4330 define amdgpu_kernel void @flat_agent_acquire_acquire_cmpxchg(
4331 ; GFX7-LABEL: flat_agent_acquire_acquire_cmpxchg:
4332 ; GFX7: ; %bb.0: ; %entry
4333 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
4334 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4335 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
4336 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
4337 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
4338 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4339 ; GFX7-NEXT: s_mov_b32 s4, s8
4340 ; GFX7-NEXT: s_mov_b32 s5, s9
4341 ; GFX7-NEXT: s_mov_b32 s9, s10
4342 ; GFX7-NEXT: s_mov_b32 s8, s11
4343 ; GFX7-NEXT: s_add_u32 s4, s4, s9
4344 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
4345 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4346 ; GFX7-NEXT: s_mov_b32 s5, s8
4347 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
4348 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
4349 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4350 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
4351 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
4352 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
4353 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4354 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4355 ; GFX7-NEXT: buffer_wbinvl1_vol
4356 ; GFX7-NEXT: s_endpgm
4357 ;
4358 ; GFX10-WGP-LABEL: flat_agent_acquire_acquire_cmpxchg:
4359 ; GFX10-WGP: ; %bb.0: ; %entry
4360 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
4361 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4362 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
4363 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
4364 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
4365 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4366 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
4367 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
4368 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
4369 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
4370 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
4371 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
4372 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4373 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
4374 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
4375 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
4376 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4377 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
4378 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
4379 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
4380 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4381 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4382 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4383 ; GFX10-WGP-NEXT: buffer_gl1_inv
4384 ; GFX10-WGP-NEXT: buffer_gl0_inv
4385 ; GFX10-WGP-NEXT: s_endpgm
4386 ;
4387 ; GFX10-CU-LABEL: flat_agent_acquire_acquire_cmpxchg:
4388 ; GFX10-CU: ; %bb.0: ; %entry
4389 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
4390 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4391 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
4392 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
4393 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
4394 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4395 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
4396 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
4397 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
4398 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
4399 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
4400 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
4401 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4402 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
4403 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
4404 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
4405 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4406 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
4407 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
4408 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
4409 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4410 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4411 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
4412 ; GFX10-CU-NEXT: buffer_gl1_inv
4413 ; GFX10-CU-NEXT: buffer_gl0_inv
4414 ; GFX10-CU-NEXT: s_endpgm
4415 ;
4416 ; SKIP-CACHE-INV-LABEL: flat_agent_acquire_acquire_cmpxchg:
4417 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
4418 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
4419 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
4420 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
4421 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
4422 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
4423 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4424 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
4425 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
4426 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
4427 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
4428 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
4429 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
4430 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
4431 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
4432 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
4433 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
4434 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4435 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
4436 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
4437 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
4438 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4439 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4440 ; SKIP-CACHE-INV-NEXT: s_endpgm
4441 ;
4442 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_acquire_cmpxchg:
4443 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
4444 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4445 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
4446 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
4447 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4448 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
4449 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
4450 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4451 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4452 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
4453 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4454 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4455 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
4456 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
4457 ;
4458 ; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_acquire_cmpxchg:
4459 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
4460 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4461 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
4462 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
4463 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4464 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
4465 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
4466 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4467 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4468 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
4469 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4470 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4471 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
4472 ; GFX90A-TGSPLIT-NEXT: s_endpgm
4473 ;
4474 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_acquire_cmpxchg:
4475 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
4476 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
4477 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
4478 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
4479 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4480 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
4481 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
4482 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4483 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4484 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4485 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4486 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4487 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
4488 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
4489 ;
4490 ; GFX940-TGSPLIT-LABEL: flat_agent_acquire_acquire_cmpxchg:
4491 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
4492 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
4493 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
4494 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
4495 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4496 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
4497 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
4498 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4499 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4500 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4501 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4502 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4503 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
4504 ; GFX940-TGSPLIT-NEXT: s_endpgm
4505 ;
4506 ; GFX11-WGP-LABEL: flat_agent_acquire_acquire_cmpxchg:
4507 ; GFX11-WGP: ; %bb.0: ; %entry
4508 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4509 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
4510 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
4511 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4512 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
4513 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
4514 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4515 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
4516 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
4517 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
4518 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4519 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4520 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4521 ; GFX11-WGP-NEXT: buffer_gl1_inv
4522 ; GFX11-WGP-NEXT: buffer_gl0_inv
4523 ; GFX11-WGP-NEXT: s_endpgm
4524 ;
4525 ; GFX11-CU-LABEL: flat_agent_acquire_acquire_cmpxchg:
4526 ; GFX11-CU: ; %bb.0: ; %entry
4527 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4528 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4529 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4530 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4531 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
4532 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
4533 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4534 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
4535 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
4536 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
4537 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4538 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4539 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
4540 ; GFX11-CU-NEXT: buffer_gl1_inv
4541 ; GFX11-CU-NEXT: buffer_gl0_inv
4542 ; GFX11-CU-NEXT: s_endpgm
4543 ;
4544 ; GFX12-WGP-LABEL: flat_agent_acquire_acquire_cmpxchg:
4545 ; GFX12-WGP: ; %bb.0: ; %entry
4546 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4547 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
4548 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
4549 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
4550 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
4551 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
4552 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4553 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
4554 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
4555 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
4556 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
4557 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
4558 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
4559 ; GFX12-WGP-NEXT: s_endpgm
4560 ;
4561 ; GFX12-CU-LABEL: flat_agent_acquire_acquire_cmpxchg:
4562 ; GFX12-CU: ; %bb.0: ; %entry
4563 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4564 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4565 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4566 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
4567 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
4568 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
4569 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4570 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
4571 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
4572 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
4573 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
4574 ; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
4575 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
4576 ; GFX12-CU-NEXT: s_endpgm
4577 ptr %out, i32 %in, i32 %old) {
4578 entry:
4579 %gep = getelementptr i32, ptr %out, i32 4
4580 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
4581 ret void
4582 }
4584 define amdgpu_kernel void @flat_agent_release_acquire_cmpxchg(
4585 ; GFX7-LABEL: flat_agent_release_acquire_cmpxchg:
4586 ; GFX7: ; %bb.0: ; %entry
4587 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
4588 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4589 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
4590 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
4591 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
4592 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4593 ; GFX7-NEXT: s_mov_b32 s4, s8
4594 ; GFX7-NEXT: s_mov_b32 s5, s9
4595 ; GFX7-NEXT: s_mov_b32 s9, s10
4596 ; GFX7-NEXT: s_mov_b32 s8, s11
4597 ; GFX7-NEXT: s_add_u32 s4, s4, s9
4598 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
4599 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4600 ; GFX7-NEXT: s_mov_b32 s5, s8
4601 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
4602 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
4603 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4604 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
4605 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
4606 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
4607 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4608 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4609 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4610 ; GFX7-NEXT: buffer_wbinvl1_vol
4611 ; GFX7-NEXT: s_endpgm
4612 ;
4613 ; GFX10-WGP-LABEL: flat_agent_release_acquire_cmpxchg:
4614 ; GFX10-WGP: ; %bb.0: ; %entry
4615 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
4616 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4617 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
4618 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
4619 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
4620 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4621 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
4622 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
4623 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
4624 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
4625 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
4626 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
4627 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4628 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
4629 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
4630 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
4631 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4632 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
4633 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
4634 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
4635 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4636 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4637 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4638 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4639 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4640 ; GFX10-WGP-NEXT: buffer_gl1_inv
4641 ; GFX10-WGP-NEXT: buffer_gl0_inv
4642 ; GFX10-WGP-NEXT: s_endpgm
4643 ;
4644 ; GFX10-CU-LABEL: flat_agent_release_acquire_cmpxchg:
4645 ; GFX10-CU: ; %bb.0: ; %entry
4646 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
4647 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4648 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
4649 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
4650 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
4651 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4652 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
4653 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
4654 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
4655 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
4656 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
4657 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
4658 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4659 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
4660 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
4661 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
4662 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4663 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
4664 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
4665 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
4666 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4667 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
4668 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4669 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4670 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
4671 ; GFX10-CU-NEXT: buffer_gl1_inv
4672 ; GFX10-CU-NEXT: buffer_gl0_inv
4673 ; GFX10-CU-NEXT: s_endpgm
4674 ;
4675 ; SKIP-CACHE-INV-LABEL: flat_agent_release_acquire_cmpxchg:
4676 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
4677 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
4678 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
4679 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
4680 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
4681 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
4682 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4683 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
4684 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
4685 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
4686 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
4687 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
4688 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
4689 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
4690 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
4691 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
4692 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
4693 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4694 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
4695 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
4696 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
4697 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4698 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4699 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4700 ; SKIP-CACHE-INV-NEXT: s_endpgm
4701 ;
4702 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_acquire_cmpxchg:
4703 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
4704 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4705 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
4706 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
4707 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4708 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
4709 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
4710 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4711 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4712 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
4713 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4714 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4715 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4716 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
4717 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
4718 ;
4719 ; GFX90A-TGSPLIT-LABEL: flat_agent_release_acquire_cmpxchg:
4720 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
4721 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4722 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
4723 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
4724 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4725 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
4726 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
4727 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4728 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4729 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
4730 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4731 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4732 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4733 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
4734 ; GFX90A-TGSPLIT-NEXT: s_endpgm
4735 ;
4736 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_acquire_cmpxchg:
4737 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
4738 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
4739 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
4740 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
4741 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4742 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
4743 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
4744 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4745 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4746 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4747 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
4748 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4749 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4750 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4751 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
4752 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
4753 ;
4754 ; GFX940-TGSPLIT-LABEL: flat_agent_release_acquire_cmpxchg:
4755 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
4756 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
4757 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
4758 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
4759 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4760 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
4761 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
4762 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4763 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4764 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4765 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
4766 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4767 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4768 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4769 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
4770 ; GFX940-TGSPLIT-NEXT: s_endpgm
4771 ;
4772 ; GFX11-WGP-LABEL: flat_agent_release_acquire_cmpxchg:
4773 ; GFX11-WGP: ; %bb.0: ; %entry
4774 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4775 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
4776 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
4777 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4778 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
4779 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
4780 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4781 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
4782 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
4783 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
4784 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4785 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4786 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4787 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4788 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4789 ; GFX11-WGP-NEXT: buffer_gl1_inv
4790 ; GFX11-WGP-NEXT: buffer_gl0_inv
4791 ; GFX11-WGP-NEXT: s_endpgm
4792 ;
4793 ; GFX11-CU-LABEL: flat_agent_release_acquire_cmpxchg:
4794 ; GFX11-CU: ; %bb.0: ; %entry
4795 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4796 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4797 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4798 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4799 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
4800 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
4801 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4802 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
4803 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
4804 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
4805 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4806 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
4807 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4808 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4809 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
4810 ; GFX11-CU-NEXT: buffer_gl1_inv
4811 ; GFX11-CU-NEXT: buffer_gl0_inv
4812 ; GFX11-CU-NEXT: s_endpgm
4813 ;
4814 ; GFX12-WGP-LABEL: flat_agent_release_acquire_cmpxchg:
4815 ; GFX12-WGP: ; %bb.0: ; %entry
4816 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4817 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
4818 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
4819 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
4820 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
4821 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
4822 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4823 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
4824 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
4825 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
4826 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
4827 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
4828 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
4829 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
4830 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
4831 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
4832 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
4833 ; GFX12-WGP-NEXT: s_endpgm
4834 ;
4835 ; GFX12-CU-LABEL: flat_agent_release_acquire_cmpxchg:
4836 ; GFX12-CU: ; %bb.0: ; %entry
4837 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4838 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4839 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4840 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
4841 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
4842 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
4843 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4844 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
4845 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
4846 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
4847 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
4848 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
4849 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
4850 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
4851 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
4852 ; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
4853 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
4854 ; GFX12-CU-NEXT: s_endpgm
4855 ptr %out, i32 %in, i32 %old) {
4856 entry:
4857 %gep = getelementptr i32, ptr %out, i32 4
4858 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") release acquire
4859 ret void
4860 }
4862 define amdgpu_kernel void @flat_agent_acq_rel_acquire_cmpxchg(
4863 ; GFX7-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
4864 ; GFX7: ; %bb.0: ; %entry
4865 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
4866 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4867 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
4868 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
4869 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
4870 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4871 ; GFX7-NEXT: s_mov_b32 s4, s8
4872 ; GFX7-NEXT: s_mov_b32 s5, s9
4873 ; GFX7-NEXT: s_mov_b32 s9, s10
4874 ; GFX7-NEXT: s_mov_b32 s8, s11
4875 ; GFX7-NEXT: s_add_u32 s4, s4, s9
4876 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
4877 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4878 ; GFX7-NEXT: s_mov_b32 s5, s8
4879 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
4880 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
4881 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4882 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
4883 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
4884 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
4885 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4886 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4887 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4888 ; GFX7-NEXT: buffer_wbinvl1_vol
4889 ; GFX7-NEXT: s_endpgm
4890 ;
4891 ; GFX10-WGP-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
4892 ; GFX10-WGP: ; %bb.0: ; %entry
4893 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
4894 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4895 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
4896 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
4897 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
4898 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4899 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
4900 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
4901 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
4902 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
4903 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
4904 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
4905 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4906 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
4907 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
4908 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
4909 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4910 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
4911 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
4912 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
4913 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4914 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4915 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4916 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4917 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4918 ; GFX10-WGP-NEXT: buffer_gl1_inv
4919 ; GFX10-WGP-NEXT: buffer_gl0_inv
4920 ; GFX10-WGP-NEXT: s_endpgm
4921 ;
4922 ; GFX10-CU-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
4923 ; GFX10-CU: ; %bb.0: ; %entry
4924 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
4925 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4926 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
4927 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
4928 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
4929 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4930 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
4931 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
4932 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
4933 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
4934 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
4935 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
4936 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4937 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
4938 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
4939 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
4940 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4941 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
4942 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
4943 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
4944 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4945 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
4946 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4947 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4948 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
4949 ; GFX10-CU-NEXT: buffer_gl1_inv
4950 ; GFX10-CU-NEXT: buffer_gl0_inv
4951 ; GFX10-CU-NEXT: s_endpgm
4952 ;
4953 ; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
4954 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
4955 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
4956 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
4957 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
4958 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
4959 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
4960 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4961 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
4962 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
4963 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
4964 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
4965 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
4966 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
4967 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
4968 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
4969 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
4970 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
4971 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4972 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
4973 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
4974 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
4975 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4976 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4977 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4978 ; SKIP-CACHE-INV-NEXT: s_endpgm
4979 ;
4980 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
4981 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
4982 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4983 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
4984 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
4985 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4986 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
4987 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
4988 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4989 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4990 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
4991 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4992 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4993 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4994 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
4995 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
4996 ;
4997 ; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
4998 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
4999 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5000 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
5001 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
5002 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5003 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
5004 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
5005 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5006 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5007 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
5008 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5009 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5010 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5011 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
5012 ; GFX90A-TGSPLIT-NEXT: s_endpgm
5013 ;
5014 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
5015 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
5016 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5017 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5018 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5019 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5020 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5021 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
5022 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5023 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5024 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5025 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
5026 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5027 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5028 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5029 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
5030 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
5031 ;
5032 ; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
5033 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
5034 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5035 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5036 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5037 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5038 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5039 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
5040 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5041 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5042 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5043 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
5044 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5045 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5046 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5047 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
5048 ; GFX940-TGSPLIT-NEXT: s_endpgm
5049 ;
5050 ; GFX11-WGP-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
5051 ; GFX11-WGP: ; %bb.0: ; %entry
5052 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5053 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5054 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5055 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5056 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
5057 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
5058 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5059 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
5060 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
5061 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
5062 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5063 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5064 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
5065 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5066 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5067 ; GFX11-WGP-NEXT: buffer_gl1_inv
5068 ; GFX11-WGP-NEXT: buffer_gl0_inv
5069 ; GFX11-WGP-NEXT: s_endpgm
5070 ;
5071 ; GFX11-CU-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
5072 ; GFX11-CU: ; %bb.0: ; %entry
5073 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5074 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5075 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5076 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5077 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
5078 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
5079 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5080 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
5081 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
5082 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
5083 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5084 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
5085 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
5086 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5087 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
5088 ; GFX11-CU-NEXT: buffer_gl1_inv
5089 ; GFX11-CU-NEXT: buffer_gl0_inv
5090 ; GFX11-CU-NEXT: s_endpgm
5091 ;
5092 ; GFX12-WGP-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
5093 ; GFX12-WGP: ; %bb.0: ; %entry
5094 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5095 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5096 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5097 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
5098 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
5099 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
5100 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5101 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
5102 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
5103 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
5104 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
5105 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
5106 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
5107 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
5108 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
5109 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
5110 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
5111 ; GFX12-WGP-NEXT: s_endpgm
5112 ;
5113 ; GFX12-CU-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
5114 ; GFX12-CU: ; %bb.0: ; %entry
5115 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5116 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5117 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5118 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
5119 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
5120 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
5121 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5122 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
5123 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
5124 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
5125 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
5126 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
5127 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
5128 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
5129 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
5130 ; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
5131 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
5132 ; GFX12-CU-NEXT: s_endpgm
5133 ptr %out, i32 %in, i32 %old) {
5134 entry:
5135 %gep = getelementptr i32, ptr %out, i32 4
5136 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
5137 ret void
5138 }
5140 define amdgpu_kernel void @flat_agent_seq_cst_acquire_cmpxchg(
5141 ; GFX7-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
5142 ; GFX7: ; %bb.0: ; %entry
5143 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
5144 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
5145 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
5146 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
5147 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
5148 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5149 ; GFX7-NEXT: s_mov_b32 s4, s8
5150 ; GFX7-NEXT: s_mov_b32 s5, s9
5151 ; GFX7-NEXT: s_mov_b32 s9, s10
5152 ; GFX7-NEXT: s_mov_b32 s8, s11
5153 ; GFX7-NEXT: s_add_u32 s4, s4, s9
5154 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
5155 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
5156 ; GFX7-NEXT: s_mov_b32 s5, s8
5157 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
5158 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
5159 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5160 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
5161 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
5162 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
5163 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5164 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5165 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5166 ; GFX7-NEXT: buffer_wbinvl1_vol
5167 ; GFX7-NEXT: s_endpgm
5168 ;
5169 ; GFX10-WGP-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
5170 ; GFX10-WGP: ; %bb.0: ; %entry
5171 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
5172 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
5173 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
5174 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
5175 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
5176 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
5177 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
5178 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
5179 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
5180 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
5181 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
5182 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
5183 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
5184 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
5185 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
5186 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
5187 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5188 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
5189 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
5190 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
5191 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5192 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5193 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5194 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
5195 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5196 ; GFX10-WGP-NEXT: buffer_gl1_inv
5197 ; GFX10-WGP-NEXT: buffer_gl0_inv
5198 ; GFX10-WGP-NEXT: s_endpgm
5199 ;
5200 ; GFX10-CU-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
5201 ; GFX10-CU: ; %bb.0: ; %entry
5202 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
5203 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
5204 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
5205 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
5206 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
5207 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5208 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
5209 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
5210 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
5211 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
5212 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
5213 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
5214 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
5215 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
5216 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
5217 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
5218 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5219 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
5220 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
5221 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
5222 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5223 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
5224 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5225 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5226 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
5227 ; GFX10-CU-NEXT: buffer_gl1_inv
5228 ; GFX10-CU-NEXT: buffer_gl0_inv
5229 ; GFX10-CU-NEXT: s_endpgm
5230 ;
5231 ; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
5232 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
5233 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
5234 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
5235 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
5236 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
5237 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
5238 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5239 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
5240 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
5241 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
5242 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
5243 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
5244 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
5245 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
5246 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
5247 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
5248 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
5249 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5250 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
5251 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
5252 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
5253 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5254 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5255 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5256 ; SKIP-CACHE-INV-NEXT: s_endpgm
5257 ;
5258 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
5259 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
5260 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5261 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
5262 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
5263 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5264 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
5265 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
5266 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5267 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5268 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
5269 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5270 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5271 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5272 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
5273 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
5274 ;
5275 ; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
5276 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
5277 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5278 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
5279 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
5280 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5281 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
5282 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
5283 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5284 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5285 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
5286 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5287 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5288 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5289 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
5290 ; GFX90A-TGSPLIT-NEXT: s_endpgm
5291 ;
5292 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
5293 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
5294 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5295 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5296 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5297 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5298 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5299 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
5300 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5301 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5302 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5303 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
5304 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5305 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5306 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5307 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
5308 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
5309 ;
5310 ; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
5311 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
5312 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5313 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5314 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5315 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5316 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5317 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
5318 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5319 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5320 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5321 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
5322 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5323 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5324 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5325 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
5326 ; GFX940-TGSPLIT-NEXT: s_endpgm
5327 ;
5328 ; GFX11-WGP-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
5329 ; GFX11-WGP: ; %bb.0: ; %entry
5330 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5331 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5332 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5333 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5334 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
5335 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
5336 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5337 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
5338 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
5339 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
5340 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5341 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5342 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
5343 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5344 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5345 ; GFX11-WGP-NEXT: buffer_gl1_inv
5346 ; GFX11-WGP-NEXT: buffer_gl0_inv
5347 ; GFX11-WGP-NEXT: s_endpgm
5348 ;
5349 ; GFX11-CU-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
5350 ; GFX11-CU: ; %bb.0: ; %entry
5351 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5352 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5353 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5354 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5355 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
5356 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
5357 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5358 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
5359 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
5360 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
5361 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5362 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
5363 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
5364 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5365 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
5366 ; GFX11-CU-NEXT: buffer_gl1_inv
5367 ; GFX11-CU-NEXT: buffer_gl0_inv
5368 ; GFX11-CU-NEXT: s_endpgm
5369 ;
5370 ; GFX12-WGP-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
5371 ; GFX12-WGP: ; %bb.0: ; %entry
5372 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5373 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5374 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5375 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
5376 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
5377 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
5378 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5379 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
5380 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
5381 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
5382 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
5383 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
5384 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
5385 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
5386 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
5387 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
5388 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
5389 ; GFX12-WGP-NEXT: s_endpgm
5390 ;
5391 ; GFX12-CU-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
5392 ; GFX12-CU: ; %bb.0: ; %entry
5393 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5394 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5395 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5396 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
5397 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
5398 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
5399 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5400 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
5401 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
5402 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
5403 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
5404 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
5405 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
5406 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
5407 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
5408 ; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
5409 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
5410 ; GFX12-CU-NEXT: s_endpgm
5411 ptr %out, i32 %in, i32 %old) {
5412 entry:
5413 %gep = getelementptr i32, ptr %out, i32 4
5414 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
5415 ret void
5416 }
5418 define amdgpu_kernel void @flat_agent_monotonic_seq_cst_cmpxchg(
5419 ; GFX7-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
5420 ; GFX7: ; %bb.0: ; %entry
5421 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
5422 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
5423 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
5424 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
5425 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
5426 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5427 ; GFX7-NEXT: s_mov_b32 s4, s8
5428 ; GFX7-NEXT: s_mov_b32 s5, s9
5429 ; GFX7-NEXT: s_mov_b32 s9, s10
5430 ; GFX7-NEXT: s_mov_b32 s8, s11
5431 ; GFX7-NEXT: s_add_u32 s4, s4, s9
5432 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
5433 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
5434 ; GFX7-NEXT: s_mov_b32 s5, s8
5435 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
5436 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
5437 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5438 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
5439 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
5440 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
5441 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5442 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5443 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5444 ; GFX7-NEXT: buffer_wbinvl1_vol
5445 ; GFX7-NEXT: s_endpgm
5446 ;
5447 ; GFX10-WGP-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
5448 ; GFX10-WGP: ; %bb.0: ; %entry
5449 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
5450 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
5451 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
5452 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
5453 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
5454 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
5455 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
5456 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
5457 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
5458 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
5459 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
5460 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
5461 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
5462 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
5463 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
5464 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
5465 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5466 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
5467 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
5468 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
5469 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5470 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5471 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5472 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
5473 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5474 ; GFX10-WGP-NEXT: buffer_gl1_inv
5475 ; GFX10-WGP-NEXT: buffer_gl0_inv
5476 ; GFX10-WGP-NEXT: s_endpgm
5477 ;
5478 ; GFX10-CU-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
5479 ; GFX10-CU: ; %bb.0: ; %entry
5480 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
5481 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
5482 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
5483 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
5484 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
5485 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5486 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
5487 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
5488 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
5489 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
5490 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
5491 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
5492 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
5493 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
5494 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
5495 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
5496 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5497 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
5498 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
5499 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
5500 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5501 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
5502 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5503 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5504 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
5505 ; GFX10-CU-NEXT: buffer_gl1_inv
5506 ; GFX10-CU-NEXT: buffer_gl0_inv
5507 ; GFX10-CU-NEXT: s_endpgm
5508 ;
5509 ; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
5510 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
5511 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
5512 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
5513 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
5514 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
5515 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
5516 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5517 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
5518 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
5519 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
5520 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
5521 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
5522 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
5523 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
5524 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
5525 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
5526 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
5527 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5528 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
5529 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
5530 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
5531 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5532 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5533 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5534 ; SKIP-CACHE-INV-NEXT: s_endpgm
5535 ;
5536 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
5537 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
5538 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5539 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
5540 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
5541 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5542 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
5543 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
5544 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5545 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5546 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
5547 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5548 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5549 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5550 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
5551 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
5552 ;
5553 ; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
5554 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
5555 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5556 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
5557 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
5558 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5559 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
5560 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
5561 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5562 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5563 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
5564 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5565 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5566 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5567 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
5568 ; GFX90A-TGSPLIT-NEXT: s_endpgm
5569 ;
5570 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
5571 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
5572 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5573 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5574 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5575 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5576 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5577 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
5578 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5579 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5580 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5581 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
5582 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5583 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5584 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5585 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
5586 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
5587 ;
5588 ; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
5589 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
5590 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5591 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5592 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5593 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5594 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5595 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
5596 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5597 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5598 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5599 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
5600 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5601 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5602 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5603 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
5604 ; GFX940-TGSPLIT-NEXT: s_endpgm
5605 ;
5606 ; GFX11-WGP-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
5607 ; GFX11-WGP: ; %bb.0: ; %entry
5608 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5609 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5610 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5611 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5612 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
5613 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
5614 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5615 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
5616 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
5617 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
5618 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5619 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5620 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
5621 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5622 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5623 ; GFX11-WGP-NEXT: buffer_gl1_inv
5624 ; GFX11-WGP-NEXT: buffer_gl0_inv
5625 ; GFX11-WGP-NEXT: s_endpgm
5626 ;
5627 ; GFX11-CU-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
5628 ; GFX11-CU: ; %bb.0: ; %entry
5629 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5630 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5631 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5632 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5633 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
5634 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
5635 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5636 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
5637 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
5638 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
5639 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5640 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
5641 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
5642 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5643 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
5644 ; GFX11-CU-NEXT: buffer_gl1_inv
5645 ; GFX11-CU-NEXT: buffer_gl0_inv
5646 ; GFX11-CU-NEXT: s_endpgm
5647 ;
5648 ; GFX12-WGP-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
5649 ; GFX12-WGP: ; %bb.0: ; %entry
5650 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5651 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5652 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5653 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
5654 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
5655 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
5656 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5657 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
5658 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
5659 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
5660 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
5661 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
5662 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
5663 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
5664 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
5665 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
5666 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
5667 ; GFX12-WGP-NEXT: s_endpgm
5668 ;
5669 ; GFX12-CU-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
5670 ; GFX12-CU: ; %bb.0: ; %entry
5671 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5672 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5673 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5674 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
5675 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
5676 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
5677 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5678 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
5679 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
5680 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
5681 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
5682 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
5683 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
5684 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
5685 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
5686 ; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
5687 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
5688 ; GFX12-CU-NEXT: s_endpgm
5689 ptr %out, i32 %in, i32 %old) {
5690 entry:
5691 %gep = getelementptr i32, ptr %out, i32 4
5692 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") monotonic seq_cst
5693 ret void
5694 }
5696 define amdgpu_kernel void @flat_agent_acquire_seq_cst_cmpxchg(
5697 ; GFX7-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
5698 ; GFX7: ; %bb.0: ; %entry
5699 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
5700 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
5701 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
5702 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
5703 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
5704 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5705 ; GFX7-NEXT: s_mov_b32 s4, s8
5706 ; GFX7-NEXT: s_mov_b32 s5, s9
5707 ; GFX7-NEXT: s_mov_b32 s9, s10
5708 ; GFX7-NEXT: s_mov_b32 s8, s11
5709 ; GFX7-NEXT: s_add_u32 s4, s4, s9
5710 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
5711 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
5712 ; GFX7-NEXT: s_mov_b32 s5, s8
5713 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
5714 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
5715 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5716 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
5717 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
5718 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
5719 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5720 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5721 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5722 ; GFX7-NEXT: buffer_wbinvl1_vol
5723 ; GFX7-NEXT: s_endpgm
5724 ;
5725 ; GFX10-WGP-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
5726 ; GFX10-WGP: ; %bb.0: ; %entry
5727 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
5728 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
5729 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
5730 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
5731 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
5732 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
5733 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
5734 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
5735 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
5736 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
5737 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
5738 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
5739 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
5740 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
5741 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
5742 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
5743 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5744 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
5745 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
5746 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
5747 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5748 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5749 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5750 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
5751 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5752 ; GFX10-WGP-NEXT: buffer_gl1_inv
5753 ; GFX10-WGP-NEXT: buffer_gl0_inv
5754 ; GFX10-WGP-NEXT: s_endpgm
5755 ;
5756 ; GFX10-CU-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
5757 ; GFX10-CU: ; %bb.0: ; %entry
5758 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
5759 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
5760 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
5761 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
5762 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
5763 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5764 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
5765 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
5766 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
5767 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
5768 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
5769 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
5770 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
5771 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
5772 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
5773 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
5774 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5775 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
5776 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
5777 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
5778 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5779 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
5780 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5781 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5782 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
5783 ; GFX10-CU-NEXT: buffer_gl1_inv
5784 ; GFX10-CU-NEXT: buffer_gl0_inv
5785 ; GFX10-CU-NEXT: s_endpgm
5786 ;
5787 ; SKIP-CACHE-INV-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
5788 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
5789 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
5790 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
5791 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
5792 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
5793 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
5794 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5795 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
5796 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
5797 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
5798 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
5799 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
5800 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
5801 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
5802 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
5803 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
5804 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
5805 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5806 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
5807 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
5808 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
5809 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5810 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5811 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5812 ; SKIP-CACHE-INV-NEXT: s_endpgm
5813 ;
5814 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
5815 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
5816 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5817 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
5818 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
5819 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5820 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
5821 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
5822 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5823 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5824 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
5825 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5826 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5827 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5828 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
5829 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
5830 ;
5831 ; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
5832 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
5833 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5834 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
5835 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
5836 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5837 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
5838 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
5839 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5840 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5841 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
5842 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5843 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5844 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5845 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
5846 ; GFX90A-TGSPLIT-NEXT: s_endpgm
5847 ;
5848 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
5849 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
5850 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5851 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5852 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5853 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5854 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5855 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
5856 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5857 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5858 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5859 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
5860 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5861 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5862 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5863 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
5864 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
5865 ;
5866 ; GFX940-TGSPLIT-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
5867 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
5868 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5869 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5870 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5871 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5872 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5873 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
5874 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5875 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5876 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5877 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
5878 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5879 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5880 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5881 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
5882 ; GFX940-TGSPLIT-NEXT: s_endpgm
5883 ;
5884 ; GFX11-WGP-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
5885 ; GFX11-WGP: ; %bb.0: ; %entry
5886 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5887 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5888 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5889 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5890 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
5891 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
5892 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5893 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
5894 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
5895 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
5896 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5897 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5898 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
5899 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5900 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5901 ; GFX11-WGP-NEXT: buffer_gl1_inv
5902 ; GFX11-WGP-NEXT: buffer_gl0_inv
5903 ; GFX11-WGP-NEXT: s_endpgm
5904 ;
5905 ; GFX11-CU-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
5906 ; GFX11-CU: ; %bb.0: ; %entry
5907 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5908 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5909 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5910 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5911 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
5912 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
5913 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5914 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
5915 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
5916 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
5917 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5918 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
5919 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
5920 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5921 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
5922 ; GFX11-CU-NEXT: buffer_gl1_inv
5923 ; GFX11-CU-NEXT: buffer_gl0_inv
5924 ; GFX11-CU-NEXT: s_endpgm
5925 ;
5926 ; GFX12-WGP-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
5927 ; GFX12-WGP: ; %bb.0: ; %entry
5928 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5929 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5930 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5931 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
5932 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
5933 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
5934 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5935 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
5936 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
5937 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
5938 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
5939 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
5940 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
5941 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
5942 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
5943 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
5944 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
5945 ; GFX12-WGP-NEXT: s_endpgm
5946 ;
5947 ; GFX12-CU-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
5948 ; GFX12-CU: ; %bb.0: ; %entry
5949 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5950 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5951 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5952 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
5953 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
5954 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
5955 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5956 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
5957 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
5958 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
5959 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
5960 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
5961 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
5962 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
5963 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
5964 ; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
5965 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
5966 ; GFX12-CU-NEXT: s_endpgm
5967 ptr %out, i32 %in, i32 %old) {
5968 entry:
5969 %gep = getelementptr i32, ptr %out, i32 4
5970 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") acquire seq_cst
5971 ret void
5972 }
5974 define amdgpu_kernel void @flat_agent_release_seq_cst_cmpxchg(
5975 ; GFX7-LABEL: flat_agent_release_seq_cst_cmpxchg:
5976 ; GFX7: ; %bb.0: ; %entry
5977 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
5978 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
5979 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
5980 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
5981 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
5982 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5983 ; GFX7-NEXT: s_mov_b32 s4, s8
5984 ; GFX7-NEXT: s_mov_b32 s5, s9
5985 ; GFX7-NEXT: s_mov_b32 s9, s10
5986 ; GFX7-NEXT: s_mov_b32 s8, s11
5987 ; GFX7-NEXT: s_add_u32 s4, s4, s9
5988 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
5989 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
5990 ; GFX7-NEXT: s_mov_b32 s5, s8
5991 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
5992 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
5993 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5994 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
5995 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
5996 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
5997 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5998 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5999 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6000 ; GFX7-NEXT: buffer_wbinvl1_vol
6001 ; GFX7-NEXT: s_endpgm
6002 ;
6003 ; GFX10-WGP-LABEL: flat_agent_release_seq_cst_cmpxchg:
6004 ; GFX10-WGP: ; %bb.0: ; %entry
6005 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
6006 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
6007 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
6008 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
6009 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
6010 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
6011 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
6012 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
6013 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
6014 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
6015 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
6016 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
6017 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
6018 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
6019 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
6020 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
6021 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6022 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
6023 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
6024 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
6025 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6026 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6027 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
6028 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
6029 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6030 ; GFX10-WGP-NEXT: buffer_gl1_inv
6031 ; GFX10-WGP-NEXT: buffer_gl0_inv
6032 ; GFX10-WGP-NEXT: s_endpgm
6033 ;
6034 ; GFX10-CU-LABEL: flat_agent_release_seq_cst_cmpxchg:
6035 ; GFX10-CU: ; %bb.0: ; %entry
6036 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
6037 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
6038 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
6039 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
6040 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
6041 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6042 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
6043 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
6044 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
6045 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
6046 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
6047 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
6048 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
6049 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
6050 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
6051 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
6052 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6053 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
6054 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
6055 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
6056 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6057 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
6058 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
6059 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6060 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
6061 ; GFX10-CU-NEXT: buffer_gl1_inv
6062 ; GFX10-CU-NEXT: buffer_gl0_inv
6063 ; GFX10-CU-NEXT: s_endpgm
6064 ;
6065 ; SKIP-CACHE-INV-LABEL: flat_agent_release_seq_cst_cmpxchg:
6066 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
6067 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
6068 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
6069 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
6070 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
6071 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
6072 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6073 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
6074 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
6075 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
6076 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
6077 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
6078 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
6079 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
6080 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
6081 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
6082 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
6083 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6084 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
6085 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
6086 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
6087 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6088 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
6089 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6090 ; SKIP-CACHE-INV-NEXT: s_endpgm
6091 ;
6092 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_seq_cst_cmpxchg:
6093 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
6094 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6095 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
6096 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
6097 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6098 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
6099 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
6100 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6101 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6102 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
6103 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6104 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6105 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6106 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
6107 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
6108 ;
6109 ; GFX90A-TGSPLIT-LABEL: flat_agent_release_seq_cst_cmpxchg:
6110 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
6111 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6112 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
6113 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
6114 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6115 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
6116 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
6117 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6118 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6119 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
6120 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6121 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6122 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6123 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
6124 ; GFX90A-TGSPLIT-NEXT: s_endpgm
6125 ;
6126 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_seq_cst_cmpxchg:
6127 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
6128 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6129 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
6130 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
6131 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6132 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
6133 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6134 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6135 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6136 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
6137 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
6138 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6139 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6140 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6141 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
6142 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
6143 ;
6144 ; GFX940-TGSPLIT-LABEL: flat_agent_release_seq_cst_cmpxchg:
6145 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
6146 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6147 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
6148 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
6149 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6150 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
6151 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6152 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6153 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6154 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
6155 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
6156 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6157 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6158 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6159 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
6160 ; GFX940-TGSPLIT-NEXT: s_endpgm
6161 ;
6162 ; GFX11-WGP-LABEL: flat_agent_release_seq_cst_cmpxchg:
6163 ; GFX11-WGP: ; %bb.0: ; %entry
6164 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6165 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
6166 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
6167 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
6168 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
6169 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
6170 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6171 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
6172 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
6173 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
6174 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6175 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6176 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
6177 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
6178 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6179 ; GFX11-WGP-NEXT: buffer_gl1_inv
6180 ; GFX11-WGP-NEXT: buffer_gl0_inv
6181 ; GFX11-WGP-NEXT: s_endpgm
6182 ;
6183 ; GFX11-CU-LABEL: flat_agent_release_seq_cst_cmpxchg:
6184 ; GFX11-CU: ; %bb.0: ; %entry
6185 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6186 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
6187 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
6188 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6189 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
6190 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
6191 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6192 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
6193 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
6194 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
6195 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6196 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
6197 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
6198 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6199 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
6200 ; GFX11-CU-NEXT: buffer_gl1_inv
6201 ; GFX11-CU-NEXT: buffer_gl0_inv
6202 ; GFX11-CU-NEXT: s_endpgm
6203 ;
6204 ; GFX12-WGP-LABEL: flat_agent_release_seq_cst_cmpxchg:
6205 ; GFX12-WGP: ; %bb.0: ; %entry
6206 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6207 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
6208 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
6209 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
6210 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
6211 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
6212 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6213 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
6214 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
6215 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
6216 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
6217 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
6218 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
6219 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
6220 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
6221 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
6222 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
6223 ; GFX12-WGP-NEXT: s_endpgm
6224 ;
6225 ; GFX12-CU-LABEL: flat_agent_release_seq_cst_cmpxchg:
6226 ; GFX12-CU: ; %bb.0: ; %entry
6227 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6228 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
6229 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
6230 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
6231 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
6232 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
6233 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6234 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
6235 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
6236 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
6237 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
6238 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
6239 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
6240 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
6241 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
6242 ; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
6243 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
6244 ; GFX12-CU-NEXT: s_endpgm
6245 ptr %out, i32 %in, i32 %old) {
6246 entry:
6247 %gep = getelementptr i32, ptr %out, i32 4
6248 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") release seq_cst
6249 ret void
6250 }
6252 define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_cmpxchg(
6253 ; GFX7-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
6254 ; GFX7: ; %bb.0: ; %entry
6255 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
6256 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
6257 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
6258 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
6259 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
6260 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6261 ; GFX7-NEXT: s_mov_b32 s4, s8
6262 ; GFX7-NEXT: s_mov_b32 s5, s9
6263 ; GFX7-NEXT: s_mov_b32 s9, s10
6264 ; GFX7-NEXT: s_mov_b32 s8, s11
6265 ; GFX7-NEXT: s_add_u32 s4, s4, s9
6266 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
6267 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
6268 ; GFX7-NEXT: s_mov_b32 s5, s8
6269 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
6270 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
6271 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6272 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
6273 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
6274 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
6275 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6276 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
6277 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6278 ; GFX7-NEXT: buffer_wbinvl1_vol
6279 ; GFX7-NEXT: s_endpgm
6280 ;
6281 ; GFX10-WGP-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
6282 ; GFX10-WGP: ; %bb.0: ; %entry
6283 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
6284 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
6285 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
6286 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
6287 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
6288 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
6289 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
6290 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
6291 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
6292 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
6293 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
6294 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
6295 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
6296 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
6297 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
6298 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
6299 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6300 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
6301 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
6302 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
6303 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6304 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6305 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
6306 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
6307 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6308 ; GFX10-WGP-NEXT: buffer_gl1_inv
6309 ; GFX10-WGP-NEXT: buffer_gl0_inv
6310 ; GFX10-WGP-NEXT: s_endpgm
6311 ;
6312 ; GFX10-CU-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
6313 ; GFX10-CU: ; %bb.0: ; %entry
6314 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
6315 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
6316 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
6317 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
6318 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
6319 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6320 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
6321 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
6322 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
6323 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
6324 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
6325 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
6326 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
6327 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
6328 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
6329 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
6330 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6331 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
6332 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
6333 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
6334 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6335 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
6336 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
6337 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6338 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
6339 ; GFX10-CU-NEXT: buffer_gl1_inv
6340 ; GFX10-CU-NEXT: buffer_gl0_inv
6341 ; GFX10-CU-NEXT: s_endpgm
6342 ;
6343 ; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
6344 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
6345 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
6346 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
6347 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
6348 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
6349 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
6350 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6351 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
6352 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
6353 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
6354 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
6355 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
6356 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
6357 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
6358 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
6359 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
6360 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
6361 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6362 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
6363 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
6364 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
6365 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6366 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
6367 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6368 ; SKIP-CACHE-INV-NEXT: s_endpgm
6369 ;
6370 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
6371 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
6372 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6373 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
6374 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
6375 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6376 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
6377 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
6378 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6379 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6380 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
6381 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6382 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6383 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6384 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
6385 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
6386 ;
6387 ; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
6388 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
6389 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6390 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
6391 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
6392 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6393 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
6394 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
6395 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6396 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6397 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
6398 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6399 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6400 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6401 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
6402 ; GFX90A-TGSPLIT-NEXT: s_endpgm
6403 ;
6404 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
6405 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
6406 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6407 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
6408 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
6409 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6410 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
6411 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6412 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6413 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6414 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
6415 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
6416 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6417 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6418 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6419 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
6420 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
6421 ;
6422 ; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
6423 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
6424 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6425 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
6426 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
6427 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6428 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
6429 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6430 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6431 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6432 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
6433 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
6434 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6435 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6436 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6437 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
6438 ; GFX940-TGSPLIT-NEXT: s_endpgm
6439 ;
6440 ; GFX11-WGP-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
6441 ; GFX11-WGP: ; %bb.0: ; %entry
6442 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6443 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
6444 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
6445 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
6446 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
6447 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
6448 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6449 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
6450 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
6451 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
6452 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6453 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6454 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
6455 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
6456 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6457 ; GFX11-WGP-NEXT: buffer_gl1_inv
6458 ; GFX11-WGP-NEXT: buffer_gl0_inv
6459 ; GFX11-WGP-NEXT: s_endpgm
6460 ;
6461 ; GFX11-CU-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
6462 ; GFX11-CU: ; %bb.0: ; %entry
6463 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6464 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
6465 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
6466 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6467 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
6468 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
6469 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6470 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
6471 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
6472 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
6473 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6474 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
6475 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
6476 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6477 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
6478 ; GFX11-CU-NEXT: buffer_gl1_inv
6479 ; GFX11-CU-NEXT: buffer_gl0_inv
6480 ; GFX11-CU-NEXT: s_endpgm
6481 ;
6482 ; GFX12-WGP-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
6483 ; GFX12-WGP: ; %bb.0: ; %entry
6484 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6485 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
6486 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
6487 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
6488 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
6489 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
6490 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6491 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
6492 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
6493 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
6494 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
6495 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
6496 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
6497 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
6498 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
6499 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
6500 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
6501 ; GFX12-WGP-NEXT: s_endpgm
6502 ;
6503 ; GFX12-CU-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
6504 ; GFX12-CU: ; %bb.0: ; %entry
6505 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6506 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
6507 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
6508 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
6509 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
6510 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
6511 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6512 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
6513 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
6514 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
6515 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
6516 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
6517 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
6518 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
6519 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
6520 ; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
6521 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
6522 ; GFX12-CU-NEXT: s_endpgm
6523 ptr %out, i32 %in, i32 %old) {
6524 entry:
6525 %gep = getelementptr i32, ptr %out, i32 4
6526 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") acq_rel seq_cst
6527 ret void
6528 }
6530 define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_cmpxchg(
6531 ; GFX7-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
6532 ; GFX7: ; %bb.0: ; %entry
6533 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
6534 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
6535 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
6536 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
6537 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
6538 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6539 ; GFX7-NEXT: s_mov_b32 s4, s8
6540 ; GFX7-NEXT: s_mov_b32 s5, s9
6541 ; GFX7-NEXT: s_mov_b32 s9, s10
6542 ; GFX7-NEXT: s_mov_b32 s8, s11
6543 ; GFX7-NEXT: s_add_u32 s4, s4, s9
6544 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
6545 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
6546 ; GFX7-NEXT: s_mov_b32 s5, s8
6547 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
6548 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
6549 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6550 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
6551 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
6552 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
6553 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6554 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
6555 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6556 ; GFX7-NEXT: buffer_wbinvl1_vol
6557 ; GFX7-NEXT: s_endpgm
6558 ;
6559 ; GFX10-WGP-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
6560 ; GFX10-WGP: ; %bb.0: ; %entry
6561 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
6562 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
6563 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
6564 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
6565 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
6566 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
6567 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
6568 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
6569 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
6570 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
6571 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
6572 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
6573 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
6574 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
6575 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
6576 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
6577 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6578 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
6579 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
6580 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
6581 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6582 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6583 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
6584 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
6585 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6586 ; GFX10-WGP-NEXT: buffer_gl1_inv
6587 ; GFX10-WGP-NEXT: buffer_gl0_inv
6588 ; GFX10-WGP-NEXT: s_endpgm
6589 ;
6590 ; GFX10-CU-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
6591 ; GFX10-CU: ; %bb.0: ; %entry
6592 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
6593 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
6594 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
6595 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
6596 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
6597 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6598 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
6599 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
6600 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
6601 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
6602 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
6603 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
6604 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
6605 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
6606 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
6607 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
6608 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6609 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
6610 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
6611 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
6612 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6613 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
6614 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
6615 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6616 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
6617 ; GFX10-CU-NEXT: buffer_gl1_inv
6618 ; GFX10-CU-NEXT: buffer_gl0_inv
6619 ; GFX10-CU-NEXT: s_endpgm
6620 ;
6621 ; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
6622 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
6623 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
6624 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
6625 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
6626 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
6627 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
6628 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6629 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
6630 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
6631 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
6632 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
6633 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
6634 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
6635 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
6636 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
6637 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
6638 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
6639 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6640 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
6641 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
6642 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
6643 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6644 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
6645 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6646 ; SKIP-CACHE-INV-NEXT: s_endpgm
6647 ;
6648 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
6649 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
6650 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6651 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
6652 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
6653 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6654 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
6655 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
6656 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6657 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6658 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
6659 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6660 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6661 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6662 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
6663 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
6664 ;
6665 ; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
6666 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
6667 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6668 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
6669 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
6670 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6671 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
6672 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
6673 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6674 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6675 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
6676 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6677 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6678 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6679 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
6680 ; GFX90A-TGSPLIT-NEXT: s_endpgm
6681 ;
6682 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
6683 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
6684 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6685 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
6686 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
6687 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6688 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
6689 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6690 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6691 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6692 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
6693 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
6694 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6695 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6696 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6697 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
6698 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
6699 ;
6700 ; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
6701 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
6702 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6703 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
6704 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
6705 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6706 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
6707 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6708 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6709 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6710 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
6711 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
6712 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6713 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6714 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6715 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
6716 ; GFX940-TGSPLIT-NEXT: s_endpgm
6717 ;
6718 ; GFX11-WGP-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
6719 ; GFX11-WGP: ; %bb.0: ; %entry
6720 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6721 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
6722 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
6723 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
6724 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
6725 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
6726 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6727 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
6728 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
6729 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
6730 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6731 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6732 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
6733 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
6734 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6735 ; GFX11-WGP-NEXT: buffer_gl1_inv
6736 ; GFX11-WGP-NEXT: buffer_gl0_inv
6737 ; GFX11-WGP-NEXT: s_endpgm
6738 ;
6739 ; GFX11-CU-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
6740 ; GFX11-CU: ; %bb.0: ; %entry
6741 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6742 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
6743 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
6744 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6745 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
6746 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
6747 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6748 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
6749 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
6750 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
6751 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6752 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
6753 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
6754 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6755 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
6756 ; GFX11-CU-NEXT: buffer_gl1_inv
6757 ; GFX11-CU-NEXT: buffer_gl0_inv
6758 ; GFX11-CU-NEXT: s_endpgm
6759 ;
6760 ; GFX12-WGP-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
6761 ; GFX12-WGP: ; %bb.0: ; %entry
6762 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6763 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
6764 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
6765 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
6766 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
6767 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
6768 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6769 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
6770 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
6771 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
6772 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
6773 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
6774 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
6775 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
6776 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
6777 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
6778 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
6779 ; GFX12-WGP-NEXT: s_endpgm
6780 ;
6781 ; GFX12-CU-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
6782 ; GFX12-CU: ; %bb.0: ; %entry
6783 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6784 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
6785 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
6786 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
6787 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
6788 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
6789 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6790 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
6791 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
6792 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
6793 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
6794 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
6795 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
6796 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
6797 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
6798 ; GFX12-CU-NEXT: s_wait_storecnt_dscnt 0x0
6799 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
6800 ; GFX12-CU-NEXT: s_endpgm
6801 ptr %out, i32 %in, i32 %old) {
6802 entry:
6803 %gep = getelementptr i32, ptr %out, i32 4
6804 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
6805 ret void
6806 }
6808 define amdgpu_kernel void @flat_agent_monotonic_monotonic_ret_cmpxchg(
6809 ; GFX7-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
6810 ; GFX7: ; %bb.0: ; %entry
6811 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
6812 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
6813 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
6814 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
6815 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
6816 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6817 ; GFX7-NEXT: s_mov_b32 s6, s4
6818 ; GFX7-NEXT: s_mov_b32 s7, s5
6819 ; GFX7-NEXT: s_mov_b32 s11, s12
6820 ; GFX7-NEXT: s_mov_b32 s10, s13
6821 ; GFX7-NEXT: s_add_u32 s6, s6, s11
6822 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
6823 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
6824 ; GFX7-NEXT: s_mov_b32 s7, s10
6825 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
6826 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
6827 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6828 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
6829 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
6830 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
6831 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6832 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
6833 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
6834 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6835 ; GFX7-NEXT: flat_store_dword v[0:1], v2
6836 ; GFX7-NEXT: s_endpgm
6837 ;
6838 ; GFX10-WGP-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
6839 ; GFX10-WGP: ; %bb.0: ; %entry
6840 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
6841 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
6842 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
6843 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
6844 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
6845 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
6846 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
6847 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
6848 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
6849 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
6850 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
6851 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
6852 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
6853 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
6854 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
6855 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
6856 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6857 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
6858 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
6859 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
6860 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6861 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
6862 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
6863 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6864 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
6865 ; GFX10-WGP-NEXT: s_endpgm
6866 ;
6867 ; GFX10-CU-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
6868 ; GFX10-CU: ; %bb.0: ; %entry
6869 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
6870 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
6871 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
6872 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
6873 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
6874 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6875 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
6876 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
6877 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
6878 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
6879 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
6880 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
6881 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
6882 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
6883 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
6884 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
6885 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6886 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
6887 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
6888 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
6889 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6890 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
6891 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
6892 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6893 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
6894 ; GFX10-CU-NEXT: s_endpgm
6895 ;
6896 ; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
6897 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
6898 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
6899 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
6900 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
6901 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
6902 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
6903 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6904 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
6905 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
6906 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
6907 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
6908 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
6909 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
6910 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
6911 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
6912 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
6913 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
6914 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6915 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
6916 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
6917 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
6918 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6919 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
6920 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
6921 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6922 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
6923 ; SKIP-CACHE-INV-NEXT: s_endpgm
6924 ;
6925 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
6926 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
6927 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6928 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
6929 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
6930 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6931 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
6932 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
6933 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6934 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6935 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
6936 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6937 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
6938 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6939 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
6940 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
6941 ;
6942 ; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
6943 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
6944 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6945 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
6946 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
6947 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6948 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
6949 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
6950 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6951 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6952 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
6953 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6954 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
6955 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6956 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
6957 ; GFX90A-TGSPLIT-NEXT: s_endpgm
6958 ;
6959 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
6960 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
6961 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6962 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
6963 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
6964 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6965 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
6966 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6967 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6968 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6969 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
6970 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6971 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
6972 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6973 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
6974 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
6975 ;
6976 ; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
6977 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
6978 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6979 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
6980 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
6981 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6982 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
6983 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6984 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6985 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6986 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
6987 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6988 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
6989 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6990 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
6991 ; GFX940-TGSPLIT-NEXT: s_endpgm
6992 ;
6993 ; GFX11-WGP-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
6994 ; GFX11-WGP: ; %bb.0: ; %entry
6995 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6996 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
6997 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
6998 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
6999 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
7000 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
7001 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7002 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
7003 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
7004 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
7005 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7006 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
7007 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
7008 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7009 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
7010 ; GFX11-WGP-NEXT: s_endpgm
7011 ;
7012 ; GFX11-CU-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
7013 ; GFX11-CU: ; %bb.0: ; %entry
7014 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7015 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
7016 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
7017 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7018 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
7019 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
7020 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7021 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
7022 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
7023 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
7024 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7025 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
7026 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
7027 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7028 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
7029 ; GFX11-CU-NEXT: s_endpgm
7030 ;
7031 ; GFX12-WGP-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
7032 ; GFX12-WGP: ; %bb.0: ; %entry
7033 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7034 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
7035 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
7036 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
7037 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
7038 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
7039 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7040 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
7041 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
7042 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
7043 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
7044 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
7045 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
7046 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
7047 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
7048 ; GFX12-WGP-NEXT: s_endpgm
7049 ;
7050 ; GFX12-CU-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
7051 ; GFX12-CU: ; %bb.0: ; %entry
7052 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7053 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
7054 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
7055 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
7056 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
7057 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
7058 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7059 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
7060 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
7061 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
7062 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
7063 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
7064 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
7065 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
7066 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
7067 ; GFX12-CU-NEXT: s_endpgm
7068 ptr %out, i32 %in, i32 %old) {
7069 entry:
7070 %gep = getelementptr i32, ptr %out, i32 4
7071 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") monotonic monotonic
7072 %val0 = extractvalue { i32, i1 } %val, 0
7073 store i32 %val0, ptr %out, align 4
7074 ret void
7075 }
7077 define amdgpu_kernel void @flat_agent_acquire_monotonic_ret_cmpxchg(
7078 ; GFX7-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
7079 ; GFX7: ; %bb.0: ; %entry
7080 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
7081 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7082 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
7083 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
7084 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
7085 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7086 ; GFX7-NEXT: s_mov_b32 s6, s4
7087 ; GFX7-NEXT: s_mov_b32 s7, s5
7088 ; GFX7-NEXT: s_mov_b32 s11, s12
7089 ; GFX7-NEXT: s_mov_b32 s10, s13
7090 ; GFX7-NEXT: s_add_u32 s6, s6, s11
7091 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
7092 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
7093 ; GFX7-NEXT: s_mov_b32 s7, s10
7094 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
7095 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
7096 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7097 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
7098 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
7099 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
7100 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7101 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7102 ; GFX7-NEXT: buffer_wbinvl1_vol
7103 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
7104 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
7105 ; GFX7-NEXT: flat_store_dword v[0:1], v2
7106 ; GFX7-NEXT: s_endpgm
7107 ;
7108 ; GFX10-WGP-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
7109 ; GFX10-WGP: ; %bb.0: ; %entry
7110 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
7111 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7112 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
7113 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
7114 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
7115 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
7116 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
7117 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
7118 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
7119 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
7120 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
7121 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
7122 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
7123 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
7124 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
7125 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
7126 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7127 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
7128 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
7129 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
7130 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7131 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7132 ; GFX10-WGP-NEXT: buffer_gl1_inv
7133 ; GFX10-WGP-NEXT: buffer_gl0_inv
7134 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
7135 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
7136 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
7137 ; GFX10-WGP-NEXT: s_endpgm
7138 ;
7139 ; GFX10-CU-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
7140 ; GFX10-CU: ; %bb.0: ; %entry
7141 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
7142 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7143 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
7144 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
7145 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
7146 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7147 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
7148 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
7149 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
7150 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
7151 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
7152 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
7153 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
7154 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
7155 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
7156 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
7157 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7158 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
7159 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
7160 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
7161 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7162 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7163 ; GFX10-CU-NEXT: buffer_gl1_inv
7164 ; GFX10-CU-NEXT: buffer_gl0_inv
7165 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
7166 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
7167 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
7168 ; GFX10-CU-NEXT: s_endpgm
7169 ;
7170 ; SKIP-CACHE-INV-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
7171 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
7172 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
7173 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
7174 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
7175 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
7176 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
7177 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7178 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
7179 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
7180 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
7181 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
7182 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
7183 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
7184 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
7185 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
7186 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
7187 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
7188 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7189 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
7190 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
7191 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
7192 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7193 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7194 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
7195 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
7196 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
7197 ; SKIP-CACHE-INV-NEXT: s_endpgm
7198 ;
7199 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
7200 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
7201 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7202 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
7203 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
7204 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7205 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
7206 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
7207 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7208 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7209 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7210 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7211 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7212 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
7213 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7214 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
7215 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
7216 ;
7217 ; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
7218 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
7219 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7220 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
7221 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
7222 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7223 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
7224 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
7225 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7226 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7227 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7228 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7229 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7230 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
7231 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7232 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
7233 ; GFX90A-TGSPLIT-NEXT: s_endpgm
7234 ;
7235 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
7236 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
7237 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7238 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
7239 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
7240 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7241 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
7242 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
7243 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7244 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7245 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7246 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7247 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7248 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
7249 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7250 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
7251 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
7252 ;
7253 ; GFX940-TGSPLIT-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
7254 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
7255 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7256 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
7257 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
7258 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7259 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
7260 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
7261 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7262 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7263 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7264 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7265 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7266 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
7267 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7268 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
7269 ; GFX940-TGSPLIT-NEXT: s_endpgm
7270 ;
7271 ; GFX11-WGP-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
7272 ; GFX11-WGP: ; %bb.0: ; %entry
7273 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7274 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
7275 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
7276 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
7277 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
7278 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
7279 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7280 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
7281 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
7282 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
7283 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7284 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7285 ; GFX11-WGP-NEXT: buffer_gl1_inv
7286 ; GFX11-WGP-NEXT: buffer_gl0_inv
7287 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
7288 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
7289 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
7290 ; GFX11-WGP-NEXT: s_endpgm
7291 ;
7292 ; GFX11-CU-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
7293 ; GFX11-CU: ; %bb.0: ; %entry
7294 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7295 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
7296 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
7297 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7298 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
7299 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
7300 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7301 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
7302 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
7303 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
7304 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7305 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7306 ; GFX11-CU-NEXT: buffer_gl1_inv
7307 ; GFX11-CU-NEXT: buffer_gl0_inv
7308 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
7309 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
7310 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
7311 ; GFX11-CU-NEXT: s_endpgm
7312 ;
7313 ; GFX12-WGP-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
7314 ; GFX12-WGP: ; %bb.0: ; %entry
7315 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7316 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
7317 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
7318 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
7319 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
7320 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
7321 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7322 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
7323 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
7324 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
7325 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
7326 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
7327 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
7328 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
7329 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
7330 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
7331 ; GFX12-WGP-NEXT: s_endpgm
7332 ;
7333 ; GFX12-CU-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
7334 ; GFX12-CU: ; %bb.0: ; %entry
7335 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7336 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
7337 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
7338 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
7339 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
7340 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
7341 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7342 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
7343 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
7344 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
7345 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
7346 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
7347 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
7348 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
7349 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
7350 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
7351 ; GFX12-CU-NEXT: s_endpgm
7352 ptr %out, i32 %in, i32 %old) {
7353 entry:
7354 %gep = getelementptr i32, ptr %out, i32 4
7355 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
7356 %val0 = extractvalue { i32, i1 } %val, 0
7357 store i32 %val0, ptr %out, align 4
7358 ret void
7359 }
7361 define amdgpu_kernel void @flat_agent_release_monotonic_ret_cmpxchg(
7362 ; GFX7-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
7363 ; GFX7: ; %bb.0: ; %entry
7364 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
7365 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7366 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
7367 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
7368 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
7369 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7370 ; GFX7-NEXT: s_mov_b32 s6, s4
7371 ; GFX7-NEXT: s_mov_b32 s7, s5
7372 ; GFX7-NEXT: s_mov_b32 s11, s12
7373 ; GFX7-NEXT: s_mov_b32 s10, s13
7374 ; GFX7-NEXT: s_add_u32 s6, s6, s11
7375 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
7376 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
7377 ; GFX7-NEXT: s_mov_b32 s7, s10
7378 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
7379 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
7380 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7381 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
7382 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
7383 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
7384 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7385 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7386 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
7387 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
7388 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7389 ; GFX7-NEXT: flat_store_dword v[0:1], v2
7390 ; GFX7-NEXT: s_endpgm
7391 ;
7392 ; GFX10-WGP-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
7393 ; GFX10-WGP: ; %bb.0: ; %entry
7394 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
7395 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7396 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
7397 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
7398 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
7399 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
7400 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
7401 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
7402 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
7403 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
7404 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
7405 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
7406 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
7407 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
7408 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
7409 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
7410 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7411 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
7412 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
7413 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
7414 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7415 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
7416 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7417 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
7418 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
7419 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7420 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
7421 ; GFX10-WGP-NEXT: s_endpgm
7422 ;
7423 ; GFX10-CU-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
7424 ; GFX10-CU: ; %bb.0: ; %entry
7425 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
7426 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7427 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
7428 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
7429 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
7430 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7431 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
7432 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
7433 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
7434 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
7435 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
7436 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
7437 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
7438 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
7439 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
7440 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
7441 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7442 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
7443 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
7444 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
7445 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7446 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
7447 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7448 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
7449 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
7450 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7451 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
7452 ; GFX10-CU-NEXT: s_endpgm
7453 ;
7454 ; SKIP-CACHE-INV-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
7455 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
7456 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
7457 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
7458 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
7459 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
7460 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
7461 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7462 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
7463 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
7464 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
7465 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
7466 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
7467 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
7468 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
7469 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
7470 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
7471 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
7472 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7473 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
7474 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
7475 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
7476 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7477 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7478 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
7479 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
7480 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7481 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
7482 ; SKIP-CACHE-INV-NEXT: s_endpgm
7483 ;
7484 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
7485 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
7486 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7487 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
7488 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
7489 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7490 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
7491 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
7492 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7493 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7494 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7495 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7496 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7497 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7498 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7499 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
7500 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
7501 ;
7502 ; GFX90A-TGSPLIT-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
7503 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
7504 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7505 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
7506 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
7507 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7508 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
7509 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
7510 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7511 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7512 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7513 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7514 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7515 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7516 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7517 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
7518 ; GFX90A-TGSPLIT-NEXT: s_endpgm
7519 ;
7520 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
7521 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
7522 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7523 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
7524 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
7525 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7526 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
7527 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
7528 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7529 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7530 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7531 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
7532 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7533 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7534 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7535 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7536 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
7537 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
7538 ;
7539 ; GFX940-TGSPLIT-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
7540 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
7541 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7542 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
7543 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
7544 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7545 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
7546 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
7547 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7548 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7549 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7550 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
7551 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7552 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7553 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7554 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7555 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
7556 ; GFX940-TGSPLIT-NEXT: s_endpgm
7557 ;
7558 ; GFX11-WGP-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
7559 ; GFX11-WGP: ; %bb.0: ; %entry
7560 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7561 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
7562 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
7563 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
7564 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
7565 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
7566 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7567 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
7568 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
7569 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
7570 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7571 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
7572 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7573 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
7574 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
7575 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7576 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
7577 ; GFX11-WGP-NEXT: s_endpgm
7578 ;
7579 ; GFX11-CU-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
7580 ; GFX11-CU: ; %bb.0: ; %entry
7581 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7582 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
7583 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
7584 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7585 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
7586 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
7587 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7588 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
7589 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
7590 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
7591 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7592 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
7593 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7594 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
7595 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
7596 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7597 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
7598 ; GFX11-CU-NEXT: s_endpgm
7599 ;
7600 ; GFX12-WGP-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
7601 ; GFX12-WGP: ; %bb.0: ; %entry
7602 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7603 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
7604 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
7605 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
7606 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
7607 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
7608 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7609 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
7610 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
7611 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
7612 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
7613 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
7614 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
7615 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
7616 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
7617 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
7618 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
7619 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
7620 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
7621 ; GFX12-WGP-NEXT: s_endpgm
7622 ;
7623 ; GFX12-CU-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
7624 ; GFX12-CU: ; %bb.0: ; %entry
7625 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7626 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
7627 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
7628 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
7629 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
7630 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
7631 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7632 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
7633 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
7634 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
7635 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
7636 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
7637 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
7638 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
7639 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
7640 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
7641 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
7642 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
7643 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
7644 ; GFX12-CU-NEXT: s_endpgm
7645 ptr %out, i32 %in, i32 %old) {
7646 entry:
7647 %gep = getelementptr i32, ptr %out, i32 4
7648 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") release monotonic
7649 %val0 = extractvalue { i32, i1 } %val, 0
7650 store i32 %val0, ptr %out, align 4
7651 ret void
7652 }
7654 define amdgpu_kernel void @flat_agent_acq_rel_monotonic_ret_cmpxchg(
7655 ; GFX7-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
7656 ; GFX7: ; %bb.0: ; %entry
7657 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
7658 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7659 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
7660 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
7661 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
7662 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7663 ; GFX7-NEXT: s_mov_b32 s6, s4
7664 ; GFX7-NEXT: s_mov_b32 s7, s5
7665 ; GFX7-NEXT: s_mov_b32 s11, s12
7666 ; GFX7-NEXT: s_mov_b32 s10, s13
7667 ; GFX7-NEXT: s_add_u32 s6, s6, s11
7668 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
7669 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
7670 ; GFX7-NEXT: s_mov_b32 s7, s10
7671 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
7672 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
7673 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7674 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
7675 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
7676 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
7677 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7678 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7679 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7680 ; GFX7-NEXT: buffer_wbinvl1_vol
7681 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
7682 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
7683 ; GFX7-NEXT: flat_store_dword v[0:1], v2
7684 ; GFX7-NEXT: s_endpgm
7685 ;
7686 ; GFX10-WGP-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
7687 ; GFX10-WGP: ; %bb.0: ; %entry
7688 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
7689 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7690 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
7691 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
7692 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
7693 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
7694 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
7695 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
7696 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
7697 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
7698 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
7699 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
7700 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
7701 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
7702 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
7703 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
7704 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7705 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
7706 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
7707 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
7708 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7709 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
7710 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7711 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7712 ; GFX10-WGP-NEXT: buffer_gl1_inv
7713 ; GFX10-WGP-NEXT: buffer_gl0_inv
7714 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
7715 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
7716 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
7717 ; GFX10-WGP-NEXT: s_endpgm
7718 ;
7719 ; GFX10-CU-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
7720 ; GFX10-CU: ; %bb.0: ; %entry
7721 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
7722 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7723 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
7724 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
7725 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
7726 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7727 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
7728 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
7729 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
7730 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
7731 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
7732 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
7733 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
7734 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
7735 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
7736 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
7737 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7738 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
7739 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
7740 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
7741 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7742 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
7743 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7744 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7745 ; GFX10-CU-NEXT: buffer_gl1_inv
7746 ; GFX10-CU-NEXT: buffer_gl0_inv
7747 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
7748 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
7749 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
7750 ; GFX10-CU-NEXT: s_endpgm
7751 ;
7752 ; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
7753 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
7754 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
7755 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
7756 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
7757 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
7758 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
7759 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7760 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
7761 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
7762 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
7763 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
7764 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
7765 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
7766 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
7767 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
7768 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
7769 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
7770 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7771 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
7772 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
7773 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
7774 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7775 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7776 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7777 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
7778 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
7779 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
7780 ; SKIP-CACHE-INV-NEXT: s_endpgm
7781 ;
7782 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
7783 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
7784 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7785 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
7786 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
7787 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7788 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
7789 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
7790 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7791 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7792 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7793 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7794 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7795 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7796 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
7797 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7798 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
7799 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
7800 ;
7801 ; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
7802 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
7803 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7804 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
7805 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
7806 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7807 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
7808 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
7809 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7810 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7811 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7812 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7813 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7814 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7815 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
7816 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7817 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
7818 ; GFX90A-TGSPLIT-NEXT: s_endpgm
7819 ;
7820 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
7821 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
7822 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7823 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
7824 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
7825 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7826 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
7827 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
7828 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7829 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7830 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7831 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
7832 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7833 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7834 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7835 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
7836 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7837 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
7838 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
7839 ;
7840 ; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
7841 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
7842 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7843 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
7844 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
7845 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7846 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
7847 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
7848 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7849 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7850 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7851 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
7852 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7853 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7854 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7855 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
7856 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7857 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
7858 ; GFX940-TGSPLIT-NEXT: s_endpgm
7859 ;
7860 ; GFX11-WGP-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
7861 ; GFX11-WGP: ; %bb.0: ; %entry
7862 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7863 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
7864 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
7865 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
7866 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
7867 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
7868 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7869 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
7870 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
7871 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
7872 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7873 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
7874 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7875 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7876 ; GFX11-WGP-NEXT: buffer_gl1_inv
7877 ; GFX11-WGP-NEXT: buffer_gl0_inv
7878 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
7879 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
7880 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
7881 ; GFX11-WGP-NEXT: s_endpgm
7882 ;
7883 ; GFX11-CU-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
7884 ; GFX11-CU: ; %bb.0: ; %entry
7885 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7886 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
7887 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
7888 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7889 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
7890 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
7891 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7892 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
7893 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
7894 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
7895 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7896 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
7897 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7898 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7899 ; GFX11-CU-NEXT: buffer_gl1_inv
7900 ; GFX11-CU-NEXT: buffer_gl0_inv
7901 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
7902 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
7903 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
7904 ; GFX11-CU-NEXT: s_endpgm
7905 ;
7906 ; GFX12-WGP-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
7907 ; GFX12-WGP: ; %bb.0: ; %entry
7908 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7909 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
7910 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
7911 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
7912 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
7913 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
7914 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7915 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
7916 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
7917 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
7918 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
7919 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
7920 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
7921 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
7922 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
7923 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
7924 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
7925 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
7926 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
7927 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
7928 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
7929 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
7930 ; GFX12-WGP-NEXT: s_endpgm
7931 ;
7932 ; GFX12-CU-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
7933 ; GFX12-CU: ; %bb.0: ; %entry
7934 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7935 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
7936 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
7937 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
7938 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
7939 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
7940 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7941 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
7942 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
7943 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
7944 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
7945 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
7946 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
7947 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
7948 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
7949 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
7950 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
7951 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
7952 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
7953 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
7954 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
7955 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
7956 ; GFX12-CU-NEXT: s_endpgm
7957 ptr %out, i32 %in, i32 %old) {
7958 entry:
7959 %gep = getelementptr i32, ptr %out, i32 4
7960 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
7961 %val0 = extractvalue { i32, i1 } %val, 0
7962 store i32 %val0, ptr %out, align 4
7963 ret void
7964 }
7966 define amdgpu_kernel void @flat_agent_seq_cst_monotonic_ret_cmpxchg(
7967 ; GFX7-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
7968 ; GFX7: ; %bb.0: ; %entry
7969 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
7970 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7971 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
7972 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
7973 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
7974 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7975 ; GFX7-NEXT: s_mov_b32 s6, s4
7976 ; GFX7-NEXT: s_mov_b32 s7, s5
7977 ; GFX7-NEXT: s_mov_b32 s11, s12
7978 ; GFX7-NEXT: s_mov_b32 s10, s13
7979 ; GFX7-NEXT: s_add_u32 s6, s6, s11
7980 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
7981 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
7982 ; GFX7-NEXT: s_mov_b32 s7, s10
7983 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
7984 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
7985 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7986 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
7987 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
7988 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
7989 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7990 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7991 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7992 ; GFX7-NEXT: buffer_wbinvl1_vol
7993 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
7994 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
7995 ; GFX7-NEXT: flat_store_dword v[0:1], v2
7996 ; GFX7-NEXT: s_endpgm
7997 ;
7998 ; GFX10-WGP-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
7999 ; GFX10-WGP: ; %bb.0: ; %entry
8000 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
8001 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8002 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
8003 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
8004 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
8005 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
8006 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
8007 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
8008 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
8009 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
8010 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
8011 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
8012 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8013 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
8014 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
8015 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
8016 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8017 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
8018 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
8019 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
8020 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8021 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
8022 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8023 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8024 ; GFX10-WGP-NEXT: buffer_gl1_inv
8025 ; GFX10-WGP-NEXT: buffer_gl0_inv
8026 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
8027 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
8028 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
8029 ; GFX10-WGP-NEXT: s_endpgm
8030 ;
8031 ; GFX10-CU-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
8032 ; GFX10-CU: ; %bb.0: ; %entry
8033 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
8034 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8035 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
8036 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
8037 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
8038 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8039 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
8040 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
8041 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
8042 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
8043 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
8044 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
8045 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8046 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
8047 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
8048 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
8049 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8050 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
8051 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
8052 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
8053 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8054 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
8055 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8056 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8057 ; GFX10-CU-NEXT: buffer_gl1_inv
8058 ; GFX10-CU-NEXT: buffer_gl0_inv
8059 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
8060 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
8061 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
8062 ; GFX10-CU-NEXT: s_endpgm
8063 ;
8064 ; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
8065 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
8066 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
8067 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
8068 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
8069 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
8070 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
8071 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8072 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
8073 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
8074 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
8075 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
8076 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
8077 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
8078 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
8079 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
8080 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
8081 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
8082 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8083 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
8084 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
8085 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
8086 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8087 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8088 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8089 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
8090 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
8091 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
8092 ; SKIP-CACHE-INV-NEXT: s_endpgm
8093 ;
8094 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
8095 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
8096 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8097 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
8098 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
8099 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8100 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
8101 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
8102 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8103 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8104 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8105 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8106 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8107 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8108 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
8109 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8110 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
8111 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
8112 ;
8113 ; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
8114 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
8115 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8116 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
8117 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
8118 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8119 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
8120 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
8121 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8122 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8123 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8124 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8125 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8126 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
8127 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
8128 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8129 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
8130 ; GFX90A-TGSPLIT-NEXT: s_endpgm
8131 ;
8132 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
8133 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
8134 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
8135 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
8136 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
8137 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8138 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
8139 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
8140 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8141 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8142 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8143 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
8144 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8145 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
8146 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8147 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
8148 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8149 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
8150 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
8151 ;
8152 ; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
8153 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
8154 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
8155 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
8156 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
8157 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8158 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
8159 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
8160 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8161 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8162 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8163 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
8164 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8165 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
8166 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
8167 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
8168 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8169 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
8170 ; GFX940-TGSPLIT-NEXT: s_endpgm
8171 ;
8172 ; GFX11-WGP-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
8173 ; GFX11-WGP: ; %bb.0: ; %entry
8174 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8175 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
8176 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
8177 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
8178 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
8179 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
8180 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8181 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
8182 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
8183 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
8184 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8185 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
8186 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
8187 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8188 ; GFX11-WGP-NEXT: buffer_gl1_inv
8189 ; GFX11-WGP-NEXT: buffer_gl0_inv
8190 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
8191 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
8192 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
8193 ; GFX11-WGP-NEXT: s_endpgm
8194 ;
8195 ; GFX11-CU-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
8196 ; GFX11-CU: ; %bb.0: ; %entry
8197 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8198 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
8199 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
8200 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8201 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
8202 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
8203 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8204 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
8205 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
8206 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
8207 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8208 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
8209 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
8210 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8211 ; GFX11-CU-NEXT: buffer_gl1_inv
8212 ; GFX11-CU-NEXT: buffer_gl0_inv
8213 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
8214 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
8215 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
8216 ; GFX11-CU-NEXT: s_endpgm
8217 ;
8218 ; GFX12-WGP-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
8219 ; GFX12-WGP: ; %bb.0: ; %entry
8220 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8221 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
8222 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
8223 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
8224 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
8225 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
8226 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8227 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
8228 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
8229 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
8230 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
8231 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
8232 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
8233 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
8234 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
8235 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
8236 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
8237 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
8238 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
8239 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
8240 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
8241 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
8242 ; GFX12-WGP-NEXT: s_endpgm
8243 ;
8244 ; GFX12-CU-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
8245 ; GFX12-CU: ; %bb.0: ; %entry
8246 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8247 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
8248 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
8249 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
8250 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
8251 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
8252 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8253 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
8254 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
8255 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
8256 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
8257 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
8258 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
8259 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
8260 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
8261 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
8262 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
8263 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
8264 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
8265 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
8266 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
8267 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
8268 ; GFX12-CU-NEXT: s_endpgm
8269 ptr %out, i32 %in, i32 %old) {
8270 entry:
8271 %gep = getelementptr i32, ptr %out, i32 4
8272 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
8273 %val0 = extractvalue { i32, i1 } %val, 0
8274 store i32 %val0, ptr %out, align 4
8275 ret void
8276 }
8278 define amdgpu_kernel void @flat_agent_monotonic_acquire_ret_cmpxchg(
8279 ; GFX7-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
8280 ; GFX7: ; %bb.0: ; %entry
8281 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
8282 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8283 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
8284 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
8285 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
8286 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8287 ; GFX7-NEXT: s_mov_b32 s6, s4
8288 ; GFX7-NEXT: s_mov_b32 s7, s5
8289 ; GFX7-NEXT: s_mov_b32 s11, s12
8290 ; GFX7-NEXT: s_mov_b32 s10, s13
8291 ; GFX7-NEXT: s_add_u32 s6, s6, s11
8292 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
8293 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8294 ; GFX7-NEXT: s_mov_b32 s7, s10
8295 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
8296 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
8297 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8298 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
8299 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
8300 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
8301 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8302 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8303 ; GFX7-NEXT: buffer_wbinvl1_vol
8304 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
8305 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
8306 ; GFX7-NEXT: flat_store_dword v[0:1], v2
8307 ; GFX7-NEXT: s_endpgm
8308 ;
8309 ; GFX10-WGP-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
8310 ; GFX10-WGP: ; %bb.0: ; %entry
8311 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
8312 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8313 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
8314 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
8315 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
8316 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
8317 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
8318 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
8319 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
8320 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
8321 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
8322 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
8323 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8324 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
8325 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
8326 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
8327 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8328 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
8329 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
8330 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
8331 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8332 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8333 ; GFX10-WGP-NEXT: buffer_gl1_inv
8334 ; GFX10-WGP-NEXT: buffer_gl0_inv
8335 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
8336 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
8337 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
8338 ; GFX10-WGP-NEXT: s_endpgm
8339 ;
8340 ; GFX10-CU-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
8341 ; GFX10-CU: ; %bb.0: ; %entry
8342 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
8343 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8344 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
8345 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
8346 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
8347 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8348 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
8349 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
8350 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
8351 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
8352 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
8353 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
8354 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8355 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
8356 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
8357 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
8358 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8359 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
8360 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
8361 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
8362 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8363 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8364 ; GFX10-CU-NEXT: buffer_gl1_inv
8365 ; GFX10-CU-NEXT: buffer_gl0_inv
8366 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
8367 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
8368 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
8369 ; GFX10-CU-NEXT: s_endpgm
8370 ;
8371 ; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
8372 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
8373 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
8374 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
8375 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
8376 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
8377 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
8378 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8379 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
8380 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
8381 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
8382 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
8383 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
8384 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
8385 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
8386 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
8387 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
8388 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
8389 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8390 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
8391 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
8392 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
8393 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8394 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8395 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
8396 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
8397 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
8398 ; SKIP-CACHE-INV-NEXT: s_endpgm
8399 ;
8400 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
8401 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
8402 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8403 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
8404 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
8405 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8406 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
8407 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
8408 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8409 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8410 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8411 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8412 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8413 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
8414 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8415 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
8416 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
8417 ;
8418 ; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
8419 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
8420 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8421 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
8422 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
8423 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8424 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
8425 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
8426 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8427 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8428 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8429 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8430 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
8431 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
8432 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8433 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
8434 ; GFX90A-TGSPLIT-NEXT: s_endpgm
8435 ;
8436 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
8437 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
8438 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
8439 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
8440 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
8441 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8442 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
8443 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
8444 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8445 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8446 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8447 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
8448 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8449 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
8450 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8451 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
8452 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
8453 ;
8454 ; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
8455 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
8456 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
8457 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
8458 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
8459 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8460 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
8461 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
8462 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8463 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8464 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8465 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
8466 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
8467 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
8468 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8469 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
8470 ; GFX940-TGSPLIT-NEXT: s_endpgm
8471 ;
8472 ; GFX11-WGP-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
8473 ; GFX11-WGP: ; %bb.0: ; %entry
8474 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8475 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
8476 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
8477 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
8478 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
8479 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
8480 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8481 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
8482 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
8483 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
8484 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
8485 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8486 ; GFX11-WGP-NEXT: buffer_gl1_inv
8487 ; GFX11-WGP-NEXT: buffer_gl0_inv
8488 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
8489 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
8490 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
8491 ; GFX11-WGP-NEXT: s_endpgm
8492 ;
8493 ; GFX11-CU-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
8494 ; GFX11-CU: ; %bb.0: ; %entry
8495 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8496 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
8497 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
8498 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8499 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
8500 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
8501 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8502 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
8503 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
8504 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
8505 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
8506 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8507 ; GFX11-CU-NEXT: buffer_gl1_inv
8508 ; GFX11-CU-NEXT: buffer_gl0_inv
8509 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
8510 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
8511 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
8512 ; GFX11-CU-NEXT: s_endpgm
8513 ;
8514 ; GFX12-WGP-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
8515 ; GFX12-WGP: ; %bb.0: ; %entry
8516 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8517 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
8518 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
8519 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
8520 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
8521 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
8522 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8523 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
8524 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
8525 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
8526 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
8527 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
8528 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
8529 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
8530 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
8531 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
8532 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
8533 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
8534 ; GFX12-WGP-NEXT: s_endpgm
8535 ;
8536 ; GFX12-CU-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
8537 ; GFX12-CU: ; %bb.0: ; %entry
8538 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8539 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
8540 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
8541 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
8542 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
8543 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
8544 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8545 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
8546 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
8547 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
8548 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
8549 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
8550 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
8551 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
8552 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
8553 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
8554 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
8555 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
8556 ; GFX12-CU-NEXT: s_endpgm
8557 ptr %out, i32 %in, i32 %old) {
8558 entry:
8559 %gep = getelementptr i32, ptr %out, i32 4
8560 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") monotonic acquire
8561 %val0 = extractvalue { i32, i1 } %val, 0
8562 store i32 %val0, ptr %out, align 4
8563 ret void
8564 }
8566 define amdgpu_kernel void @flat_agent_acquire_acquire_ret_cmpxchg(
8567 ; GFX7-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
8568 ; GFX7: ; %bb.0: ; %entry
8569 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
8570 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8571 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
8572 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
8573 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
8574 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8575 ; GFX7-NEXT: s_mov_b32 s6, s4
8576 ; GFX7-NEXT: s_mov_b32 s7, s5
8577 ; GFX7-NEXT: s_mov_b32 s11, s12
8578 ; GFX7-NEXT: s_mov_b32 s10, s13
8579 ; GFX7-NEXT: s_add_u32 s6, s6, s11
8580 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
8581 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8582 ; GFX7-NEXT: s_mov_b32 s7, s10
8583 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
8584 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
8585 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8586 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
8587 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
8588 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
8589 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8590 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8591 ; GFX7-NEXT: buffer_wbinvl1_vol
8592 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
8593 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
8594 ; GFX7-NEXT: flat_store_dword v[0:1], v2
8595 ; GFX7-NEXT: s_endpgm
8596 ;
8597 ; GFX10-WGP-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
8598 ; GFX10-WGP: ; %bb.0: ; %entry
8599 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
8600 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8601 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
8602 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
8603 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
8604 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
8605 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
8606 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
8607 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
8608 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
8609 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
8610 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
8611 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8612 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
8613 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
8614 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
8615 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8616 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
8617 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
8618 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
8619 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8620 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8621 ; GFX10-WGP-NEXT: buffer_gl1_inv
8622 ; GFX10-WGP-NEXT: buffer_gl0_inv
8623 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
8624 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
8625 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
8626 ; GFX10-WGP-NEXT: s_endpgm
8627 ;
8628 ; GFX10-CU-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
8629 ; GFX10-CU: ; %bb.0: ; %entry
8630 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
8631 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8632 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
8633 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
8634 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
8635 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8636 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
8637 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
8638 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
8639 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
8640 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
8641 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
8642 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8643 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
8644 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
8645 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
8646 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8647 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
8648 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
8649 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
8650 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8651 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8652 ; GFX10-CU-NEXT: buffer_gl1_inv
8653 ; GFX10-CU-NEXT: buffer_gl0_inv
8654 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
8655 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
8656 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
8657 ; GFX10-CU-NEXT: s_endpgm
8658 ;
8659 ; SKIP-CACHE-INV-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
8660 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
8661 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
8662 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
8663 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
8664 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
8665 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
8666 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8667 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
8668 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
8669 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
8670 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
8671 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
8672 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
8673 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
8674 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
8675 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
8676 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
8677 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8678 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
8679 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
8680 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
8681 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8682 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8683 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
8684 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
8685 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
8686 ; SKIP-CACHE-INV-NEXT: s_endpgm
8687 ;
8688 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
8689 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
8690 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8691 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
8692 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
8693 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8694 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
8695 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
8696 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8697 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8698 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8699 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8700 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8701 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
8702 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8703 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
8704 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
8705 ;
8706 ; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
8707 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
8708 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8709 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
8710 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
8711 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8712 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
8713 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
8714 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8715 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8716 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8717 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8718 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
8719 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
8720 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8721 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
8722 ; GFX90A-TGSPLIT-NEXT: s_endpgm
8723 ;
8724 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
8725 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
8726 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
8727 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
8728 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
8729 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8730 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
8731 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
8732 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8733 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8734 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8735 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
8736 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8737 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
8738 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8739 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
8740 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
8741 ;
8742 ; GFX940-TGSPLIT-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
8743 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
8744 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
8745 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
8746 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
8747 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8748 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
8749 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
8750 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8751 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8752 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8753 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
8754 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
8755 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
8756 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8757 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
8758 ; GFX940-TGSPLIT-NEXT: s_endpgm
8759 ;
8760 ; GFX11-WGP-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
8761 ; GFX11-WGP: ; %bb.0: ; %entry
8762 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8763 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
8764 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
8765 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
8766 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
8767 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
8768 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8769 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
8770 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
8771 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
8772 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
8773 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8774 ; GFX11-WGP-NEXT: buffer_gl1_inv
8775 ; GFX11-WGP-NEXT: buffer_gl0_inv
8776 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
8777 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
8778 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
8779 ; GFX11-WGP-NEXT: s_endpgm
8780 ;
8781 ; GFX11-CU-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
8782 ; GFX11-CU: ; %bb.0: ; %entry
8783 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8784 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
8785 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
8786 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8787 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
8788 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
8789 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8790 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
8791 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
8792 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
8793 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
8794 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8795 ; GFX11-CU-NEXT: buffer_gl1_inv
8796 ; GFX11-CU-NEXT: buffer_gl0_inv
8797 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
8798 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
8799 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
8800 ; GFX11-CU-NEXT: s_endpgm
8801 ;
8802 ; GFX12-WGP-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
8803 ; GFX12-WGP: ; %bb.0: ; %entry
8804 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8805 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
8806 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
8807 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
8808 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
8809 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
8810 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8811 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
8812 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
8813 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
8814 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
8815 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
8816 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
8817 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
8818 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
8819 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
8820 ; GFX12-WGP-NEXT: s_endpgm
8821 ;
8822 ; GFX12-CU-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
8823 ; GFX12-CU: ; %bb.0: ; %entry
8824 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8825 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
8826 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
8827 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
8828 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
8829 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
8830 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8831 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
8832 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
8833 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
8834 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
8835 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
8836 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
8837 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
8838 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
8839 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
8840 ; GFX12-CU-NEXT: s_endpgm
8841 ptr %out, i32 %in, i32 %old) {
8842 entry:
8843 %gep = getelementptr i32, ptr %out, i32 4
8844 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
8845 %val0 = extractvalue { i32, i1 } %val, 0
8846 store i32 %val0, ptr %out, align 4
8847 ret void
8848 }
8850 define amdgpu_kernel void @flat_agent_release_acquire_ret_cmpxchg(
8851 ; GFX7-LABEL: flat_agent_release_acquire_ret_cmpxchg:
8852 ; GFX7: ; %bb.0: ; %entry
8853 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
8854 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8855 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
8856 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
8857 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
8858 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8859 ; GFX7-NEXT: s_mov_b32 s6, s4
8860 ; GFX7-NEXT: s_mov_b32 s7, s5
8861 ; GFX7-NEXT: s_mov_b32 s11, s12
8862 ; GFX7-NEXT: s_mov_b32 s10, s13
8863 ; GFX7-NEXT: s_add_u32 s6, s6, s11
8864 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
8865 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8866 ; GFX7-NEXT: s_mov_b32 s7, s10
8867 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
8868 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
8869 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8870 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
8871 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
8872 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
8873 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8874 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8875 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8876 ; GFX7-NEXT: buffer_wbinvl1_vol
8877 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
8878 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
8879 ; GFX7-NEXT: flat_store_dword v[0:1], v2
8880 ; GFX7-NEXT: s_endpgm
8881 ;
8882 ; GFX10-WGP-LABEL: flat_agent_release_acquire_ret_cmpxchg:
8883 ; GFX10-WGP: ; %bb.0: ; %entry
8884 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
8885 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8886 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
8887 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
8888 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
8889 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
8890 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
8891 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
8892 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
8893 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
8894 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
8895 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
8896 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8897 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
8898 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
8899 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
8900 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8901 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
8902 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
8903 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
8904 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8905 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
8906 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8907 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8908 ; GFX10-WGP-NEXT: buffer_gl1_inv
8909 ; GFX10-WGP-NEXT: buffer_gl0_inv
8910 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
8911 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
8912 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
8913 ; GFX10-WGP-NEXT: s_endpgm
8914 ;
8915 ; GFX10-CU-LABEL: flat_agent_release_acquire_ret_cmpxchg:
8916 ; GFX10-CU: ; %bb.0: ; %entry
8917 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
8918 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8919 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
8920 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
8921 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
8922 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8923 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
8924 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
8925 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
8926 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
8927 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
8928 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
8929 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8930 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
8931 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
8932 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
8933 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8934 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
8935 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
8936 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
8937 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8938 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
8939 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8940 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8941 ; GFX10-CU-NEXT: buffer_gl1_inv
8942 ; GFX10-CU-NEXT: buffer_gl0_inv
8943 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
8944 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
8945 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
8946 ; GFX10-CU-NEXT: s_endpgm
8947 ;
8948 ; SKIP-CACHE-INV-LABEL: flat_agent_release_acquire_ret_cmpxchg:
8949 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
8950 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
8951 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
8952 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
8953 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
8954 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
8955 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8956 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
8957 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
8958 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
8959 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
8960 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
8961 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
8962 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
8963 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
8964 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
8965 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
8966 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8967 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
8968 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
8969 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
8970 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8971 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8972 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8973 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
8974 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
8975 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
8976 ; SKIP-CACHE-INV-NEXT: s_endpgm
8977 ;
8978 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_acquire_ret_cmpxchg:
8979 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
8980 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8981 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
8982 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
8983 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8984 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
8985 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
8986 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8987 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8988 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8989 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8990 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8991 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8992 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
8993 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8994 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
8995 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
8996 ;
8997 ; GFX90A-TGSPLIT-LABEL: flat_agent_release_acquire_ret_cmpxchg:
8998 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
8999 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9000 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
9001 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
9002 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9003 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
9004 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
9005 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9006 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9007 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9008 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9009 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9010 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9011 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
9012 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9013 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
9014 ; GFX90A-TGSPLIT-NEXT: s_endpgm
9015 ;
9016 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_acquire_ret_cmpxchg:
9017 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
9018 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
9019 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
9020 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
9021 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9022 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
9023 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
9024 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9025 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9026 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9027 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
9028 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9029 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9030 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9031 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
9032 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9033 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
9034 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
9035 ;
9036 ; GFX940-TGSPLIT-LABEL: flat_agent_release_acquire_ret_cmpxchg:
9037 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
9038 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
9039 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
9040 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
9041 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9042 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
9043 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
9044 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9045 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9046 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9047 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
9048 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9049 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9050 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9051 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
9052 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9053 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
9054 ; GFX940-TGSPLIT-NEXT: s_endpgm
9055 ;
9056 ; GFX11-WGP-LABEL: flat_agent_release_acquire_ret_cmpxchg:
9057 ; GFX11-WGP: ; %bb.0: ; %entry
9058 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9059 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
9060 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
9061 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
9062 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
9063 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
9064 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9065 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
9066 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
9067 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
9068 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9069 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
9070 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
9071 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9072 ; GFX11-WGP-NEXT: buffer_gl1_inv
9073 ; GFX11-WGP-NEXT: buffer_gl0_inv
9074 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
9075 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
9076 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
9077 ; GFX11-WGP-NEXT: s_endpgm
9078 ;
9079 ; GFX11-CU-LABEL: flat_agent_release_acquire_ret_cmpxchg:
9080 ; GFX11-CU: ; %bb.0: ; %entry
9081 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9082 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
9083 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
9084 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9085 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
9086 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
9087 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9088 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
9089 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
9090 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
9091 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9092 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
9093 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
9094 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9095 ; GFX11-CU-NEXT: buffer_gl1_inv
9096 ; GFX11-CU-NEXT: buffer_gl0_inv
9097 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
9098 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
9099 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
9100 ; GFX11-CU-NEXT: s_endpgm
9101 ;
9102 ; GFX12-WGP-LABEL: flat_agent_release_acquire_ret_cmpxchg:
9103 ; GFX12-WGP: ; %bb.0: ; %entry
9104 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9105 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
9106 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
9107 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
9108 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
9109 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
9110 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9111 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
9112 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
9113 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
9114 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
9115 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
9116 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
9117 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
9118 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
9119 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
9120 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
9121 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
9122 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
9123 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
9124 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
9125 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
9126 ; GFX12-WGP-NEXT: s_endpgm
9127 ;
9128 ; GFX12-CU-LABEL: flat_agent_release_acquire_ret_cmpxchg:
9129 ; GFX12-CU: ; %bb.0: ; %entry
9130 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9131 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
9132 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
9133 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
9134 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
9135 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
9136 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9137 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
9138 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
9139 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
9140 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
9141 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
9142 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
9143 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
9144 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
9145 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
9146 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
9147 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
9148 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
9149 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
9150 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
9151 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
9152 ; GFX12-CU-NEXT: s_endpgm
9153 ptr %out, i32 %in, i32 %old) {
9154 entry:
9155 %gep = getelementptr i32, ptr %out, i32 4
9156 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") release acquire
9157 %val0 = extractvalue { i32, i1 } %val, 0
9158 store i32 %val0, ptr %out, align 4
9159 ret void
9160 }
9162 define amdgpu_kernel void @flat_agent_acq_rel_acquire_ret_cmpxchg(
9163 ; GFX7-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
9164 ; GFX7: ; %bb.0: ; %entry
9165 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
9166 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9167 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
9168 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
9169 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
9170 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9171 ; GFX7-NEXT: s_mov_b32 s6, s4
9172 ; GFX7-NEXT: s_mov_b32 s7, s5
9173 ; GFX7-NEXT: s_mov_b32 s11, s12
9174 ; GFX7-NEXT: s_mov_b32 s10, s13
9175 ; GFX7-NEXT: s_add_u32 s6, s6, s11
9176 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
9177 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9178 ; GFX7-NEXT: s_mov_b32 s7, s10
9179 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
9180 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
9181 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9182 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
9183 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
9184 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
9185 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9186 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9187 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9188 ; GFX7-NEXT: buffer_wbinvl1_vol
9189 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
9190 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
9191 ; GFX7-NEXT: flat_store_dword v[0:1], v2
9192 ; GFX7-NEXT: s_endpgm
9193 ;
9194 ; GFX10-WGP-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
9195 ; GFX10-WGP: ; %bb.0: ; %entry
9196 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
9197 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9198 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
9199 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
9200 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
9201 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
9202 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
9203 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
9204 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
9205 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
9206 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
9207 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
9208 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9209 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
9210 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
9211 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
9212 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9213 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
9214 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
9215 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
9216 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9217 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
9218 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9219 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9220 ; GFX10-WGP-NEXT: buffer_gl1_inv
9221 ; GFX10-WGP-NEXT: buffer_gl0_inv
9222 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
9223 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
9224 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
9225 ; GFX10-WGP-NEXT: s_endpgm
9226 ;
9227 ; GFX10-CU-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
9228 ; GFX10-CU: ; %bb.0: ; %entry
9229 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
9230 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9231 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
9232 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
9233 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
9234 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9235 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
9236 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
9237 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
9238 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
9239 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
9240 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
9241 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9242 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
9243 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
9244 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
9245 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9246 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
9247 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
9248 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
9249 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9250 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
9251 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9252 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9253 ; GFX10-CU-NEXT: buffer_gl1_inv
9254 ; GFX10-CU-NEXT: buffer_gl0_inv
9255 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
9256 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
9257 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
9258 ; GFX10-CU-NEXT: s_endpgm
9259 ;
9260 ; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
9261 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
9262 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
9263 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
9264 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
9265 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
9266 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
9267 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9268 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
9269 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
9270 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
9271 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
9272 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
9273 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
9274 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
9275 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
9276 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
9277 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
9278 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9279 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
9280 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
9281 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
9282 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9283 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9284 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9285 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
9286 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
9287 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
9288 ; SKIP-CACHE-INV-NEXT: s_endpgm
9289 ;
9290 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
9291 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
9292 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9293 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
9294 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
9295 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9296 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
9297 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
9298 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9299 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9300 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9301 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9302 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9303 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9304 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
9305 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9306 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
9307 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
9308 ;
9309 ; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
9310 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
9311 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9312 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
9313 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
9314 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9315 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
9316 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
9317 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9318 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9319 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9320 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9321 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9322 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9323 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
9324 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9325 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
9326 ; GFX90A-TGSPLIT-NEXT: s_endpgm
9327 ;
9328 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
9329 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
9330 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
9331 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
9332 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
9333 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9334 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
9335 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
9336 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9337 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9338 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9339 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
9340 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9341 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9342 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9343 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
9344 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9345 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
9346 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
9347 ;
9348 ; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
9349 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
9350 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
9351 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
9352 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
9353 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9354 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
9355 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
9356 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9357 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9358 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9359 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
9360 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9361 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9362 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9363 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
9364 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9365 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
9366 ; GFX940-TGSPLIT-NEXT: s_endpgm
9367 ;
9368 ; GFX11-WGP-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
9369 ; GFX11-WGP: ; %bb.0: ; %entry
9370 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9371 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
9372 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
9373 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
9374 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
9375 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
9376 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9377 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
9378 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
9379 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
9380 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9381 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
9382 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
9383 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9384 ; GFX11-WGP-NEXT: buffer_gl1_inv
9385 ; GFX11-WGP-NEXT: buffer_gl0_inv
9386 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
9387 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
9388 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
9389 ; GFX11-WGP-NEXT: s_endpgm
9390 ;
9391 ; GFX11-CU-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
9392 ; GFX11-CU: ; %bb.0: ; %entry
9393 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9394 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
9395 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
9396 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9397 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
9398 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
9399 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9400 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
9401 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
9402 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
9403 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9404 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
9405 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
9406 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9407 ; GFX11-CU-NEXT: buffer_gl1_inv
9408 ; GFX11-CU-NEXT: buffer_gl0_inv
9409 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
9410 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
9411 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
9412 ; GFX11-CU-NEXT: s_endpgm
9413 ;
9414 ; GFX12-WGP-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
9415 ; GFX12-WGP: ; %bb.0: ; %entry
9416 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9417 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
9418 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
9419 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
9420 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
9421 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
9422 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9423 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
9424 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
9425 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
9426 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
9427 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
9428 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
9429 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
9430 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
9431 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
9432 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
9433 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
9434 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
9435 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
9436 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
9437 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
9438 ; GFX12-WGP-NEXT: s_endpgm
9439 ;
9440 ; GFX12-CU-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
9441 ; GFX12-CU: ; %bb.0: ; %entry
9442 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9443 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
9444 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
9445 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
9446 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
9447 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
9448 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9449 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
9450 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
9451 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
9452 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
9453 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
9454 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
9455 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
9456 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
9457 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
9458 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
9459 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
9460 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
9461 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
9462 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
9463 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
9464 ; GFX12-CU-NEXT: s_endpgm
9465 ptr %out, i32 %in, i32 %old) {
9466 entry:
9467 %gep = getelementptr i32, ptr %out, i32 4
9468 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
9469 %val0 = extractvalue { i32, i1 } %val, 0
9470 store i32 %val0, ptr %out, align 4
9471 ret void
9472 }
9474 define amdgpu_kernel void @flat_agent_seq_cst_acquire_ret_cmpxchg(
9475 ; GFX7-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
9476 ; GFX7: ; %bb.0: ; %entry
9477 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
9478 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9479 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
9480 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
9481 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
9482 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9483 ; GFX7-NEXT: s_mov_b32 s6, s4
9484 ; GFX7-NEXT: s_mov_b32 s7, s5
9485 ; GFX7-NEXT: s_mov_b32 s11, s12
9486 ; GFX7-NEXT: s_mov_b32 s10, s13
9487 ; GFX7-NEXT: s_add_u32 s6, s6, s11
9488 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
9489 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9490 ; GFX7-NEXT: s_mov_b32 s7, s10
9491 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
9492 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
9493 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9494 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
9495 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
9496 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
9497 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9498 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9499 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9500 ; GFX7-NEXT: buffer_wbinvl1_vol
9501 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
9502 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
9503 ; GFX7-NEXT: flat_store_dword v[0:1], v2
9504 ; GFX7-NEXT: s_endpgm
9505 ;
9506 ; GFX10-WGP-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
9507 ; GFX10-WGP: ; %bb.0: ; %entry
9508 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
9509 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9510 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
9511 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
9512 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
9513 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
9514 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
9515 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
9516 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
9517 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
9518 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
9519 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
9520 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9521 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
9522 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
9523 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
9524 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9525 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
9526 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
9527 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
9528 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9529 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
9530 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9531 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9532 ; GFX10-WGP-NEXT: buffer_gl1_inv
9533 ; GFX10-WGP-NEXT: buffer_gl0_inv
9534 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
9535 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
9536 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
9537 ; GFX10-WGP-NEXT: s_endpgm
9538 ;
9539 ; GFX10-CU-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
9540 ; GFX10-CU: ; %bb.0: ; %entry
9541 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
9542 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9543 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
9544 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
9545 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
9546 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9547 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
9548 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
9549 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
9550 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
9551 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
9552 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
9553 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9554 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
9555 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
9556 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
9557 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9558 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
9559 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
9560 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
9561 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9562 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
9563 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9564 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9565 ; GFX10-CU-NEXT: buffer_gl1_inv
9566 ; GFX10-CU-NEXT: buffer_gl0_inv
9567 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
9568 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
9569 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
9570 ; GFX10-CU-NEXT: s_endpgm
9571 ;
9572 ; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
9573 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
9574 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
9575 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
9576 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
9577 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
9578 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
9579 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9580 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
9581 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
9582 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
9583 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
9584 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
9585 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
9586 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
9587 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
9588 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
9589 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
9590 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9591 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
9592 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
9593 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
9594 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9595 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9596 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9597 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
9598 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
9599 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
9600 ; SKIP-CACHE-INV-NEXT: s_endpgm
9601 ;
9602 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
9603 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
9604 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9605 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
9606 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
9607 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9608 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
9609 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
9610 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9611 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9612 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9613 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9614 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9615 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9616 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
9617 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9618 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
9619 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
9620 ;
9621 ; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
9622 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
9623 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9624 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
9625 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
9626 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9627 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
9628 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
9629 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9630 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9631 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9632 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9633 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9634 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9635 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
9636 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9637 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
9638 ; GFX90A-TGSPLIT-NEXT: s_endpgm
9639 ;
9640 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
9641 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
9642 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
9643 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
9644 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
9645 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9646 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
9647 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
9648 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9649 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9650 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9651 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
9652 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9653 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9654 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9655 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
9656 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9657 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
9658 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
9659 ;
9660 ; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
9661 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
9662 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
9663 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
9664 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
9665 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9666 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
9667 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
9668 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9669 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9670 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9671 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
9672 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9673 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9674 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9675 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
9676 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9677 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
9678 ; GFX940-TGSPLIT-NEXT: s_endpgm
9679 ;
9680 ; GFX11-WGP-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
9681 ; GFX11-WGP: ; %bb.0: ; %entry
9682 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9683 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
9684 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
9685 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
9686 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
9687 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
9688 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9689 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
9690 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
9691 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
9692 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9693 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
9694 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
9695 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9696 ; GFX11-WGP-NEXT: buffer_gl1_inv
9697 ; GFX11-WGP-NEXT: buffer_gl0_inv
9698 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
9699 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
9700 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
9701 ; GFX11-WGP-NEXT: s_endpgm
9702 ;
9703 ; GFX11-CU-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
9704 ; GFX11-CU: ; %bb.0: ; %entry
9705 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9706 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
9707 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
9708 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9709 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
9710 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
9711 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9712 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
9713 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
9714 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
9715 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9716 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
9717 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
9718 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9719 ; GFX11-CU-NEXT: buffer_gl1_inv
9720 ; GFX11-CU-NEXT: buffer_gl0_inv
9721 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
9722 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
9723 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
9724 ; GFX11-CU-NEXT: s_endpgm
9725 ;
9726 ; GFX12-WGP-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
9727 ; GFX12-WGP: ; %bb.0: ; %entry
9728 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9729 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
9730 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
9731 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
9732 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
9733 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
9734 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9735 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
9736 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
9737 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
9738 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
9739 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
9740 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
9741 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
9742 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
9743 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
9744 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
9745 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
9746 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
9747 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
9748 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
9749 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
9750 ; GFX12-WGP-NEXT: s_endpgm
9751 ;
9752 ; GFX12-CU-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
9753 ; GFX12-CU: ; %bb.0: ; %entry
9754 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9755 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
9756 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
9757 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
9758 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
9759 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
9760 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9761 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
9762 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
9763 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
9764 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
9765 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
9766 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
9767 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
9768 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
9769 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
9770 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
9771 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
9772 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
9773 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
9774 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
9775 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
9776 ; GFX12-CU-NEXT: s_endpgm
9777 ptr %out, i32 %in, i32 %old) {
9778 entry:
9779 %gep = getelementptr i32, ptr %out, i32 4
9780 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
9781 %val0 = extractvalue { i32, i1 } %val, 0
9782 store i32 %val0, ptr %out, align 4
9783 ret void
9784 }
9786 define amdgpu_kernel void @flat_agent_monotonic_seq_cst_ret_cmpxchg(
9787 ; GFX7-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
9788 ; GFX7: ; %bb.0: ; %entry
9789 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
9790 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9791 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
9792 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
9793 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
9794 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9795 ; GFX7-NEXT: s_mov_b32 s6, s4
9796 ; GFX7-NEXT: s_mov_b32 s7, s5
9797 ; GFX7-NEXT: s_mov_b32 s11, s12
9798 ; GFX7-NEXT: s_mov_b32 s10, s13
9799 ; GFX7-NEXT: s_add_u32 s6, s6, s11
9800 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
9801 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9802 ; GFX7-NEXT: s_mov_b32 s7, s10
9803 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
9804 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
9805 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9806 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
9807 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
9808 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
9809 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9810 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9811 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9812 ; GFX7-NEXT: buffer_wbinvl1_vol
9813 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
9814 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
9815 ; GFX7-NEXT: flat_store_dword v[0:1], v2
9816 ; GFX7-NEXT: s_endpgm
9817 ;
9818 ; GFX10-WGP-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
9819 ; GFX10-WGP: ; %bb.0: ; %entry
9820 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
9821 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9822 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
9823 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
9824 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
9825 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
9826 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
9827 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
9828 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
9829 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
9830 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
9831 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
9832 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9833 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
9834 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
9835 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
9836 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9837 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
9838 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
9839 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
9840 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9841 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
9842 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9843 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9844 ; GFX10-WGP-NEXT: buffer_gl1_inv
9845 ; GFX10-WGP-NEXT: buffer_gl0_inv
9846 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
9847 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
9848 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
9849 ; GFX10-WGP-NEXT: s_endpgm
9850 ;
9851 ; GFX10-CU-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
9852 ; GFX10-CU: ; %bb.0: ; %entry
9853 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
9854 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9855 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
9856 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
9857 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
9858 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9859 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
9860 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
9861 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
9862 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
9863 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
9864 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
9865 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9866 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
9867 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
9868 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
9869 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9870 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
9871 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
9872 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
9873 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9874 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
9875 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9876 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9877 ; GFX10-CU-NEXT: buffer_gl1_inv
9878 ; GFX10-CU-NEXT: buffer_gl0_inv
9879 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
9880 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
9881 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
9882 ; GFX10-CU-NEXT: s_endpgm
9883 ;
9884 ; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
9885 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
9886 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
9887 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
9888 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
9889 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
9890 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
9891 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9892 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
9893 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
9894 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
9895 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
9896 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
9897 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
9898 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
9899 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
9900 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
9901 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
9902 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9903 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
9904 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
9905 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
9906 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9907 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9908 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9909 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
9910 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
9911 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
9912 ; SKIP-CACHE-INV-NEXT: s_endpgm
9913 ;
9914 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
9915 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
9916 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9917 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
9918 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
9919 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9920 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
9921 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
9922 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9923 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9924 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9925 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9926 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9927 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9928 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
9929 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9930 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
9931 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
9932 ;
9933 ; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
9934 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
9935 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9936 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
9937 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
9938 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9939 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
9940 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
9941 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9942 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9943 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9944 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9945 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9946 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9947 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
9948 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9949 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
9950 ; GFX90A-TGSPLIT-NEXT: s_endpgm
9951 ;
9952 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
9953 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
9954 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
9955 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
9956 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
9957 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9958 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
9959 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
9960 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9961 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9962 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9963 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
9964 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9965 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9966 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9967 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
9968 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9969 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
9970 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
9971 ;
9972 ; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
9973 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
9974 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
9975 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
9976 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
9977 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9978 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
9979 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
9980 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9981 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9982 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9983 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
9984 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9985 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9986 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9987 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
9988 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9989 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
9990 ; GFX940-TGSPLIT-NEXT: s_endpgm
9991 ;
9992 ; GFX11-WGP-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
9993 ; GFX11-WGP: ; %bb.0: ; %entry
9994 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9995 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
9996 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
9997 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
9998 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
9999 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
10000 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10001 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
10002 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
10003 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
10004 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10005 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
10006 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10007 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10008 ; GFX11-WGP-NEXT: buffer_gl1_inv
10009 ; GFX11-WGP-NEXT: buffer_gl0_inv
10010 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
10011 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
10012 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
10013 ; GFX11-WGP-NEXT: s_endpgm
10014 ;
10015 ; GFX11-CU-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
10016 ; GFX11-CU: ; %bb.0: ; %entry
10017 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10018 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
10019 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
10020 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10021 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
10022 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
10023 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10024 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
10025 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
10026 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
10027 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10028 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
10029 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10030 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10031 ; GFX11-CU-NEXT: buffer_gl1_inv
10032 ; GFX11-CU-NEXT: buffer_gl0_inv
10033 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
10034 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
10035 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
10036 ; GFX11-CU-NEXT: s_endpgm
10037 ;
10038 ; GFX12-WGP-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
10039 ; GFX12-WGP: ; %bb.0: ; %entry
10040 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10041 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
10042 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
10043 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
10044 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
10045 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
10046 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10047 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
10048 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
10049 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
10050 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
10051 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
10052 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
10053 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
10054 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
10055 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
10056 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
10057 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
10058 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
10059 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
10060 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
10061 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
10062 ; GFX12-WGP-NEXT: s_endpgm
10063 ;
10064 ; GFX12-CU-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
10065 ; GFX12-CU: ; %bb.0: ; %entry
10066 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10067 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
10068 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
10069 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
10070 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
10071 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
10072 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10073 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
10074 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
10075 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
10076 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
10077 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
10078 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
10079 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
10080 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
10081 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
10082 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
10083 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
10084 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
10085 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
10086 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
10087 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
10088 ; GFX12-CU-NEXT: s_endpgm
10089 ptr %out, i32 %in, i32 %old) {
10090 entry:
10091 %gep = getelementptr i32, ptr %out, i32 4
10092 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") monotonic seq_cst
10093 %val0 = extractvalue { i32, i1 } %val, 0
10094 store i32 %val0, ptr %out, align 4
10095 ret void
10096 }
10098 define amdgpu_kernel void @flat_agent_acquire_seq_cst_ret_cmpxchg(
10099 ; GFX7-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
10100 ; GFX7: ; %bb.0: ; %entry
10101 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
10102 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
10103 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
10104 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
10105 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
10106 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
10107 ; GFX7-NEXT: s_mov_b32 s6, s4
10108 ; GFX7-NEXT: s_mov_b32 s7, s5
10109 ; GFX7-NEXT: s_mov_b32 s11, s12
10110 ; GFX7-NEXT: s_mov_b32 s10, s13
10111 ; GFX7-NEXT: s_add_u32 s6, s6, s11
10112 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
10113 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
10114 ; GFX7-NEXT: s_mov_b32 s7, s10
10115 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
10116 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
10117 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10118 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
10119 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
10120 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
10121 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10122 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10123 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10124 ; GFX7-NEXT: buffer_wbinvl1_vol
10125 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
10126 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
10127 ; GFX7-NEXT: flat_store_dword v[0:1], v2
10128 ; GFX7-NEXT: s_endpgm
10129 ;
10130 ; GFX10-WGP-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
10131 ; GFX10-WGP: ; %bb.0: ; %entry
10132 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
10133 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
10134 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
10135 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
10136 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
10137 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
10138 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
10139 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
10140 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
10141 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
10142 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
10143 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
10144 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
10145 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
10146 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
10147 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
10148 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10149 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
10150 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
10151 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
10152 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10153 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
10154 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10155 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10156 ; GFX10-WGP-NEXT: buffer_gl1_inv
10157 ; GFX10-WGP-NEXT: buffer_gl0_inv
10158 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
10159 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
10160 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
10161 ; GFX10-WGP-NEXT: s_endpgm
10162 ;
10163 ; GFX10-CU-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
10164 ; GFX10-CU: ; %bb.0: ; %entry
10165 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
10166 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
10167 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
10168 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
10169 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
10170 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
10171 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
10172 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
10173 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
10174 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
10175 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
10176 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
10177 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
10178 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
10179 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
10180 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
10181 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10182 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
10183 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
10184 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
10185 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10186 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
10187 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10188 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10189 ; GFX10-CU-NEXT: buffer_gl1_inv
10190 ; GFX10-CU-NEXT: buffer_gl0_inv
10191 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
10192 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
10193 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
10194 ; GFX10-CU-NEXT: s_endpgm
10195 ;
10196 ; SKIP-CACHE-INV-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
10197 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
10198 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
10199 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
10200 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
10201 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
10202 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
10203 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10204 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
10205 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
10206 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
10207 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
10208 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
10209 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
10210 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
10211 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
10212 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
10213 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
10214 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10215 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
10216 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
10217 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
10218 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10219 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10220 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10221 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
10222 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
10223 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
10224 ; SKIP-CACHE-INV-NEXT: s_endpgm
10225 ;
10226 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
10227 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
10228 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
10229 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
10230 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
10231 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10232 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
10233 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
10234 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10235 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
10236 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10237 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10238 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10239 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10240 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
10241 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10242 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
10243 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
10244 ;
10245 ; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
10246 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
10247 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
10248 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
10249 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
10250 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10251 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
10252 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
10253 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10254 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
10255 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10256 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10257 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10258 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10259 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
10260 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10261 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
10262 ; GFX90A-TGSPLIT-NEXT: s_endpgm
10263 ;
10264 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
10265 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
10266 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
10267 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
10268 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
10269 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10270 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
10271 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
10272 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10273 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
10274 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10275 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
10276 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10277 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10278 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10279 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
10280 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10281 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10282 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
10283 ;
10284 ; GFX940-TGSPLIT-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
10285 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
10286 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
10287 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
10288 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
10289 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10290 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
10291 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
10292 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10293 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
10294 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10295 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
10296 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10297 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10298 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10299 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
10300 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10301 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10302 ; GFX940-TGSPLIT-NEXT: s_endpgm
10303 ;
10304 ; GFX11-WGP-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
10305 ; GFX11-WGP: ; %bb.0: ; %entry
10306 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10307 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
10308 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
10309 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
10310 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
10311 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
10312 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10313 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
10314 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
10315 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
10316 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10317 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
10318 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10319 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10320 ; GFX11-WGP-NEXT: buffer_gl1_inv
10321 ; GFX11-WGP-NEXT: buffer_gl0_inv
10322 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
10323 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
10324 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
10325 ; GFX11-WGP-NEXT: s_endpgm
10326 ;
10327 ; GFX11-CU-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
10328 ; GFX11-CU: ; %bb.0: ; %entry
10329 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10330 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
10331 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
10332 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10333 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
10334 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
10335 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10336 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
10337 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
10338 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
10339 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10340 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
10341 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10342 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10343 ; GFX11-CU-NEXT: buffer_gl1_inv
10344 ; GFX11-CU-NEXT: buffer_gl0_inv
10345 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
10346 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
10347 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
10348 ; GFX11-CU-NEXT: s_endpgm
10349 ;
10350 ; GFX12-WGP-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
10351 ; GFX12-WGP: ; %bb.0: ; %entry
10352 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10353 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
10354 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
10355 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
10356 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
10357 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
10358 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10359 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
10360 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
10361 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
10362 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
10363 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
10364 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
10365 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
10366 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
10367 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
10368 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
10369 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
10370 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
10371 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
10372 ; GFX12-WGP-NEXT: s_endpgm
10373 ;
10374 ; GFX12-CU-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
10375 ; GFX12-CU: ; %bb.0: ; %entry
10376 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10377 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
10378 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
10379 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
10380 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
10381 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
10382 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10383 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
10384 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
10385 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
10386 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
10387 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
10388 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
10389 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
10390 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
10391 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
10392 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
10393 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
10394 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
10395 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
10396 ; GFX12-CU-NEXT: s_endpgm
10397 ptr %out, i32 %in, i32 %old) {
10398 entry:
10399 %gep = getelementptr i32, ptr %out, i32 4
10400 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") acquire seq_cst
10401 %val0 = extractvalue { i32, i1 } %val, 0
10402 store i32 %val0, ptr %out, align 4
10403 ret void
10404 }
10406 define amdgpu_kernel void @flat_agent_release_seq_cst_ret_cmpxchg(
10407 ; GFX7-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
10408 ; GFX7: ; %bb.0: ; %entry
10409 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
10410 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
10411 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
10412 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
10413 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
10414 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
10415 ; GFX7-NEXT: s_mov_b32 s6, s4
10416 ; GFX7-NEXT: s_mov_b32 s7, s5
10417 ; GFX7-NEXT: s_mov_b32 s11, s12
10418 ; GFX7-NEXT: s_mov_b32 s10, s13
10419 ; GFX7-NEXT: s_add_u32 s6, s6, s11
10420 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
10421 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
10422 ; GFX7-NEXT: s_mov_b32 s7, s10
10423 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
10424 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
10425 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10426 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
10427 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
10428 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
10429 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10430 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10431 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10432 ; GFX7-NEXT: buffer_wbinvl1_vol
10433 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
10434 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
10435 ; GFX7-NEXT: flat_store_dword v[0:1], v2
10436 ; GFX7-NEXT: s_endpgm
10437 ;
10438 ; GFX10-WGP-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
10439 ; GFX10-WGP: ; %bb.0: ; %entry
10440 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
10441 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
10442 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
10443 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
10444 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
10445 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
10446 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
10447 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
10448 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
10449 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
10450 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
10451 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
10452 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
10453 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
10454 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
10455 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
10456 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10457 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
10458 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
10459 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
10460 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10461 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
10462 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10463 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10464 ; GFX10-WGP-NEXT: buffer_gl1_inv
10465 ; GFX10-WGP-NEXT: buffer_gl0_inv
10466 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
10467 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
10468 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
10469 ; GFX10-WGP-NEXT: s_endpgm
10470 ;
10471 ; GFX10-CU-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
10472 ; GFX10-CU: ; %bb.0: ; %entry
10473 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
10474 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
10475 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
10476 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
10477 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
10478 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
10479 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
10480 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
10481 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
10482 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
10483 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
10484 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
10485 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
10486 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
10487 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
10488 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
10489 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10490 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
10491 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
10492 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
10493 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10494 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
10495 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10496 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10497 ; GFX10-CU-NEXT: buffer_gl1_inv
10498 ; GFX10-CU-NEXT: buffer_gl0_inv
10499 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
10500 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
10501 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
10502 ; GFX10-CU-NEXT: s_endpgm
10503 ;
10504 ; SKIP-CACHE-INV-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
10505 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
10506 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
10507 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
10508 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
10509 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
10510 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
10511 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10512 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
10513 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
10514 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
10515 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
10516 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
10517 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
10518 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
10519 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
10520 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
10521 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
10522 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10523 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
10524 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
10525 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
10526 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10527 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10528 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10529 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
10530 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
10531 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
10532 ; SKIP-CACHE-INV-NEXT: s_endpgm
10533 ;
10534 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
10535 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
10536 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
10537 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
10538 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
10539 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10540 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
10541 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
10542 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10543 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
10544 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10545 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10546 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10547 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10548 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
10549 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10550 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
10551 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
10552 ;
10553 ; GFX90A-TGSPLIT-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
10554 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
10555 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
10556 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
10557 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
10558 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10559 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
10560 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
10561 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10562 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
10563 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10564 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10565 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10566 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10567 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
10568 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10569 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
10570 ; GFX90A-TGSPLIT-NEXT: s_endpgm
10571 ;
10572 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
10573 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
10574 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
10575 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
10576 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
10577 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10578 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
10579 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
10580 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10581 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
10582 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10583 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
10584 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10585 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10586 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10587 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
10588 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10589 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10590 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
10591 ;
10592 ; GFX940-TGSPLIT-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
10593 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
10594 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
10595 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
10596 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
10597 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10598 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
10599 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
10600 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10601 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
10602 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10603 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
10604 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10605 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10606 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10607 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
10608 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10609 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10610 ; GFX940-TGSPLIT-NEXT: s_endpgm
10611 ;
10612 ; GFX11-WGP-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
10613 ; GFX11-WGP: ; %bb.0: ; %entry
10614 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10615 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
10616 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
10617 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
10618 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
10619 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
10620 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10621 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
10622 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
10623 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
10624 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10625 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
10626 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10627 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10628 ; GFX11-WGP-NEXT: buffer_gl1_inv
10629 ; GFX11-WGP-NEXT: buffer_gl0_inv
10630 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
10631 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
10632 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
10633 ; GFX11-WGP-NEXT: s_endpgm
10634 ;
10635 ; GFX11-CU-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
10636 ; GFX11-CU: ; %bb.0: ; %entry
10637 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10638 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
10639 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
10640 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10641 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
10642 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
10643 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10644 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
10645 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
10646 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
10647 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10648 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
10649 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10650 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10651 ; GFX11-CU-NEXT: buffer_gl1_inv
10652 ; GFX11-CU-NEXT: buffer_gl0_inv
10653 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
10654 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
10655 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
10656 ; GFX11-CU-NEXT: s_endpgm
10657 ;
10658 ; GFX12-WGP-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
10659 ; GFX12-WGP: ; %bb.0: ; %entry
10660 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10661 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
10662 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
10663 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
10664 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
10665 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
10666 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10667 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
10668 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
10669 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
10670 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
10671 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
10672 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
10673 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
10674 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
10675 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
10676 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
10677 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
10678 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
10679 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
10680 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
10681 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
10682 ; GFX12-WGP-NEXT: s_endpgm
10683 ;
10684 ; GFX12-CU-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
10685 ; GFX12-CU: ; %bb.0: ; %entry
10686 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10687 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
10688 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
10689 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
10690 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
10691 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
10692 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10693 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
10694 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
10695 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
10696 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
10697 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
10698 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
10699 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
10700 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
10701 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
10702 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
10703 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
10704 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
10705 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
10706 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
10707 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
10708 ; GFX12-CU-NEXT: s_endpgm
10709 ptr %out, i32 %in, i32 %old) {
10710 entry:
10711 %gep = getelementptr i32, ptr %out, i32 4
10712 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") release seq_cst
10713 %val0 = extractvalue { i32, i1 } %val, 0
10714 store i32 %val0, ptr %out, align 4
10715 ret void
10716 }
10718 define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_ret_cmpxchg(
10719 ; GFX7-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
10720 ; GFX7: ; %bb.0: ; %entry
10721 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
10722 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
10723 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
10724 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
10725 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
10726 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
10727 ; GFX7-NEXT: s_mov_b32 s6, s4
10728 ; GFX7-NEXT: s_mov_b32 s7, s5
10729 ; GFX7-NEXT: s_mov_b32 s11, s12
10730 ; GFX7-NEXT: s_mov_b32 s10, s13
10731 ; GFX7-NEXT: s_add_u32 s6, s6, s11
10732 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
10733 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
10734 ; GFX7-NEXT: s_mov_b32 s7, s10
10735 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
10736 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
10737 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10738 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
10739 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
10740 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
10741 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10742 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10743 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10744 ; GFX7-NEXT: buffer_wbinvl1_vol
10745 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
10746 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
10747 ; GFX7-NEXT: flat_store_dword v[0:1], v2
10748 ; GFX7-NEXT: s_endpgm
10749 ;
10750 ; GFX10-WGP-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
10751 ; GFX10-WGP: ; %bb.0: ; %entry
10752 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
10753 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
10754 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
10755 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
10756 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
10757 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
10758 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
10759 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
10760 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
10761 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
10762 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
10763 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
10764 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
10765 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
10766 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
10767 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
10768 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10769 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
10770 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
10771 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
10772 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10773 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
10774 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10775 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10776 ; GFX10-WGP-NEXT: buffer_gl1_inv
10777 ; GFX10-WGP-NEXT: buffer_gl0_inv
10778 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
10779 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
10780 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
10781 ; GFX10-WGP-NEXT: s_endpgm
10782 ;
10783 ; GFX10-CU-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
10784 ; GFX10-CU: ; %bb.0: ; %entry
10785 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
10786 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
10787 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
10788 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
10789 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
10790 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
10791 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
10792 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
10793 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
10794 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
10795 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
10796 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
10797 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
10798 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
10799 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
10800 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
10801 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10802 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
10803 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
10804 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
10805 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10806 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
10807 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10808 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10809 ; GFX10-CU-NEXT: buffer_gl1_inv
10810 ; GFX10-CU-NEXT: buffer_gl0_inv
10811 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
10812 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
10813 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
10814 ; GFX10-CU-NEXT: s_endpgm
10815 ;
10816 ; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
10817 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
10818 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
10819 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
10820 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
10821 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
10822 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
10823 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10824 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
10825 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
10826 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
10827 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
10828 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
10829 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
10830 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
10831 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
10832 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
10833 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
10834 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10835 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
10836 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
10837 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
10838 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10839 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10840 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10841 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
10842 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
10843 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
10844 ; SKIP-CACHE-INV-NEXT: s_endpgm
10845 ;
10846 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
10847 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
10848 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
10849 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
10850 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
10851 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10852 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
10853 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
10854 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10855 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
10856 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10857 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10858 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10859 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10860 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
10861 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10862 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
10863 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
10864 ;
10865 ; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
10866 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
10867 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
10868 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
10869 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
10870 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10871 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
10872 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
10873 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10874 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
10875 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10876 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10877 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10878 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10879 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
10880 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10881 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
10882 ; GFX90A-TGSPLIT-NEXT: s_endpgm
10883 ;
10884 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
10885 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
10886 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
10887 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
10888 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
10889 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10890 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
10891 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
10892 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10893 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
10894 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10895 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
10896 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10897 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10898 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10899 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
10900 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10901 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10902 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
10903 ;
10904 ; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
10905 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
10906 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
10907 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
10908 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
10909 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10910 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
10911 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
10912 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10913 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
10914 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10915 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
10916 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10917 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10918 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10919 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
10920 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10921 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10922 ; GFX940-TGSPLIT-NEXT: s_endpgm
10923 ;
10924 ; GFX11-WGP-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
10925 ; GFX11-WGP: ; %bb.0: ; %entry
10926 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10927 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
10928 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
10929 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
10930 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
10931 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
10932 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10933 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
10934 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
10935 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
10936 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10937 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
10938 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10939 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10940 ; GFX11-WGP-NEXT: buffer_gl1_inv
10941 ; GFX11-WGP-NEXT: buffer_gl0_inv
10942 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
10943 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
10944 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
10945 ; GFX11-WGP-NEXT: s_endpgm
10946 ;
10947 ; GFX11-CU-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
10948 ; GFX11-CU: ; %bb.0: ; %entry
10949 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10950 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
10951 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
10952 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10953 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
10954 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
10955 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10956 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
10957 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
10958 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
10959 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10960 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
10961 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10962 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10963 ; GFX11-CU-NEXT: buffer_gl1_inv
10964 ; GFX11-CU-NEXT: buffer_gl0_inv
10965 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
10966 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
10967 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
10968 ; GFX11-CU-NEXT: s_endpgm
10969 ;
10970 ; GFX12-WGP-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
10971 ; GFX12-WGP: ; %bb.0: ; %entry
10972 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10973 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
10974 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
10975 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
10976 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
10977 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
10978 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10979 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
10980 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
10981 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
10982 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
10983 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
10984 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
10985 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
10986 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
10987 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
10988 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
10989 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
10990 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
10991 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
10992 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
10993 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
10994 ; GFX12-WGP-NEXT: s_endpgm
10995 ;
10996 ; GFX12-CU-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
10997 ; GFX12-CU: ; %bb.0: ; %entry
10998 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10999 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
11000 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
11001 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
11002 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
11003 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
11004 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
11005 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
11006 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
11007 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
11008 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
11009 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
11010 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
11011 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
11012 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
11013 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
11014 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
11015 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
11016 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
11017 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
11018 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
11019 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
11020 ; GFX12-CU-NEXT: s_endpgm
11021 ptr %out, i32 %in, i32 %old) {
11022 entry:
11023 %gep = getelementptr i32, ptr %out, i32 4
11024 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") acq_rel seq_cst
11025 %val0 = extractvalue { i32, i1 } %val, 0
11026 store i32 %val0, ptr %out, align 4
11027 ret void
11028 }
11030 define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_ret_cmpxchg(
11031 ; GFX7-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
11032 ; GFX7: ; %bb.0: ; %entry
11033 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
11034 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
11035 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
11036 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
11037 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
11038 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
11039 ; GFX7-NEXT: s_mov_b32 s6, s4
11040 ; GFX7-NEXT: s_mov_b32 s7, s5
11041 ; GFX7-NEXT: s_mov_b32 s11, s12
11042 ; GFX7-NEXT: s_mov_b32 s10, s13
11043 ; GFX7-NEXT: s_add_u32 s6, s6, s11
11044 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
11045 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
11046 ; GFX7-NEXT: s_mov_b32 s7, s10
11047 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
11048 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
11049 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
11050 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
11051 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
11052 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
11053 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11054 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11055 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11056 ; GFX7-NEXT: buffer_wbinvl1_vol
11057 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
11058 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
11059 ; GFX7-NEXT: flat_store_dword v[0:1], v2
11060 ; GFX7-NEXT: s_endpgm
11061 ;
11062 ; GFX10-WGP-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
11063 ; GFX10-WGP: ; %bb.0: ; %entry
11064 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
11065 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
11066 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
11067 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
11068 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
11069 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
11070 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
11071 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
11072 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
11073 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
11074 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
11075 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
11076 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
11077 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
11078 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
11079 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
11080 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
11081 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
11082 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
11083 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
11084 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11085 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
11086 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11087 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11088 ; GFX10-WGP-NEXT: buffer_gl1_inv
11089 ; GFX10-WGP-NEXT: buffer_gl0_inv
11090 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
11091 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
11092 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
11093 ; GFX10-WGP-NEXT: s_endpgm
11094 ;
11095 ; GFX10-CU-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
11096 ; GFX10-CU: ; %bb.0: ; %entry
11097 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
11098 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
11099 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
11100 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
11101 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
11102 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
11103 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
11104 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
11105 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
11106 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
11107 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
11108 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
11109 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
11110 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
11111 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
11112 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
11113 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
11114 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
11115 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
11116 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
11117 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11118 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
11119 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11120 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11121 ; GFX10-CU-NEXT: buffer_gl1_inv
11122 ; GFX10-CU-NEXT: buffer_gl0_inv
11123 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
11124 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
11125 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
11126 ; GFX10-CU-NEXT: s_endpgm
11127 ;
11128 ; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
11129 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
11130 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
11131 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
11132 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
11133 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
11134 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
11135 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
11136 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
11137 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
11138 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
11139 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
11140 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
11141 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
11142 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
11143 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
11144 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
11145 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
11146 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
11147 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
11148 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
11149 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
11150 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11151 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11152 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11153 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
11154 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
11155 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
11156 ; SKIP-CACHE-INV-NEXT: s_endpgm
11157 ;
11158 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
11159 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
11160 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
11161 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
11162 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
11163 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11164 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
11165 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
11166 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
11167 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
11168 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
11169 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11170 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11171 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11172 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
11173 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
11174 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
11175 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
11176 ;
11177 ; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
11178 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
11179 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
11180 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
11181 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
11182 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11183 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
11184 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
11185 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
11186 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
11187 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
11188 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11189 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11190 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11191 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
11192 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
11193 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
11194 ; GFX90A-TGSPLIT-NEXT: s_endpgm
11195 ;
11196 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
11197 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
11198 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
11199 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
11200 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
11201 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11202 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
11203 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
11204 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
11205 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
11206 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
11207 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
11208 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11209 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11210 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11211 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
11212 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
11213 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11214 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
11215 ;
11216 ; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
11217 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
11218 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
11219 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
11220 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
11221 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11222 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
11223 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
11224 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
11225 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
11226 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
11227 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
11228 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11229 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11230 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11231 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
11232 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
11233 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11234 ; GFX940-TGSPLIT-NEXT: s_endpgm
11235 ;
11236 ; GFX11-WGP-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
11237 ; GFX11-WGP: ; %bb.0: ; %entry
11238 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
11239 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
11240 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
11241 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
11242 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
11243 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
11244 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
11245 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
11246 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
11247 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
11248 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11249 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
11250 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11251 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11252 ; GFX11-WGP-NEXT: buffer_gl1_inv
11253 ; GFX11-WGP-NEXT: buffer_gl0_inv
11254 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
11255 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
11256 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
11257 ; GFX11-WGP-NEXT: s_endpgm
11258 ;
11259 ; GFX11-CU-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
11260 ; GFX11-CU: ; %bb.0: ; %entry
11261 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
11262 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
11263 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
11264 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
11265 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
11266 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
11267 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
11268 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
11269 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
11270 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
11271 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11272 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
11273 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11274 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11275 ; GFX11-CU-NEXT: buffer_gl1_inv
11276 ; GFX11-CU-NEXT: buffer_gl0_inv
11277 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
11278 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
11279 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
11280 ; GFX11-CU-NEXT: s_endpgm
11281 ;
11282 ; GFX12-WGP-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
11283 ; GFX12-WGP: ; %bb.0: ; %entry
11284 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
11285 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
11286 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
11287 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
11288 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
11289 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
11290 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
11291 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
11292 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
11293 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
11294 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
11295 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
11296 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
11297 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
11298 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
11299 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
11300 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
11301 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
11302 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
11303 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
11304 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
11305 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
11306 ; GFX12-WGP-NEXT: s_endpgm
11307 ;
11308 ; GFX12-CU-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
11309 ; GFX12-CU: ; %bb.0: ; %entry
11310 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
11311 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
11312 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
11313 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
11314 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
11315 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
11316 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
11317 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
11318 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
11319 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
11320 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
11321 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
11322 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
11323 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
11324 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
11325 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
11326 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
11327 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
11328 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
11329 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
11330 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
11331 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
11332 ; GFX12-CU-NEXT: s_endpgm
11333 ptr %out, i32 %in, i32 %old) {
11334 entry:
11335 %gep = getelementptr i32, ptr %out, i32 4
11336 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
11337 %val0 = extractvalue { i32, i1 } %val, 0
11338 store i32 %val0, ptr %out, align 4
11339 ret void
11340 }
11342 define amdgpu_kernel void @flat_agent_one_as_unordered_load(
11343 ; GFX7-LABEL: flat_agent_one_as_unordered_load:
11344 ; GFX7: ; %bb.0: ; %entry
11345 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
11346 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x2
11347 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
11348 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
11349 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
11350 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
11351 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
11352 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
11353 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11354 ; GFX7-NEXT: flat_store_dword v[0:1], v2
11355 ; GFX7-NEXT: s_endpgm
11356 ;
11357 ; GFX10-WGP-LABEL: flat_agent_one_as_unordered_load:
11358 ; GFX10-WGP: ; %bb.0: ; %entry
11359 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
11360 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
11361 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
11362 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
11363 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
11364 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1]
11365 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
11366 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
11367 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11368 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
11369 ; GFX10-WGP-NEXT: s_endpgm
11370 ;
11371 ; GFX10-CU-LABEL: flat_agent_one_as_unordered_load:
11372 ; GFX10-CU: ; %bb.0: ; %entry
11373 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
11374 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
11375 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
11376 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
11377 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
11378 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]
11379 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
11380 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
11381 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11382 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
11383 ; GFX10-CU-NEXT: s_endpgm
11384 ;
11385 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_unordered_load:
11386 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
11387 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
11388 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
11389 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
11390 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
11391 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
11392 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]
11393 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
11394 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
11395 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11396 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
11397 ; SKIP-CACHE-INV-NEXT: s_endpgm
11398 ;
11399 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_unordered_load:
11400 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
11401 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
11402 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
11403 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11404 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
11405 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
11406 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
11407 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11408 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
11409 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
11410 ;
11411 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_unordered_load:
11412 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
11413 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
11414 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
11415 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11416 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
11417 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
11418 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
11419 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11420 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
11421 ; GFX90A-TGSPLIT-NEXT: s_endpgm
11422 ;
11423 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_unordered_load:
11424 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
11425 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
11426 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
11427 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11428 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
11429 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
11430 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
11431 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11432 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11433 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
11434 ;
11435 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_unordered_load:
11436 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
11437 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
11438 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
11439 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11440 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
11441 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
11442 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
11443 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11444 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11445 ; GFX940-TGSPLIT-NEXT: s_endpgm
11446 ;
11447 ; GFX11-WGP-LABEL: flat_agent_one_as_unordered_load:
11448 ; GFX11-WGP: ; %bb.0: ; %entry
11449 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
11450 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
11451 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
11452 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
11453 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
11454 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1]
11455 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
11456 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
11457 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11458 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
11459 ; GFX11-WGP-NEXT: s_endpgm
11460 ;
11461 ; GFX11-CU-LABEL: flat_agent_one_as_unordered_load:
11462 ; GFX11-CU: ; %bb.0: ; %entry
11463 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
11464 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
11465 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
11466 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
11467 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
11468 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]
11469 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
11470 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
11471 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11472 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
11473 ; GFX11-CU-NEXT: s_endpgm
11474 ;
11475 ; GFX12-WGP-LABEL: flat_agent_one_as_unordered_load:
11476 ; GFX12-WGP: ; %bb.0: ; %entry
11477 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
11478 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
11479 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
11480 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
11481 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
11482 ; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
11483 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
11484 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
11485 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
11486 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
11487 ; GFX12-WGP-NEXT: s_endpgm
11488 ;
11489 ; GFX12-CU-LABEL: flat_agent_one_as_unordered_load:
11490 ; GFX12-CU: ; %bb.0: ; %entry
11491 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
11492 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
11493 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
11494 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
11495 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
11496 ; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
11497 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
11498 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
11499 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
11500 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
11501 ; GFX12-CU-NEXT: s_endpgm
11502 ptr %in, ptr %out) {
11503 entry:
11504 %val = load atomic i32, ptr %in syncscope("agent-one-as") unordered, align 4
11505 store i32 %val, ptr %out
11506 ret void
11507 }
11509 define amdgpu_kernel void @flat_agent_one_as_monotonic_load(
11510 ; GFX7-LABEL: flat_agent_one_as_monotonic_load:
11511 ; GFX7: ; %bb.0: ; %entry
11512 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
11513 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x2
11514 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
11515 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
11516 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
11517 ; GFX7-NEXT: flat_load_dword v2, v[0:1] glc
11518 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
11519 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
11520 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11521 ; GFX7-NEXT: flat_store_dword v[0:1], v2
11522 ; GFX7-NEXT: s_endpgm
11523 ;
11524 ; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_load:
11525 ; GFX10-WGP: ; %bb.0: ; %entry
11526 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
11527 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
11528 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
11529 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
11530 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
11531 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc dlc
11532 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
11533 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
11534 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11535 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
11536 ; GFX10-WGP-NEXT: s_endpgm
11537 ;
11538 ; GFX10-CU-LABEL: flat_agent_one_as_monotonic_load:
11539 ; GFX10-CU: ; %bb.0: ; %entry
11540 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
11541 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
11542 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
11543 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
11544 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
11545 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] glc dlc
11546 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
11547 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
11548 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11549 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
11550 ; GFX10-CU-NEXT: s_endpgm
11551 ;
11552 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_load:
11553 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
11554 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
11555 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
11556 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
11557 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
11558 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
11559 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1] glc
11560 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
11561 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
11562 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11563 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
11564 ; SKIP-CACHE-INV-NEXT: s_endpgm
11565 ;
11566 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_load:
11567 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
11568 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
11569 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
11570 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11571 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
11572 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc
11573 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
11574 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11575 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
11576 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
11577 ;
11578 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_load:
11579 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
11580 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
11581 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
11582 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11583 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
11584 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc
11585 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
11586 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11587 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
11588 ; GFX90A-TGSPLIT-NEXT: s_endpgm
11589 ;
11590 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_load:
11591 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
11592 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
11593 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
11594 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11595 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
11596 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc1
11597 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
11598 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11599 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11600 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
11601 ;
11602 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_load:
11603 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
11604 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
11605 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
11606 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11607 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
11608 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc1
11609 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
11610 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11611 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11612 ; GFX940-TGSPLIT-NEXT: s_endpgm
11613 ;
11614 ; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_load:
11615 ; GFX11-WGP: ; %bb.0: ; %entry
11616 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
11617 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
11618 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
11619 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
11620 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
11621 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc
11622 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
11623 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
11624 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11625 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
11626 ; GFX11-WGP-NEXT: s_endpgm
11627 ;
11628 ; GFX11-CU-LABEL: flat_agent_one_as_monotonic_load:
11629 ; GFX11-CU: ; %bb.0: ; %entry
11630 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
11631 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
11632 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
11633 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
11634 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
11635 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1] glc
11636 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
11637 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
11638 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11639 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
11640 ; GFX11-CU-NEXT: s_endpgm
11641 ;
11642 ; GFX12-WGP-LABEL: flat_agent_one_as_monotonic_load:
11643 ; GFX12-WGP: ; %bb.0: ; %entry
11644 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
11645 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
11646 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
11647 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
11648 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
11649 ; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_DEV
11650 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
11651 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
11652 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
11653 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
11654 ; GFX12-WGP-NEXT: s_endpgm
11655 ;
11656 ; GFX12-CU-LABEL: flat_agent_one_as_monotonic_load:
11657 ; GFX12-CU: ; %bb.0: ; %entry
11658 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
11659 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
11660 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
11661 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
11662 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
11663 ; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_DEV
11664 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
11665 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
11666 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
11667 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
11668 ; GFX12-CU-NEXT: s_endpgm
11669 ptr %in, ptr %out) {
11670 entry:
11671 %val = load atomic i32, ptr %in syncscope("agent-one-as") monotonic, align 4
11672 store i32 %val, ptr %out
11673 ret void
11674 }
11676 define amdgpu_kernel void @flat_agent_one_as_acquire_load(
11677 ; GFX7-LABEL: flat_agent_one_as_acquire_load:
11678 ; GFX7: ; %bb.0: ; %entry
11679 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
11680 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x2
11681 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
11682 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
11683 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
11684 ; GFX7-NEXT: flat_load_dword v2, v[0:1] glc
11685 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11686 ; GFX7-NEXT: buffer_wbinvl1_vol
11687 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
11688 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
11689 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
11690 ; GFX7-NEXT: flat_store_dword v[0:1], v2
11691 ; GFX7-NEXT: s_endpgm
11692 ;
11693 ; GFX10-WGP-LABEL: flat_agent_one_as_acquire_load:
11694 ; GFX10-WGP: ; %bb.0: ; %entry
11695 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
11696 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
11697 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
11698 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
11699 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
11700 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc dlc
11701 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
11702 ; GFX10-WGP-NEXT: buffer_gl1_inv
11703 ; GFX10-WGP-NEXT: buffer_gl0_inv
11704 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
11705 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
11706 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
11707 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
11708 ; GFX10-WGP-NEXT: s_endpgm
11709 ;
11710 ; GFX10-CU-LABEL: flat_agent_one_as_acquire_load:
11711 ; GFX10-CU: ; %bb.0: ; %entry
11712 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
11713 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
11714 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
11715 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
11716 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
11717 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] glc dlc
11718 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
11719 ; GFX10-CU-NEXT: buffer_gl1_inv
11720 ; GFX10-CU-NEXT: buffer_gl0_inv
11721 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
11722 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
11723 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
11724 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
11725 ; GFX10-CU-NEXT: s_endpgm
11726 ;
11727 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_load:
11728 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
11729 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
11730 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
11731 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
11732 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
11733 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
11734 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1] glc
11735 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
11736 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
11737 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
11738 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
11739 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
11740 ; SKIP-CACHE-INV-NEXT: s_endpgm
11741 ;
11742 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_load:
11743 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
11744 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
11745 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
11746 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11747 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
11748 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc
11749 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
11750 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
11751 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
11752 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11753 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
11754 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
11755 ;
11756 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_load:
11757 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
11758 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
11759 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
11760 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11761 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
11762 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc
11763 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11764 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
11765 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
11766 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
11767 ; GFX90A-TGSPLIT-NEXT: s_endpgm
11768 ;
11769 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_load:
11770 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
11771 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
11772 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
11773 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11774 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
11775 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc1
11776 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
11777 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
11778 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
11779 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11780 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11781 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
11782 ;
11783 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_load:
11784 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
11785 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
11786 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
11787 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11788 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
11789 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc1
11790 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11791 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
11792 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
11793 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11794 ; GFX940-TGSPLIT-NEXT: s_endpgm
11795 ;
11796 ; GFX11-WGP-LABEL: flat_agent_one_as_acquire_load:
11797 ; GFX11-WGP: ; %bb.0: ; %entry
11798 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
11799 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
11800 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
11801 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
11802 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
11803 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc
11804 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
11805 ; GFX11-WGP-NEXT: buffer_gl1_inv
11806 ; GFX11-WGP-NEXT: buffer_gl0_inv
11807 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
11808 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
11809 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
11810 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
11811 ; GFX11-WGP-NEXT: s_endpgm
11812 ;
11813 ; GFX11-CU-LABEL: flat_agent_one_as_acquire_load:
11814 ; GFX11-CU: ; %bb.0: ; %entry
11815 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
11816 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
11817 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
11818 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
11819 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
11820 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1] glc
11821 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
11822 ; GFX11-CU-NEXT: buffer_gl1_inv
11823 ; GFX11-CU-NEXT: buffer_gl0_inv
11824 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
11825 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
11826 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
11827 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
11828 ; GFX11-CU-NEXT: s_endpgm
11829 ;
11830 ; GFX12-WGP-LABEL: flat_agent_one_as_acquire_load:
11831 ; GFX12-WGP: ; %bb.0: ; %entry
11832 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
11833 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
11834 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
11835 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
11836 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
11837 ; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_DEV
11838 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
11839 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
11840 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
11841 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
11842 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
11843 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
11844 ; GFX12-WGP-NEXT: s_endpgm
11845 ;
11846 ; GFX12-CU-LABEL: flat_agent_one_as_acquire_load:
11847 ; GFX12-CU: ; %bb.0: ; %entry
11848 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
11849 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
11850 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
11851 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
11852 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
11853 ; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_DEV
11854 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
11855 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
11856 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
11857 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
11858 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
11859 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
11860 ; GFX12-CU-NEXT: s_endpgm
11861 ptr %in, ptr %out) {
11862 entry:
11863 %val = load atomic i32, ptr %in syncscope("agent-one-as") acquire, align 4
11864 store i32 %val, ptr %out
11865 ret void
11866 }
11868 define amdgpu_kernel void @flat_agent_one_as_seq_cst_load(
11869 ; GFX7-LABEL: flat_agent_one_as_seq_cst_load:
11870 ; GFX7: ; %bb.0: ; %entry
11871 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
11872 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x2
11873 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
11874 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
11875 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
11876 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11877 ; GFX7-NEXT: flat_load_dword v2, v[0:1] glc
11878 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11879 ; GFX7-NEXT: buffer_wbinvl1_vol
11880 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
11881 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
11882 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
11883 ; GFX7-NEXT: flat_store_dword v[0:1], v2
11884 ; GFX7-NEXT: s_endpgm
11885 ;
11886 ; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_load:
11887 ; GFX10-WGP: ; %bb.0: ; %entry
11888 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
11889 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
11890 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
11891 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
11892 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
11893 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
11894 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
11895 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc dlc
11896 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
11897 ; GFX10-WGP-NEXT: buffer_gl1_inv
11898 ; GFX10-WGP-NEXT: buffer_gl0_inv
11899 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
11900 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
11901 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
11902 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
11903 ; GFX10-WGP-NEXT: s_endpgm
11904 ;
11905 ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_load:
11906 ; GFX10-CU: ; %bb.0: ; %entry
11907 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
11908 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
11909 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
11910 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
11911 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
11912 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
11913 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
11914 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] glc dlc
11915 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
11916 ; GFX10-CU-NEXT: buffer_gl1_inv
11917 ; GFX10-CU-NEXT: buffer_gl0_inv
11918 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
11919 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
11920 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
11921 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
11922 ; GFX10-CU-NEXT: s_endpgm
11923 ;
11924 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_load:
11925 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
11926 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
11927 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
11928 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
11929 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
11930 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
11931 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
11932 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1] glc
11933 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
11934 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
11935 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
11936 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
11937 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
11938 ; SKIP-CACHE-INV-NEXT: s_endpgm
11939 ;
11940 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_load:
11941 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
11942 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
11943 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
11944 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11945 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
11946 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
11947 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc
11948 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
11949 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
11950 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
11951 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11952 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
11953 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
11954 ;
11955 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_load:
11956 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
11957 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
11958 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
11959 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11960 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
11961 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11962 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc
11963 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11964 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
11965 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
11966 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
11967 ; GFX90A-TGSPLIT-NEXT: s_endpgm
11968 ;
11969 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_load:
11970 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
11971 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
11972 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
11973 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11974 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
11975 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
11976 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc1
11977 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
11978 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
11979 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
11980 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11981 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11982 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
11983 ;
11984 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_load:
11985 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
11986 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
11987 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
11988 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11989 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
11990 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11991 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc1
11992 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11993 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
11994 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
11995 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11996 ; GFX940-TGSPLIT-NEXT: s_endpgm
11997 ;
11998 ; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_load:
11999 ; GFX11-WGP: ; %bb.0: ; %entry
12000 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
12001 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
12002 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
12003 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
12004 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
12005 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
12006 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12007 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc
12008 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
12009 ; GFX11-WGP-NEXT: buffer_gl1_inv
12010 ; GFX11-WGP-NEXT: buffer_gl0_inv
12011 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
12012 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
12013 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
12014 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
12015 ; GFX11-WGP-NEXT: s_endpgm
12016 ;
12017 ; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_load:
12018 ; GFX11-CU: ; %bb.0: ; %entry
12019 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
12020 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
12021 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
12022 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
12023 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
12024 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
12025 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
12026 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1] glc
12027 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
12028 ; GFX11-CU-NEXT: buffer_gl1_inv
12029 ; GFX11-CU-NEXT: buffer_gl0_inv
12030 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
12031 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
12032 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
12033 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
12034 ; GFX11-CU-NEXT: s_endpgm
12035 ;
12036 ; GFX12-WGP-LABEL: flat_agent_one_as_seq_cst_load:
12037 ; GFX12-WGP: ; %bb.0: ; %entry
12038 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
12039 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
12040 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
12041 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
12042 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
12043 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
12044 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
12045 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
12046 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
12047 ; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_DEV
12048 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
12049 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
12050 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
12051 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
12052 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
12053 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
12054 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
12055 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
12056 ; GFX12-WGP-NEXT: s_endpgm
12057 ;
12058 ; GFX12-CU-LABEL: flat_agent_one_as_seq_cst_load:
12059 ; GFX12-CU: ; %bb.0: ; %entry
12060 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
12061 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
12062 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
12063 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
12064 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
12065 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
12066 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
12067 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
12068 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
12069 ; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_DEV
12070 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
12071 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
12072 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
12073 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
12074 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
12075 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
12076 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
12077 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
12078 ; GFX12-CU-NEXT: s_endpgm
12079 ptr %in, ptr %out) {
12080 entry:
12081 %val = load atomic i32, ptr %in syncscope("agent-one-as") seq_cst, align 4
12082 store i32 %val, ptr %out
12083 ret void
12084 }
12086 define amdgpu_kernel void @flat_agent_one_as_unordered_store(
12087 ; GFX7-LABEL: flat_agent_one_as_unordered_store:
12088 ; GFX7: ; %bb.0: ; %entry
12089 ; GFX7-NEXT: s_load_dword s4, s[8:9], 0x0
12090 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x2
12091 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
12092 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
12093 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
12094 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
12095 ; GFX7-NEXT: flat_store_dword v[0:1], v2
12096 ; GFX7-NEXT: s_endpgm
12097 ;
12098 ; GFX10-WGP-LABEL: flat_agent_one_as_unordered_store:
12099 ; GFX10-WGP: ; %bb.0: ; %entry
12100 ; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x0
12101 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
12102 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
12103 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
12104 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
12105 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
12106 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
12107 ; GFX10-WGP-NEXT: s_endpgm
12108 ;
12109 ; GFX10-CU-LABEL: flat_agent_one_as_unordered_store:
12110 ; GFX10-CU: ; %bb.0: ; %entry
12111 ; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x0
12112 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
12113 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
12114 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
12115 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
12116 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
12117 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
12118 ; GFX10-CU-NEXT: s_endpgm
12119 ;
12120 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_unordered_store:
12121 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
12122 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x0
12123 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
12124 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
12125 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
12126 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
12127 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
12128 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
12129 ; SKIP-CACHE-INV-NEXT: s_endpgm
12130 ;
12131 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_unordered_store:
12132 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
12133 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x0
12134 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
12135 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12136 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
12137 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
12138 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
12139 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
12140 ;
12141 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_unordered_store:
12142 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
12143 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x0
12144 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
12145 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12146 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
12147 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
12148 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
12149 ; GFX90A-TGSPLIT-NEXT: s_endpgm
12150 ;
12151 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_unordered_store:
12152 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
12153 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x0
12154 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
12155 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12156 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
12157 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
12158 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
12159 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
12160 ;
12161 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_unordered_store:
12162 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
12163 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x0
12164 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
12165 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12166 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
12167 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
12168 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
12169 ; GFX940-TGSPLIT-NEXT: s_endpgm
12170 ;
12171 ; GFX11-WGP-LABEL: flat_agent_one_as_unordered_store:
12172 ; GFX11-WGP: ; %bb.0: ; %entry
12173 ; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
12174 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
12175 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
12176 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
12177 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
12178 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
12179 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
12180 ; GFX11-WGP-NEXT: s_endpgm
12181 ;
12182 ; GFX11-CU-LABEL: flat_agent_one_as_unordered_store:
12183 ; GFX11-CU: ; %bb.0: ; %entry
12184 ; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
12185 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
12186 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
12187 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
12188 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
12189 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
12190 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
12191 ; GFX11-CU-NEXT: s_endpgm
12192 ;
12193 ; GFX12-WGP-LABEL: flat_agent_one_as_unordered_store:
12194 ; GFX12-WGP: ; %bb.0: ; %entry
12195 ; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
12196 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
12197 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
12198 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
12199 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
12200 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
12201 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
12202 ; GFX12-WGP-NEXT: s_endpgm
12203 ;
12204 ; GFX12-CU-LABEL: flat_agent_one_as_unordered_store:
12205 ; GFX12-CU: ; %bb.0: ; %entry
12206 ; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
12207 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
12208 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
12209 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
12210 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
12211 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
12212 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
12213 ; GFX12-CU-NEXT: s_endpgm
12214 i32 %in, ptr %out) {
12215 entry:
12216 store atomic i32 %in, ptr %out syncscope("agent-one-as") unordered, align 4
12217 ret void
12218 }
12220 define amdgpu_kernel void @flat_agent_one_as_monotonic_store(
12221 ; GFX7-LABEL: flat_agent_one_as_monotonic_store:
12222 ; GFX7: ; %bb.0: ; %entry
12223 ; GFX7-NEXT: s_load_dword s4, s[8:9], 0x0
12224 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x2
12225 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
12226 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
12227 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
12228 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
12229 ; GFX7-NEXT: flat_store_dword v[0:1], v2
12230 ; GFX7-NEXT: s_endpgm
12231 ;
12232 ; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_store:
12233 ; GFX10-WGP: ; %bb.0: ; %entry
12234 ; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x0
12235 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
12236 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
12237 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
12238 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
12239 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
12240 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
12241 ; GFX10-WGP-NEXT: s_endpgm
12242 ;
12243 ; GFX10-CU-LABEL: flat_agent_one_as_monotonic_store:
12244 ; GFX10-CU: ; %bb.0: ; %entry
12245 ; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x0
12246 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
12247 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
12248 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
12249 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
12250 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
12251 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
12252 ; GFX10-CU-NEXT: s_endpgm
12253 ;
12254 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_store:
12255 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
12256 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x0
12257 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
12258 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
12259 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
12260 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
12261 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
12262 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
12263 ; SKIP-CACHE-INV-NEXT: s_endpgm
12264 ;
12265 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_store:
12266 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
12267 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x0
12268 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
12269 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12270 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
12271 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
12272 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
12273 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
12274 ;
12275 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_store:
12276 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
12277 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x0
12278 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
12279 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12280 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
12281 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
12282 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
12283 ; GFX90A-TGSPLIT-NEXT: s_endpgm
12284 ;
12285 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_store:
12286 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
12287 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x0
12288 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
12289 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12290 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
12291 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
12292 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
12293 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
12294 ;
12295 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_store:
12296 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
12297 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x0
12298 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
12299 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12300 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
12301 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
12302 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
12303 ; GFX940-TGSPLIT-NEXT: s_endpgm
12304 ;
12305 ; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_store:
12306 ; GFX11-WGP: ; %bb.0: ; %entry
12307 ; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
12308 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
12309 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
12310 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
12311 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
12312 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
12313 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
12314 ; GFX11-WGP-NEXT: s_endpgm
12315 ;
12316 ; GFX11-CU-LABEL: flat_agent_one_as_monotonic_store:
12317 ; GFX11-CU: ; %bb.0: ; %entry
12318 ; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
12319 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
12320 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
12321 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
12322 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
12323 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
12324 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
12325 ; GFX11-CU-NEXT: s_endpgm
12326 ;
12327 ; GFX12-WGP-LABEL: flat_agent_one_as_monotonic_store:
12328 ; GFX12-WGP: ; %bb.0: ; %entry
12329 ; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
12330 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
12331 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
12332 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
12333 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
12334 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
12335 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_DEV
12336 ; GFX12-WGP-NEXT: s_endpgm
12337 ;
12338 ; GFX12-CU-LABEL: flat_agent_one_as_monotonic_store:
12339 ; GFX12-CU: ; %bb.0: ; %entry
12340 ; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
12341 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
12342 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
12343 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
12344 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
12345 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
12346 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_DEV
12347 ; GFX12-CU-NEXT: s_endpgm
12348 i32 %in, ptr %out) {
12349 entry:
12350 store atomic i32 %in, ptr %out syncscope("agent-one-as") monotonic, align 4
12351 ret void
12352 }
12354 define amdgpu_kernel void @flat_agent_one_as_release_store(
12355 ; GFX7-LABEL: flat_agent_one_as_release_store:
12356 ; GFX7: ; %bb.0: ; %entry
12357 ; GFX7-NEXT: s_load_dword s4, s[8:9], 0x0
12358 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x2
12359 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
12360 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
12361 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
12362 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
12363 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12364 ; GFX7-NEXT: flat_store_dword v[0:1], v2
12365 ; GFX7-NEXT: s_endpgm
12366 ;
12367 ; GFX10-WGP-LABEL: flat_agent_one_as_release_store:
12368 ; GFX10-WGP: ; %bb.0: ; %entry
12369 ; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x0
12370 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
12371 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
12372 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
12373 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
12374 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
12375 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
12376 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12377 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
12378 ; GFX10-WGP-NEXT: s_endpgm
12379 ;
12380 ; GFX10-CU-LABEL: flat_agent_one_as_release_store:
12381 ; GFX10-CU: ; %bb.0: ; %entry
12382 ; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x0
12383 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
12384 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
12385 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
12386 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
12387 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
12388 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
12389 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
12390 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
12391 ; GFX10-CU-NEXT: s_endpgm
12392 ;
12393 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_store:
12394 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
12395 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x0
12396 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
12397 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
12398 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
12399 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
12400 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
12401 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
12402 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
12403 ; SKIP-CACHE-INV-NEXT: s_endpgm
12404 ;
12405 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_store:
12406 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
12407 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x0
12408 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
12409 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12410 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
12411 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
12412 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
12413 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
12414 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
12415 ;
12416 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_store:
12417 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
12418 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x0
12419 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
12420 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12421 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
12422 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
12423 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12424 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
12425 ; GFX90A-TGSPLIT-NEXT: s_endpgm
12426 ;
12427 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_store:
12428 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
12429 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x0
12430 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
12431 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12432 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
12433 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
12434 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
12435 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
12436 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
12437 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
12438 ;
12439 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_store:
12440 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
12441 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x0
12442 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
12443 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12444 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
12445 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
12446 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
12447 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12448 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
12449 ; GFX940-TGSPLIT-NEXT: s_endpgm
12450 ;
12451 ; GFX11-WGP-LABEL: flat_agent_one_as_release_store:
12452 ; GFX11-WGP: ; %bb.0: ; %entry
12453 ; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
12454 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
12455 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
12456 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
12457 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
12458 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
12459 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
12460 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12461 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
12462 ; GFX11-WGP-NEXT: s_endpgm
12463 ;
12464 ; GFX11-CU-LABEL: flat_agent_one_as_release_store:
12465 ; GFX11-CU: ; %bb.0: ; %entry
12466 ; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
12467 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
12468 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
12469 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
12470 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
12471 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
12472 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
12473 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
12474 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
12475 ; GFX11-CU-NEXT: s_endpgm
12476 ;
12477 ; GFX12-WGP-LABEL: flat_agent_one_as_release_store:
12478 ; GFX12-WGP: ; %bb.0: ; %entry
12479 ; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
12480 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
12481 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
12482 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
12483 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
12484 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
12485 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
12486 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
12487 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
12488 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
12489 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_DEV
12490 ; GFX12-WGP-NEXT: s_endpgm
12491 ;
12492 ; GFX12-CU-LABEL: flat_agent_one_as_release_store:
12493 ; GFX12-CU: ; %bb.0: ; %entry
12494 ; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
12495 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
12496 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
12497 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
12498 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
12499 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
12500 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
12501 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
12502 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
12503 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
12504 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_DEV
12505 ; GFX12-CU-NEXT: s_endpgm
12506 i32 %in, ptr %out) {
12507 entry:
12508 store atomic i32 %in, ptr %out syncscope("agent-one-as") release, align 4
12509 ret void
12510 }
12512 define amdgpu_kernel void @flat_agent_one_as_seq_cst_store(
12513 ; GFX7-LABEL: flat_agent_one_as_seq_cst_store:
12514 ; GFX7: ; %bb.0: ; %entry
12515 ; GFX7-NEXT: s_load_dword s4, s[8:9], 0x0
12516 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x2
12517 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
12518 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
12519 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
12520 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
12521 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12522 ; GFX7-NEXT: flat_store_dword v[0:1], v2
12523 ; GFX7-NEXT: s_endpgm
12524 ;
12525 ; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_store:
12526 ; GFX10-WGP: ; %bb.0: ; %entry
12527 ; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x0
12528 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
12529 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
12530 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
12531 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
12532 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
12533 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
12534 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12535 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
12536 ; GFX10-WGP-NEXT: s_endpgm
12537 ;
12538 ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_store:
12539 ; GFX10-CU: ; %bb.0: ; %entry
12540 ; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x0
12541 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
12542 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
12543 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
12544 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
12545 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
12546 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
12547 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
12548 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
12549 ; GFX10-CU-NEXT: s_endpgm
12550 ;
12551 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_store:
12552 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
12553 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x0
12554 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
12555 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
12556 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
12557 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
12558 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
12559 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
12560 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
12561 ; SKIP-CACHE-INV-NEXT: s_endpgm
12562 ;
12563 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_store:
12564 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
12565 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x0
12566 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
12567 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12568 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
12569 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
12570 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
12571 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
12572 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
12573 ;
12574 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_store:
12575 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
12576 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x0
12577 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
12578 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12579 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
12580 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
12581 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12582 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
12583 ; GFX90A-TGSPLIT-NEXT: s_endpgm
12584 ;
12585 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_store:
12586 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
12587 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x0
12588 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
12589 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12590 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
12591 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
12592 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
12593 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
12594 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
12595 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
12596 ;
12597 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_store:
12598 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
12599 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x0
12600 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
12601 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12602 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
12603 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
12604 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
12605 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12606 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
12607 ; GFX940-TGSPLIT-NEXT: s_endpgm
12608 ;
12609 ; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_store:
12610 ; GFX11-WGP: ; %bb.0: ; %entry
12611 ; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
12612 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
12613 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
12614 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
12615 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
12616 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
12617 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
12618 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12619 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
12620 ; GFX11-WGP-NEXT: s_endpgm
12621 ;
12622 ; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_store:
12623 ; GFX11-CU: ; %bb.0: ; %entry
12624 ; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
12625 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
12626 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
12627 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
12628 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
12629 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
12630 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
12631 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
12632 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
12633 ; GFX11-CU-NEXT: s_endpgm
12634 ;
12635 ; GFX12-WGP-LABEL: flat_agent_one_as_seq_cst_store:
12636 ; GFX12-WGP: ; %bb.0: ; %entry
12637 ; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
12638 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
12639 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
12640 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
12641 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
12642 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
12643 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
12644 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
12645 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
12646 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
12647 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_DEV
12648 ; GFX12-WGP-NEXT: s_endpgm
12649 ;
12650 ; GFX12-CU-LABEL: flat_agent_one_as_seq_cst_store:
12651 ; GFX12-CU: ; %bb.0: ; %entry
12652 ; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
12653 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
12654 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
12655 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
12656 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
12657 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
12658 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
12659 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
12660 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
12661 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
12662 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_DEV
12663 ; GFX12-CU-NEXT: s_endpgm
12664 i32 %in, ptr %out) {
12665 entry:
12666 store atomic i32 %in, ptr %out syncscope("agent-one-as") seq_cst, align 4
12667 ret void
12668 }
12670 define amdgpu_kernel void @flat_agent_one_as_monotonic_atomicrmw(
12671 ; GFX7-LABEL: flat_agent_one_as_monotonic_atomicrmw:
12672 ; GFX7: ; %bb.0: ; %entry
12673 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
12674 ; GFX7-NEXT: s_load_dword s4, s[8:9], 0x2
12675 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
12676 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
12677 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
12678 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
12679 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
12680 ; GFX7-NEXT: s_endpgm
12681 ;
12682 ; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_atomicrmw:
12683 ; GFX10-WGP: ; %bb.0: ; %entry
12684 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
12685 ; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x8
12686 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
12687 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
12688 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
12689 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
12690 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
12691 ; GFX10-WGP-NEXT: s_endpgm
12692 ;
12693 ; GFX10-CU-LABEL: flat_agent_one_as_monotonic_atomicrmw:
12694 ; GFX10-CU: ; %bb.0: ; %entry
12695 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
12696 ; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x8
12697 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
12698 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
12699 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
12700 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
12701 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
12702 ; GFX10-CU-NEXT: s_endpgm
12703 ;
12704 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_atomicrmw:
12705 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
12706 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
12707 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x2
12708 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
12709 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
12710 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
12711 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
12712 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
12713 ; SKIP-CACHE-INV-NEXT: s_endpgm
12714 ;
12715 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_atomicrmw:
12716 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
12717 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
12718 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
12719 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12720 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
12721 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
12722 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
12723 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
12724 ;
12725 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_atomicrmw:
12726 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
12727 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
12728 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
12729 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12730 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
12731 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
12732 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
12733 ; GFX90A-TGSPLIT-NEXT: s_endpgm
12734 ;
12735 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_atomicrmw:
12736 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
12737 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
12738 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
12739 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12740 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
12741 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
12742 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
12743 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
12744 ;
12745 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_atomicrmw:
12746 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
12747 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
12748 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
12749 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12750 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
12751 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
12752 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
12753 ; GFX940-TGSPLIT-NEXT: s_endpgm
12754 ;
12755 ; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_atomicrmw:
12756 ; GFX11-WGP: ; %bb.0: ; %entry
12757 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
12758 ; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
12759 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
12760 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
12761 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
12762 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
12763 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
12764 ; GFX11-WGP-NEXT: s_endpgm
12765 ;
12766 ; GFX11-CU-LABEL: flat_agent_one_as_monotonic_atomicrmw:
12767 ; GFX11-CU: ; %bb.0: ; %entry
12768 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
12769 ; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
12770 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
12771 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
12772 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
12773 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
12774 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
12775 ; GFX11-CU-NEXT: s_endpgm
12776 ;
12777 ; GFX12-WGP-LABEL: flat_agent_one_as_monotonic_atomicrmw:
12778 ; GFX12-WGP: ; %bb.0: ; %entry
12779 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
12780 ; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
12781 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
12782 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
12783 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
12784 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
12785 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
12786 ; GFX12-WGP-NEXT: s_endpgm
12787 ;
12788 ; GFX12-CU-LABEL: flat_agent_one_as_monotonic_atomicrmw:
12789 ; GFX12-CU: ; %bb.0: ; %entry
12790 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
12791 ; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
12792 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
12793 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
12794 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
12795 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
12796 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
12797 ; GFX12-CU-NEXT: s_endpgm
12798 ptr %out, i32 %in) {
12799 entry:
12800 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") monotonic
12801 ret void
12802 }
12804 define amdgpu_kernel void @flat_agent_one_as_acquire_atomicrmw(
12805 ; GFX7-LABEL: flat_agent_one_as_acquire_atomicrmw:
12806 ; GFX7: ; %bb.0: ; %entry
12807 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
12808 ; GFX7-NEXT: s_load_dword s4, s[8:9], 0x2
12809 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
12810 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
12811 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
12812 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
12813 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
12814 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12815 ; GFX7-NEXT: buffer_wbinvl1_vol
12816 ; GFX7-NEXT: s_endpgm
12817 ;
12818 ; GFX10-WGP-LABEL: flat_agent_one_as_acquire_atomicrmw:
12819 ; GFX10-WGP: ; %bb.0: ; %entry
12820 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
12821 ; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x8
12822 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
12823 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
12824 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
12825 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
12826 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
12827 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12828 ; GFX10-WGP-NEXT: buffer_gl1_inv
12829 ; GFX10-WGP-NEXT: buffer_gl0_inv
12830 ; GFX10-WGP-NEXT: s_endpgm
12831 ;
12832 ; GFX10-CU-LABEL: flat_agent_one_as_acquire_atomicrmw:
12833 ; GFX10-CU: ; %bb.0: ; %entry
12834 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
12835 ; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x8
12836 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
12837 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
12838 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
12839 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
12840 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
12841 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
12842 ; GFX10-CU-NEXT: buffer_gl1_inv
12843 ; GFX10-CU-NEXT: buffer_gl0_inv
12844 ; GFX10-CU-NEXT: s_endpgm
12845 ;
12846 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_atomicrmw:
12847 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
12848 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
12849 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x2
12850 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
12851 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
12852 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
12853 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
12854 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
12855 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
12856 ; SKIP-CACHE-INV-NEXT: s_endpgm
12857 ;
12858 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_atomicrmw:
12859 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
12860 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
12861 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
12862 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12863 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
12864 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
12865 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
12866 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
12867 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
12868 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
12869 ;
12870 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_atomicrmw:
12871 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
12872 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
12873 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
12874 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12875 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
12876 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
12877 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
12878 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12879 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
12880 ; GFX90A-TGSPLIT-NEXT: s_endpgm
12881 ;
12882 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_atomicrmw:
12883 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
12884 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
12885 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
12886 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12887 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
12888 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
12889 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
12890 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
12891 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
12892 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
12893 ;
12894 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_atomicrmw:
12895 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
12896 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
12897 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
12898 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12899 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
12900 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
12901 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
12902 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12903 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
12904 ; GFX940-TGSPLIT-NEXT: s_endpgm
12905 ;
12906 ; GFX11-WGP-LABEL: flat_agent_one_as_acquire_atomicrmw:
12907 ; GFX11-WGP: ; %bb.0: ; %entry
12908 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
12909 ; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
12910 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
12911 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
12912 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
12913 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
12914 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
12915 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12916 ; GFX11-WGP-NEXT: buffer_gl1_inv
12917 ; GFX11-WGP-NEXT: buffer_gl0_inv
12918 ; GFX11-WGP-NEXT: s_endpgm
12919 ;
12920 ; GFX11-CU-LABEL: flat_agent_one_as_acquire_atomicrmw:
12921 ; GFX11-CU: ; %bb.0: ; %entry
12922 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
12923 ; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
12924 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
12925 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
12926 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
12927 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
12928 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
12929 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
12930 ; GFX11-CU-NEXT: buffer_gl1_inv
12931 ; GFX11-CU-NEXT: buffer_gl0_inv
12932 ; GFX11-CU-NEXT: s_endpgm
12933 ;
12934 ; GFX12-WGP-LABEL: flat_agent_one_as_acquire_atomicrmw:
12935 ; GFX12-WGP: ; %bb.0: ; %entry
12936 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
12937 ; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
12938 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
12939 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
12940 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
12941 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
12942 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
12943 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
12944 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
12945 ; GFX12-WGP-NEXT: s_endpgm
12946 ;
12947 ; GFX12-CU-LABEL: flat_agent_one_as_acquire_atomicrmw:
12948 ; GFX12-CU: ; %bb.0: ; %entry
12949 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
12950 ; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
12951 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
12952 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
12953 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
12954 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
12955 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
12956 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
12957 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
12958 ; GFX12-CU-NEXT: s_endpgm
12959 ptr %out, i32 %in) {
12960 entry:
12961 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") acquire
12962 ret void
12963 }
12965 define amdgpu_kernel void @flat_agent_one_as_release_atomicrmw(
12966 ; GFX7-LABEL: flat_agent_one_as_release_atomicrmw:
12967 ; GFX7: ; %bb.0: ; %entry
12968 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
12969 ; GFX7-NEXT: s_load_dword s4, s[8:9], 0x2
12970 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
12971 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
12972 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
12973 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
12974 ; GFX7-NEXT: s_waitcnt vmcnt(0)
12975 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
12976 ; GFX7-NEXT: s_endpgm
12977 ;
12978 ; GFX10-WGP-LABEL: flat_agent_one_as_release_atomicrmw:
12979 ; GFX10-WGP: ; %bb.0: ; %entry
12980 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
12981 ; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x8
12982 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
12983 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
12984 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
12985 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
12986 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
12987 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12988 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
12989 ; GFX10-WGP-NEXT: s_endpgm
12990 ;
12991 ; GFX10-CU-LABEL: flat_agent_one_as_release_atomicrmw:
12992 ; GFX10-CU: ; %bb.0: ; %entry
12993 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
12994 ; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x8
12995 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
12996 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
12997 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
12998 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
12999 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
13000 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
13001 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
13002 ; GFX10-CU-NEXT: s_endpgm
13003 ;
13004 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_atomicrmw:
13005 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
13006 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
13007 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x2
13008 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
13009 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
13010 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
13011 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
13012 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
13013 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
13014 ; SKIP-CACHE-INV-NEXT: s_endpgm
13015 ;
13016 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_atomicrmw:
13017 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
13018 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
13019 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
13020 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13021 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
13022 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
13023 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
13024 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
13025 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
13026 ;
13027 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_atomicrmw:
13028 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
13029 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
13030 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
13031 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13032 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
13033 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
13034 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13035 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
13036 ; GFX90A-TGSPLIT-NEXT: s_endpgm
13037 ;
13038 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_atomicrmw:
13039 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
13040 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
13041 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
13042 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13043 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
13044 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
13045 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
13046 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
13047 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
13048 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
13049 ;
13050 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_atomicrmw:
13051 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
13052 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
13053 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
13054 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13055 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
13056 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
13057 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
13058 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13059 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
13060 ; GFX940-TGSPLIT-NEXT: s_endpgm
13061 ;
13062 ; GFX11-WGP-LABEL: flat_agent_one_as_release_atomicrmw:
13063 ; GFX11-WGP: ; %bb.0: ; %entry
13064 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
13065 ; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
13066 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
13067 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
13068 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
13069 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
13070 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
13071 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13072 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
13073 ; GFX11-WGP-NEXT: s_endpgm
13074 ;
13075 ; GFX11-CU-LABEL: flat_agent_one_as_release_atomicrmw:
13076 ; GFX11-CU: ; %bb.0: ; %entry
13077 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
13078 ; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
13079 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
13080 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
13081 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
13082 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
13083 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
13084 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
13085 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
13086 ; GFX11-CU-NEXT: s_endpgm
13087 ;
13088 ; GFX12-WGP-LABEL: flat_agent_one_as_release_atomicrmw:
13089 ; GFX12-WGP: ; %bb.0: ; %entry
13090 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
13091 ; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
13092 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
13093 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
13094 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
13095 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
13096 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
13097 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
13098 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
13099 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
13100 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
13101 ; GFX12-WGP-NEXT: s_endpgm
13102 ;
13103 ; GFX12-CU-LABEL: flat_agent_one_as_release_atomicrmw:
13104 ; GFX12-CU: ; %bb.0: ; %entry
13105 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
13106 ; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
13107 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
13108 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
13109 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
13110 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
13111 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
13112 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
13113 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
13114 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
13115 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
13116 ; GFX12-CU-NEXT: s_endpgm
13117 ptr %out, i32 %in) {
13118 entry:
13119 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") release
13120 ret void
13121 }
13123 define amdgpu_kernel void @flat_agent_one_as_acq_rel_atomicrmw(
13124 ; GFX7-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
13125 ; GFX7: ; %bb.0: ; %entry
13126 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
13127 ; GFX7-NEXT: s_load_dword s4, s[8:9], 0x2
13128 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
13129 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
13130 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
13131 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
13132 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13133 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
13134 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13135 ; GFX7-NEXT: buffer_wbinvl1_vol
13136 ; GFX7-NEXT: s_endpgm
13137 ;
13138 ; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
13139 ; GFX10-WGP: ; %bb.0: ; %entry
13140 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
13141 ; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x8
13142 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
13143 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
13144 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
13145 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
13146 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
13147 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13148 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
13149 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13150 ; GFX10-WGP-NEXT: buffer_gl1_inv
13151 ; GFX10-WGP-NEXT: buffer_gl0_inv
13152 ; GFX10-WGP-NEXT: s_endpgm
13153 ;
13154 ; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
13155 ; GFX10-CU: ; %bb.0: ; %entry
13156 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
13157 ; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x8
13158 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
13159 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
13160 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
13161 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
13162 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
13163 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
13164 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
13165 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
13166 ; GFX10-CU-NEXT: buffer_gl1_inv
13167 ; GFX10-CU-NEXT: buffer_gl0_inv
13168 ; GFX10-CU-NEXT: s_endpgm
13169 ;
13170 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
13171 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
13172 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
13173 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x2
13174 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
13175 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
13176 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
13177 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
13178 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
13179 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
13180 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
13181 ; SKIP-CACHE-INV-NEXT: s_endpgm
13182 ;
13183 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
13184 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
13185 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
13186 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
13187 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13188 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
13189 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
13190 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
13191 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
13192 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
13193 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
13194 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
13195 ;
13196 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
13197 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
13198 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
13199 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
13200 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13201 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
13202 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
13203 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13204 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
13205 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13206 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
13207 ; GFX90A-TGSPLIT-NEXT: s_endpgm
13208 ;
13209 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
13210 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
13211 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
13212 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
13213 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13214 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
13215 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
13216 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
13217 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
13218 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
13219 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
13220 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
13221 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
13222 ;
13223 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
13224 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
13225 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
13226 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
13227 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13228 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
13229 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
13230 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
13231 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13232 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
13233 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13234 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
13235 ; GFX940-TGSPLIT-NEXT: s_endpgm
13236 ;
13237 ; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
13238 ; GFX11-WGP: ; %bb.0: ; %entry
13239 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
13240 ; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
13241 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
13242 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
13243 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
13244 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
13245 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
13246 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13247 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
13248 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13249 ; GFX11-WGP-NEXT: buffer_gl1_inv
13250 ; GFX11-WGP-NEXT: buffer_gl0_inv
13251 ; GFX11-WGP-NEXT: s_endpgm
13252 ;
13253 ; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
13254 ; GFX11-CU: ; %bb.0: ; %entry
13255 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
13256 ; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
13257 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
13258 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
13259 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
13260 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
13261 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
13262 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
13263 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
13264 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
13265 ; GFX11-CU-NEXT: buffer_gl1_inv
13266 ; GFX11-CU-NEXT: buffer_gl0_inv
13267 ; GFX11-CU-NEXT: s_endpgm
13268 ;
13269 ; GFX12-WGP-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
13270 ; GFX12-WGP: ; %bb.0: ; %entry
13271 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
13272 ; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
13273 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
13274 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
13275 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
13276 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
13277 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
13278 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
13279 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
13280 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
13281 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
13282 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
13283 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
13284 ; GFX12-WGP-NEXT: s_endpgm
13285 ;
13286 ; GFX12-CU-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
13287 ; GFX12-CU: ; %bb.0: ; %entry
13288 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
13289 ; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
13290 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
13291 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
13292 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
13293 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
13294 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
13295 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
13296 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
13297 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
13298 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
13299 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
13300 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
13301 ; GFX12-CU-NEXT: s_endpgm
13302 ptr %out, i32 %in) {
13303 entry:
13304 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") acq_rel
13305 ret void
13306 }
13308 define amdgpu_kernel void @flat_agent_one_as_seq_cst_atomicrmw(
13309 ; GFX7-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
13310 ; GFX7: ; %bb.0: ; %entry
13311 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
13312 ; GFX7-NEXT: s_load_dword s4, s[8:9], 0x2
13313 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
13314 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
13315 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
13316 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
13317 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13318 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
13319 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13320 ; GFX7-NEXT: buffer_wbinvl1_vol
13321 ; GFX7-NEXT: s_endpgm
13322 ;
13323 ; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
13324 ; GFX10-WGP: ; %bb.0: ; %entry
13325 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
13326 ; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x8
13327 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
13328 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
13329 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
13330 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
13331 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
13332 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13333 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
13334 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13335 ; GFX10-WGP-NEXT: buffer_gl1_inv
13336 ; GFX10-WGP-NEXT: buffer_gl0_inv
13337 ; GFX10-WGP-NEXT: s_endpgm
13338 ;
13339 ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
13340 ; GFX10-CU: ; %bb.0: ; %entry
13341 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
13342 ; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x8
13343 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
13344 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
13345 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
13346 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
13347 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
13348 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
13349 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
13350 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
13351 ; GFX10-CU-NEXT: buffer_gl1_inv
13352 ; GFX10-CU-NEXT: buffer_gl0_inv
13353 ; GFX10-CU-NEXT: s_endpgm
13354 ;
13355 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
13356 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
13357 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
13358 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x2
13359 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
13360 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
13361 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
13362 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
13363 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
13364 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
13365 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
13366 ; SKIP-CACHE-INV-NEXT: s_endpgm
13367 ;
13368 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
13369 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
13370 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
13371 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
13372 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13373 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
13374 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
13375 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
13376 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
13377 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
13378 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
13379 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
13380 ;
13381 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
13382 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
13383 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
13384 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
13385 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13386 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
13387 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
13388 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13389 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
13390 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13391 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
13392 ; GFX90A-TGSPLIT-NEXT: s_endpgm
13393 ;
13394 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
13395 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
13396 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
13397 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
13398 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13399 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
13400 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
13401 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
13402 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
13403 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
13404 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
13405 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
13406 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
13407 ;
13408 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
13409 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
13410 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
13411 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
13412 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13413 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
13414 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
13415 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
13416 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13417 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
13418 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13419 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
13420 ; GFX940-TGSPLIT-NEXT: s_endpgm
13421 ;
13422 ; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
13423 ; GFX11-WGP: ; %bb.0: ; %entry
13424 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
13425 ; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
13426 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
13427 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
13428 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
13429 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
13430 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
13431 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13432 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
13433 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13434 ; GFX11-WGP-NEXT: buffer_gl1_inv
13435 ; GFX11-WGP-NEXT: buffer_gl0_inv
13436 ; GFX11-WGP-NEXT: s_endpgm
13437 ;
13438 ; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
13439 ; GFX11-CU: ; %bb.0: ; %entry
13440 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
13441 ; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
13442 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
13443 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
13444 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
13445 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
13446 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
13447 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
13448 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
13449 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
13450 ; GFX11-CU-NEXT: buffer_gl1_inv
13451 ; GFX11-CU-NEXT: buffer_gl0_inv
13452 ; GFX11-CU-NEXT: s_endpgm
13453 ;
13454 ; GFX12-WGP-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
13455 ; GFX12-WGP: ; %bb.0: ; %entry
13456 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
13457 ; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
13458 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
13459 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
13460 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
13461 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
13462 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
13463 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
13464 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
13465 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
13466 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
13467 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
13468 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
13469 ; GFX12-WGP-NEXT: s_endpgm
13470 ;
13471 ; GFX12-CU-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
13472 ; GFX12-CU: ; %bb.0: ; %entry
13473 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
13474 ; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
13475 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
13476 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
13477 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
13478 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
13479 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
13480 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
13481 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
13482 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
13483 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
13484 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
13485 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
13486 ; GFX12-CU-NEXT: s_endpgm
13487 ptr %out, i32 %in) {
13488 entry:
13489 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") seq_cst
13490 ret void
13491 }
13493 define amdgpu_kernel void @flat_agent_one_as_acquire_ret_atomicrmw(
13494 ; GFX7-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
13495 ; GFX7: ; %bb.0: ; %entry
13496 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13497 ; GFX7-NEXT: s_load_dword s6, s[8:9], 0x2
13498 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
13499 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
13500 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
13501 ; GFX7-NEXT: v_mov_b32_e32 v2, s6
13502 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
13503 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13504 ; GFX7-NEXT: buffer_wbinvl1_vol
13505 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
13506 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
13507 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
13508 ; GFX7-NEXT: flat_store_dword v[0:1], v2
13509 ; GFX7-NEXT: s_endpgm
13510 ;
13511 ; GFX10-WGP-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
13512 ; GFX10-WGP: ; %bb.0: ; %entry
13513 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13514 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x8
13515 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
13516 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
13517 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
13518 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s6
13519 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
13520 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
13521 ; GFX10-WGP-NEXT: buffer_gl1_inv
13522 ; GFX10-WGP-NEXT: buffer_gl0_inv
13523 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
13524 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
13525 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
13526 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
13527 ; GFX10-WGP-NEXT: s_endpgm
13528 ;
13529 ; GFX10-CU-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
13530 ; GFX10-CU: ; %bb.0: ; %entry
13531 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13532 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x8
13533 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
13534 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
13535 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
13536 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s6
13537 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
13538 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
13539 ; GFX10-CU-NEXT: buffer_gl1_inv
13540 ; GFX10-CU-NEXT: buffer_gl0_inv
13541 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
13542 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
13543 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
13544 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
13545 ; GFX10-CU-NEXT: s_endpgm
13546 ;
13547 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
13548 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
13549 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
13550 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x2
13551 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
13552 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
13553 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
13554 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
13555 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
13556 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
13557 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
13558 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
13559 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
13560 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
13561 ; SKIP-CACHE-INV-NEXT: s_endpgm
13562 ;
13563 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
13564 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
13565 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13566 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x8
13567 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13568 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
13569 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s6
13570 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
13571 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
13572 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
13573 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
13574 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13575 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
13576 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
13577 ;
13578 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
13579 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
13580 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13581 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x8
13582 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13583 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
13584 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s6
13585 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
13586 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13587 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
13588 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
13589 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
13590 ; GFX90A-TGSPLIT-NEXT: s_endpgm
13591 ;
13592 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
13593 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
13594 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
13595 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
13596 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13597 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
13598 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
13599 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
13600 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
13601 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
13602 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
13603 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13604 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
13605 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
13606 ;
13607 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
13608 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
13609 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
13610 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
13611 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13612 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
13613 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
13614 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
13615 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13616 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
13617 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
13618 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
13619 ; GFX940-TGSPLIT-NEXT: s_endpgm
13620 ;
13621 ; GFX11-WGP-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
13622 ; GFX11-WGP: ; %bb.0: ; %entry
13623 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13624 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x8
13625 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
13626 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
13627 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
13628 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s2
13629 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
13630 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
13631 ; GFX11-WGP-NEXT: buffer_gl1_inv
13632 ; GFX11-WGP-NEXT: buffer_gl0_inv
13633 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
13634 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
13635 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
13636 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
13637 ; GFX11-WGP-NEXT: s_endpgm
13638 ;
13639 ; GFX11-CU-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
13640 ; GFX11-CU: ; %bb.0: ; %entry
13641 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13642 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x8
13643 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
13644 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
13645 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
13646 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
13647 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
13648 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
13649 ; GFX11-CU-NEXT: buffer_gl1_inv
13650 ; GFX11-CU-NEXT: buffer_gl0_inv
13651 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
13652 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
13653 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
13654 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
13655 ; GFX11-CU-NEXT: s_endpgm
13656 ;
13657 ; GFX12-WGP-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
13658 ; GFX12-WGP: ; %bb.0: ; %entry
13659 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13660 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x8
13661 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
13662 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
13663 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
13664 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
13665 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
13666 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
13667 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
13668 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
13669 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
13670 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
13671 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
13672 ; GFX12-WGP-NEXT: s_endpgm
13673 ;
13674 ; GFX12-CU-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
13675 ; GFX12-CU: ; %bb.0: ; %entry
13676 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13677 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x8
13678 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
13679 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
13680 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
13681 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
13682 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
13683 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
13684 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
13685 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
13686 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
13687 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
13688 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
13689 ; GFX12-CU-NEXT: s_endpgm
13690 ptr %out, i32 %in) {
13691 entry:
13692 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") acquire
13693 store i32 %val, ptr %out, align 4
13694 ret void
13695 }
13697 define amdgpu_kernel void @flat_agent_one_as_acq_rel_ret_atomicrmw(
13698 ; GFX7-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
13699 ; GFX7: ; %bb.0: ; %entry
13700 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13701 ; GFX7-NEXT: s_load_dword s6, s[8:9], 0x2
13702 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
13703 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
13704 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
13705 ; GFX7-NEXT: v_mov_b32_e32 v2, s6
13706 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13707 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
13708 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13709 ; GFX7-NEXT: buffer_wbinvl1_vol
13710 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
13711 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
13712 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
13713 ; GFX7-NEXT: flat_store_dword v[0:1], v2
13714 ; GFX7-NEXT: s_endpgm
13715 ;
13716 ; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
13717 ; GFX10-WGP: ; %bb.0: ; %entry
13718 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13719 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x8
13720 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
13721 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
13722 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
13723 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s6
13724 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
13725 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13726 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
13727 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
13728 ; GFX10-WGP-NEXT: buffer_gl1_inv
13729 ; GFX10-WGP-NEXT: buffer_gl0_inv
13730 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
13731 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
13732 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
13733 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
13734 ; GFX10-WGP-NEXT: s_endpgm
13735 ;
13736 ; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
13737 ; GFX10-CU: ; %bb.0: ; %entry
13738 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13739 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x8
13740 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
13741 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
13742 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
13743 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s6
13744 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
13745 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
13746 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
13747 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
13748 ; GFX10-CU-NEXT: buffer_gl1_inv
13749 ; GFX10-CU-NEXT: buffer_gl0_inv
13750 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
13751 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
13752 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
13753 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
13754 ; GFX10-CU-NEXT: s_endpgm
13755 ;
13756 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
13757 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
13758 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
13759 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x2
13760 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
13761 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
13762 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
13763 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
13764 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
13765 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
13766 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
13767 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
13768 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
13769 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
13770 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
13771 ; SKIP-CACHE-INV-NEXT: s_endpgm
13772 ;
13773 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
13774 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
13775 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13776 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x8
13777 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13778 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
13779 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s6
13780 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
13781 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
13782 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
13783 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
13784 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
13785 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13786 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
13787 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
13788 ;
13789 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
13790 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
13791 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13792 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x8
13793 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13794 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
13795 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s6
13796 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13797 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
13798 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13799 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
13800 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
13801 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
13802 ; GFX90A-TGSPLIT-NEXT: s_endpgm
13803 ;
13804 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
13805 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
13806 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
13807 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
13808 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13809 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
13810 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
13811 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
13812 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
13813 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
13814 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
13815 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
13816 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
13817 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13818 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
13819 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
13820 ;
13821 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
13822 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
13823 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
13824 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
13825 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13826 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
13827 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
13828 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
13829 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13830 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
13831 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13832 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
13833 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
13834 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
13835 ; GFX940-TGSPLIT-NEXT: s_endpgm
13836 ;
13837 ; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
13838 ; GFX11-WGP: ; %bb.0: ; %entry
13839 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13840 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x8
13841 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
13842 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
13843 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
13844 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s2
13845 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
13846 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13847 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
13848 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
13849 ; GFX11-WGP-NEXT: buffer_gl1_inv
13850 ; GFX11-WGP-NEXT: buffer_gl0_inv
13851 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
13852 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
13853 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
13854 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
13855 ; GFX11-WGP-NEXT: s_endpgm
13856 ;
13857 ; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
13858 ; GFX11-CU: ; %bb.0: ; %entry
13859 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13860 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x8
13861 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
13862 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
13863 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
13864 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
13865 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
13866 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
13867 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
13868 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
13869 ; GFX11-CU-NEXT: buffer_gl1_inv
13870 ; GFX11-CU-NEXT: buffer_gl0_inv
13871 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
13872 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
13873 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
13874 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
13875 ; GFX11-CU-NEXT: s_endpgm
13876 ;
13877 ; GFX12-WGP-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
13878 ; GFX12-WGP: ; %bb.0: ; %entry
13879 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13880 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x8
13881 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
13882 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
13883 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
13884 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
13885 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
13886 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
13887 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
13888 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
13889 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
13890 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
13891 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
13892 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
13893 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
13894 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
13895 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
13896 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
13897 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
13898 ; GFX12-WGP-NEXT: s_endpgm
13899 ;
13900 ; GFX12-CU-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
13901 ; GFX12-CU: ; %bb.0: ; %entry
13902 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13903 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x8
13904 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
13905 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
13906 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
13907 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
13908 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
13909 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
13910 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
13911 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
13912 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
13913 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
13914 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
13915 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
13916 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
13917 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
13918 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
13919 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
13920 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
13921 ; GFX12-CU-NEXT: s_endpgm
13922 ptr %out, i32 %in) {
13923 entry:
13924 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") acq_rel
13925 store i32 %val, ptr %out, align 4
13926 ret void
13927 }
13929 define amdgpu_kernel void @flat_agent_one_as_seq_cst_ret_atomicrmw(
13930 ; GFX7-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
13931 ; GFX7: ; %bb.0: ; %entry
13932 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13933 ; GFX7-NEXT: s_load_dword s6, s[8:9], 0x2
13934 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
13935 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
13936 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
13937 ; GFX7-NEXT: v_mov_b32_e32 v2, s6
13938 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13939 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
13940 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13941 ; GFX7-NEXT: buffer_wbinvl1_vol
13942 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
13943 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
13944 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
13945 ; GFX7-NEXT: flat_store_dword v[0:1], v2
13946 ; GFX7-NEXT: s_endpgm
13947 ;
13948 ; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
13949 ; GFX10-WGP: ; %bb.0: ; %entry
13950 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13951 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x8
13952 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
13953 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
13954 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
13955 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s6
13956 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
13957 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13958 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
13959 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
13960 ; GFX10-WGP-NEXT: buffer_gl1_inv
13961 ; GFX10-WGP-NEXT: buffer_gl0_inv
13962 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
13963 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
13964 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
13965 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
13966 ; GFX10-WGP-NEXT: s_endpgm
13967 ;
13968 ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
13969 ; GFX10-CU: ; %bb.0: ; %entry
13970 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13971 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x8
13972 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
13973 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
13974 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
13975 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s6
13976 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
13977 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
13978 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
13979 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
13980 ; GFX10-CU-NEXT: buffer_gl1_inv
13981 ; GFX10-CU-NEXT: buffer_gl0_inv
13982 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
13983 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
13984 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
13985 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
13986 ; GFX10-CU-NEXT: s_endpgm
13987 ;
13988 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
13989 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
13990 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
13991 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x2
13992 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
13993 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
13994 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
13995 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
13996 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
13997 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
13998 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
13999 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
14000 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
14001 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
14002 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
14003 ; SKIP-CACHE-INV-NEXT: s_endpgm
14004 ;
14005 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
14006 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
14007 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14008 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x8
14009 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14010 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
14011 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s6
14012 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
14013 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
14014 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
14015 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
14016 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
14017 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14018 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
14019 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
14020 ;
14021 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
14022 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
14023 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14024 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x8
14025 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14026 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
14027 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s6
14028 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14029 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
14030 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14031 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
14032 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
14033 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
14034 ; GFX90A-TGSPLIT-NEXT: s_endpgm
14035 ;
14036 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
14037 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
14038 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14039 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
14040 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14041 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
14042 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
14043 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
14044 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
14045 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
14046 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
14047 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
14048 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
14049 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14050 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
14051 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
14052 ;
14053 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
14054 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
14055 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14056 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
14057 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14058 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
14059 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
14060 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
14061 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14062 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
14063 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14064 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
14065 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
14066 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
14067 ; GFX940-TGSPLIT-NEXT: s_endpgm
14068 ;
14069 ; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
14070 ; GFX11-WGP: ; %bb.0: ; %entry
14071 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14072 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x8
14073 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
14074 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
14075 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
14076 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s2
14077 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
14078 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14079 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
14080 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
14081 ; GFX11-WGP-NEXT: buffer_gl1_inv
14082 ; GFX11-WGP-NEXT: buffer_gl0_inv
14083 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
14084 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
14085 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
14086 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
14087 ; GFX11-WGP-NEXT: s_endpgm
14088 ;
14089 ; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
14090 ; GFX11-CU: ; %bb.0: ; %entry
14091 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14092 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x8
14093 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
14094 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
14095 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
14096 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
14097 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
14098 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
14099 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
14100 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
14101 ; GFX11-CU-NEXT: buffer_gl1_inv
14102 ; GFX11-CU-NEXT: buffer_gl0_inv
14103 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
14104 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
14105 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
14106 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
14107 ; GFX11-CU-NEXT: s_endpgm
14108 ;
14109 ; GFX12-WGP-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
14110 ; GFX12-WGP: ; %bb.0: ; %entry
14111 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14112 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x8
14113 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
14114 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
14115 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
14116 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
14117 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
14118 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
14119 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
14120 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
14121 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
14122 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
14123 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
14124 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
14125 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
14126 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
14127 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
14128 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
14129 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
14130 ; GFX12-WGP-NEXT: s_endpgm
14131 ;
14132 ; GFX12-CU-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
14133 ; GFX12-CU: ; %bb.0: ; %entry
14134 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14135 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x8
14136 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
14137 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
14138 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
14139 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
14140 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
14141 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
14142 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
14143 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
14144 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
14145 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
14146 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
14147 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
14148 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
14149 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
14150 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
14151 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
14152 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
14153 ; GFX12-CU-NEXT: s_endpgm
14154 ptr %out, i32 %in) {
14155 entry:
14156 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") seq_cst
14157 store i32 %val, ptr %out, align 4
14158 ret void
14159 }
14161 define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_cmpxchg(
14162 ; GFX7-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
14163 ; GFX7: ; %bb.0: ; %entry
14164 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
14165 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14166 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
14167 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
14168 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
14169 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
14170 ; GFX7-NEXT: s_mov_b32 s4, s8
14171 ; GFX7-NEXT: s_mov_b32 s5, s9
14172 ; GFX7-NEXT: s_mov_b32 s9, s10
14173 ; GFX7-NEXT: s_mov_b32 s8, s11
14174 ; GFX7-NEXT: s_add_u32 s4, s4, s9
14175 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
14176 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14177 ; GFX7-NEXT: s_mov_b32 s5, s8
14178 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
14179 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
14180 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14181 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
14182 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
14183 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
14184 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14185 ; GFX7-NEXT: s_endpgm
14186 ;
14187 ; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
14188 ; GFX10-WGP: ; %bb.0: ; %entry
14189 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
14190 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14191 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
14192 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
14193 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
14194 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
14195 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
14196 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
14197 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
14198 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
14199 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
14200 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
14201 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14202 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
14203 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
14204 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
14205 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14206 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
14207 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
14208 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
14209 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14210 ; GFX10-WGP-NEXT: s_endpgm
14211 ;
14212 ; GFX10-CU-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
14213 ; GFX10-CU: ; %bb.0: ; %entry
14214 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
14215 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14216 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
14217 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
14218 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
14219 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
14220 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
14221 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
14222 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
14223 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
14224 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
14225 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
14226 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14227 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
14228 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
14229 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
14230 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14231 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
14232 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
14233 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
14234 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14235 ; GFX10-CU-NEXT: s_endpgm
14236 ;
14237 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
14238 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
14239 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
14240 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
14241 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
14242 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
14243 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
14244 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
14245 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
14246 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
14247 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
14248 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
14249 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
14250 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
14251 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
14252 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
14253 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
14254 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
14255 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14256 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
14257 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
14258 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
14259 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14260 ; SKIP-CACHE-INV-NEXT: s_endpgm
14261 ;
14262 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
14263 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
14264 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14265 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
14266 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
14267 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14268 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
14269 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
14270 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14271 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14272 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
14273 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14274 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
14275 ;
14276 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
14277 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
14278 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14279 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
14280 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
14281 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14282 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
14283 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
14284 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14285 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14286 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
14287 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14288 ; GFX90A-TGSPLIT-NEXT: s_endpgm
14289 ;
14290 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
14291 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
14292 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14293 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
14294 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
14295 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14296 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
14297 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
14298 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14299 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14300 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
14301 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14302 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
14303 ;
14304 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
14305 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
14306 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14307 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
14308 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
14309 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14310 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
14311 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
14312 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14313 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14314 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
14315 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14316 ; GFX940-TGSPLIT-NEXT: s_endpgm
14317 ;
14318 ; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
14319 ; GFX11-WGP: ; %bb.0: ; %entry
14320 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14321 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
14322 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
14323 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
14324 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
14325 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
14326 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14327 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
14328 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
14329 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
14330 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
14331 ; GFX11-WGP-NEXT: s_endpgm
14332 ;
14333 ; GFX11-CU-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
14334 ; GFX11-CU: ; %bb.0: ; %entry
14335 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14336 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
14337 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
14338 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
14339 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
14340 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
14341 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14342 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
14343 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
14344 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
14345 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
14346 ; GFX11-CU-NEXT: s_endpgm
14347 ;
14348 ; GFX12-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
14349 ; GFX12-WGP: ; %bb.0: ; %entry
14350 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14351 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
14352 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
14353 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
14354 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
14355 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
14356 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14357 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
14358 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
14359 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
14360 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
14361 ; GFX12-WGP-NEXT: s_endpgm
14362 ;
14363 ; GFX12-CU-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
14364 ; GFX12-CU: ; %bb.0: ; %entry
14365 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14366 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
14367 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
14368 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
14369 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
14370 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
14371 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14372 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
14373 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
14374 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
14375 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
14376 ; GFX12-CU-NEXT: s_endpgm
14377 ptr %out, i32 %in, i32 %old) {
14378 entry:
14379 %gep = getelementptr i32, ptr %out, i32 4
14380 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic monotonic
14381 ret void
14382 }
14384 define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_cmpxchg(
14385 ; GFX7-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
14386 ; GFX7: ; %bb.0: ; %entry
14387 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
14388 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14389 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
14390 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
14391 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
14392 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
14393 ; GFX7-NEXT: s_mov_b32 s4, s8
14394 ; GFX7-NEXT: s_mov_b32 s5, s9
14395 ; GFX7-NEXT: s_mov_b32 s9, s10
14396 ; GFX7-NEXT: s_mov_b32 s8, s11
14397 ; GFX7-NEXT: s_add_u32 s4, s4, s9
14398 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
14399 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14400 ; GFX7-NEXT: s_mov_b32 s5, s8
14401 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
14402 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
14403 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14404 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
14405 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
14406 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
14407 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14408 ; GFX7-NEXT: s_waitcnt vmcnt(0)
14409 ; GFX7-NEXT: buffer_wbinvl1_vol
14410 ; GFX7-NEXT: s_endpgm
14411 ;
14412 ; GFX10-WGP-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
14413 ; GFX10-WGP: ; %bb.0: ; %entry
14414 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
14415 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14416 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
14417 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
14418 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
14419 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
14420 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
14421 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
14422 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
14423 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
14424 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
14425 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
14426 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14427 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
14428 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
14429 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
14430 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14431 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
14432 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
14433 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
14434 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14435 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14436 ; GFX10-WGP-NEXT: buffer_gl1_inv
14437 ; GFX10-WGP-NEXT: buffer_gl0_inv
14438 ; GFX10-WGP-NEXT: s_endpgm
14439 ;
14440 ; GFX10-CU-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
14441 ; GFX10-CU: ; %bb.0: ; %entry
14442 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
14443 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14444 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
14445 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
14446 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
14447 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
14448 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
14449 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
14450 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
14451 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
14452 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
14453 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
14454 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14455 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
14456 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
14457 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
14458 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14459 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
14460 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
14461 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
14462 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14463 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
14464 ; GFX10-CU-NEXT: buffer_gl1_inv
14465 ; GFX10-CU-NEXT: buffer_gl0_inv
14466 ; GFX10-CU-NEXT: s_endpgm
14467 ;
14468 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
14469 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
14470 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
14471 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
14472 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
14473 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
14474 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
14475 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
14476 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
14477 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
14478 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
14479 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
14480 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
14481 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
14482 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
14483 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
14484 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
14485 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
14486 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14487 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
14488 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
14489 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
14490 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14491 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
14492 ; SKIP-CACHE-INV-NEXT: s_endpgm
14493 ;
14494 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
14495 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
14496 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14497 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
14498 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
14499 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14500 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
14501 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
14502 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14503 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14504 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
14505 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14506 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
14507 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
14508 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
14509 ;
14510 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
14511 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
14512 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14513 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
14514 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
14515 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14516 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
14517 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
14518 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14519 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14520 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
14521 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14522 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14523 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
14524 ; GFX90A-TGSPLIT-NEXT: s_endpgm
14525 ;
14526 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
14527 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
14528 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14529 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
14530 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
14531 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14532 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
14533 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
14534 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14535 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14536 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
14537 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14538 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
14539 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
14540 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
14541 ;
14542 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
14543 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
14544 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14545 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
14546 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
14547 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14548 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
14549 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
14550 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14551 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14552 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
14553 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14554 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14555 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
14556 ; GFX940-TGSPLIT-NEXT: s_endpgm
14557 ;
14558 ; GFX11-WGP-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
14559 ; GFX11-WGP: ; %bb.0: ; %entry
14560 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14561 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
14562 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
14563 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
14564 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
14565 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
14566 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14567 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
14568 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
14569 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
14570 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
14571 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14572 ; GFX11-WGP-NEXT: buffer_gl1_inv
14573 ; GFX11-WGP-NEXT: buffer_gl0_inv
14574 ; GFX11-WGP-NEXT: s_endpgm
14575 ;
14576 ; GFX11-CU-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
14577 ; GFX11-CU: ; %bb.0: ; %entry
14578 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14579 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
14580 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
14581 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
14582 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
14583 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
14584 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14585 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
14586 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
14587 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
14588 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
14589 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
14590 ; GFX11-CU-NEXT: buffer_gl1_inv
14591 ; GFX11-CU-NEXT: buffer_gl0_inv
14592 ; GFX11-CU-NEXT: s_endpgm
14593 ;
14594 ; GFX12-WGP-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
14595 ; GFX12-WGP: ; %bb.0: ; %entry
14596 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14597 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
14598 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
14599 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
14600 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
14601 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
14602 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14603 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
14604 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
14605 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
14606 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
14607 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
14608 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
14609 ; GFX12-WGP-NEXT: s_endpgm
14610 ;
14611 ; GFX12-CU-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
14612 ; GFX12-CU: ; %bb.0: ; %entry
14613 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14614 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
14615 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
14616 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
14617 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
14618 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
14619 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14620 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
14621 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
14622 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
14623 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
14624 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
14625 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
14626 ; GFX12-CU-NEXT: s_endpgm
14627 ptr %out, i32 %in, i32 %old) {
14628 entry:
14629 %gep = getelementptr i32, ptr %out, i32 4
14630 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire monotonic
14631 ret void
14632 }
14634 define amdgpu_kernel void @flat_agent_one_as_release_monotonic_cmpxchg(
14635 ; GFX7-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
14636 ; GFX7: ; %bb.0: ; %entry
14637 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
14638 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14639 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
14640 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
14641 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
14642 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
14643 ; GFX7-NEXT: s_mov_b32 s4, s8
14644 ; GFX7-NEXT: s_mov_b32 s5, s9
14645 ; GFX7-NEXT: s_mov_b32 s9, s10
14646 ; GFX7-NEXT: s_mov_b32 s8, s11
14647 ; GFX7-NEXT: s_add_u32 s4, s4, s9
14648 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
14649 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14650 ; GFX7-NEXT: s_mov_b32 s5, s8
14651 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
14652 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
14653 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14654 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
14655 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
14656 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
14657 ; GFX7-NEXT: s_waitcnt vmcnt(0)
14658 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14659 ; GFX7-NEXT: s_endpgm
14660 ;
14661 ; GFX10-WGP-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
14662 ; GFX10-WGP: ; %bb.0: ; %entry
14663 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
14664 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14665 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
14666 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
14667 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
14668 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
14669 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
14670 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
14671 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
14672 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
14673 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
14674 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
14675 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14676 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
14677 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
14678 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
14679 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14680 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
14681 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
14682 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
14683 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
14684 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14685 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14686 ; GFX10-WGP-NEXT: s_endpgm
14687 ;
14688 ; GFX10-CU-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
14689 ; GFX10-CU: ; %bb.0: ; %entry
14690 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
14691 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14692 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
14693 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
14694 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
14695 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
14696 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
14697 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
14698 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
14699 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
14700 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
14701 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
14702 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14703 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
14704 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
14705 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
14706 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14707 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
14708 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
14709 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
14710 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
14711 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
14712 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14713 ; GFX10-CU-NEXT: s_endpgm
14714 ;
14715 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
14716 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
14717 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
14718 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
14719 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
14720 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
14721 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
14722 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
14723 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
14724 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
14725 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
14726 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
14727 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
14728 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
14729 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
14730 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
14731 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
14732 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
14733 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14734 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
14735 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
14736 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
14737 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
14738 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14739 ; SKIP-CACHE-INV-NEXT: s_endpgm
14740 ;
14741 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
14742 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
14743 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14744 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
14745 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
14746 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14747 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
14748 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
14749 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14750 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14751 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
14752 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
14753 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14754 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
14755 ;
14756 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
14757 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
14758 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14759 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
14760 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
14761 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14762 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
14763 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
14764 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14765 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14766 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
14767 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14768 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14769 ; GFX90A-TGSPLIT-NEXT: s_endpgm
14770 ;
14771 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
14772 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
14773 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14774 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
14775 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
14776 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14777 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
14778 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
14779 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14780 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14781 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
14782 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
14783 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
14784 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14785 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
14786 ;
14787 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
14788 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
14789 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14790 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
14791 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
14792 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14793 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
14794 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
14795 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14796 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14797 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
14798 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
14799 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14800 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14801 ; GFX940-TGSPLIT-NEXT: s_endpgm
14802 ;
14803 ; GFX11-WGP-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
14804 ; GFX11-WGP: ; %bb.0: ; %entry
14805 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14806 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
14807 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
14808 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
14809 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
14810 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
14811 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14812 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
14813 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
14814 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
14815 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
14816 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14817 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
14818 ; GFX11-WGP-NEXT: s_endpgm
14819 ;
14820 ; GFX11-CU-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
14821 ; GFX11-CU: ; %bb.0: ; %entry
14822 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14823 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
14824 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
14825 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
14826 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
14827 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
14828 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14829 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
14830 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
14831 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
14832 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
14833 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
14834 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
14835 ; GFX11-CU-NEXT: s_endpgm
14836 ;
14837 ; GFX12-WGP-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
14838 ; GFX12-WGP: ; %bb.0: ; %entry
14839 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14840 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
14841 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
14842 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
14843 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
14844 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
14845 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14846 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
14847 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
14848 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
14849 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
14850 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
14851 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
14852 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
14853 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
14854 ; GFX12-WGP-NEXT: s_endpgm
14855 ;
14856 ; GFX12-CU-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
14857 ; GFX12-CU: ; %bb.0: ; %entry
14858 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14859 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
14860 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
14861 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
14862 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
14863 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
14864 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14865 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
14866 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
14867 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
14868 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
14869 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
14870 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
14871 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
14872 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
14873 ; GFX12-CU-NEXT: s_endpgm
14874 ptr %out, i32 %in, i32 %old) {
14875 entry:
14876 %gep = getelementptr i32, ptr %out, i32 4
14877 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") release monotonic
14878 ret void
14879 }
14881 define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_cmpxchg(
14882 ; GFX7-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
14883 ; GFX7: ; %bb.0: ; %entry
14884 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
14885 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14886 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
14887 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
14888 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
14889 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
14890 ; GFX7-NEXT: s_mov_b32 s4, s8
14891 ; GFX7-NEXT: s_mov_b32 s5, s9
14892 ; GFX7-NEXT: s_mov_b32 s9, s10
14893 ; GFX7-NEXT: s_mov_b32 s8, s11
14894 ; GFX7-NEXT: s_add_u32 s4, s4, s9
14895 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
14896 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14897 ; GFX7-NEXT: s_mov_b32 s5, s8
14898 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
14899 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
14900 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14901 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
14902 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
14903 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
14904 ; GFX7-NEXT: s_waitcnt vmcnt(0)
14905 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14906 ; GFX7-NEXT: s_waitcnt vmcnt(0)
14907 ; GFX7-NEXT: buffer_wbinvl1_vol
14908 ; GFX7-NEXT: s_endpgm
14909 ;
14910 ; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
14911 ; GFX10-WGP: ; %bb.0: ; %entry
14912 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
14913 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14914 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
14915 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
14916 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
14917 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
14918 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
14919 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
14920 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
14921 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
14922 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
14923 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
14924 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14925 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
14926 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
14927 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
14928 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14929 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
14930 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
14931 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
14932 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
14933 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14934 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14935 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14936 ; GFX10-WGP-NEXT: buffer_gl1_inv
14937 ; GFX10-WGP-NEXT: buffer_gl0_inv
14938 ; GFX10-WGP-NEXT: s_endpgm
14939 ;
14940 ; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
14941 ; GFX10-CU: ; %bb.0: ; %entry
14942 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
14943 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14944 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
14945 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
14946 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
14947 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
14948 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
14949 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
14950 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
14951 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
14952 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
14953 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
14954 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14955 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
14956 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
14957 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
14958 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14959 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
14960 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
14961 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
14962 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
14963 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
14964 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14965 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
14966 ; GFX10-CU-NEXT: buffer_gl1_inv
14967 ; GFX10-CU-NEXT: buffer_gl0_inv
14968 ; GFX10-CU-NEXT: s_endpgm
14969 ;
14970 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
14971 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
14972 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
14973 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
14974 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
14975 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
14976 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
14977 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
14978 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
14979 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
14980 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
14981 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
14982 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
14983 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
14984 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
14985 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
14986 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
14987 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
14988 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14989 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
14990 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
14991 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
14992 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
14993 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14994 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
14995 ; SKIP-CACHE-INV-NEXT: s_endpgm
14996 ;
14997 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
14998 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
14999 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15000 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15001 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15002 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15003 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15004 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
15005 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15006 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15007 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
15008 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
15009 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15010 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
15011 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
15012 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
15013 ;
15014 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
15015 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
15016 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15017 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15018 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15019 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15020 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15021 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
15022 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15023 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15024 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
15025 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15026 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15027 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15028 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
15029 ; GFX90A-TGSPLIT-NEXT: s_endpgm
15030 ;
15031 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
15032 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
15033 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15034 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15035 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15036 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15037 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15038 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
15039 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15040 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15041 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
15042 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
15043 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
15044 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15045 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
15046 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
15047 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
15048 ;
15049 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
15050 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
15051 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15052 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15053 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15054 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15055 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15056 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
15057 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15058 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15059 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
15060 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
15061 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15062 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15063 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15064 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
15065 ; GFX940-TGSPLIT-NEXT: s_endpgm
15066 ;
15067 ; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
15068 ; GFX11-WGP: ; %bb.0: ; %entry
15069 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15070 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15071 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15072 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
15073 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
15074 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
15075 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15076 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
15077 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
15078 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
15079 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
15080 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15081 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
15082 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15083 ; GFX11-WGP-NEXT: buffer_gl1_inv
15084 ; GFX11-WGP-NEXT: buffer_gl0_inv
15085 ; GFX11-WGP-NEXT: s_endpgm
15086 ;
15087 ; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
15088 ; GFX11-CU: ; %bb.0: ; %entry
15089 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15090 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15091 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15092 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
15093 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
15094 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
15095 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15096 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
15097 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
15098 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
15099 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
15100 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
15101 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
15102 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
15103 ; GFX11-CU-NEXT: buffer_gl1_inv
15104 ; GFX11-CU-NEXT: buffer_gl0_inv
15105 ; GFX11-CU-NEXT: s_endpgm
15106 ;
15107 ; GFX12-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
15108 ; GFX12-WGP: ; %bb.0: ; %entry
15109 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15110 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15111 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15112 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
15113 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
15114 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
15115 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15116 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
15117 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
15118 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
15119 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
15120 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
15121 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
15122 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
15123 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
15124 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
15125 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
15126 ; GFX12-WGP-NEXT: s_endpgm
15127 ;
15128 ; GFX12-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
15129 ; GFX12-CU: ; %bb.0: ; %entry
15130 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15131 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15132 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15133 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
15134 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
15135 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
15136 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15137 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
15138 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
15139 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
15140 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
15141 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
15142 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
15143 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
15144 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
15145 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
15146 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
15147 ; GFX12-CU-NEXT: s_endpgm
15148 ptr %out, i32 %in, i32 %old) {
15149 entry:
15150 %gep = getelementptr i32, ptr %out, i32 4
15151 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel monotonic
15152 ret void
15153 }
15155 define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_cmpxchg(
15156 ; GFX7-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
15157 ; GFX7: ; %bb.0: ; %entry
15158 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
15159 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15160 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
15161 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
15162 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
15163 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
15164 ; GFX7-NEXT: s_mov_b32 s4, s8
15165 ; GFX7-NEXT: s_mov_b32 s5, s9
15166 ; GFX7-NEXT: s_mov_b32 s9, s10
15167 ; GFX7-NEXT: s_mov_b32 s8, s11
15168 ; GFX7-NEXT: s_add_u32 s4, s4, s9
15169 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
15170 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15171 ; GFX7-NEXT: s_mov_b32 s5, s8
15172 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
15173 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
15174 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15175 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
15176 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
15177 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
15178 ; GFX7-NEXT: s_waitcnt vmcnt(0)
15179 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15180 ; GFX7-NEXT: s_waitcnt vmcnt(0)
15181 ; GFX7-NEXT: buffer_wbinvl1_vol
15182 ; GFX7-NEXT: s_endpgm
15183 ;
15184 ; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
15185 ; GFX10-WGP: ; %bb.0: ; %entry
15186 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
15187 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15188 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
15189 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
15190 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
15191 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
15192 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
15193 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
15194 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
15195 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
15196 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
15197 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
15198 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15199 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
15200 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
15201 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
15202 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15203 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
15204 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
15205 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
15206 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
15207 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15208 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15209 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15210 ; GFX10-WGP-NEXT: buffer_gl1_inv
15211 ; GFX10-WGP-NEXT: buffer_gl0_inv
15212 ; GFX10-WGP-NEXT: s_endpgm
15213 ;
15214 ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
15215 ; GFX10-CU: ; %bb.0: ; %entry
15216 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
15217 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15218 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
15219 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
15220 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
15221 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
15222 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
15223 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
15224 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
15225 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
15226 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
15227 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
15228 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15229 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
15230 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
15231 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
15232 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15233 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
15234 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
15235 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
15236 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
15237 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
15238 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15239 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
15240 ; GFX10-CU-NEXT: buffer_gl1_inv
15241 ; GFX10-CU-NEXT: buffer_gl0_inv
15242 ; GFX10-CU-NEXT: s_endpgm
15243 ;
15244 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
15245 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
15246 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
15247 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
15248 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
15249 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
15250 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
15251 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
15252 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
15253 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
15254 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
15255 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
15256 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
15257 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
15258 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
15259 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
15260 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
15261 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
15262 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15263 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
15264 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
15265 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
15266 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
15267 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15268 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
15269 ; SKIP-CACHE-INV-NEXT: s_endpgm
15270 ;
15271 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
15272 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
15273 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15274 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15275 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15276 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15277 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15278 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
15279 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15280 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15281 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
15282 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
15283 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15284 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
15285 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
15286 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
15287 ;
15288 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
15289 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
15290 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15291 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15292 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15293 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15294 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15295 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
15296 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15297 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15298 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
15299 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15300 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15301 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15302 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
15303 ; GFX90A-TGSPLIT-NEXT: s_endpgm
15304 ;
15305 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
15306 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
15307 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15308 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15309 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15310 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15311 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15312 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
15313 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15314 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15315 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
15316 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
15317 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
15318 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15319 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
15320 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
15321 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
15322 ;
15323 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
15324 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
15325 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15326 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15327 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15328 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15329 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15330 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
15331 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15332 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15333 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
15334 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
15335 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15336 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15337 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15338 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
15339 ; GFX940-TGSPLIT-NEXT: s_endpgm
15340 ;
15341 ; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
15342 ; GFX11-WGP: ; %bb.0: ; %entry
15343 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15344 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15345 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15346 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
15347 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
15348 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
15349 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15350 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
15351 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
15352 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
15353 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
15354 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15355 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
15356 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15357 ; GFX11-WGP-NEXT: buffer_gl1_inv
15358 ; GFX11-WGP-NEXT: buffer_gl0_inv
15359 ; GFX11-WGP-NEXT: s_endpgm
15360 ;
15361 ; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
15362 ; GFX11-CU: ; %bb.0: ; %entry
15363 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15364 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15365 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15366 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
15367 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
15368 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
15369 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15370 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
15371 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
15372 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
15373 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
15374 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
15375 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
15376 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
15377 ; GFX11-CU-NEXT: buffer_gl1_inv
15378 ; GFX11-CU-NEXT: buffer_gl0_inv
15379 ; GFX11-CU-NEXT: s_endpgm
15380 ;
15381 ; GFX12-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
15382 ; GFX12-WGP: ; %bb.0: ; %entry
15383 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15384 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15385 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15386 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
15387 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
15388 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
15389 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15390 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
15391 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
15392 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
15393 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
15394 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
15395 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
15396 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
15397 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
15398 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
15399 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
15400 ; GFX12-WGP-NEXT: s_endpgm
15401 ;
15402 ; GFX12-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
15403 ; GFX12-CU: ; %bb.0: ; %entry
15404 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15405 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15406 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15407 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
15408 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
15409 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
15410 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15411 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
15412 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
15413 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
15414 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
15415 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
15416 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
15417 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
15418 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
15419 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
15420 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
15421 ; GFX12-CU-NEXT: s_endpgm
15422 ptr %out, i32 %in, i32 %old) {
15423 entry:
15424 %gep = getelementptr i32, ptr %out, i32 4
15425 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst monotonic
15426 ret void
15427 }
15429 define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_cmpxchg(
15430 ; GFX7-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
15431 ; GFX7: ; %bb.0: ; %entry
15432 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
15433 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15434 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
15435 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
15436 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
15437 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
15438 ; GFX7-NEXT: s_mov_b32 s4, s8
15439 ; GFX7-NEXT: s_mov_b32 s5, s9
15440 ; GFX7-NEXT: s_mov_b32 s9, s10
15441 ; GFX7-NEXT: s_mov_b32 s8, s11
15442 ; GFX7-NEXT: s_add_u32 s4, s4, s9
15443 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
15444 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15445 ; GFX7-NEXT: s_mov_b32 s5, s8
15446 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
15447 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
15448 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15449 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
15450 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
15451 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
15452 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15453 ; GFX7-NEXT: s_waitcnt vmcnt(0)
15454 ; GFX7-NEXT: buffer_wbinvl1_vol
15455 ; GFX7-NEXT: s_endpgm
15456 ;
15457 ; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
15458 ; GFX10-WGP: ; %bb.0: ; %entry
15459 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
15460 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15461 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
15462 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
15463 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
15464 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
15465 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
15466 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
15467 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
15468 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
15469 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
15470 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
15471 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15472 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
15473 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
15474 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
15475 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15476 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
15477 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
15478 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
15479 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15480 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15481 ; GFX10-WGP-NEXT: buffer_gl1_inv
15482 ; GFX10-WGP-NEXT: buffer_gl0_inv
15483 ; GFX10-WGP-NEXT: s_endpgm
15484 ;
15485 ; GFX10-CU-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
15486 ; GFX10-CU: ; %bb.0: ; %entry
15487 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
15488 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15489 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
15490 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
15491 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
15492 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
15493 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
15494 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
15495 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
15496 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
15497 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
15498 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
15499 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15500 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
15501 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
15502 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
15503 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15504 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
15505 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
15506 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
15507 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15508 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
15509 ; GFX10-CU-NEXT: buffer_gl1_inv
15510 ; GFX10-CU-NEXT: buffer_gl0_inv
15511 ; GFX10-CU-NEXT: s_endpgm
15512 ;
15513 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
15514 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
15515 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
15516 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
15517 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
15518 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
15519 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
15520 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
15521 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
15522 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
15523 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
15524 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
15525 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
15526 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
15527 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
15528 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
15529 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
15530 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
15531 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15532 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
15533 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
15534 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
15535 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15536 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
15537 ; SKIP-CACHE-INV-NEXT: s_endpgm
15538 ;
15539 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
15540 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
15541 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15542 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15543 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15544 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15545 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15546 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
15547 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15548 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15549 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
15550 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15551 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
15552 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
15553 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
15554 ;
15555 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
15556 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
15557 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15558 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15559 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15560 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15561 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15562 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
15563 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15564 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15565 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
15566 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15567 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15568 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
15569 ; GFX90A-TGSPLIT-NEXT: s_endpgm
15570 ;
15571 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
15572 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
15573 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15574 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15575 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15576 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15577 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15578 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
15579 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15580 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15581 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
15582 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15583 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
15584 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
15585 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
15586 ;
15587 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
15588 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
15589 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15590 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15591 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15592 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15593 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15594 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
15595 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15596 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15597 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
15598 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15599 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15600 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
15601 ; GFX940-TGSPLIT-NEXT: s_endpgm
15602 ;
15603 ; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
15604 ; GFX11-WGP: ; %bb.0: ; %entry
15605 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15606 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15607 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15608 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
15609 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
15610 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
15611 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15612 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
15613 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
15614 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
15615 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
15616 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15617 ; GFX11-WGP-NEXT: buffer_gl1_inv
15618 ; GFX11-WGP-NEXT: buffer_gl0_inv
15619 ; GFX11-WGP-NEXT: s_endpgm
15620 ;
15621 ; GFX11-CU-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
15622 ; GFX11-CU: ; %bb.0: ; %entry
15623 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15624 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15625 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15626 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
15627 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
15628 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
15629 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15630 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
15631 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
15632 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
15633 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
15634 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
15635 ; GFX11-CU-NEXT: buffer_gl1_inv
15636 ; GFX11-CU-NEXT: buffer_gl0_inv
15637 ; GFX11-CU-NEXT: s_endpgm
15638 ;
15639 ; GFX12-WGP-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
15640 ; GFX12-WGP: ; %bb.0: ; %entry
15641 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15642 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15643 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15644 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
15645 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
15646 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
15647 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15648 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
15649 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
15650 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
15651 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
15652 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
15653 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
15654 ; GFX12-WGP-NEXT: s_endpgm
15655 ;
15656 ; GFX12-CU-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
15657 ; GFX12-CU: ; %bb.0: ; %entry
15658 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15659 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15660 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15661 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
15662 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
15663 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
15664 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15665 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
15666 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
15667 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
15668 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
15669 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
15670 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
15671 ; GFX12-CU-NEXT: s_endpgm
15672 ptr %out, i32 %in, i32 %old) {
15673 entry:
15674 %gep = getelementptr i32, ptr %out, i32 4
15675 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic acquire
15676 ret void
15677 }
15679 define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_cmpxchg(
15680 ; GFX7-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
15681 ; GFX7: ; %bb.0: ; %entry
15682 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
15683 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15684 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
15685 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
15686 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
15687 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
15688 ; GFX7-NEXT: s_mov_b32 s4, s8
15689 ; GFX7-NEXT: s_mov_b32 s5, s9
15690 ; GFX7-NEXT: s_mov_b32 s9, s10
15691 ; GFX7-NEXT: s_mov_b32 s8, s11
15692 ; GFX7-NEXT: s_add_u32 s4, s4, s9
15693 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
15694 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15695 ; GFX7-NEXT: s_mov_b32 s5, s8
15696 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
15697 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
15698 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15699 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
15700 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
15701 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
15702 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15703 ; GFX7-NEXT: s_waitcnt vmcnt(0)
15704 ; GFX7-NEXT: buffer_wbinvl1_vol
15705 ; GFX7-NEXT: s_endpgm
15706 ;
15707 ; GFX10-WGP-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
15708 ; GFX10-WGP: ; %bb.0: ; %entry
15709 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
15710 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15711 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
15712 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
15713 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
15714 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
15715 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
15716 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
15717 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
15718 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
15719 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
15720 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
15721 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15722 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
15723 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
15724 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
15725 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15726 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
15727 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
15728 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
15729 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15730 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15731 ; GFX10-WGP-NEXT: buffer_gl1_inv
15732 ; GFX10-WGP-NEXT: buffer_gl0_inv
15733 ; GFX10-WGP-NEXT: s_endpgm
15734 ;
15735 ; GFX10-CU-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
15736 ; GFX10-CU: ; %bb.0: ; %entry
15737 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
15738 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15739 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
15740 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
15741 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
15742 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
15743 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
15744 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
15745 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
15746 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
15747 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
15748 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
15749 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15750 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
15751 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
15752 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
15753 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15754 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
15755 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
15756 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
15757 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15758 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
15759 ; GFX10-CU-NEXT: buffer_gl1_inv
15760 ; GFX10-CU-NEXT: buffer_gl0_inv
15761 ; GFX10-CU-NEXT: s_endpgm
15762 ;
15763 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
15764 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
15765 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
15766 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
15767 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
15768 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
15769 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
15770 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
15771 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
15772 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
15773 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
15774 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
15775 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
15776 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
15777 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
15778 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
15779 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
15780 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
15781 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15782 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
15783 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
15784 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
15785 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15786 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
15787 ; SKIP-CACHE-INV-NEXT: s_endpgm
15788 ;
15789 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
15790 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
15791 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15792 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15793 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15794 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15795 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15796 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
15797 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15798 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15799 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
15800 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15801 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
15802 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
15803 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
15804 ;
15805 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
15806 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
15807 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15808 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15809 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15810 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15811 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15812 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
15813 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15814 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15815 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
15816 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15817 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15818 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
15819 ; GFX90A-TGSPLIT-NEXT: s_endpgm
15820 ;
15821 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
15822 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
15823 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15824 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15825 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15826 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15827 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15828 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
15829 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15830 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15831 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
15832 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15833 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
15834 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
15835 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
15836 ;
15837 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
15838 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
15839 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15840 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15841 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15842 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15843 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15844 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
15845 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15846 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15847 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
15848 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15849 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15850 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
15851 ; GFX940-TGSPLIT-NEXT: s_endpgm
15852 ;
15853 ; GFX11-WGP-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
15854 ; GFX11-WGP: ; %bb.0: ; %entry
15855 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15856 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15857 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15858 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
15859 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
15860 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
15861 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15862 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
15863 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
15864 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
15865 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
15866 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15867 ; GFX11-WGP-NEXT: buffer_gl1_inv
15868 ; GFX11-WGP-NEXT: buffer_gl0_inv
15869 ; GFX11-WGP-NEXT: s_endpgm
15870 ;
15871 ; GFX11-CU-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
15872 ; GFX11-CU: ; %bb.0: ; %entry
15873 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15874 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15875 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15876 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
15877 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
15878 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
15879 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15880 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
15881 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
15882 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
15883 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
15884 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
15885 ; GFX11-CU-NEXT: buffer_gl1_inv
15886 ; GFX11-CU-NEXT: buffer_gl0_inv
15887 ; GFX11-CU-NEXT: s_endpgm
15888 ;
15889 ; GFX12-WGP-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
15890 ; GFX12-WGP: ; %bb.0: ; %entry
15891 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15892 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15893 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15894 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
15895 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
15896 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
15897 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15898 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
15899 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
15900 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
15901 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
15902 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
15903 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
15904 ; GFX12-WGP-NEXT: s_endpgm
15905 ;
15906 ; GFX12-CU-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
15907 ; GFX12-CU: ; %bb.0: ; %entry
15908 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15909 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15910 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15911 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
15912 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
15913 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
15914 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15915 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
15916 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
15917 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
15918 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
15919 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
15920 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
15921 ; GFX12-CU-NEXT: s_endpgm
15922 ptr %out, i32 %in, i32 %old) {
15923 entry:
15924 %gep = getelementptr i32, ptr %out, i32 4
15925 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire acquire
15926 ret void
15927 }
15929 define amdgpu_kernel void @flat_agent_one_as_release_acquire_cmpxchg(
15930 ; GFX7-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
15931 ; GFX7: ; %bb.0: ; %entry
15932 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
15933 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15934 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
15935 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
15936 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
15937 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
15938 ; GFX7-NEXT: s_mov_b32 s4, s8
15939 ; GFX7-NEXT: s_mov_b32 s5, s9
15940 ; GFX7-NEXT: s_mov_b32 s9, s10
15941 ; GFX7-NEXT: s_mov_b32 s8, s11
15942 ; GFX7-NEXT: s_add_u32 s4, s4, s9
15943 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
15944 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15945 ; GFX7-NEXT: s_mov_b32 s5, s8
15946 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
15947 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
15948 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15949 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
15950 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
15951 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
15952 ; GFX7-NEXT: s_waitcnt vmcnt(0)
15953 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15954 ; GFX7-NEXT: s_waitcnt vmcnt(0)
15955 ; GFX7-NEXT: buffer_wbinvl1_vol
15956 ; GFX7-NEXT: s_endpgm
15957 ;
15958 ; GFX10-WGP-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
15959 ; GFX10-WGP: ; %bb.0: ; %entry
15960 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
15961 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15962 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
15963 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
15964 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
15965 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
15966 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
15967 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
15968 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
15969 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
15970 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
15971 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
15972 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15973 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
15974 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
15975 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
15976 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15977 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
15978 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
15979 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
15980 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
15981 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15982 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15983 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15984 ; GFX10-WGP-NEXT: buffer_gl1_inv
15985 ; GFX10-WGP-NEXT: buffer_gl0_inv
15986 ; GFX10-WGP-NEXT: s_endpgm
15987 ;
15988 ; GFX10-CU-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
15989 ; GFX10-CU: ; %bb.0: ; %entry
15990 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
15991 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15992 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
15993 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
15994 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
15995 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
15996 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
15997 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
15998 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
15999 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
16000 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
16001 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
16002 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
16003 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
16004 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
16005 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
16006 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16007 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
16008 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
16009 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
16010 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
16011 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
16012 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16013 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
16014 ; GFX10-CU-NEXT: buffer_gl1_inv
16015 ; GFX10-CU-NEXT: buffer_gl0_inv
16016 ; GFX10-CU-NEXT: s_endpgm
16017 ;
16018 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
16019 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
16020 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
16021 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
16022 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
16023 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
16024 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
16025 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
16026 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
16027 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
16028 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
16029 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
16030 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
16031 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
16032 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
16033 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
16034 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
16035 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
16036 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16037 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
16038 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
16039 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
16040 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
16041 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16042 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
16043 ; SKIP-CACHE-INV-NEXT: s_endpgm
16044 ;
16045 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
16046 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
16047 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16048 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16049 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16050 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16051 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16052 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
16053 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16054 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16055 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
16056 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
16057 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16058 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
16059 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
16060 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
16061 ;
16062 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
16063 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
16064 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16065 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16066 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16067 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16068 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16069 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
16070 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16071 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16072 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
16073 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16074 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16075 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16076 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
16077 ; GFX90A-TGSPLIT-NEXT: s_endpgm
16078 ;
16079 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
16080 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
16081 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16082 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16083 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16084 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16085 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16086 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
16087 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16088 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16089 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
16090 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
16091 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
16092 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16093 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
16094 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
16095 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
16096 ;
16097 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
16098 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
16099 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16100 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16101 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16102 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16103 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16104 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
16105 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16106 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16107 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
16108 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
16109 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16110 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16111 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16112 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
16113 ; GFX940-TGSPLIT-NEXT: s_endpgm
16114 ;
16115 ; GFX11-WGP-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
16116 ; GFX11-WGP: ; %bb.0: ; %entry
16117 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16118 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16119 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16120 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
16121 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
16122 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
16123 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16124 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
16125 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
16126 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
16127 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
16128 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16129 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
16130 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16131 ; GFX11-WGP-NEXT: buffer_gl1_inv
16132 ; GFX11-WGP-NEXT: buffer_gl0_inv
16133 ; GFX11-WGP-NEXT: s_endpgm
16134 ;
16135 ; GFX11-CU-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
16136 ; GFX11-CU: ; %bb.0: ; %entry
16137 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16138 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
16139 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
16140 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
16141 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
16142 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
16143 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16144 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
16145 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
16146 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
16147 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
16148 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
16149 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
16150 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
16151 ; GFX11-CU-NEXT: buffer_gl1_inv
16152 ; GFX11-CU-NEXT: buffer_gl0_inv
16153 ; GFX11-CU-NEXT: s_endpgm
16154 ;
16155 ; GFX12-WGP-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
16156 ; GFX12-WGP: ; %bb.0: ; %entry
16157 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16158 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16159 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16160 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
16161 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
16162 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
16163 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16164 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
16165 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
16166 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
16167 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
16168 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
16169 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
16170 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
16171 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
16172 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
16173 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
16174 ; GFX12-WGP-NEXT: s_endpgm
16175 ;
16176 ; GFX12-CU-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
16177 ; GFX12-CU: ; %bb.0: ; %entry
16178 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16179 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
16180 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
16181 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
16182 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
16183 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
16184 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16185 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
16186 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
16187 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
16188 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
16189 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
16190 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
16191 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
16192 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
16193 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
16194 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
16195 ; GFX12-CU-NEXT: s_endpgm
16196 ptr %out, i32 %in, i32 %old) {
16197 entry:
16198 %gep = getelementptr i32, ptr %out, i32 4
16199 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") release acquire
16200 ret void
16201 }
16203 define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_cmpxchg(
16204 ; GFX7-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
16205 ; GFX7: ; %bb.0: ; %entry
16206 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
16207 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
16208 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
16209 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
16210 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
16211 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
16212 ; GFX7-NEXT: s_mov_b32 s4, s8
16213 ; GFX7-NEXT: s_mov_b32 s5, s9
16214 ; GFX7-NEXT: s_mov_b32 s9, s10
16215 ; GFX7-NEXT: s_mov_b32 s8, s11
16216 ; GFX7-NEXT: s_add_u32 s4, s4, s9
16217 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
16218 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
16219 ; GFX7-NEXT: s_mov_b32 s5, s8
16220 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
16221 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
16222 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16223 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
16224 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
16225 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
16226 ; GFX7-NEXT: s_waitcnt vmcnt(0)
16227 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16228 ; GFX7-NEXT: s_waitcnt vmcnt(0)
16229 ; GFX7-NEXT: buffer_wbinvl1_vol
16230 ; GFX7-NEXT: s_endpgm
16231 ;
16232 ; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
16233 ; GFX10-WGP: ; %bb.0: ; %entry
16234 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
16235 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
16236 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
16237 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
16238 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
16239 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
16240 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
16241 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
16242 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
16243 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
16244 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
16245 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
16246 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
16247 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
16248 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
16249 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
16250 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16251 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
16252 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
16253 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
16254 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
16255 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16256 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16257 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16258 ; GFX10-WGP-NEXT: buffer_gl1_inv
16259 ; GFX10-WGP-NEXT: buffer_gl0_inv
16260 ; GFX10-WGP-NEXT: s_endpgm
16261 ;
16262 ; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
16263 ; GFX10-CU: ; %bb.0: ; %entry
16264 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
16265 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
16266 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
16267 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
16268 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
16269 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
16270 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
16271 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
16272 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
16273 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
16274 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
16275 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
16276 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
16277 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
16278 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
16279 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
16280 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16281 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
16282 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
16283 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
16284 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
16285 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
16286 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16287 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
16288 ; GFX10-CU-NEXT: buffer_gl1_inv
16289 ; GFX10-CU-NEXT: buffer_gl0_inv
16290 ; GFX10-CU-NEXT: s_endpgm
16291 ;
16292 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
16293 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
16294 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
16295 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
16296 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
16297 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
16298 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
16299 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
16300 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
16301 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
16302 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
16303 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
16304 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
16305 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
16306 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
16307 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
16308 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
16309 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
16310 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16311 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
16312 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
16313 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
16314 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
16315 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16316 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
16317 ; SKIP-CACHE-INV-NEXT: s_endpgm
16318 ;
16319 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
16320 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
16321 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16322 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16323 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16324 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16325 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16326 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
16327 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16328 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16329 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
16330 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
16331 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16332 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
16333 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
16334 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
16335 ;
16336 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
16337 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
16338 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16339 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16340 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16341 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16342 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16343 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
16344 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16345 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16346 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
16347 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16348 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16349 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16350 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
16351 ; GFX90A-TGSPLIT-NEXT: s_endpgm
16352 ;
16353 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
16354 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
16355 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16356 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16357 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16358 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16359 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16360 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
16361 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16362 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16363 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
16364 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
16365 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
16366 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16367 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
16368 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
16369 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
16370 ;
16371 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
16372 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
16373 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16374 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16375 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16376 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16377 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16378 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
16379 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16380 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16381 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
16382 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
16383 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16384 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16385 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16386 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
16387 ; GFX940-TGSPLIT-NEXT: s_endpgm
16388 ;
16389 ; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
16390 ; GFX11-WGP: ; %bb.0: ; %entry
16391 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16392 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16393 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16394 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
16395 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
16396 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
16397 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16398 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
16399 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
16400 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
16401 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
16402 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16403 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
16404 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16405 ; GFX11-WGP-NEXT: buffer_gl1_inv
16406 ; GFX11-WGP-NEXT: buffer_gl0_inv
16407 ; GFX11-WGP-NEXT: s_endpgm
16408 ;
16409 ; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
16410 ; GFX11-CU: ; %bb.0: ; %entry
16411 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16412 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
16413 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
16414 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
16415 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
16416 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
16417 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16418 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
16419 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
16420 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
16421 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
16422 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
16423 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
16424 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
16425 ; GFX11-CU-NEXT: buffer_gl1_inv
16426 ; GFX11-CU-NEXT: buffer_gl0_inv
16427 ; GFX11-CU-NEXT: s_endpgm
16428 ;
16429 ; GFX12-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
16430 ; GFX12-WGP: ; %bb.0: ; %entry
16431 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16432 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16433 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16434 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
16435 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
16436 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
16437 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16438 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
16439 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
16440 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
16441 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
16442 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
16443 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
16444 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
16445 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
16446 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
16447 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
16448 ; GFX12-WGP-NEXT: s_endpgm
16449 ;
16450 ; GFX12-CU-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
16451 ; GFX12-CU: ; %bb.0: ; %entry
16452 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16453 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
16454 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
16455 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
16456 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
16457 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
16458 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16459 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
16460 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
16461 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
16462 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
16463 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
16464 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
16465 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
16466 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
16467 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
16468 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
16469 ; GFX12-CU-NEXT: s_endpgm
16470 ptr %out, i32 %in, i32 %old) {
16471 entry:
16472 %gep = getelementptr i32, ptr %out, i32 4
16473 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel acquire
16474 ret void
16475 }
16477 define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_cmpxchg(
16478 ; GFX7-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
16479 ; GFX7: ; %bb.0: ; %entry
16480 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
16481 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
16482 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
16483 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
16484 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
16485 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
16486 ; GFX7-NEXT: s_mov_b32 s4, s8
16487 ; GFX7-NEXT: s_mov_b32 s5, s9
16488 ; GFX7-NEXT: s_mov_b32 s9, s10
16489 ; GFX7-NEXT: s_mov_b32 s8, s11
16490 ; GFX7-NEXT: s_add_u32 s4, s4, s9
16491 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
16492 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
16493 ; GFX7-NEXT: s_mov_b32 s5, s8
16494 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
16495 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
16496 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16497 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
16498 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
16499 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
16500 ; GFX7-NEXT: s_waitcnt vmcnt(0)
16501 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16502 ; GFX7-NEXT: s_waitcnt vmcnt(0)
16503 ; GFX7-NEXT: buffer_wbinvl1_vol
16504 ; GFX7-NEXT: s_endpgm
16505 ;
16506 ; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
16507 ; GFX10-WGP: ; %bb.0: ; %entry
16508 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
16509 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
16510 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
16511 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
16512 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
16513 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
16514 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
16515 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
16516 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
16517 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
16518 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
16519 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
16520 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
16521 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
16522 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
16523 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
16524 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16525 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
16526 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
16527 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
16528 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
16529 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16530 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16531 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16532 ; GFX10-WGP-NEXT: buffer_gl1_inv
16533 ; GFX10-WGP-NEXT: buffer_gl0_inv
16534 ; GFX10-WGP-NEXT: s_endpgm
16535 ;
16536 ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
16537 ; GFX10-CU: ; %bb.0: ; %entry
16538 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
16539 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
16540 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
16541 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
16542 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
16543 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
16544 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
16545 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
16546 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
16547 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
16548 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
16549 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
16550 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
16551 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
16552 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
16553 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
16554 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16555 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
16556 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
16557 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
16558 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
16559 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
16560 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16561 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
16562 ; GFX10-CU-NEXT: buffer_gl1_inv
16563 ; GFX10-CU-NEXT: buffer_gl0_inv
16564 ; GFX10-CU-NEXT: s_endpgm
16565 ;
16566 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
16567 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
16568 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
16569 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
16570 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
16571 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
16572 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
16573 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
16574 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
16575 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
16576 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
16577 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
16578 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
16579 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
16580 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
16581 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
16582 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
16583 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
16584 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16585 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
16586 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
16587 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
16588 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
16589 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16590 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
16591 ; SKIP-CACHE-INV-NEXT: s_endpgm
16592 ;
16593 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
16594 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
16595 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16596 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16597 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16598 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16599 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16600 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
16601 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16602 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16603 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
16604 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
16605 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16606 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
16607 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
16608 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
16609 ;
16610 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
16611 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
16612 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16613 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16614 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16615 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16616 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16617 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
16618 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16619 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16620 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
16621 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16622 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16623 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16624 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
16625 ; GFX90A-TGSPLIT-NEXT: s_endpgm
16626 ;
16627 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
16628 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
16629 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16630 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16631 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16632 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16633 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16634 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
16635 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16636 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16637 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
16638 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
16639 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
16640 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16641 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
16642 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
16643 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
16644 ;
16645 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
16646 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
16647 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16648 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16649 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16650 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16651 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16652 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
16653 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16654 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16655 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
16656 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
16657 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16658 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16659 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16660 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
16661 ; GFX940-TGSPLIT-NEXT: s_endpgm
16662 ;
16663 ; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
16664 ; GFX11-WGP: ; %bb.0: ; %entry
16665 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16666 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16667 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16668 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
16669 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
16670 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
16671 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16672 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
16673 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
16674 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
16675 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
16676 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16677 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
16678 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16679 ; GFX11-WGP-NEXT: buffer_gl1_inv
16680 ; GFX11-WGP-NEXT: buffer_gl0_inv
16681 ; GFX11-WGP-NEXT: s_endpgm
16682 ;
16683 ; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
16684 ; GFX11-CU: ; %bb.0: ; %entry
16685 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16686 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
16687 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
16688 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
16689 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
16690 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
16691 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16692 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
16693 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
16694 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
16695 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
16696 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
16697 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
16698 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
16699 ; GFX11-CU-NEXT: buffer_gl1_inv
16700 ; GFX11-CU-NEXT: buffer_gl0_inv
16701 ; GFX11-CU-NEXT: s_endpgm
16702 ;
16703 ; GFX12-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
16704 ; GFX12-WGP: ; %bb.0: ; %entry
16705 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16706 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16707 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16708 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
16709 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
16710 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
16711 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16712 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
16713 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
16714 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
16715 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
16716 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
16717 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
16718 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
16719 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
16720 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
16721 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
16722 ; GFX12-WGP-NEXT: s_endpgm
16723 ;
16724 ; GFX12-CU-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
16725 ; GFX12-CU: ; %bb.0: ; %entry
16726 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16727 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
16728 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
16729 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
16730 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
16731 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
16732 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16733 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
16734 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
16735 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
16736 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
16737 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
16738 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
16739 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
16740 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
16741 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
16742 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
16743 ; GFX12-CU-NEXT: s_endpgm
16744 ptr %out, i32 %in, i32 %old) {
16745 entry:
16746 %gep = getelementptr i32, ptr %out, i32 4
16747 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst acquire
16748 ret void
16749 }
16751 define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_cmpxchg(
16752 ; GFX7-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
16753 ; GFX7: ; %bb.0: ; %entry
16754 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
16755 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
16756 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
16757 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
16758 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
16759 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
16760 ; GFX7-NEXT: s_mov_b32 s4, s8
16761 ; GFX7-NEXT: s_mov_b32 s5, s9
16762 ; GFX7-NEXT: s_mov_b32 s9, s10
16763 ; GFX7-NEXT: s_mov_b32 s8, s11
16764 ; GFX7-NEXT: s_add_u32 s4, s4, s9
16765 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
16766 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
16767 ; GFX7-NEXT: s_mov_b32 s5, s8
16768 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
16769 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
16770 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16771 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
16772 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
16773 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
16774 ; GFX7-NEXT: s_waitcnt vmcnt(0)
16775 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16776 ; GFX7-NEXT: s_waitcnt vmcnt(0)
16777 ; GFX7-NEXT: buffer_wbinvl1_vol
16778 ; GFX7-NEXT: s_endpgm
16779 ;
16780 ; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
16781 ; GFX10-WGP: ; %bb.0: ; %entry
16782 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
16783 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
16784 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
16785 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
16786 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
16787 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
16788 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
16789 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
16790 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
16791 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
16792 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
16793 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
16794 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
16795 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
16796 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
16797 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
16798 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16799 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
16800 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
16801 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
16802 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
16803 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16804 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16805 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16806 ; GFX10-WGP-NEXT: buffer_gl1_inv
16807 ; GFX10-WGP-NEXT: buffer_gl0_inv
16808 ; GFX10-WGP-NEXT: s_endpgm
16809 ;
16810 ; GFX10-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
16811 ; GFX10-CU: ; %bb.0: ; %entry
16812 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
16813 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
16814 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
16815 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
16816 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
16817 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
16818 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
16819 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
16820 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
16821 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
16822 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
16823 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
16824 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
16825 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
16826 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
16827 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
16828 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16829 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
16830 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
16831 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
16832 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
16833 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
16834 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16835 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
16836 ; GFX10-CU-NEXT: buffer_gl1_inv
16837 ; GFX10-CU-NEXT: buffer_gl0_inv
16838 ; GFX10-CU-NEXT: s_endpgm
16839 ;
16840 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
16841 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
16842 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
16843 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
16844 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
16845 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
16846 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
16847 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
16848 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
16849 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
16850 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
16851 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
16852 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
16853 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
16854 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
16855 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
16856 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
16857 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
16858 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16859 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
16860 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
16861 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
16862 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
16863 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16864 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
16865 ; SKIP-CACHE-INV-NEXT: s_endpgm
16866 ;
16867 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
16868 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
16869 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16870 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16871 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16872 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16873 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16874 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
16875 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16876 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16877 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
16878 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
16879 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16880 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
16881 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
16882 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
16883 ;
16884 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
16885 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
16886 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16887 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16888 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16889 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16890 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16891 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
16892 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16893 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16894 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
16895 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16896 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16897 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16898 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
16899 ; GFX90A-TGSPLIT-NEXT: s_endpgm
16900 ;
16901 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
16902 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
16903 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16904 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16905 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16906 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16907 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16908 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
16909 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16910 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16911 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
16912 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
16913 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
16914 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16915 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
16916 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
16917 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
16918 ;
16919 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
16920 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
16921 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16922 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16923 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16924 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16925 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16926 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
16927 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16928 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16929 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
16930 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
16931 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16932 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16933 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16934 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
16935 ; GFX940-TGSPLIT-NEXT: s_endpgm
16936 ;
16937 ; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
16938 ; GFX11-WGP: ; %bb.0: ; %entry
16939 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16940 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16941 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16942 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
16943 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
16944 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
16945 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16946 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
16947 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
16948 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
16949 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
16950 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16951 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
16952 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16953 ; GFX11-WGP-NEXT: buffer_gl1_inv
16954 ; GFX11-WGP-NEXT: buffer_gl0_inv
16955 ; GFX11-WGP-NEXT: s_endpgm
16956 ;
16957 ; GFX11-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
16958 ; GFX11-CU: ; %bb.0: ; %entry
16959 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16960 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
16961 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
16962 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
16963 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
16964 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
16965 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16966 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
16967 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
16968 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
16969 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
16970 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
16971 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
16972 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
16973 ; GFX11-CU-NEXT: buffer_gl1_inv
16974 ; GFX11-CU-NEXT: buffer_gl0_inv
16975 ; GFX11-CU-NEXT: s_endpgm
16976 ;
16977 ; GFX12-WGP-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
16978 ; GFX12-WGP: ; %bb.0: ; %entry
16979 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16980 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16981 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16982 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
16983 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
16984 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
16985 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16986 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
16987 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
16988 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
16989 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
16990 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
16991 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
16992 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
16993 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
16994 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
16995 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
16996 ; GFX12-WGP-NEXT: s_endpgm
16997 ;
16998 ; GFX12-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
16999 ; GFX12-CU: ; %bb.0: ; %entry
17000 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17001 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
17002 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
17003 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
17004 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
17005 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
17006 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17007 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
17008 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
17009 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
17010 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
17011 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
17012 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
17013 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
17014 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
17015 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
17016 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
17017 ; GFX12-CU-NEXT: s_endpgm
17018 ptr %out, i32 %in, i32 %old) {
17019 entry:
17020 %gep = getelementptr i32, ptr %out, i32 4
17021 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic seq_cst
17022 ret void
17023 }
17025 define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_cmpxchg(
17026 ; GFX7-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
17027 ; GFX7: ; %bb.0: ; %entry
17028 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
17029 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
17030 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
17031 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
17032 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
17033 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
17034 ; GFX7-NEXT: s_mov_b32 s4, s8
17035 ; GFX7-NEXT: s_mov_b32 s5, s9
17036 ; GFX7-NEXT: s_mov_b32 s9, s10
17037 ; GFX7-NEXT: s_mov_b32 s8, s11
17038 ; GFX7-NEXT: s_add_u32 s4, s4, s9
17039 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
17040 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
17041 ; GFX7-NEXT: s_mov_b32 s5, s8
17042 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
17043 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
17044 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17045 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
17046 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
17047 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
17048 ; GFX7-NEXT: s_waitcnt vmcnt(0)
17049 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
17050 ; GFX7-NEXT: s_waitcnt vmcnt(0)
17051 ; GFX7-NEXT: buffer_wbinvl1_vol
17052 ; GFX7-NEXT: s_endpgm
17053 ;
17054 ; GFX10-WGP-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
17055 ; GFX10-WGP: ; %bb.0: ; %entry
17056 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
17057 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
17058 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
17059 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
17060 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
17061 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
17062 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
17063 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
17064 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
17065 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
17066 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
17067 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
17068 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
17069 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
17070 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
17071 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
17072 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17073 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
17074 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
17075 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
17076 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
17077 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17078 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
17079 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17080 ; GFX10-WGP-NEXT: buffer_gl1_inv
17081 ; GFX10-WGP-NEXT: buffer_gl0_inv
17082 ; GFX10-WGP-NEXT: s_endpgm
17083 ;
17084 ; GFX10-CU-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
17085 ; GFX10-CU: ; %bb.0: ; %entry
17086 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
17087 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
17088 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
17089 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
17090 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
17091 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
17092 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
17093 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
17094 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
17095 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
17096 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
17097 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
17098 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
17099 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
17100 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
17101 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
17102 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17103 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
17104 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
17105 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
17106 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
17107 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
17108 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
17109 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
17110 ; GFX10-CU-NEXT: buffer_gl1_inv
17111 ; GFX10-CU-NEXT: buffer_gl0_inv
17112 ; GFX10-CU-NEXT: s_endpgm
17113 ;
17114 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
17115 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
17116 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
17117 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
17118 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
17119 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
17120 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
17121 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
17122 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
17123 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
17124 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
17125 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
17126 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
17127 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
17128 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
17129 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
17130 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
17131 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
17132 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17133 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
17134 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
17135 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
17136 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
17137 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
17138 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
17139 ; SKIP-CACHE-INV-NEXT: s_endpgm
17140 ;
17141 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
17142 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
17143 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17144 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17145 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17146 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17147 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17148 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
17149 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17150 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17151 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17152 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
17153 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
17154 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
17155 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
17156 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
17157 ;
17158 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
17159 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
17160 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17161 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17162 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17163 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17164 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17165 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
17166 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17167 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17168 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17169 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17170 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
17171 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17172 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
17173 ; GFX90A-TGSPLIT-NEXT: s_endpgm
17174 ;
17175 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
17176 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
17177 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
17178 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
17179 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
17180 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17181 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
17182 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
17183 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17184 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17185 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
17186 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
17187 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
17188 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
17189 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
17190 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
17191 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
17192 ;
17193 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
17194 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
17195 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
17196 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
17197 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
17198 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17199 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
17200 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
17201 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17202 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17203 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
17204 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
17205 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17206 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
17207 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17208 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
17209 ; GFX940-TGSPLIT-NEXT: s_endpgm
17210 ;
17211 ; GFX11-WGP-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
17212 ; GFX11-WGP: ; %bb.0: ; %entry
17213 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17214 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
17215 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
17216 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
17217 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
17218 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
17219 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17220 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
17221 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
17222 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
17223 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
17224 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17225 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
17226 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17227 ; GFX11-WGP-NEXT: buffer_gl1_inv
17228 ; GFX11-WGP-NEXT: buffer_gl0_inv
17229 ; GFX11-WGP-NEXT: s_endpgm
17230 ;
17231 ; GFX11-CU-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
17232 ; GFX11-CU: ; %bb.0: ; %entry
17233 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17234 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
17235 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
17236 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
17237 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
17238 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
17239 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17240 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
17241 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
17242 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
17243 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
17244 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
17245 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
17246 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
17247 ; GFX11-CU-NEXT: buffer_gl1_inv
17248 ; GFX11-CU-NEXT: buffer_gl0_inv
17249 ; GFX11-CU-NEXT: s_endpgm
17250 ;
17251 ; GFX12-WGP-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
17252 ; GFX12-WGP: ; %bb.0: ; %entry
17253 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17254 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
17255 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
17256 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
17257 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
17258 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
17259 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17260 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
17261 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
17262 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
17263 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
17264 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
17265 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
17266 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
17267 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
17268 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
17269 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
17270 ; GFX12-WGP-NEXT: s_endpgm
17271 ;
17272 ; GFX12-CU-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
17273 ; GFX12-CU: ; %bb.0: ; %entry
17274 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17275 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
17276 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
17277 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
17278 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
17279 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
17280 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17281 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
17282 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
17283 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
17284 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
17285 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
17286 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
17287 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
17288 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
17289 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
17290 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
17291 ; GFX12-CU-NEXT: s_endpgm
17292 ptr %out, i32 %in, i32 %old) {
17293 entry:
17294 %gep = getelementptr i32, ptr %out, i32 4
17295 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire seq_cst
17296 ret void
17297 }
17299 define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_cmpxchg(
17300 ; GFX7-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
17301 ; GFX7: ; %bb.0: ; %entry
17302 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
17303 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
17304 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
17305 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
17306 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
17307 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
17308 ; GFX7-NEXT: s_mov_b32 s4, s8
17309 ; GFX7-NEXT: s_mov_b32 s5, s9
17310 ; GFX7-NEXT: s_mov_b32 s9, s10
17311 ; GFX7-NEXT: s_mov_b32 s8, s11
17312 ; GFX7-NEXT: s_add_u32 s4, s4, s9
17313 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
17314 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
17315 ; GFX7-NEXT: s_mov_b32 s5, s8
17316 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
17317 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
17318 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17319 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
17320 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
17321 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
17322 ; GFX7-NEXT: s_waitcnt vmcnt(0)
17323 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
17324 ; GFX7-NEXT: s_waitcnt vmcnt(0)
17325 ; GFX7-NEXT: buffer_wbinvl1_vol
17326 ; GFX7-NEXT: s_endpgm
17327 ;
17328 ; GFX10-WGP-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
17329 ; GFX10-WGP: ; %bb.0: ; %entry
17330 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
17331 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
17332 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
17333 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
17334 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
17335 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
17336 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
17337 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
17338 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
17339 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
17340 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
17341 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
17342 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
17343 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
17344 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
17345 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
17346 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17347 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
17348 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
17349 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
17350 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
17351 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17352 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
17353 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17354 ; GFX10-WGP-NEXT: buffer_gl1_inv
17355 ; GFX10-WGP-NEXT: buffer_gl0_inv
17356 ; GFX10-WGP-NEXT: s_endpgm
17357 ;
17358 ; GFX10-CU-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
17359 ; GFX10-CU: ; %bb.0: ; %entry
17360 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
17361 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
17362 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
17363 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
17364 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
17365 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
17366 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
17367 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
17368 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
17369 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
17370 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
17371 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
17372 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
17373 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
17374 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
17375 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
17376 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17377 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
17378 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
17379 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
17380 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
17381 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
17382 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
17383 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
17384 ; GFX10-CU-NEXT: buffer_gl1_inv
17385 ; GFX10-CU-NEXT: buffer_gl0_inv
17386 ; GFX10-CU-NEXT: s_endpgm
17387 ;
17388 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
17389 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
17390 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
17391 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
17392 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
17393 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
17394 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
17395 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
17396 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
17397 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
17398 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
17399 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
17400 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
17401 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
17402 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
17403 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
17404 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
17405 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
17406 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17407 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
17408 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
17409 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
17410 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
17411 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
17412 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
17413 ; SKIP-CACHE-INV-NEXT: s_endpgm
17414 ;
17415 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
17416 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
17417 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17418 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17419 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17420 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17421 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17422 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
17423 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17424 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17425 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17426 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
17427 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
17428 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
17429 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
17430 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
17431 ;
17432 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
17433 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
17434 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17435 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17436 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17437 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17438 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17439 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
17440 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17441 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17442 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17443 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17444 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
17445 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17446 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
17447 ; GFX90A-TGSPLIT-NEXT: s_endpgm
17448 ;
17449 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
17450 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
17451 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
17452 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
17453 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
17454 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17455 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
17456 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
17457 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17458 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17459 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
17460 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
17461 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
17462 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
17463 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
17464 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
17465 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
17466 ;
17467 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
17468 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
17469 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
17470 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
17471 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
17472 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17473 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
17474 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
17475 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17476 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17477 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
17478 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
17479 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17480 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
17481 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17482 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
17483 ; GFX940-TGSPLIT-NEXT: s_endpgm
17484 ;
17485 ; GFX11-WGP-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
17486 ; GFX11-WGP: ; %bb.0: ; %entry
17487 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17488 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
17489 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
17490 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
17491 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
17492 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
17493 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17494 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
17495 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
17496 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
17497 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
17498 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17499 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
17500 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17501 ; GFX11-WGP-NEXT: buffer_gl1_inv
17502 ; GFX11-WGP-NEXT: buffer_gl0_inv
17503 ; GFX11-WGP-NEXT: s_endpgm
17504 ;
17505 ; GFX11-CU-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
17506 ; GFX11-CU: ; %bb.0: ; %entry
17507 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17508 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
17509 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
17510 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
17511 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
17512 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
17513 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17514 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
17515 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
17516 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
17517 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
17518 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
17519 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
17520 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
17521 ; GFX11-CU-NEXT: buffer_gl1_inv
17522 ; GFX11-CU-NEXT: buffer_gl0_inv
17523 ; GFX11-CU-NEXT: s_endpgm
17524 ;
17525 ; GFX12-WGP-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
17526 ; GFX12-WGP: ; %bb.0: ; %entry
17527 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17528 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
17529 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
17530 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
17531 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
17532 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
17533 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17534 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
17535 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
17536 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
17537 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
17538 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
17539 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
17540 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
17541 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
17542 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
17543 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
17544 ; GFX12-WGP-NEXT: s_endpgm
17545 ;
17546 ; GFX12-CU-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
17547 ; GFX12-CU: ; %bb.0: ; %entry
17548 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17549 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
17550 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
17551 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
17552 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
17553 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
17554 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17555 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
17556 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
17557 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
17558 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
17559 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
17560 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
17561 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
17562 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
17563 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
17564 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
17565 ; GFX12-CU-NEXT: s_endpgm
17566 ptr %out, i32 %in, i32 %old) {
17567 entry:
17568 %gep = getelementptr i32, ptr %out, i32 4
17569 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") release seq_cst
17570 ret void
17571 }
17573 define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_cmpxchg(
17574 ; GFX7-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
17575 ; GFX7: ; %bb.0: ; %entry
17576 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
17577 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
17578 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
17579 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
17580 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
17581 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
17582 ; GFX7-NEXT: s_mov_b32 s4, s8
17583 ; GFX7-NEXT: s_mov_b32 s5, s9
17584 ; GFX7-NEXT: s_mov_b32 s9, s10
17585 ; GFX7-NEXT: s_mov_b32 s8, s11
17586 ; GFX7-NEXT: s_add_u32 s4, s4, s9
17587 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
17588 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
17589 ; GFX7-NEXT: s_mov_b32 s5, s8
17590 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
17591 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
17592 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17593 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
17594 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
17595 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
17596 ; GFX7-NEXT: s_waitcnt vmcnt(0)
17597 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
17598 ; GFX7-NEXT: s_waitcnt vmcnt(0)
17599 ; GFX7-NEXT: buffer_wbinvl1_vol
17600 ; GFX7-NEXT: s_endpgm
17601 ;
17602 ; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
17603 ; GFX10-WGP: ; %bb.0: ; %entry
17604 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
17605 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
17606 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
17607 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
17608 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
17609 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
17610 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
17611 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
17612 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
17613 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
17614 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
17615 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
17616 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
17617 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
17618 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
17619 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
17620 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17621 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
17622 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
17623 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
17624 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
17625 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17626 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
17627 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17628 ; GFX10-WGP-NEXT: buffer_gl1_inv
17629 ; GFX10-WGP-NEXT: buffer_gl0_inv
17630 ; GFX10-WGP-NEXT: s_endpgm
17631 ;
17632 ; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
17633 ; GFX10-CU: ; %bb.0: ; %entry
17634 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
17635 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
17636 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
17637 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
17638 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
17639 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
17640 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
17641 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
17642 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
17643 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
17644 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
17645 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
17646 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
17647 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
17648 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
17649 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
17650 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17651 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
17652 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
17653 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
17654 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
17655 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
17656 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
17657 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
17658 ; GFX10-CU-NEXT: buffer_gl1_inv
17659 ; GFX10-CU-NEXT: buffer_gl0_inv
17660 ; GFX10-CU-NEXT: s_endpgm
17661 ;
17662 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
17663 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
17664 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
17665 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
17666 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
17667 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
17668 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
17669 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
17670 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
17671 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
17672 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
17673 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
17674 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
17675 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
17676 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
17677 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
17678 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
17679 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
17680 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17681 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
17682 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
17683 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
17684 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
17685 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
17686 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
17687 ; SKIP-CACHE-INV-NEXT: s_endpgm
17688 ;
17689 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
17690 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
17691 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17692 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17693 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17694 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17695 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17696 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
17697 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17698 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17699 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17700 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
17701 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
17702 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
17703 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
17704 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
17705 ;
17706 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
17707 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
17708 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17709 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17710 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17711 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17712 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17713 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
17714 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17715 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17716 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17717 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17718 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
17719 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17720 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
17721 ; GFX90A-TGSPLIT-NEXT: s_endpgm
17722 ;
17723 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
17724 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
17725 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
17726 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
17727 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
17728 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17729 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
17730 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
17731 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17732 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17733 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
17734 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
17735 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
17736 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
17737 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
17738 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
17739 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
17740 ;
17741 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
17742 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
17743 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
17744 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
17745 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
17746 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17747 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
17748 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
17749 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17750 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17751 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
17752 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
17753 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17754 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
17755 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17756 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
17757 ; GFX940-TGSPLIT-NEXT: s_endpgm
17758 ;
17759 ; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
17760 ; GFX11-WGP: ; %bb.0: ; %entry
17761 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17762 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
17763 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
17764 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
17765 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
17766 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
17767 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17768 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
17769 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
17770 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
17771 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
17772 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17773 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
17774 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17775 ; GFX11-WGP-NEXT: buffer_gl1_inv
17776 ; GFX11-WGP-NEXT: buffer_gl0_inv
17777 ; GFX11-WGP-NEXT: s_endpgm
17778 ;
17779 ; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
17780 ; GFX11-CU: ; %bb.0: ; %entry
17781 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17782 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
17783 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
17784 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
17785 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
17786 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
17787 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17788 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
17789 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
17790 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
17791 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
17792 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
17793 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
17794 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
17795 ; GFX11-CU-NEXT: buffer_gl1_inv
17796 ; GFX11-CU-NEXT: buffer_gl0_inv
17797 ; GFX11-CU-NEXT: s_endpgm
17798 ;
17799 ; GFX12-WGP-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
17800 ; GFX12-WGP: ; %bb.0: ; %entry
17801 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17802 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
17803 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
17804 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
17805 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
17806 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
17807 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17808 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
17809 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
17810 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
17811 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
17812 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
17813 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
17814 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
17815 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
17816 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
17817 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
17818 ; GFX12-WGP-NEXT: s_endpgm
17819 ;
17820 ; GFX12-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
17821 ; GFX12-CU: ; %bb.0: ; %entry
17822 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17823 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
17824 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
17825 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
17826 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
17827 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
17828 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17829 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
17830 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
17831 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
17832 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
17833 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
17834 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
17835 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
17836 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
17837 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
17838 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
17839 ; GFX12-CU-NEXT: s_endpgm
17840 ptr %out, i32 %in, i32 %old) {
17841 entry:
17842 %gep = getelementptr i32, ptr %out, i32 4
17843 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel seq_cst
17844 ret void
17845 }
17847 define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_cmpxchg(
17848 ; GFX7-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
17849 ; GFX7: ; %bb.0: ; %entry
17850 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
17851 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
17852 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
17853 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
17854 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
17855 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
17856 ; GFX7-NEXT: s_mov_b32 s4, s8
17857 ; GFX7-NEXT: s_mov_b32 s5, s9
17858 ; GFX7-NEXT: s_mov_b32 s9, s10
17859 ; GFX7-NEXT: s_mov_b32 s8, s11
17860 ; GFX7-NEXT: s_add_u32 s4, s4, s9
17861 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
17862 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
17863 ; GFX7-NEXT: s_mov_b32 s5, s8
17864 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
17865 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
17866 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17867 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
17868 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
17869 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
17870 ; GFX7-NEXT: s_waitcnt vmcnt(0)
17871 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
17872 ; GFX7-NEXT: s_waitcnt vmcnt(0)
17873 ; GFX7-NEXT: buffer_wbinvl1_vol
17874 ; GFX7-NEXT: s_endpgm
17875 ;
17876 ; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
17877 ; GFX10-WGP: ; %bb.0: ; %entry
17878 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
17879 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
17880 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
17881 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
17882 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
17883 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
17884 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
17885 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
17886 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
17887 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
17888 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
17889 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
17890 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
17891 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
17892 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
17893 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
17894 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17895 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
17896 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
17897 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
17898 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
17899 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17900 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
17901 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17902 ; GFX10-WGP-NEXT: buffer_gl1_inv
17903 ; GFX10-WGP-NEXT: buffer_gl0_inv
17904 ; GFX10-WGP-NEXT: s_endpgm
17905 ;
17906 ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
17907 ; GFX10-CU: ; %bb.0: ; %entry
17908 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
17909 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
17910 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
17911 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
17912 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
17913 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
17914 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
17915 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
17916 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
17917 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
17918 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
17919 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
17920 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
17921 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
17922 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
17923 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
17924 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17925 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
17926 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
17927 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
17928 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
17929 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
17930 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
17931 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
17932 ; GFX10-CU-NEXT: buffer_gl1_inv
17933 ; GFX10-CU-NEXT: buffer_gl0_inv
17934 ; GFX10-CU-NEXT: s_endpgm
17935 ;
17936 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
17937 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
17938 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
17939 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
17940 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
17941 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
17942 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
17943 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
17944 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
17945 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
17946 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
17947 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
17948 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
17949 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
17950 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
17951 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
17952 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
17953 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
17954 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17955 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
17956 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
17957 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
17958 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
17959 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
17960 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
17961 ; SKIP-CACHE-INV-NEXT: s_endpgm
17962 ;
17963 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
17964 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
17965 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17966 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17967 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17968 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17969 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17970 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
17971 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17972 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17973 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17974 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
17975 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
17976 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
17977 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
17978 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
17979 ;
17980 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
17981 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
17982 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17983 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17984 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17985 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17986 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17987 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
17988 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17989 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17990 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17991 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17992 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
17993 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17994 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
17995 ; GFX90A-TGSPLIT-NEXT: s_endpgm
17996 ;
17997 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
17998 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
17999 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18000 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18001 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18002 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18003 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18004 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
18005 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18006 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18007 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18008 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
18009 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
18010 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
18011 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
18012 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
18013 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
18014 ;
18015 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
18016 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
18017 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18018 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18019 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18020 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18021 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18022 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
18023 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18024 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18025 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18026 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
18027 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18028 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
18029 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18030 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
18031 ; GFX940-TGSPLIT-NEXT: s_endpgm
18032 ;
18033 ; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
18034 ; GFX11-WGP: ; %bb.0: ; %entry
18035 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18036 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18037 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18038 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
18039 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
18040 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
18041 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18042 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
18043 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
18044 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
18045 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
18046 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
18047 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
18048 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
18049 ; GFX11-WGP-NEXT: buffer_gl1_inv
18050 ; GFX11-WGP-NEXT: buffer_gl0_inv
18051 ; GFX11-WGP-NEXT: s_endpgm
18052 ;
18053 ; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
18054 ; GFX11-CU: ; %bb.0: ; %entry
18055 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18056 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18057 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18058 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
18059 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
18060 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
18061 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18062 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
18063 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
18064 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
18065 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
18066 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
18067 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
18068 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
18069 ; GFX11-CU-NEXT: buffer_gl1_inv
18070 ; GFX11-CU-NEXT: buffer_gl0_inv
18071 ; GFX11-CU-NEXT: s_endpgm
18072 ;
18073 ; GFX12-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
18074 ; GFX12-WGP: ; %bb.0: ; %entry
18075 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18076 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18077 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18078 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
18079 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
18080 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
18081 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18082 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
18083 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
18084 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
18085 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
18086 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
18087 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
18088 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
18089 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
18090 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
18091 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
18092 ; GFX12-WGP-NEXT: s_endpgm
18093 ;
18094 ; GFX12-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
18095 ; GFX12-CU: ; %bb.0: ; %entry
18096 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18097 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18098 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18099 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
18100 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
18101 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
18102 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18103 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
18104 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
18105 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
18106 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
18107 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
18108 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
18109 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
18110 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
18111 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
18112 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
18113 ; GFX12-CU-NEXT: s_endpgm
18114 ptr %out, i32 %in, i32 %old) {
18115 entry:
18116 %gep = getelementptr i32, ptr %out, i32 4
18117 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst seq_cst
18118 ret void
18119 }
18121 define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_ret_cmpxchg(
18122 ; GFX7-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
18123 ; GFX7: ; %bb.0: ; %entry
18124 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
18125 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18126 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
18127 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
18128 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
18129 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
18130 ; GFX7-NEXT: s_mov_b32 s6, s4
18131 ; GFX7-NEXT: s_mov_b32 s7, s5
18132 ; GFX7-NEXT: s_mov_b32 s11, s12
18133 ; GFX7-NEXT: s_mov_b32 s10, s13
18134 ; GFX7-NEXT: s_add_u32 s6, s6, s11
18135 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
18136 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
18137 ; GFX7-NEXT: s_mov_b32 s7, s10
18138 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
18139 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
18140 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18141 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
18142 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
18143 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
18144 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18145 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
18146 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
18147 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18148 ; GFX7-NEXT: flat_store_dword v[0:1], v2
18149 ; GFX7-NEXT: s_endpgm
18150 ;
18151 ; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
18152 ; GFX10-WGP: ; %bb.0: ; %entry
18153 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
18154 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18155 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
18156 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
18157 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
18158 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
18159 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
18160 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
18161 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
18162 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
18163 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
18164 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
18165 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
18166 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
18167 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
18168 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
18169 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18170 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
18171 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
18172 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
18173 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18174 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
18175 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
18176 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18177 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
18178 ; GFX10-WGP-NEXT: s_endpgm
18179 ;
18180 ; GFX10-CU-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
18181 ; GFX10-CU: ; %bb.0: ; %entry
18182 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
18183 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18184 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
18185 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
18186 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
18187 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
18188 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
18189 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
18190 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
18191 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
18192 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
18193 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
18194 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
18195 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
18196 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
18197 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
18198 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18199 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
18200 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
18201 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
18202 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18203 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
18204 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
18205 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18206 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
18207 ; GFX10-CU-NEXT: s_endpgm
18208 ;
18209 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
18210 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
18211 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
18212 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
18213 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
18214 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
18215 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
18216 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
18217 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
18218 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
18219 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
18220 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
18221 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
18222 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
18223 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
18224 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
18225 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
18226 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
18227 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18228 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
18229 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
18230 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
18231 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18232 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
18233 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
18234 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18235 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
18236 ; SKIP-CACHE-INV-NEXT: s_endpgm
18237 ;
18238 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
18239 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
18240 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18241 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
18242 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
18243 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18244 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
18245 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
18246 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18247 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18248 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18249 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
18250 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18251 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18252 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
18253 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
18254 ;
18255 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
18256 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
18257 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18258 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
18259 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
18260 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18261 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
18262 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
18263 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18264 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18265 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18266 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
18267 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18268 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18269 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
18270 ; GFX90A-TGSPLIT-NEXT: s_endpgm
18271 ;
18272 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
18273 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
18274 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18275 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18276 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18277 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18278 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18279 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
18280 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18281 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18282 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18283 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
18284 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18285 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18286 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
18287 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
18288 ;
18289 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
18290 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
18291 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18292 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18293 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18294 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18295 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18296 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
18297 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18298 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18299 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18300 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
18301 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18302 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18303 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
18304 ; GFX940-TGSPLIT-NEXT: s_endpgm
18305 ;
18306 ; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
18307 ; GFX11-WGP: ; %bb.0: ; %entry
18308 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18309 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18310 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18311 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
18312 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
18313 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
18314 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18315 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
18316 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
18317 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
18318 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
18319 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
18320 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
18321 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18322 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
18323 ; GFX11-WGP-NEXT: s_endpgm
18324 ;
18325 ; GFX11-CU-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
18326 ; GFX11-CU: ; %bb.0: ; %entry
18327 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18328 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18329 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18330 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
18331 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
18332 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
18333 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18334 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
18335 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
18336 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
18337 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
18338 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
18339 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
18340 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18341 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
18342 ; GFX11-CU-NEXT: s_endpgm
18343 ;
18344 ; GFX12-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
18345 ; GFX12-WGP: ; %bb.0: ; %entry
18346 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18347 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18348 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18349 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
18350 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
18351 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
18352 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18353 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
18354 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
18355 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
18356 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
18357 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
18358 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
18359 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
18360 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
18361 ; GFX12-WGP-NEXT: s_endpgm
18362 ;
18363 ; GFX12-CU-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
18364 ; GFX12-CU: ; %bb.0: ; %entry
18365 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18366 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18367 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18368 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
18369 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
18370 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
18371 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18372 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
18373 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
18374 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
18375 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
18376 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
18377 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
18378 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
18379 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
18380 ; GFX12-CU-NEXT: s_endpgm
18381 ptr %out, i32 %in, i32 %old) {
18382 entry:
18383 %gep = getelementptr i32, ptr %out, i32 4
18384 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic monotonic
18385 %val0 = extractvalue { i32, i1 } %val, 0
18386 store i32 %val0, ptr %out, align 4
18387 ret void
18388 }
18390 define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_ret_cmpxchg(
18391 ; GFX7-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
18392 ; GFX7: ; %bb.0: ; %entry
18393 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
18394 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18395 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
18396 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
18397 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
18398 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
18399 ; GFX7-NEXT: s_mov_b32 s6, s4
18400 ; GFX7-NEXT: s_mov_b32 s7, s5
18401 ; GFX7-NEXT: s_mov_b32 s11, s12
18402 ; GFX7-NEXT: s_mov_b32 s10, s13
18403 ; GFX7-NEXT: s_add_u32 s6, s6, s11
18404 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
18405 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
18406 ; GFX7-NEXT: s_mov_b32 s7, s10
18407 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
18408 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
18409 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18410 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
18411 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
18412 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
18413 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18414 ; GFX7-NEXT: s_waitcnt vmcnt(0)
18415 ; GFX7-NEXT: buffer_wbinvl1_vol
18416 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
18417 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
18418 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
18419 ; GFX7-NEXT: flat_store_dword v[0:1], v2
18420 ; GFX7-NEXT: s_endpgm
18421 ;
18422 ; GFX10-WGP-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
18423 ; GFX10-WGP: ; %bb.0: ; %entry
18424 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
18425 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18426 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
18427 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
18428 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
18429 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
18430 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
18431 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
18432 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
18433 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
18434 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
18435 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
18436 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
18437 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
18438 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
18439 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
18440 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18441 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
18442 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
18443 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
18444 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18445 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
18446 ; GFX10-WGP-NEXT: buffer_gl1_inv
18447 ; GFX10-WGP-NEXT: buffer_gl0_inv
18448 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
18449 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
18450 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
18451 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
18452 ; GFX10-WGP-NEXT: s_endpgm
18453 ;
18454 ; GFX10-CU-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
18455 ; GFX10-CU: ; %bb.0: ; %entry
18456 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
18457 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18458 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
18459 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
18460 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
18461 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
18462 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
18463 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
18464 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
18465 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
18466 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
18467 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
18468 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
18469 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
18470 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
18471 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
18472 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18473 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
18474 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
18475 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
18476 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18477 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
18478 ; GFX10-CU-NEXT: buffer_gl1_inv
18479 ; GFX10-CU-NEXT: buffer_gl0_inv
18480 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
18481 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
18482 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
18483 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
18484 ; GFX10-CU-NEXT: s_endpgm
18485 ;
18486 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
18487 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
18488 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
18489 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
18490 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
18491 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
18492 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
18493 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
18494 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
18495 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
18496 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
18497 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
18498 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
18499 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
18500 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
18501 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
18502 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
18503 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
18504 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18505 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
18506 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
18507 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
18508 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18509 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
18510 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
18511 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
18512 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
18513 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
18514 ; SKIP-CACHE-INV-NEXT: s_endpgm
18515 ;
18516 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
18517 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
18518 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18519 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
18520 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
18521 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18522 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
18523 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
18524 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18525 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18526 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18527 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
18528 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
18529 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
18530 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18531 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18532 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
18533 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
18534 ;
18535 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
18536 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
18537 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18538 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
18539 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
18540 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18541 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
18542 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
18543 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18544 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18545 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18546 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
18547 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18548 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
18549 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18550 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
18551 ; GFX90A-TGSPLIT-NEXT: s_endpgm
18552 ;
18553 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
18554 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
18555 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18556 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18557 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18558 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18559 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18560 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
18561 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18562 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18563 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18564 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
18565 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
18566 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
18567 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18568 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18569 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
18570 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
18571 ;
18572 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
18573 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
18574 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18575 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18576 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18577 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18578 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18579 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
18580 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18581 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18582 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18583 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
18584 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18585 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
18586 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18587 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
18588 ; GFX940-TGSPLIT-NEXT: s_endpgm
18589 ;
18590 ; GFX11-WGP-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
18591 ; GFX11-WGP: ; %bb.0: ; %entry
18592 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18593 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18594 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18595 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
18596 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
18597 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
18598 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18599 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
18600 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
18601 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
18602 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
18603 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
18604 ; GFX11-WGP-NEXT: buffer_gl1_inv
18605 ; GFX11-WGP-NEXT: buffer_gl0_inv
18606 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
18607 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
18608 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
18609 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
18610 ; GFX11-WGP-NEXT: s_endpgm
18611 ;
18612 ; GFX11-CU-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
18613 ; GFX11-CU: ; %bb.0: ; %entry
18614 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18615 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18616 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18617 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
18618 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
18619 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
18620 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18621 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
18622 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
18623 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
18624 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
18625 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
18626 ; GFX11-CU-NEXT: buffer_gl1_inv
18627 ; GFX11-CU-NEXT: buffer_gl0_inv
18628 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
18629 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
18630 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
18631 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
18632 ; GFX11-CU-NEXT: s_endpgm
18633 ;
18634 ; GFX12-WGP-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
18635 ; GFX12-WGP: ; %bb.0: ; %entry
18636 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18637 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18638 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18639 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
18640 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
18641 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
18642 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18643 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
18644 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
18645 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
18646 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
18647 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
18648 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
18649 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
18650 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
18651 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
18652 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
18653 ; GFX12-WGP-NEXT: s_endpgm
18654 ;
18655 ; GFX12-CU-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
18656 ; GFX12-CU: ; %bb.0: ; %entry
18657 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18658 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18659 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18660 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
18661 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
18662 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
18663 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18664 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
18665 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
18666 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
18667 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
18668 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
18669 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
18670 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
18671 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
18672 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
18673 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
18674 ; GFX12-CU-NEXT: s_endpgm
18675 ptr %out, i32 %in, i32 %old) {
18676 entry:
18677 %gep = getelementptr i32, ptr %out, i32 4
18678 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire monotonic
18679 %val0 = extractvalue { i32, i1 } %val, 0
18680 store i32 %val0, ptr %out, align 4
18681 ret void
18682 }
18684 define amdgpu_kernel void @flat_agent_one_as_release_monotonic_ret_cmpxchg(
18685 ; GFX7-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
18686 ; GFX7: ; %bb.0: ; %entry
18687 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
18688 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18689 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
18690 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
18691 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
18692 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
18693 ; GFX7-NEXT: s_mov_b32 s6, s4
18694 ; GFX7-NEXT: s_mov_b32 s7, s5
18695 ; GFX7-NEXT: s_mov_b32 s11, s12
18696 ; GFX7-NEXT: s_mov_b32 s10, s13
18697 ; GFX7-NEXT: s_add_u32 s6, s6, s11
18698 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
18699 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
18700 ; GFX7-NEXT: s_mov_b32 s7, s10
18701 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
18702 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
18703 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18704 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
18705 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
18706 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
18707 ; GFX7-NEXT: s_waitcnt vmcnt(0)
18708 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18709 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
18710 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
18711 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18712 ; GFX7-NEXT: flat_store_dword v[0:1], v2
18713 ; GFX7-NEXT: s_endpgm
18714 ;
18715 ; GFX10-WGP-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
18716 ; GFX10-WGP: ; %bb.0: ; %entry
18717 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
18718 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18719 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
18720 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
18721 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
18722 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
18723 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
18724 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
18725 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
18726 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
18727 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
18728 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
18729 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
18730 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
18731 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
18732 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
18733 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18734 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
18735 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
18736 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
18737 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
18738 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
18739 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18740 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
18741 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
18742 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18743 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
18744 ; GFX10-WGP-NEXT: s_endpgm
18745 ;
18746 ; GFX10-CU-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
18747 ; GFX10-CU: ; %bb.0: ; %entry
18748 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
18749 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18750 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
18751 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
18752 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
18753 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
18754 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
18755 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
18756 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
18757 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
18758 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
18759 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
18760 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
18761 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
18762 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
18763 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
18764 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18765 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
18766 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
18767 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
18768 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
18769 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
18770 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18771 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
18772 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
18773 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18774 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
18775 ; GFX10-CU-NEXT: s_endpgm
18776 ;
18777 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
18778 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
18779 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
18780 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
18781 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
18782 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
18783 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
18784 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
18785 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
18786 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
18787 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
18788 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
18789 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
18790 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
18791 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
18792 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
18793 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
18794 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
18795 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18796 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
18797 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
18798 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
18799 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
18800 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18801 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
18802 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
18803 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18804 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
18805 ; SKIP-CACHE-INV-NEXT: s_endpgm
18806 ;
18807 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
18808 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
18809 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18810 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
18811 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
18812 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18813 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
18814 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
18815 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18816 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18817 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18818 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
18819 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
18820 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18821 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18822 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
18823 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
18824 ;
18825 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
18826 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
18827 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18828 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
18829 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
18830 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18831 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
18832 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
18833 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18834 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18835 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18836 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18837 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
18838 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18839 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18840 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
18841 ; GFX90A-TGSPLIT-NEXT: s_endpgm
18842 ;
18843 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
18844 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
18845 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18846 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18847 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18848 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18849 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18850 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
18851 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18852 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18853 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18854 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
18855 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
18856 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
18857 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18858 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18859 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
18860 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
18861 ;
18862 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
18863 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
18864 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18865 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18866 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18867 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18868 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18869 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
18870 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18871 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18872 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18873 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
18874 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18875 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
18876 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18877 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18878 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
18879 ; GFX940-TGSPLIT-NEXT: s_endpgm
18880 ;
18881 ; GFX11-WGP-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
18882 ; GFX11-WGP: ; %bb.0: ; %entry
18883 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18884 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18885 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18886 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
18887 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
18888 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
18889 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18890 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
18891 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
18892 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
18893 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
18894 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
18895 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
18896 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
18897 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
18898 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18899 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
18900 ; GFX11-WGP-NEXT: s_endpgm
18901 ;
18902 ; GFX11-CU-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
18903 ; GFX11-CU: ; %bb.0: ; %entry
18904 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18905 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18906 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18907 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
18908 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
18909 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
18910 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18911 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
18912 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
18913 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
18914 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
18915 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
18916 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
18917 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
18918 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
18919 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18920 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
18921 ; GFX11-CU-NEXT: s_endpgm
18922 ;
18923 ; GFX12-WGP-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
18924 ; GFX12-WGP: ; %bb.0: ; %entry
18925 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18926 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18927 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18928 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
18929 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
18930 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
18931 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18932 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
18933 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
18934 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
18935 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
18936 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
18937 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
18938 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
18939 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
18940 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
18941 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
18942 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
18943 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
18944 ; GFX12-WGP-NEXT: s_endpgm
18945 ;
18946 ; GFX12-CU-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
18947 ; GFX12-CU: ; %bb.0: ; %entry
18948 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18949 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18950 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18951 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
18952 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
18953 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
18954 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18955 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
18956 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
18957 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
18958 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
18959 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
18960 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
18961 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
18962 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
18963 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
18964 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
18965 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
18966 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
18967 ; GFX12-CU-NEXT: s_endpgm
18968 ptr %out, i32 %in, i32 %old) {
18969 entry:
18970 %gep = getelementptr i32, ptr %out, i32 4
18971 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") release monotonic
18972 %val0 = extractvalue { i32, i1 } %val, 0
18973 store i32 %val0, ptr %out, align 4
18974 ret void
18975 }
18977 define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
18978 ; GFX7-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
18979 ; GFX7: ; %bb.0: ; %entry
18980 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
18981 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18982 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
18983 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
18984 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
18985 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
18986 ; GFX7-NEXT: s_mov_b32 s6, s4
18987 ; GFX7-NEXT: s_mov_b32 s7, s5
18988 ; GFX7-NEXT: s_mov_b32 s11, s12
18989 ; GFX7-NEXT: s_mov_b32 s10, s13
18990 ; GFX7-NEXT: s_add_u32 s6, s6, s11
18991 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
18992 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
18993 ; GFX7-NEXT: s_mov_b32 s7, s10
18994 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
18995 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
18996 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18997 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
18998 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
18999 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
19000 ; GFX7-NEXT: s_waitcnt vmcnt(0)
19001 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19002 ; GFX7-NEXT: s_waitcnt vmcnt(0)
19003 ; GFX7-NEXT: buffer_wbinvl1_vol
19004 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
19005 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
19006 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
19007 ; GFX7-NEXT: flat_store_dword v[0:1], v2
19008 ; GFX7-NEXT: s_endpgm
19009 ;
19010 ; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
19011 ; GFX10-WGP: ; %bb.0: ; %entry
19012 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
19013 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19014 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
19015 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
19016 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
19017 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
19018 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
19019 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
19020 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
19021 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
19022 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
19023 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
19024 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19025 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
19026 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
19027 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
19028 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19029 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
19030 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
19031 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
19032 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
19033 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
19034 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19035 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
19036 ; GFX10-WGP-NEXT: buffer_gl1_inv
19037 ; GFX10-WGP-NEXT: buffer_gl0_inv
19038 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
19039 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
19040 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
19041 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
19042 ; GFX10-WGP-NEXT: s_endpgm
19043 ;
19044 ; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
19045 ; GFX10-CU: ; %bb.0: ; %entry
19046 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
19047 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19048 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
19049 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
19050 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
19051 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
19052 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
19053 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
19054 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
19055 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
19056 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
19057 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
19058 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19059 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
19060 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
19061 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
19062 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19063 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
19064 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
19065 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
19066 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
19067 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
19068 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19069 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
19070 ; GFX10-CU-NEXT: buffer_gl1_inv
19071 ; GFX10-CU-NEXT: buffer_gl0_inv
19072 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
19073 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
19074 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
19075 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
19076 ; GFX10-CU-NEXT: s_endpgm
19077 ;
19078 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
19079 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
19080 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
19081 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
19082 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
19083 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
19084 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
19085 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
19086 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
19087 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
19088 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
19089 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
19090 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
19091 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
19092 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
19093 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
19094 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
19095 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
19096 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19097 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
19098 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
19099 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
19100 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
19101 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19102 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
19103 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
19104 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
19105 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
19106 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
19107 ; SKIP-CACHE-INV-NEXT: s_endpgm
19108 ;
19109 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
19110 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
19111 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19112 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
19113 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
19114 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19115 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
19116 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
19117 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19118 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19119 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19120 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
19121 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
19122 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
19123 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
19124 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19125 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19126 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
19127 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
19128 ;
19129 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
19130 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
19131 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19132 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
19133 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
19134 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19135 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
19136 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
19137 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19138 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19139 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19140 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19141 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
19142 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19143 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
19144 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19145 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
19146 ; GFX90A-TGSPLIT-NEXT: s_endpgm
19147 ;
19148 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
19149 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
19150 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
19151 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
19152 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
19153 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19154 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
19155 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
19156 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19157 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19158 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19159 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
19160 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
19161 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
19162 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
19163 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
19164 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19165 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19166 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
19167 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
19168 ;
19169 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
19170 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
19171 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
19172 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
19173 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
19174 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19175 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
19176 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
19177 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19178 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19179 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19180 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
19181 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19182 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
19183 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19184 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
19185 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19186 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
19187 ; GFX940-TGSPLIT-NEXT: s_endpgm
19188 ;
19189 ; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
19190 ; GFX11-WGP: ; %bb.0: ; %entry
19191 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19192 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
19193 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
19194 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
19195 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
19196 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
19197 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19198 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
19199 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
19200 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
19201 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
19202 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
19203 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
19204 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
19205 ; GFX11-WGP-NEXT: buffer_gl1_inv
19206 ; GFX11-WGP-NEXT: buffer_gl0_inv
19207 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
19208 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
19209 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
19210 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
19211 ; GFX11-WGP-NEXT: s_endpgm
19212 ;
19213 ; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
19214 ; GFX11-CU: ; %bb.0: ; %entry
19215 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19216 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
19217 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
19218 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
19219 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
19220 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
19221 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19222 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
19223 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
19224 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
19225 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
19226 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
19227 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
19228 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
19229 ; GFX11-CU-NEXT: buffer_gl1_inv
19230 ; GFX11-CU-NEXT: buffer_gl0_inv
19231 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
19232 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
19233 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
19234 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
19235 ; GFX11-CU-NEXT: s_endpgm
19236 ;
19237 ; GFX12-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
19238 ; GFX12-WGP: ; %bb.0: ; %entry
19239 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19240 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
19241 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
19242 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
19243 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
19244 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
19245 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19246 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
19247 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
19248 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
19249 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
19250 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
19251 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
19252 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
19253 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
19254 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
19255 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
19256 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
19257 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
19258 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
19259 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
19260 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
19261 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
19262 ; GFX12-WGP-NEXT: s_endpgm
19263 ;
19264 ; GFX12-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
19265 ; GFX12-CU: ; %bb.0: ; %entry
19266 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19267 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
19268 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
19269 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
19270 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
19271 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
19272 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19273 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
19274 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
19275 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
19276 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
19277 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
19278 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
19279 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
19280 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
19281 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
19282 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
19283 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
19284 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
19285 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
19286 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
19287 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
19288 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
19289 ; GFX12-CU-NEXT: s_endpgm
19290 ptr %out, i32 %in, i32 %old) {
19291 entry:
19292 %gep = getelementptr i32, ptr %out, i32 4
19293 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel monotonic
19294 %val0 = extractvalue { i32, i1 } %val, 0
19295 store i32 %val0, ptr %out, align 4
19296 ret void
19297 }
19299 define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
19300 ; GFX7-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
19301 ; GFX7: ; %bb.0: ; %entry
19302 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
19303 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19304 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
19305 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
19306 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
19307 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
19308 ; GFX7-NEXT: s_mov_b32 s6, s4
19309 ; GFX7-NEXT: s_mov_b32 s7, s5
19310 ; GFX7-NEXT: s_mov_b32 s11, s12
19311 ; GFX7-NEXT: s_mov_b32 s10, s13
19312 ; GFX7-NEXT: s_add_u32 s6, s6, s11
19313 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
19314 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19315 ; GFX7-NEXT: s_mov_b32 s7, s10
19316 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
19317 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
19318 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19319 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
19320 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
19321 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
19322 ; GFX7-NEXT: s_waitcnt vmcnt(0)
19323 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19324 ; GFX7-NEXT: s_waitcnt vmcnt(0)
19325 ; GFX7-NEXT: buffer_wbinvl1_vol
19326 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
19327 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
19328 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
19329 ; GFX7-NEXT: flat_store_dword v[0:1], v2
19330 ; GFX7-NEXT: s_endpgm
19331 ;
19332 ; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
19333 ; GFX10-WGP: ; %bb.0: ; %entry
19334 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
19335 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19336 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
19337 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
19338 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
19339 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
19340 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
19341 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
19342 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
19343 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
19344 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
19345 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
19346 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19347 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
19348 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
19349 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
19350 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19351 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
19352 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
19353 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
19354 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
19355 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
19356 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19357 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
19358 ; GFX10-WGP-NEXT: buffer_gl1_inv
19359 ; GFX10-WGP-NEXT: buffer_gl0_inv
19360 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
19361 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
19362 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
19363 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
19364 ; GFX10-WGP-NEXT: s_endpgm
19365 ;
19366 ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
19367 ; GFX10-CU: ; %bb.0: ; %entry
19368 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
19369 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19370 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
19371 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
19372 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
19373 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
19374 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
19375 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
19376 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
19377 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
19378 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
19379 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
19380 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19381 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
19382 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
19383 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
19384 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19385 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
19386 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
19387 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
19388 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
19389 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
19390 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19391 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
19392 ; GFX10-CU-NEXT: buffer_gl1_inv
19393 ; GFX10-CU-NEXT: buffer_gl0_inv
19394 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
19395 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
19396 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
19397 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
19398 ; GFX10-CU-NEXT: s_endpgm
19399 ;
19400 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
19401 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
19402 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
19403 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
19404 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
19405 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
19406 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
19407 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
19408 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
19409 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
19410 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
19411 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
19412 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
19413 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
19414 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
19415 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
19416 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
19417 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
19418 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19419 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
19420 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
19421 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
19422 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
19423 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19424 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
19425 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
19426 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
19427 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
19428 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
19429 ; SKIP-CACHE-INV-NEXT: s_endpgm
19430 ;
19431 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
19432 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
19433 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19434 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
19435 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
19436 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19437 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
19438 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
19439 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19440 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19441 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19442 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
19443 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
19444 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
19445 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
19446 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19447 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19448 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
19449 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
19450 ;
19451 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
19452 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
19453 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19454 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
19455 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
19456 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19457 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
19458 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
19459 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19460 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19461 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19462 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19463 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
19464 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19465 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
19466 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19467 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
19468 ; GFX90A-TGSPLIT-NEXT: s_endpgm
19469 ;
19470 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
19471 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
19472 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
19473 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
19474 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
19475 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19476 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
19477 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
19478 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19479 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19480 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19481 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
19482 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
19483 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
19484 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
19485 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
19486 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19487 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19488 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
19489 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
19490 ;
19491 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
19492 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
19493 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
19494 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
19495 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
19496 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19497 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
19498 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
19499 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19500 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19501 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19502 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
19503 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19504 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
19505 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19506 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
19507 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19508 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
19509 ; GFX940-TGSPLIT-NEXT: s_endpgm
19510 ;
19511 ; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
19512 ; GFX11-WGP: ; %bb.0: ; %entry
19513 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19514 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
19515 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
19516 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
19517 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
19518 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
19519 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19520 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
19521 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
19522 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
19523 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
19524 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
19525 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
19526 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
19527 ; GFX11-WGP-NEXT: buffer_gl1_inv
19528 ; GFX11-WGP-NEXT: buffer_gl0_inv
19529 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
19530 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
19531 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
19532 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
19533 ; GFX11-WGP-NEXT: s_endpgm
19534 ;
19535 ; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
19536 ; GFX11-CU: ; %bb.0: ; %entry
19537 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19538 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
19539 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
19540 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
19541 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
19542 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
19543 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19544 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
19545 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
19546 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
19547 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
19548 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
19549 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
19550 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
19551 ; GFX11-CU-NEXT: buffer_gl1_inv
19552 ; GFX11-CU-NEXT: buffer_gl0_inv
19553 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
19554 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
19555 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
19556 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
19557 ; GFX11-CU-NEXT: s_endpgm
19558 ;
19559 ; GFX12-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
19560 ; GFX12-WGP: ; %bb.0: ; %entry
19561 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19562 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
19563 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
19564 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
19565 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
19566 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
19567 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19568 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
19569 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
19570 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
19571 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
19572 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
19573 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
19574 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
19575 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
19576 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
19577 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
19578 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
19579 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
19580 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
19581 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
19582 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
19583 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
19584 ; GFX12-WGP-NEXT: s_endpgm
19585 ;
19586 ; GFX12-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
19587 ; GFX12-CU: ; %bb.0: ; %entry
19588 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19589 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
19590 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
19591 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
19592 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
19593 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
19594 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19595 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
19596 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
19597 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
19598 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
19599 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
19600 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
19601 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
19602 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
19603 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
19604 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
19605 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
19606 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
19607 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
19608 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
19609 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
19610 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
19611 ; GFX12-CU-NEXT: s_endpgm
19612 ptr %out, i32 %in, i32 %old) {
19613 entry:
19614 %gep = getelementptr i32, ptr %out, i32 4
19615 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst monotonic
19616 %val0 = extractvalue { i32, i1 } %val, 0
19617 store i32 %val0, ptr %out, align 4
19618 ret void
19619 }
19621 define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_ret_cmpxchg(
19622 ; GFX7-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
19623 ; GFX7: ; %bb.0: ; %entry
19624 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
19625 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19626 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
19627 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
19628 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
19629 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
19630 ; GFX7-NEXT: s_mov_b32 s6, s4
19631 ; GFX7-NEXT: s_mov_b32 s7, s5
19632 ; GFX7-NEXT: s_mov_b32 s11, s12
19633 ; GFX7-NEXT: s_mov_b32 s10, s13
19634 ; GFX7-NEXT: s_add_u32 s6, s6, s11
19635 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
19636 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19637 ; GFX7-NEXT: s_mov_b32 s7, s10
19638 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
19639 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
19640 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19641 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
19642 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
19643 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
19644 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19645 ; GFX7-NEXT: s_waitcnt vmcnt(0)
19646 ; GFX7-NEXT: buffer_wbinvl1_vol
19647 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
19648 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
19649 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
19650 ; GFX7-NEXT: flat_store_dword v[0:1], v2
19651 ; GFX7-NEXT: s_endpgm
19652 ;
19653 ; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
19654 ; GFX10-WGP: ; %bb.0: ; %entry
19655 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
19656 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19657 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
19658 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
19659 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
19660 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
19661 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
19662 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
19663 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
19664 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
19665 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
19666 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
19667 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19668 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
19669 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
19670 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
19671 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19672 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
19673 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
19674 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
19675 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19676 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
19677 ; GFX10-WGP-NEXT: buffer_gl1_inv
19678 ; GFX10-WGP-NEXT: buffer_gl0_inv
19679 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
19680 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
19681 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
19682 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
19683 ; GFX10-WGP-NEXT: s_endpgm
19684 ;
19685 ; GFX10-CU-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
19686 ; GFX10-CU: ; %bb.0: ; %entry
19687 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
19688 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19689 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
19690 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
19691 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
19692 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
19693 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
19694 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
19695 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
19696 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
19697 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
19698 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
19699 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19700 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
19701 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
19702 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
19703 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19704 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
19705 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
19706 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
19707 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19708 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
19709 ; GFX10-CU-NEXT: buffer_gl1_inv
19710 ; GFX10-CU-NEXT: buffer_gl0_inv
19711 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
19712 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
19713 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
19714 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
19715 ; GFX10-CU-NEXT: s_endpgm
19716 ;
19717 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
19718 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
19719 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
19720 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
19721 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
19722 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
19723 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
19724 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
19725 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
19726 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
19727 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
19728 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
19729 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
19730 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
19731 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
19732 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
19733 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
19734 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
19735 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19736 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
19737 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
19738 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
19739 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19740 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
19741 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
19742 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
19743 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
19744 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
19745 ; SKIP-CACHE-INV-NEXT: s_endpgm
19746 ;
19747 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
19748 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
19749 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19750 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
19751 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
19752 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19753 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
19754 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
19755 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19756 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19757 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19758 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
19759 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
19760 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
19761 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19762 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19763 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
19764 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
19765 ;
19766 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
19767 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
19768 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19769 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
19770 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
19771 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19772 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
19773 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
19774 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19775 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19776 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19777 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
19778 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19779 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
19780 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19781 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
19782 ; GFX90A-TGSPLIT-NEXT: s_endpgm
19783 ;
19784 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
19785 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
19786 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
19787 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
19788 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
19789 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19790 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
19791 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
19792 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19793 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19794 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19795 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
19796 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
19797 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
19798 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19799 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19800 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
19801 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
19802 ;
19803 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
19804 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
19805 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
19806 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
19807 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
19808 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19809 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
19810 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
19811 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19812 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19813 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19814 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
19815 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19816 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
19817 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19818 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
19819 ; GFX940-TGSPLIT-NEXT: s_endpgm
19820 ;
19821 ; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
19822 ; GFX11-WGP: ; %bb.0: ; %entry
19823 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19824 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
19825 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
19826 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
19827 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
19828 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
19829 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19830 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
19831 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
19832 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
19833 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
19834 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
19835 ; GFX11-WGP-NEXT: buffer_gl1_inv
19836 ; GFX11-WGP-NEXT: buffer_gl0_inv
19837 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
19838 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
19839 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
19840 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
19841 ; GFX11-WGP-NEXT: s_endpgm
19842 ;
19843 ; GFX11-CU-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
19844 ; GFX11-CU: ; %bb.0: ; %entry
19845 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19846 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
19847 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
19848 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
19849 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
19850 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
19851 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19852 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
19853 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
19854 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
19855 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
19856 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
19857 ; GFX11-CU-NEXT: buffer_gl1_inv
19858 ; GFX11-CU-NEXT: buffer_gl0_inv
19859 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
19860 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
19861 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
19862 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
19863 ; GFX11-CU-NEXT: s_endpgm
19864 ;
19865 ; GFX12-WGP-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
19866 ; GFX12-WGP: ; %bb.0: ; %entry
19867 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19868 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
19869 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
19870 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
19871 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
19872 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
19873 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19874 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
19875 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
19876 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
19877 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
19878 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
19879 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
19880 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
19881 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
19882 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
19883 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
19884 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
19885 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
19886 ; GFX12-WGP-NEXT: s_endpgm
19887 ;
19888 ; GFX12-CU-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
19889 ; GFX12-CU: ; %bb.0: ; %entry
19890 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19891 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
19892 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
19893 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
19894 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
19895 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
19896 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19897 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
19898 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
19899 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
19900 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
19901 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
19902 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
19903 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
19904 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
19905 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
19906 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
19907 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
19908 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
19909 ; GFX12-CU-NEXT: s_endpgm
19910 ptr %out, i32 %in, i32 %old) {
19911 entry:
19912 %gep = getelementptr i32, ptr %out, i32 4
19913 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic acquire
19914 %val0 = extractvalue { i32, i1 } %val, 0
19915 store i32 %val0, ptr %out, align 4
19916 ret void
19917 }
19919 define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_ret_cmpxchg(
19920 ; GFX7-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
19921 ; GFX7: ; %bb.0: ; %entry
19922 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
19923 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19924 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
19925 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
19926 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
19927 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
19928 ; GFX7-NEXT: s_mov_b32 s6, s4
19929 ; GFX7-NEXT: s_mov_b32 s7, s5
19930 ; GFX7-NEXT: s_mov_b32 s11, s12
19931 ; GFX7-NEXT: s_mov_b32 s10, s13
19932 ; GFX7-NEXT: s_add_u32 s6, s6, s11
19933 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
19934 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19935 ; GFX7-NEXT: s_mov_b32 s7, s10
19936 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
19937 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
19938 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19939 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
19940 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
19941 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
19942 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19943 ; GFX7-NEXT: s_waitcnt vmcnt(0)
19944 ; GFX7-NEXT: buffer_wbinvl1_vol
19945 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
19946 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
19947 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
19948 ; GFX7-NEXT: flat_store_dword v[0:1], v2
19949 ; GFX7-NEXT: s_endpgm
19950 ;
19951 ; GFX10-WGP-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
19952 ; GFX10-WGP: ; %bb.0: ; %entry
19953 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
19954 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19955 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
19956 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
19957 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
19958 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
19959 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
19960 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
19961 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
19962 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
19963 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
19964 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
19965 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19966 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
19967 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
19968 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
19969 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19970 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
19971 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
19972 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
19973 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19974 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
19975 ; GFX10-WGP-NEXT: buffer_gl1_inv
19976 ; GFX10-WGP-NEXT: buffer_gl0_inv
19977 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
19978 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
19979 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
19980 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
19981 ; GFX10-WGP-NEXT: s_endpgm
19982 ;
19983 ; GFX10-CU-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
19984 ; GFX10-CU: ; %bb.0: ; %entry
19985 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
19986 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19987 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
19988 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
19989 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
19990 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
19991 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
19992 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
19993 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
19994 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
19995 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
19996 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
19997 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19998 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
19999 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
20000 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
20001 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20002 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
20003 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
20004 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
20005 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20006 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
20007 ; GFX10-CU-NEXT: buffer_gl1_inv
20008 ; GFX10-CU-NEXT: buffer_gl0_inv
20009 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
20010 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
20011 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
20012 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
20013 ; GFX10-CU-NEXT: s_endpgm
20014 ;
20015 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
20016 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
20017 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
20018 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
20019 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
20020 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
20021 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
20022 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
20023 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
20024 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
20025 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
20026 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
20027 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
20028 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
20029 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
20030 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
20031 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
20032 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
20033 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20034 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
20035 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
20036 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
20037 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20038 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
20039 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
20040 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
20041 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
20042 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
20043 ; SKIP-CACHE-INV-NEXT: s_endpgm
20044 ;
20045 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
20046 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
20047 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20048 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
20049 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
20050 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20051 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
20052 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
20053 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20054 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20055 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20056 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
20057 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
20058 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
20059 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20060 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20061 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
20062 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
20063 ;
20064 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
20065 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
20066 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20067 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
20068 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
20069 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20070 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
20071 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
20072 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20073 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20074 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20075 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
20076 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20077 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
20078 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20079 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
20080 ; GFX90A-TGSPLIT-NEXT: s_endpgm
20081 ;
20082 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
20083 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
20084 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
20085 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
20086 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
20087 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20088 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
20089 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
20090 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20091 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20092 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20093 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
20094 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
20095 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
20096 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20097 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20098 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
20099 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
20100 ;
20101 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
20102 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
20103 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
20104 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
20105 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
20106 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20107 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
20108 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
20109 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20110 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20111 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20112 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
20113 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20114 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
20115 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20116 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
20117 ; GFX940-TGSPLIT-NEXT: s_endpgm
20118 ;
20119 ; GFX11-WGP-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
20120 ; GFX11-WGP: ; %bb.0: ; %entry
20121 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20122 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
20123 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
20124 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
20125 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
20126 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
20127 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20128 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
20129 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
20130 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
20131 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
20132 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
20133 ; GFX11-WGP-NEXT: buffer_gl1_inv
20134 ; GFX11-WGP-NEXT: buffer_gl0_inv
20135 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
20136 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
20137 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
20138 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
20139 ; GFX11-WGP-NEXT: s_endpgm
20140 ;
20141 ; GFX11-CU-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
20142 ; GFX11-CU: ; %bb.0: ; %entry
20143 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20144 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
20145 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
20146 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
20147 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
20148 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
20149 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20150 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
20151 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
20152 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
20153 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
20154 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
20155 ; GFX11-CU-NEXT: buffer_gl1_inv
20156 ; GFX11-CU-NEXT: buffer_gl0_inv
20157 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
20158 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
20159 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
20160 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
20161 ; GFX11-CU-NEXT: s_endpgm
20162 ;
20163 ; GFX12-WGP-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
20164 ; GFX12-WGP: ; %bb.0: ; %entry
20165 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20166 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
20167 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
20168 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
20169 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
20170 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
20171 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20172 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
20173 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
20174 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
20175 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
20176 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
20177 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
20178 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
20179 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
20180 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
20181 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
20182 ; GFX12-WGP-NEXT: s_endpgm
20183 ;
20184 ; GFX12-CU-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
20185 ; GFX12-CU: ; %bb.0: ; %entry
20186 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20187 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
20188 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
20189 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
20190 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
20191 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
20192 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20193 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
20194 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
20195 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
20196 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
20197 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
20198 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
20199 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
20200 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
20201 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
20202 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
20203 ; GFX12-CU-NEXT: s_endpgm
20204 ptr %out, i32 %in, i32 %old) {
20205 entry:
20206 %gep = getelementptr i32, ptr %out, i32 4
20207 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire acquire
20208 %val0 = extractvalue { i32, i1 } %val, 0
20209 store i32 %val0, ptr %out, align 4
20210 ret void
20211 }
20213 define amdgpu_kernel void @flat_agent_one_as_release_acquire_ret_cmpxchg(
20214 ; GFX7-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
20215 ; GFX7: ; %bb.0: ; %entry
20216 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
20217 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20218 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
20219 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
20220 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
20221 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
20222 ; GFX7-NEXT: s_mov_b32 s6, s4
20223 ; GFX7-NEXT: s_mov_b32 s7, s5
20224 ; GFX7-NEXT: s_mov_b32 s11, s12
20225 ; GFX7-NEXT: s_mov_b32 s10, s13
20226 ; GFX7-NEXT: s_add_u32 s6, s6, s11
20227 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
20228 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
20229 ; GFX7-NEXT: s_mov_b32 s7, s10
20230 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
20231 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
20232 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20233 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
20234 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
20235 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
20236 ; GFX7-NEXT: s_waitcnt vmcnt(0)
20237 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20238 ; GFX7-NEXT: s_waitcnt vmcnt(0)
20239 ; GFX7-NEXT: buffer_wbinvl1_vol
20240 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
20241 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
20242 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
20243 ; GFX7-NEXT: flat_store_dword v[0:1], v2
20244 ; GFX7-NEXT: s_endpgm
20245 ;
20246 ; GFX10-WGP-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
20247 ; GFX10-WGP: ; %bb.0: ; %entry
20248 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
20249 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20250 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
20251 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
20252 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
20253 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
20254 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
20255 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
20256 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
20257 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
20258 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
20259 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
20260 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
20261 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
20262 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
20263 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
20264 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20265 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
20266 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
20267 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
20268 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
20269 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
20270 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20271 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
20272 ; GFX10-WGP-NEXT: buffer_gl1_inv
20273 ; GFX10-WGP-NEXT: buffer_gl0_inv
20274 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
20275 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
20276 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
20277 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
20278 ; GFX10-WGP-NEXT: s_endpgm
20279 ;
20280 ; GFX10-CU-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
20281 ; GFX10-CU: ; %bb.0: ; %entry
20282 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
20283 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20284 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
20285 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
20286 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
20287 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
20288 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
20289 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
20290 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
20291 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
20292 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
20293 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
20294 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
20295 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
20296 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
20297 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
20298 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20299 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
20300 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
20301 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
20302 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
20303 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
20304 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20305 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
20306 ; GFX10-CU-NEXT: buffer_gl1_inv
20307 ; GFX10-CU-NEXT: buffer_gl0_inv
20308 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
20309 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
20310 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
20311 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
20312 ; GFX10-CU-NEXT: s_endpgm
20313 ;
20314 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
20315 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
20316 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
20317 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
20318 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
20319 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
20320 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
20321 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
20322 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
20323 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
20324 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
20325 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
20326 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
20327 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
20328 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
20329 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
20330 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
20331 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
20332 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20333 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
20334 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
20335 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
20336 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
20337 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20338 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
20339 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
20340 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
20341 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
20342 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
20343 ; SKIP-CACHE-INV-NEXT: s_endpgm
20344 ;
20345 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
20346 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
20347 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20348 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
20349 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
20350 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20351 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
20352 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
20353 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20354 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20355 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20356 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
20357 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
20358 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
20359 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
20360 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20361 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20362 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
20363 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
20364 ;
20365 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
20366 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
20367 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20368 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
20369 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
20370 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20371 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
20372 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
20373 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20374 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20375 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20376 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20377 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
20378 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20379 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
20380 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20381 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
20382 ; GFX90A-TGSPLIT-NEXT: s_endpgm
20383 ;
20384 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
20385 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
20386 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
20387 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
20388 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
20389 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20390 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
20391 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
20392 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20393 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20394 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20395 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
20396 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
20397 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
20398 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
20399 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
20400 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20401 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20402 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
20403 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
20404 ;
20405 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
20406 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
20407 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
20408 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
20409 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
20410 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20411 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
20412 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
20413 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20414 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20415 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20416 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
20417 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20418 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
20419 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20420 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
20421 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20422 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
20423 ; GFX940-TGSPLIT-NEXT: s_endpgm
20424 ;
20425 ; GFX11-WGP-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
20426 ; GFX11-WGP: ; %bb.0: ; %entry
20427 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20428 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
20429 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
20430 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
20431 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
20432 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
20433 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20434 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
20435 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
20436 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
20437 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
20438 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
20439 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
20440 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
20441 ; GFX11-WGP-NEXT: buffer_gl1_inv
20442 ; GFX11-WGP-NEXT: buffer_gl0_inv
20443 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
20444 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
20445 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
20446 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
20447 ; GFX11-WGP-NEXT: s_endpgm
20448 ;
20449 ; GFX11-CU-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
20450 ; GFX11-CU: ; %bb.0: ; %entry
20451 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20452 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
20453 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
20454 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
20455 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
20456 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
20457 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20458 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
20459 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
20460 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
20461 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
20462 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
20463 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
20464 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
20465 ; GFX11-CU-NEXT: buffer_gl1_inv
20466 ; GFX11-CU-NEXT: buffer_gl0_inv
20467 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
20468 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
20469 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
20470 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
20471 ; GFX11-CU-NEXT: s_endpgm
20472 ;
20473 ; GFX12-WGP-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
20474 ; GFX12-WGP: ; %bb.0: ; %entry
20475 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20476 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
20477 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
20478 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
20479 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
20480 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
20481 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20482 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
20483 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
20484 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
20485 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
20486 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
20487 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
20488 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
20489 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
20490 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
20491 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
20492 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
20493 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
20494 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
20495 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
20496 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
20497 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
20498 ; GFX12-WGP-NEXT: s_endpgm
20499 ;
20500 ; GFX12-CU-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
20501 ; GFX12-CU: ; %bb.0: ; %entry
20502 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20503 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
20504 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
20505 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
20506 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
20507 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
20508 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20509 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
20510 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
20511 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
20512 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
20513 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
20514 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
20515 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
20516 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
20517 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
20518 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
20519 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
20520 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
20521 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
20522 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
20523 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
20524 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
20525 ; GFX12-CU-NEXT: s_endpgm
20526 ptr %out, i32 %in, i32 %old) {
20527 entry:
20528 %gep = getelementptr i32, ptr %out, i32 4
20529 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") release acquire
20530 %val0 = extractvalue { i32, i1 } %val, 0
20531 store i32 %val0, ptr %out, align 4
20532 ret void
20533 }
20535 define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_ret_cmpxchg(
20536 ; GFX7-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
20537 ; GFX7: ; %bb.0: ; %entry
20538 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
20539 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20540 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
20541 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
20542 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
20543 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
20544 ; GFX7-NEXT: s_mov_b32 s6, s4
20545 ; GFX7-NEXT: s_mov_b32 s7, s5
20546 ; GFX7-NEXT: s_mov_b32 s11, s12
20547 ; GFX7-NEXT: s_mov_b32 s10, s13
20548 ; GFX7-NEXT: s_add_u32 s6, s6, s11
20549 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
20550 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
20551 ; GFX7-NEXT: s_mov_b32 s7, s10
20552 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
20553 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
20554 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20555 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
20556 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
20557 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
20558 ; GFX7-NEXT: s_waitcnt vmcnt(0)
20559 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20560 ; GFX7-NEXT: s_waitcnt vmcnt(0)
20561 ; GFX7-NEXT: buffer_wbinvl1_vol
20562 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
20563 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
20564 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
20565 ; GFX7-NEXT: flat_store_dword v[0:1], v2
20566 ; GFX7-NEXT: s_endpgm
20567 ;
20568 ; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
20569 ; GFX10-WGP: ; %bb.0: ; %entry
20570 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
20571 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20572 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
20573 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
20574 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
20575 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
20576 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
20577 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
20578 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
20579 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
20580 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
20581 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
20582 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
20583 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
20584 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
20585 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
20586 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20587 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
20588 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
20589 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
20590 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
20591 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
20592 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20593 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
20594 ; GFX10-WGP-NEXT: buffer_gl1_inv
20595 ; GFX10-WGP-NEXT: buffer_gl0_inv
20596 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
20597 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
20598 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
20599 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
20600 ; GFX10-WGP-NEXT: s_endpgm
20601 ;
20602 ; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
20603 ; GFX10-CU: ; %bb.0: ; %entry
20604 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
20605 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20606 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
20607 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
20608 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
20609 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
20610 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
20611 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
20612 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
20613 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
20614 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
20615 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
20616 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
20617 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
20618 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
20619 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
20620 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20621 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
20622 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
20623 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
20624 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
20625 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
20626 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20627 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
20628 ; GFX10-CU-NEXT: buffer_gl1_inv
20629 ; GFX10-CU-NEXT: buffer_gl0_inv
20630 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
20631 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
20632 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
20633 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
20634 ; GFX10-CU-NEXT: s_endpgm
20635 ;
20636 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
20637 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
20638 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
20639 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
20640 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
20641 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
20642 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
20643 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
20644 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
20645 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
20646 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
20647 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
20648 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
20649 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
20650 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
20651 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
20652 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
20653 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
20654 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20655 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
20656 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
20657 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
20658 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
20659 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20660 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
20661 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
20662 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
20663 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
20664 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
20665 ; SKIP-CACHE-INV-NEXT: s_endpgm
20666 ;
20667 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
20668 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
20669 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20670 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
20671 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
20672 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20673 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
20674 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
20675 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20676 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20677 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20678 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
20679 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
20680 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
20681 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
20682 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20683 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20684 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
20685 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
20686 ;
20687 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
20688 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
20689 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20690 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
20691 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
20692 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20693 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
20694 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
20695 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20696 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20697 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20698 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20699 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
20700 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20701 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
20702 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20703 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
20704 ; GFX90A-TGSPLIT-NEXT: s_endpgm
20705 ;
20706 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
20707 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
20708 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
20709 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
20710 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
20711 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20712 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
20713 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
20714 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20715 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20716 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20717 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
20718 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
20719 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
20720 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
20721 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
20722 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20723 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20724 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
20725 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
20726 ;
20727 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
20728 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
20729 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
20730 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
20731 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
20732 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20733 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
20734 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
20735 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20736 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20737 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20738 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
20739 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20740 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
20741 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20742 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
20743 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20744 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
20745 ; GFX940-TGSPLIT-NEXT: s_endpgm
20746 ;
20747 ; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
20748 ; GFX11-WGP: ; %bb.0: ; %entry
20749 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20750 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
20751 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
20752 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
20753 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
20754 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
20755 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20756 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
20757 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
20758 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
20759 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
20760 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
20761 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
20762 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
20763 ; GFX11-WGP-NEXT: buffer_gl1_inv
20764 ; GFX11-WGP-NEXT: buffer_gl0_inv
20765 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
20766 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
20767 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
20768 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
20769 ; GFX11-WGP-NEXT: s_endpgm
20770 ;
20771 ; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
20772 ; GFX11-CU: ; %bb.0: ; %entry
20773 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20774 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
20775 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
20776 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
20777 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
20778 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
20779 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20780 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
20781 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
20782 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
20783 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
20784 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
20785 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
20786 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
20787 ; GFX11-CU-NEXT: buffer_gl1_inv
20788 ; GFX11-CU-NEXT: buffer_gl0_inv
20789 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
20790 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
20791 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
20792 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
20793 ; GFX11-CU-NEXT: s_endpgm
20794 ;
20795 ; GFX12-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
20796 ; GFX12-WGP: ; %bb.0: ; %entry
20797 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20798 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
20799 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
20800 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
20801 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
20802 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
20803 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20804 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
20805 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
20806 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
20807 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
20808 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
20809 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
20810 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
20811 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
20812 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
20813 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
20814 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
20815 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
20816 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
20817 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
20818 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
20819 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
20820 ; GFX12-WGP-NEXT: s_endpgm
20821 ;
20822 ; GFX12-CU-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
20823 ; GFX12-CU: ; %bb.0: ; %entry
20824 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20825 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
20826 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
20827 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
20828 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
20829 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
20830 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20831 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
20832 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
20833 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
20834 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
20835 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
20836 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
20837 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
20838 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
20839 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
20840 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
20841 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
20842 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
20843 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
20844 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
20845 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
20846 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
20847 ; GFX12-CU-NEXT: s_endpgm
20848 ptr %out, i32 %in, i32 %old) {
20849 entry:
20850 %gep = getelementptr i32, ptr %out, i32 4
20851 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel acquire
20852 %val0 = extractvalue { i32, i1 } %val, 0
20853 store i32 %val0, ptr %out, align 4
20854 ret void
20855 }
20857 define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_ret_cmpxchg(
20858 ; GFX7-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
20859 ; GFX7: ; %bb.0: ; %entry
20860 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
20861 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20862 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
20863 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
20864 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
20865 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
20866 ; GFX7-NEXT: s_mov_b32 s6, s4
20867 ; GFX7-NEXT: s_mov_b32 s7, s5
20868 ; GFX7-NEXT: s_mov_b32 s11, s12
20869 ; GFX7-NEXT: s_mov_b32 s10, s13
20870 ; GFX7-NEXT: s_add_u32 s6, s6, s11
20871 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
20872 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
20873 ; GFX7-NEXT: s_mov_b32 s7, s10
20874 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
20875 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
20876 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20877 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
20878 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
20879 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
20880 ; GFX7-NEXT: s_waitcnt vmcnt(0)
20881 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20882 ; GFX7-NEXT: s_waitcnt vmcnt(0)
20883 ; GFX7-NEXT: buffer_wbinvl1_vol
20884 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
20885 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
20886 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
20887 ; GFX7-NEXT: flat_store_dword v[0:1], v2
20888 ; GFX7-NEXT: s_endpgm
20889 ;
20890 ; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
20891 ; GFX10-WGP: ; %bb.0: ; %entry
20892 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
20893 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20894 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
20895 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
20896 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
20897 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
20898 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
20899 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
20900 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
20901 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
20902 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
20903 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
20904 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
20905 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
20906 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
20907 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
20908 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20909 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
20910 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
20911 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
20912 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
20913 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
20914 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20915 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
20916 ; GFX10-WGP-NEXT: buffer_gl1_inv
20917 ; GFX10-WGP-NEXT: buffer_gl0_inv
20918 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
20919 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
20920 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
20921 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
20922 ; GFX10-WGP-NEXT: s_endpgm
20923 ;
20924 ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
20925 ; GFX10-CU: ; %bb.0: ; %entry
20926 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
20927 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20928 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
20929 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
20930 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
20931 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
20932 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
20933 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
20934 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
20935 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
20936 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
20937 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
20938 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
20939 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
20940 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
20941 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
20942 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20943 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
20944 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
20945 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
20946 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
20947 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
20948 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20949 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
20950 ; GFX10-CU-NEXT: buffer_gl1_inv
20951 ; GFX10-CU-NEXT: buffer_gl0_inv
20952 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
20953 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
20954 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
20955 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
20956 ; GFX10-CU-NEXT: s_endpgm
20957 ;
20958 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
20959 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
20960 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
20961 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
20962 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
20963 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
20964 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
20965 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
20966 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
20967 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
20968 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
20969 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
20970 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
20971 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
20972 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
20973 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
20974 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
20975 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
20976 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20977 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
20978 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
20979 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
20980 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
20981 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20982 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
20983 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
20984 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
20985 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
20986 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
20987 ; SKIP-CACHE-INV-NEXT: s_endpgm
20988 ;
20989 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
20990 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
20991 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20992 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
20993 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
20994 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20995 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
20996 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
20997 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20998 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20999 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
21000 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
21001 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
21002 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
21003 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
21004 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
21005 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21006 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
21007 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
21008 ;
21009 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
21010 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
21011 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
21012 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
21013 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
21014 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21015 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
21016 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
21017 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21018 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
21019 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
21020 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
21021 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
21022 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
21023 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
21024 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
21025 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
21026 ; GFX90A-TGSPLIT-NEXT: s_endpgm
21027 ;
21028 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
21029 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
21030 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
21031 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
21032 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
21033 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21034 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
21035 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
21036 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21037 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
21038 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
21039 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
21040 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
21041 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
21042 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
21043 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
21044 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
21045 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21046 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
21047 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
21048 ;
21049 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
21050 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
21051 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
21052 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
21053 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
21054 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21055 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
21056 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
21057 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21058 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
21059 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
21060 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
21061 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
21062 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
21063 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
21064 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
21065 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
21066 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
21067 ; GFX940-TGSPLIT-NEXT: s_endpgm
21068 ;
21069 ; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
21070 ; GFX11-WGP: ; %bb.0: ; %entry
21071 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
21072 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
21073 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
21074 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
21075 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
21076 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
21077 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21078 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
21079 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
21080 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
21081 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
21082 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
21083 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
21084 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
21085 ; GFX11-WGP-NEXT: buffer_gl1_inv
21086 ; GFX11-WGP-NEXT: buffer_gl0_inv
21087 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
21088 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
21089 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
21090 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
21091 ; GFX11-WGP-NEXT: s_endpgm
21092 ;
21093 ; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
21094 ; GFX11-CU: ; %bb.0: ; %entry
21095 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
21096 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
21097 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
21098 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
21099 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
21100 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
21101 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21102 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
21103 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
21104 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
21105 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
21106 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
21107 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
21108 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
21109 ; GFX11-CU-NEXT: buffer_gl1_inv
21110 ; GFX11-CU-NEXT: buffer_gl0_inv
21111 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
21112 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
21113 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
21114 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
21115 ; GFX11-CU-NEXT: s_endpgm
21116 ;
21117 ; GFX12-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
21118 ; GFX12-WGP: ; %bb.0: ; %entry
21119 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
21120 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
21121 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
21122 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
21123 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
21124 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
21125 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21126 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
21127 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
21128 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
21129 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
21130 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
21131 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
21132 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
21133 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
21134 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
21135 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
21136 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
21137 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
21138 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
21139 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
21140 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
21141 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
21142 ; GFX12-WGP-NEXT: s_endpgm
21143 ;
21144 ; GFX12-CU-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
21145 ; GFX12-CU: ; %bb.0: ; %entry
21146 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
21147 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
21148 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
21149 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
21150 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
21151 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
21152 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21153 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
21154 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
21155 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
21156 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
21157 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
21158 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
21159 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
21160 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
21161 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
21162 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
21163 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
21164 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
21165 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
21166 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
21167 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
21168 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
21169 ; GFX12-CU-NEXT: s_endpgm
21170 ptr %out, i32 %in, i32 %old) {
21171 entry:
21172 %gep = getelementptr i32, ptr %out, i32 4
21173 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst acquire
21174 %val0 = extractvalue { i32, i1 } %val, 0
21175 store i32 %val0, ptr %out, align 4
21176 ret void
21177 }
21179 define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
21180 ; GFX7-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
21181 ; GFX7: ; %bb.0: ; %entry
21182 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
21183 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
21184 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
21185 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
21186 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
21187 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
21188 ; GFX7-NEXT: s_mov_b32 s6, s4
21189 ; GFX7-NEXT: s_mov_b32 s7, s5
21190 ; GFX7-NEXT: s_mov_b32 s11, s12
21191 ; GFX7-NEXT: s_mov_b32 s10, s13
21192 ; GFX7-NEXT: s_add_u32 s6, s6, s11
21193 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
21194 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
21195 ; GFX7-NEXT: s_mov_b32 s7, s10
21196 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
21197 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
21198 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21199 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
21200 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
21201 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
21202 ; GFX7-NEXT: s_waitcnt vmcnt(0)
21203 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
21204 ; GFX7-NEXT: s_waitcnt vmcnt(0)
21205 ; GFX7-NEXT: buffer_wbinvl1_vol
21206 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
21207 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
21208 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
21209 ; GFX7-NEXT: flat_store_dword v[0:1], v2
21210 ; GFX7-NEXT: s_endpgm
21211 ;
21212 ; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
21213 ; GFX10-WGP: ; %bb.0: ; %entry
21214 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
21215 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
21216 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
21217 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
21218 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
21219 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
21220 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
21221 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
21222 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
21223 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
21224 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
21225 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
21226 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
21227 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
21228 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
21229 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
21230 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21231 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
21232 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
21233 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
21234 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
21235 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
21236 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
21237 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
21238 ; GFX10-WGP-NEXT: buffer_gl1_inv
21239 ; GFX10-WGP-NEXT: buffer_gl0_inv
21240 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
21241 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
21242 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
21243 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
21244 ; GFX10-WGP-NEXT: s_endpgm
21245 ;
21246 ; GFX10-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
21247 ; GFX10-CU: ; %bb.0: ; %entry
21248 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
21249 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
21250 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
21251 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
21252 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
21253 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
21254 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
21255 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
21256 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
21257 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
21258 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
21259 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
21260 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
21261 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
21262 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
21263 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
21264 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21265 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
21266 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
21267 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
21268 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
21269 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
21270 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
21271 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
21272 ; GFX10-CU-NEXT: buffer_gl1_inv
21273 ; GFX10-CU-NEXT: buffer_gl0_inv
21274 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
21275 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
21276 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
21277 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
21278 ; GFX10-CU-NEXT: s_endpgm
21279 ;
21280 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
21281 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
21282 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
21283 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
21284 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
21285 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
21286 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
21287 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
21288 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
21289 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
21290 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
21291 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
21292 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
21293 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
21294 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
21295 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
21296 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
21297 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
21298 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21299 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
21300 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
21301 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
21302 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
21303 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
21304 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
21305 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
21306 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
21307 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
21308 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
21309 ; SKIP-CACHE-INV-NEXT: s_endpgm
21310 ;
21311 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
21312 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
21313 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
21314 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
21315 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
21316 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21317 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
21318 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
21319 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21320 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
21321 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
21322 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
21323 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
21324 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
21325 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
21326 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
21327 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21328 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
21329 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
21330 ;
21331 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
21332 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
21333 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
21334 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
21335 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
21336 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21337 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
21338 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
21339 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21340 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
21341 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
21342 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
21343 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
21344 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
21345 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
21346 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
21347 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
21348 ; GFX90A-TGSPLIT-NEXT: s_endpgm
21349 ;
21350 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
21351 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
21352 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
21353 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
21354 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
21355 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21356 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
21357 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
21358 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21359 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
21360 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
21361 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
21362 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
21363 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
21364 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
21365 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
21366 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
21367 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21368 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
21369 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
21370 ;
21371 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
21372 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
21373 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
21374 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
21375 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
21376 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21377 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
21378 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
21379 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21380 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
21381 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
21382 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
21383 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
21384 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
21385 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
21386 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
21387 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
21388 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
21389 ; GFX940-TGSPLIT-NEXT: s_endpgm
21390 ;
21391 ; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
21392 ; GFX11-WGP: ; %bb.0: ; %entry
21393 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
21394 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
21395 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
21396 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
21397 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
21398 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
21399 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21400 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
21401 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
21402 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
21403 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
21404 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
21405 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
21406 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
21407 ; GFX11-WGP-NEXT: buffer_gl1_inv
21408 ; GFX11-WGP-NEXT: buffer_gl0_inv
21409 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
21410 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
21411 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
21412 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
21413 ; GFX11-WGP-NEXT: s_endpgm
21414 ;
21415 ; GFX11-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
21416 ; GFX11-CU: ; %bb.0: ; %entry
21417 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
21418 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
21419 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
21420 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
21421 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
21422 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
21423 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21424 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
21425 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
21426 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
21427 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
21428 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
21429 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
21430 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
21431 ; GFX11-CU-NEXT: buffer_gl1_inv
21432 ; GFX11-CU-NEXT: buffer_gl0_inv
21433 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
21434 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
21435 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
21436 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
21437 ; GFX11-CU-NEXT: s_endpgm
21438 ;
21439 ; GFX12-WGP-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
21440 ; GFX12-WGP: ; %bb.0: ; %entry
21441 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
21442 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
21443 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
21444 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
21445 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
21446 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
21447 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21448 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
21449 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
21450 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
21451 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
21452 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
21453 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
21454 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
21455 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
21456 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
21457 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
21458 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
21459 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
21460 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
21461 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
21462 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
21463 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
21464 ; GFX12-WGP-NEXT: s_endpgm
21465 ;
21466 ; GFX12-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
21467 ; GFX12-CU: ; %bb.0: ; %entry
21468 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
21469 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
21470 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
21471 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
21472 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
21473 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
21474 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21475 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
21476 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
21477 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
21478 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
21479 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
21480 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
21481 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
21482 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
21483 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
21484 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
21485 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
21486 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
21487 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
21488 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
21489 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
21490 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
21491 ; GFX12-CU-NEXT: s_endpgm
21492 ptr %out, i32 %in, i32 %old) {
21493 entry:
21494 %gep = getelementptr i32, ptr %out, i32 4
21495 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic seq_cst
21496 %val0 = extractvalue { i32, i1 } %val, 0
21497 store i32 %val0, ptr %out, align 4
21498 ret void
21499 }
21501 define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_ret_cmpxchg(
21502 ; GFX7-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
21503 ; GFX7: ; %bb.0: ; %entry
21504 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
21505 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
21506 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
21507 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
21508 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
21509 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
21510 ; GFX7-NEXT: s_mov_b32 s6, s4
21511 ; GFX7-NEXT: s_mov_b32 s7, s5
21512 ; GFX7-NEXT: s_mov_b32 s11, s12
21513 ; GFX7-NEXT: s_mov_b32 s10, s13
21514 ; GFX7-NEXT: s_add_u32 s6, s6, s11
21515 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
21516 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
21517 ; GFX7-NEXT: s_mov_b32 s7, s10
21518 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
21519 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
21520 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21521 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
21522 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
21523 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
21524 ; GFX7-NEXT: s_waitcnt vmcnt(0)
21525 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
21526 ; GFX7-NEXT: s_waitcnt vmcnt(0)
21527 ; GFX7-NEXT: buffer_wbinvl1_vol
21528 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
21529 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
21530 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
21531 ; GFX7-NEXT: flat_store_dword v[0:1], v2
21532 ; GFX7-NEXT: s_endpgm
21533 ;
21534 ; GFX10-WGP-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
21535 ; GFX10-WGP: ; %bb.0: ; %entry
21536 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
21537 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
21538 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
21539 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
21540 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
21541 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
21542 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
21543 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
21544 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
21545 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
21546 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
21547 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
21548 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
21549 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
21550 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
21551 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
21552 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21553 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
21554 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
21555 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
21556 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
21557 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
21558 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
21559 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
21560 ; GFX10-WGP-NEXT: buffer_gl1_inv
21561 ; GFX10-WGP-NEXT: buffer_gl0_inv
21562 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
21563 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
21564 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
21565 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
21566 ; GFX10-WGP-NEXT: s_endpgm
21567 ;
21568 ; GFX10-CU-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
21569 ; GFX10-CU: ; %bb.0: ; %entry
21570 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
21571 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
21572 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
21573 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
21574 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
21575 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
21576 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
21577 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
21578 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
21579 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
21580 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
21581 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
21582 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
21583 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
21584 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
21585 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
21586 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21587 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
21588 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
21589 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
21590 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
21591 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
21592 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
21593 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
21594 ; GFX10-CU-NEXT: buffer_gl1_inv
21595 ; GFX10-CU-NEXT: buffer_gl0_inv
21596 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
21597 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
21598 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
21599 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
21600 ; GFX10-CU-NEXT: s_endpgm
21601 ;
21602 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
21603 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
21604 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
21605 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
21606 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
21607 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
21608 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
21609 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
21610 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
21611 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
21612 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
21613 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
21614 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
21615 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
21616 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
21617 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
21618 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
21619 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
21620 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21621 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
21622 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
21623 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
21624 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
21625 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
21626 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
21627 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
21628 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
21629 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
21630 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
21631 ; SKIP-CACHE-INV-NEXT: s_endpgm
21632 ;
21633 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
21634 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
21635 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
21636 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
21637 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
21638 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21639 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
21640 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
21641 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21642 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
21643 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
21644 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
21645 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
21646 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
21647 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
21648 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
21649 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21650 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
21651 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
21652 ;
21653 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
21654 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
21655 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
21656 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
21657 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
21658 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21659 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
21660 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
21661 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21662 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
21663 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
21664 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
21665 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
21666 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
21667 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
21668 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
21669 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
21670 ; GFX90A-TGSPLIT-NEXT: s_endpgm
21671 ;
21672 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
21673 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
21674 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
21675 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
21676 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
21677 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21678 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
21679 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
21680 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21681 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
21682 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
21683 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
21684 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
21685 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
21686 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
21687 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
21688 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
21689 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21690 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
21691 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
21692 ;
21693 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
21694 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
21695 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
21696 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
21697 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
21698 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21699 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
21700 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
21701 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21702 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
21703 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
21704 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
21705 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
21706 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
21707 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
21708 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
21709 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
21710 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
21711 ; GFX940-TGSPLIT-NEXT: s_endpgm
21712 ;
21713 ; GFX11-WGP-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
21714 ; GFX11-WGP: ; %bb.0: ; %entry
21715 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
21716 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
21717 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
21718 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
21719 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
21720 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
21721 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21722 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
21723 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
21724 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
21725 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
21726 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
21727 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
21728 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
21729 ; GFX11-WGP-NEXT: buffer_gl1_inv
21730 ; GFX11-WGP-NEXT: buffer_gl0_inv
21731 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
21732 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
21733 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
21734 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
21735 ; GFX11-WGP-NEXT: s_endpgm
21736 ;
21737 ; GFX11-CU-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
21738 ; GFX11-CU: ; %bb.0: ; %entry
21739 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
21740 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
21741 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
21742 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
21743 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
21744 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
21745 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21746 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
21747 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
21748 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
21749 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
21750 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
21751 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
21752 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
21753 ; GFX11-CU-NEXT: buffer_gl1_inv
21754 ; GFX11-CU-NEXT: buffer_gl0_inv
21755 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
21756 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
21757 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
21758 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
21759 ; GFX11-CU-NEXT: s_endpgm
21760 ;
21761 ; GFX12-WGP-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
21762 ; GFX12-WGP: ; %bb.0: ; %entry
21763 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
21764 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
21765 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
21766 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
21767 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
21768 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
21769 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21770 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
21771 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
21772 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
21773 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
21774 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
21775 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
21776 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
21777 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
21778 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
21779 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
21780 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
21781 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
21782 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
21783 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
21784 ; GFX12-WGP-NEXT: s_endpgm
21785 ;
21786 ; GFX12-CU-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
21787 ; GFX12-CU: ; %bb.0: ; %entry
21788 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
21789 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
21790 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
21791 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
21792 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
21793 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
21794 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21795 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
21796 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
21797 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
21798 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
21799 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
21800 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
21801 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
21802 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
21803 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
21804 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
21805 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
21806 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
21807 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
21808 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
21809 ; GFX12-CU-NEXT: s_endpgm
21810 ptr %out, i32 %in, i32 %old) {
21811 entry:
21812 %gep = getelementptr i32, ptr %out, i32 4
21813 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire seq_cst
21814 %val0 = extractvalue { i32, i1 } %val, 0
21815 store i32 %val0, ptr %out, align 4
21816 ret void
21817 }
21819 define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_ret_cmpxchg(
21820 ; GFX7-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
21821 ; GFX7: ; %bb.0: ; %entry
21822 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
21823 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
21824 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
21825 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
21826 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
21827 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
21828 ; GFX7-NEXT: s_mov_b32 s6, s4
21829 ; GFX7-NEXT: s_mov_b32 s7, s5
21830 ; GFX7-NEXT: s_mov_b32 s11, s12
21831 ; GFX7-NEXT: s_mov_b32 s10, s13
21832 ; GFX7-NEXT: s_add_u32 s6, s6, s11
21833 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
21834 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
21835 ; GFX7-NEXT: s_mov_b32 s7, s10
21836 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
21837 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
21838 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21839 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
21840 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
21841 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
21842 ; GFX7-NEXT: s_waitcnt vmcnt(0)
21843 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
21844 ; GFX7-NEXT: s_waitcnt vmcnt(0)
21845 ; GFX7-NEXT: buffer_wbinvl1_vol
21846 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
21847 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
21848 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
21849 ; GFX7-NEXT: flat_store_dword v[0:1], v2
21850 ; GFX7-NEXT: s_endpgm
21851 ;
21852 ; GFX10-WGP-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
21853 ; GFX10-WGP: ; %bb.0: ; %entry
21854 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
21855 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
21856 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
21857 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
21858 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
21859 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
21860 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
21861 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
21862 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
21863 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
21864 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
21865 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
21866 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
21867 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
21868 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
21869 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
21870 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21871 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
21872 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
21873 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
21874 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
21875 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
21876 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
21877 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
21878 ; GFX10-WGP-NEXT: buffer_gl1_inv
21879 ; GFX10-WGP-NEXT: buffer_gl0_inv
21880 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
21881 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
21882 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
21883 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
21884 ; GFX10-WGP-NEXT: s_endpgm
21885 ;
21886 ; GFX10-CU-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
21887 ; GFX10-CU: ; %bb.0: ; %entry
21888 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
21889 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
21890 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
21891 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
21892 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
21893 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
21894 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
21895 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
21896 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
21897 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
21898 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
21899 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
21900 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
21901 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
21902 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
21903 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
21904 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21905 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
21906 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
21907 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
21908 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
21909 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
21910 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
21911 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
21912 ; GFX10-CU-NEXT: buffer_gl1_inv
21913 ; GFX10-CU-NEXT: buffer_gl0_inv
21914 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
21915 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
21916 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
21917 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
21918 ; GFX10-CU-NEXT: s_endpgm
21919 ;
21920 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
21921 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
21922 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
21923 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
21924 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
21925 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
21926 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
21927 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
21928 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
21929 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
21930 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
21931 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
21932 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
21933 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
21934 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
21935 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
21936 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
21937 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
21938 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21939 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
21940 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
21941 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
21942 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
21943 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
21944 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
21945 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
21946 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
21947 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
21948 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
21949 ; SKIP-CACHE-INV-NEXT: s_endpgm
21950 ;
21951 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
21952 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
21953 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
21954 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
21955 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
21956 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21957 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
21958 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
21959 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21960 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
21961 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
21962 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
21963 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
21964 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
21965 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
21966 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
21967 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21968 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
21969 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
21970 ;
21971 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
21972 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
21973 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
21974 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
21975 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
21976 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21977 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
21978 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
21979 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21980 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
21981 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
21982 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
21983 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
21984 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
21985 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
21986 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
21987 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
21988 ; GFX90A-TGSPLIT-NEXT: s_endpgm
21989 ;
21990 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
21991 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
21992 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
21993 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
21994 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
21995 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21996 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
21997 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
21998 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21999 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
22000 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
22001 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
22002 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
22003 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
22004 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
22005 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
22006 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
22007 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
22008 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
22009 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
22010 ;
22011 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
22012 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
22013 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
22014 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
22015 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
22016 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
22017 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
22018 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
22019 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22020 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
22021 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
22022 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
22023 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
22024 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
22025 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
22026 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
22027 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
22028 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
22029 ; GFX940-TGSPLIT-NEXT: s_endpgm
22030 ;
22031 ; GFX11-WGP-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
22032 ; GFX11-WGP: ; %bb.0: ; %entry
22033 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
22034 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
22035 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
22036 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
22037 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
22038 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
22039 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22040 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
22041 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
22042 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
22043 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
22044 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
22045 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
22046 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
22047 ; GFX11-WGP-NEXT: buffer_gl1_inv
22048 ; GFX11-WGP-NEXT: buffer_gl0_inv
22049 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
22050 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
22051 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
22052 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
22053 ; GFX11-WGP-NEXT: s_endpgm
22054 ;
22055 ; GFX11-CU-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
22056 ; GFX11-CU: ; %bb.0: ; %entry
22057 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
22058 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
22059 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
22060 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
22061 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
22062 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
22063 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22064 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
22065 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
22066 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
22067 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
22068 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
22069 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
22070 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
22071 ; GFX11-CU-NEXT: buffer_gl1_inv
22072 ; GFX11-CU-NEXT: buffer_gl0_inv
22073 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
22074 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
22075 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
22076 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
22077 ; GFX11-CU-NEXT: s_endpgm
22078 ;
22079 ; GFX12-WGP-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
22080 ; GFX12-WGP: ; %bb.0: ; %entry
22081 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
22082 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
22083 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
22084 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
22085 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
22086 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
22087 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22088 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
22089 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
22090 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
22091 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
22092 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
22093 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
22094 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
22095 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
22096 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
22097 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
22098 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
22099 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
22100 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
22101 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
22102 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
22103 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
22104 ; GFX12-WGP-NEXT: s_endpgm
22105 ;
22106 ; GFX12-CU-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
22107 ; GFX12-CU: ; %bb.0: ; %entry
22108 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
22109 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
22110 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
22111 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
22112 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
22113 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
22114 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22115 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
22116 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
22117 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
22118 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
22119 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
22120 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
22121 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
22122 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
22123 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
22124 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
22125 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
22126 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
22127 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
22128 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
22129 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
22130 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
22131 ; GFX12-CU-NEXT: s_endpgm
22132 ptr %out, i32 %in, i32 %old) {
22133 entry:
22134 %gep = getelementptr i32, ptr %out, i32 4
22135 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") release seq_cst
22136 %val0 = extractvalue { i32, i1 } %val, 0
22137 store i32 %val0, ptr %out, align 4
22138 ret void
22139 }
22141 define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
22142 ; GFX7-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
22143 ; GFX7: ; %bb.0: ; %entry
22144 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
22145 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
22146 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
22147 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
22148 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
22149 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
22150 ; GFX7-NEXT: s_mov_b32 s6, s4
22151 ; GFX7-NEXT: s_mov_b32 s7, s5
22152 ; GFX7-NEXT: s_mov_b32 s11, s12
22153 ; GFX7-NEXT: s_mov_b32 s10, s13
22154 ; GFX7-NEXT: s_add_u32 s6, s6, s11
22155 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
22156 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
22157 ; GFX7-NEXT: s_mov_b32 s7, s10
22158 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
22159 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
22160 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22161 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
22162 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
22163 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
22164 ; GFX7-NEXT: s_waitcnt vmcnt(0)
22165 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
22166 ; GFX7-NEXT: s_waitcnt vmcnt(0)
22167 ; GFX7-NEXT: buffer_wbinvl1_vol
22168 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
22169 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
22170 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
22171 ; GFX7-NEXT: flat_store_dword v[0:1], v2
22172 ; GFX7-NEXT: s_endpgm
22173 ;
22174 ; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
22175 ; GFX10-WGP: ; %bb.0: ; %entry
22176 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
22177 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
22178 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
22179 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
22180 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
22181 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
22182 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
22183 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
22184 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
22185 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
22186 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
22187 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
22188 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
22189 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
22190 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
22191 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
22192 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22193 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
22194 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
22195 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
22196 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
22197 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
22198 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
22199 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
22200 ; GFX10-WGP-NEXT: buffer_gl1_inv
22201 ; GFX10-WGP-NEXT: buffer_gl0_inv
22202 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
22203 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
22204 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
22205 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
22206 ; GFX10-WGP-NEXT: s_endpgm
22207 ;
22208 ; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
22209 ; GFX10-CU: ; %bb.0: ; %entry
22210 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
22211 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
22212 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
22213 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
22214 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
22215 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
22216 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
22217 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
22218 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
22219 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
22220 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
22221 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
22222 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
22223 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
22224 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
22225 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
22226 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22227 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
22228 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
22229 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
22230 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
22231 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
22232 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
22233 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
22234 ; GFX10-CU-NEXT: buffer_gl1_inv
22235 ; GFX10-CU-NEXT: buffer_gl0_inv
22236 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
22237 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
22238 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
22239 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
22240 ; GFX10-CU-NEXT: s_endpgm
22241 ;
22242 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
22243 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
22244 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
22245 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
22246 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
22247 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
22248 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
22249 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
22250 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
22251 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
22252 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
22253 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
22254 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
22255 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
22256 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
22257 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
22258 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
22259 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
22260 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22261 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
22262 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
22263 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
22264 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
22265 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
22266 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
22267 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
22268 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
22269 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
22270 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
22271 ; SKIP-CACHE-INV-NEXT: s_endpgm
22272 ;
22273 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
22274 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
22275 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
22276 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
22277 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
22278 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
22279 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
22280 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
22281 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22282 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
22283 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
22284 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
22285 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
22286 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
22287 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
22288 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
22289 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
22290 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
22291 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
22292 ;
22293 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
22294 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
22295 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
22296 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
22297 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
22298 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
22299 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
22300 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
22301 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22302 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
22303 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
22304 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
22305 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
22306 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
22307 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
22308 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
22309 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
22310 ; GFX90A-TGSPLIT-NEXT: s_endpgm
22311 ;
22312 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
22313 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
22314 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
22315 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
22316 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
22317 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
22318 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
22319 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
22320 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22321 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
22322 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
22323 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
22324 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
22325 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
22326 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
22327 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
22328 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
22329 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
22330 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
22331 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
22332 ;
22333 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
22334 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
22335 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
22336 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
22337 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
22338 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
22339 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
22340 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
22341 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22342 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
22343 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
22344 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
22345 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
22346 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
22347 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
22348 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
22349 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
22350 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
22351 ; GFX940-TGSPLIT-NEXT: s_endpgm
22352 ;
22353 ; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
22354 ; GFX11-WGP: ; %bb.0: ; %entry
22355 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
22356 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
22357 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
22358 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
22359 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
22360 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
22361 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22362 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
22363 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
22364 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
22365 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
22366 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
22367 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
22368 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
22369 ; GFX11-WGP-NEXT: buffer_gl1_inv
22370 ; GFX11-WGP-NEXT: buffer_gl0_inv
22371 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
22372 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
22373 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
22374 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
22375 ; GFX11-WGP-NEXT: s_endpgm
22376 ;
22377 ; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
22378 ; GFX11-CU: ; %bb.0: ; %entry
22379 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
22380 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
22381 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
22382 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
22383 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
22384 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
22385 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22386 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
22387 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
22388 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
22389 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
22390 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
22391 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
22392 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
22393 ; GFX11-CU-NEXT: buffer_gl1_inv
22394 ; GFX11-CU-NEXT: buffer_gl0_inv
22395 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
22396 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
22397 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
22398 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
22399 ; GFX11-CU-NEXT: s_endpgm
22400 ;
22401 ; GFX12-WGP-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
22402 ; GFX12-WGP: ; %bb.0: ; %entry
22403 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
22404 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
22405 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
22406 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
22407 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
22408 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
22409 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22410 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
22411 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
22412 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
22413 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
22414 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
22415 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
22416 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
22417 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
22418 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
22419 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
22420 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
22421 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
22422 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
22423 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
22424 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
22425 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
22426 ; GFX12-WGP-NEXT: s_endpgm
22427 ;
22428 ; GFX12-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
22429 ; GFX12-CU: ; %bb.0: ; %entry
22430 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
22431 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
22432 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
22433 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
22434 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
22435 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
22436 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22437 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
22438 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
22439 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
22440 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
22441 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
22442 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
22443 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
22444 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
22445 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
22446 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
22447 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
22448 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
22449 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
22450 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
22451 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
22452 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
22453 ; GFX12-CU-NEXT: s_endpgm
22454 ptr %out, i32 %in, i32 %old) {
22455 entry:
22456 %gep = getelementptr i32, ptr %out, i32 4
22457 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel seq_cst
22458 %val0 = extractvalue { i32, i1 } %val, 0
22459 store i32 %val0, ptr %out, align 4
22460 ret void
22461 }
22463 define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
22464 ; GFX7-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
22465 ; GFX7: ; %bb.0: ; %entry
22466 ; GFX7-NEXT: s_mov_b64 s[6:7], s[8:9]
22467 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
22468 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
22469 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
22470 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
22471 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
22472 ; GFX7-NEXT: s_mov_b32 s6, s4
22473 ; GFX7-NEXT: s_mov_b32 s7, s5
22474 ; GFX7-NEXT: s_mov_b32 s11, s12
22475 ; GFX7-NEXT: s_mov_b32 s10, s13
22476 ; GFX7-NEXT: s_add_u32 s6, s6, s11
22477 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
22478 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
22479 ; GFX7-NEXT: s_mov_b32 s7, s10
22480 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
22481 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
22482 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22483 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
22484 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
22485 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
22486 ; GFX7-NEXT: s_waitcnt vmcnt(0)
22487 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
22488 ; GFX7-NEXT: s_waitcnt vmcnt(0)
22489 ; GFX7-NEXT: buffer_wbinvl1_vol
22490 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
22491 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
22492 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
22493 ; GFX7-NEXT: flat_store_dword v[0:1], v2
22494 ; GFX7-NEXT: s_endpgm
22495 ;
22496 ; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
22497 ; GFX10-WGP: ; %bb.0: ; %entry
22498 ; GFX10-WGP-NEXT: s_mov_b64 s[6:7], s[8:9]
22499 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
22500 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
22501 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
22502 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
22503 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
22504 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
22505 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
22506 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
22507 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
22508 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
22509 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
22510 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
22511 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
22512 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
22513 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
22514 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22515 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
22516 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
22517 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
22518 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
22519 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
22520 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
22521 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
22522 ; GFX10-WGP-NEXT: buffer_gl1_inv
22523 ; GFX10-WGP-NEXT: buffer_gl0_inv
22524 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
22525 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
22526 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
22527 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
22528 ; GFX10-WGP-NEXT: s_endpgm
22529 ;
22530 ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
22531 ; GFX10-CU: ; %bb.0: ; %entry
22532 ; GFX10-CU-NEXT: s_mov_b64 s[6:7], s[8:9]
22533 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
22534 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
22535 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
22536 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
22537 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
22538 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
22539 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
22540 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
22541 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
22542 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
22543 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
22544 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
22545 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
22546 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
22547 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
22548 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22549 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
22550 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
22551 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
22552 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
22553 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
22554 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
22555 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
22556 ; GFX10-CU-NEXT: buffer_gl1_inv
22557 ; GFX10-CU-NEXT: buffer_gl0_inv
22558 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
22559 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
22560 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
22561 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
22562 ; GFX10-CU-NEXT: s_endpgm
22563 ;
22564 ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
22565 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
22566 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[2:3], s[4:5]
22567 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
22568 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
22569 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
22570 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
22571 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
22572 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
22573 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
22574 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
22575 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
22576 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
22577 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
22578 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
22579 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
22580 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
22581 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
22582 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22583 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
22584 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
22585 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
22586 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
22587 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
22588 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
22589 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
22590 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
22591 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
22592 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
22593 ; SKIP-CACHE-INV-NEXT: s_endpgm
22594 ;
22595 ; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
22596 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
22597 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
22598 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
22599 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
22600 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
22601 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
22602 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
22603 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22604 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
22605 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
22606 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
22607 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
22608 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
22609 ; GFX90A-NOTTGSPLIT-NEXT: buffer_wbinvl1_vol
22610 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
22611 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
22612 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
22613 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
22614 ;
22615 ; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
22616 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
22617 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
22618 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
22619 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
22620 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
22621 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
22622 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
22623 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22624 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
22625 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
22626 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
22627 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
22628 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
22629 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
22630 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
22631 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
22632 ; GFX90A-TGSPLIT-NEXT: s_endpgm
22633 ;
22634 ; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
22635 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
22636 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
22637 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
22638 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
22639 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
22640 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
22641 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
22642 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22643 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
22644 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
22645 ; GFX940-NOTTGSPLIT-NEXT: buffer_wbl2 sc1
22646 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
22647 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
22648 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
22649 ; GFX940-NOTTGSPLIT-NEXT: buffer_inv sc1
22650 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
22651 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
22652 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
22653 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
22654 ;
22655 ; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
22656 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
22657 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
22658 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
22659 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
22660 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
22661 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
22662 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
22663 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22664 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
22665 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
22666 ; GFX940-TGSPLIT-NEXT: buffer_wbl2 sc1
22667 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
22668 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
22669 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
22670 ; GFX940-TGSPLIT-NEXT: buffer_inv sc1
22671 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
22672 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
22673 ; GFX940-TGSPLIT-NEXT: s_endpgm
22674 ;
22675 ; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
22676 ; GFX11-WGP: ; %bb.0: ; %entry
22677 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
22678 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
22679 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
22680 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
22681 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
22682 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
22683 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22684 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
22685 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
22686 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
22687 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
22688 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
22689 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
22690 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
22691 ; GFX11-WGP-NEXT: buffer_gl1_inv
22692 ; GFX11-WGP-NEXT: buffer_gl0_inv
22693 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
22694 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
22695 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
22696 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
22697 ; GFX11-WGP-NEXT: s_endpgm
22698 ;
22699 ; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
22700 ; GFX11-CU: ; %bb.0: ; %entry
22701 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
22702 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
22703 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
22704 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
22705 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
22706 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
22707 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22708 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
22709 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
22710 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
22711 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
22712 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
22713 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
22714 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
22715 ; GFX11-CU-NEXT: buffer_gl1_inv
22716 ; GFX11-CU-NEXT: buffer_gl0_inv
22717 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
22718 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
22719 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
22720 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
22721 ; GFX11-CU-NEXT: s_endpgm
22722 ;
22723 ; GFX12-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
22724 ; GFX12-WGP: ; %bb.0: ; %entry
22725 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
22726 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
22727 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
22728 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
22729 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
22730 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
22731 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22732 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
22733 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
22734 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
22735 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
22736 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
22737 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
22738 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
22739 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
22740 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
22741 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
22742 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
22743 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_DEV
22744 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
22745 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
22746 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
22747 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
22748 ; GFX12-WGP-NEXT: s_endpgm
22749 ;
22750 ; GFX12-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
22751 ; GFX12-CU: ; %bb.0: ; %entry
22752 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
22753 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
22754 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
22755 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
22756 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
22757 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
22758 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
22759 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
22760 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
22761 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
22762 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
22763 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
22764 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
22765 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
22766 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
22767 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
22768 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
22769 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
22770 ; GFX12-CU-NEXT: global_inv scope:SCOPE_DEV
22771 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
22772 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
22773 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
22774 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
22775 ; GFX12-CU-NEXT: s_endpgm
22776 ptr %out, i32 %in, i32 %old) {
22777 entry:
22778 %gep = getelementptr i32, ptr %out, i32 4
22779 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst seq_cst
22780 %val0 = extractvalue { i32, i1 } %val, 0
22781 store i32 %val0, ptr %out, align 4
22782 ret void
22783 }